diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index d065762d..2cbe9068 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -110,3 +110,54 @@ jobs: platforms: linux/amd64 build-args: | INSTALL_EXTRAS=qualitative-voice + + build-worker-harbor: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU (for multi-platform builds) + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name == 'workflow_dispatch' && inputs.publish == true + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Harbor worker image + id: meta + uses: docker/metadata-action@v5 + with: + # Same GHCR package as worker; tags end with -harbor (matches compose + release-and-publish). + images: ${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }} + tags: | + type=ref,event=branch,suffix=-harbor + type=ref,event=pr,suffix=-harbor + type=raw,value=latest-harbor,enable={{is_default_branch}} + + - name: Build Harbor worker image (amd64 only for speed) + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.worker + push: ${{ github.event_name == 'workflow_dispatch' && inputs.publish == true }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=worker-harbor-check + cache-to: type=gha,scope=worker-harbor-check,mode=max + platforms: linux/amd64 + build-args: | + PYTHON_VERSION=3.12 + INSTALL_EXTRAS=harbor + TORCH_CPU_ONLY=false diff --git a/.github/workflows/pr-dev-images.yml b/.github/workflows/pr-dev-images.yml index 2f090b1e..1fe0f542 100644 --- a/.github/workflows/pr-dev-images.yml +++ b/.github/workflows/pr-dev-images.yml @@ -99,14 +99,26 @@ jobs: - service: api dockerfile: docker/Dockerfile.api image_name_var: API_IMAGE_NAME + tag_suffix: "" cache_scope: api-pr-dev build_args: "" - service: worker dockerfile: docker/Dockerfile.worker image_name_var: WORKER_IMAGE_NAME + tag_suffix: "" cache_scope: worker-pr-dev build_args: | INSTALL_EXTRAS=qualitative-voice + # Same GHCR package as worker; tags end with -harbor (avoids creating a new package). + - service: worker-harbor + dockerfile: docker/Dockerfile.worker + image_name_var: WORKER_IMAGE_NAME + tag_suffix: "-harbor" + cache_scope: worker-harbor-pr-dev + build_args: | + PYTHON_VERSION=3.12 + INSTALL_EXTRAS=harbor + TORCH_CPU_ONLY=false steps: - name: Checkout repository @@ -141,10 +153,10 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} tags: | - type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }} - type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}-${{ needs.compute-metadata.outputs.short_sha }} - type=raw,value=${{ needs.compute-metadata.outputs.dev_version }} - type=raw,value=dev-latest-pr-${{ needs.compute-metadata.outputs.pr_number }} + type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}${{ matrix.tag_suffix }} + type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}-${{ needs.compute-metadata.outputs.short_sha }}${{ matrix.tag_suffix }} + type=raw,value=${{ needs.compute-metadata.outputs.dev_version }}${{ matrix.tag_suffix }} + type=raw,value=dev-latest-pr-${{ needs.compute-metadata.outputs.pr_number }}${{ matrix.tag_suffix }} - name: Build and optionally push image uses: docker/build-push-action@v5 diff --git a/.github/workflows/release-and-publish.yml b/.github/workflows/release-and-publish.yml index 7d1c2b1f..c241f492 100644 --- a/.github/workflows/release-and-publish.yml +++ b/.github/workflows/release-and-publish.yml @@ -9,6 +9,7 @@ env: REGISTRY: ghcr.io API_IMAGE_NAME: ${{ github.repository }}-api WORKER_IMAGE_NAME: ${{ github.repository }}-worker + WORKER_HARBOR_IMAGE_NAME: ${{ github.repository }}-worker-harbor permissions: contents: write @@ -180,6 +181,7 @@ jobs: run: | echo "API_IMAGE_NAME=${API_IMAGE_NAME,,}" >> $GITHUB_ENV echo "WORKER_IMAGE_NAME=${WORKER_IMAGE_NAME,,}" >> $GITHUB_ENV + echo "WORKER_HARBOR_IMAGE_NAME=${WORKER_HARBOR_IMAGE_NAME,,}" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -215,13 +217,30 @@ jobs: build-args: | INSTALL_EXTRAS=qualitative-voice + - name: Build and push Harbor worker image (by digest) + id: build-worker-harbor + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.worker + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=worker-harbor-${{ matrix.slug }} + cache-to: type=gha,scope=worker-harbor-${{ matrix.slug }},mode=max + build-args: | + PYTHON_VERSION=3.12 + INSTALL_EXTRAS=harbor + TORCH_CPU_ONLY=false + - name: Export digests run: | - mkdir -p /tmp/digests/api /tmp/digests/worker + mkdir -p /tmp/digests/api /tmp/digests/worker /tmp/digests/worker-harbor api_digest="${{ steps.build-api.outputs.digest }}" worker_digest="${{ steps.build-worker.outputs.digest }}" + harbor_digest="${{ steps.build-worker-harbor.outputs.digest }}" touch "/tmp/digests/api/${api_digest#sha256:}" touch "/tmp/digests/worker/${worker_digest#sha256:}" + touch "/tmp/digests/worker-harbor/${harbor_digest#sha256:}" - name: Upload API digest uses: actions/upload-artifact@v4 @@ -239,6 +258,14 @@ jobs: if-no-files-found: error retention-days: 1 + - name: Upload Harbor worker digest + uses: actions/upload-artifact@v4 + with: + name: worker-harbor-digest-${{ matrix.slug }} + path: /tmp/digests/worker-harbor/* + if-no-files-found: error + retention-days: 1 + # ── Step 3: Merge platform digests into multi-arch manifests ─ merge: needs: [release, build] @@ -252,6 +279,7 @@ jobs: run: | echo "API_IMAGE_NAME=${API_IMAGE_NAME,,}" >> $GITHUB_ENV echo "WORKER_IMAGE_NAME=${WORKER_IMAGE_NAME,,}" >> $GITHUB_ENV + echo "WORKER_HARBOR_IMAGE_NAME=${WORKER_HARBOR_IMAGE_NAME,,}" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -277,6 +305,13 @@ jobs: pattern: worker-digest-* merge-multiple: true + - name: Download Harbor worker digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests/worker-harbor + pattern: worker-harbor-digest-* + merge-multiple: true + - name: Create API multi-arch manifest env: VERSION: ${{ needs.release.outputs.version }} @@ -303,6 +338,19 @@ jobs: -t "$IMAGE:latest" \ $(printf "$IMAGE@sha256:%s " *) + - name: Create Harbor worker multi-arch manifest + env: + VERSION: ${{ needs.release.outputs.version }} + MAJOR_MINOR: ${{ needs.release.outputs.major_minor }} + IMAGE: ${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }} + run: | + cd /tmp/digests/worker-harbor + docker buildx imagetools create \ + -t "$IMAGE:${VERSION}-harbor" \ + -t "$IMAGE:${MAJOR_MINOR}-harbor" \ + -t "$IMAGE:latest-harbor" \ + $(printf "$IMAGE@sha256:%s " *) + - name: Create GitHub release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index cd501589..ecc2528d 100644 --- a/.gitignore +++ b/.gitignore @@ -137,5 +137,8 @@ enterprise/keys/*.pem celerybeat-schedule celerybeat-schedule-* celerybeat-schedule.* +.claude/ +.cursor/ + # \ No newline at end of file diff --git a/README.md b/README.md index f3e36623..d0151bb7 100644 --- a/README.md +++ b/README.md @@ -520,8 +520,8 @@ gcs: # CORS Settings cors: origins: - - "http://localhost:3000" - "http://localhost:8000" + - "https://sandbox.efficientai.cloud" # API Settings api: diff --git a/app/api/v1/api.py b/app/api/v1/api.py index b5f651c8..7dd2ca52 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -1,127 +1,137 @@ -"""API v1 router aggregation.""" - -from fastapi import APIRouter -from app.api.v1.routes import ( - auth, - audio, - evaluations, - results, - agents, - personas, - scenarios, - iam, - profile, - integrations, - data_sources, - voicebundles, - aiproviders, - model_config, - manual_evaluations, - test_agents, - conversation_evaluations, - voice_agent, - evaluators, - evaluator_suites, - metrics, - evaluator_results, - evaluator_result_metric_clusters, - chat, - chat_messaging_webhooks, - meta_whatsapp_webhooks, - twilio_sms_webhooks, - twilio_voice_webhooks, - telnyx_sms_webhooks, - telnyx_voice_webhooks, - playground, - settings, - observability, - alerts, - alert_settings, - cron_jobs, - voice_playground, - public_blind_test, - prompt_partials, - prompt_optimization, - telephony, - vobiz_telephony, - call_imports, - platform_call_imports, - call_import_schemas, - call_import_tags, - call_import_evaluations, - judge_alignment, - metric_studio, - workspaces, - workspace_iam, - dashboard, - llm_gateway, - platform_admin, - org_usage, - usage_pricing, - synthetic_traces, -) - -api_router = APIRouter() - -# Include all route routers -api_router.include_router(auth.router) -api_router.include_router(audio.router) -api_router.include_router(evaluations.router) -api_router.include_router(results.router) -api_router.include_router(agents.router) -api_router.include_router(personas.router) -api_router.include_router(scenarios.router) -api_router.include_router(iam.router) -api_router.include_router(profile.router) -api_router.include_router(integrations.router) -api_router.include_router(data_sources.router) -api_router.include_router(voicebundles.router) -api_router.include_router(aiproviders.router) -api_router.include_router(model_config.router) -api_router.include_router(manual_evaluations.router) -api_router.include_router(test_agents.router) -api_router.include_router(conversation_evaluations.router) -api_router.include_router(voice_agent.router) -api_router.include_router(evaluators.router) -api_router.include_router(evaluator_suites.router) -api_router.include_router(metrics.router) -# Register before evaluator_results so `/evaluator-results/metric-clusters` -# is not captured by `/evaluator-results/{id}` with id="metric-clusters". -api_router.include_router(evaluator_result_metric_clusters.router) -api_router.include_router(evaluator_results.router) -api_router.include_router(chat.router) -api_router.include_router(chat_messaging_webhooks.router) -api_router.include_router(meta_whatsapp_webhooks.router) -api_router.include_router(twilio_sms_webhooks.router) -api_router.include_router(twilio_voice_webhooks.router) -api_router.include_router(telnyx_sms_webhooks.router) -api_router.include_router(telnyx_voice_webhooks.router) -api_router.include_router(playground.router) -api_router.include_router(settings.router) -api_router.include_router(observability.router) -api_router.include_router(alerts.router) -api_router.include_router(alert_settings.settings_router) -api_router.include_router(alert_settings.webhook_router) -api_router.include_router(cron_jobs.router) -api_router.include_router(voice_playground.router) -api_router.include_router(public_blind_test.router) -api_router.include_router(prompt_partials.router) -api_router.include_router(prompt_optimization.router) -api_router.include_router(telephony.router) -api_router.include_router(telephony.plivo_webhook_router) -api_router.include_router(vobiz_telephony.router) -api_router.include_router(call_imports.router) -api_router.include_router(platform_call_imports.router) -api_router.include_router(call_import_schemas.router) -api_router.include_router(call_import_tags.router) -api_router.include_router(call_import_evaluations.router) -api_router.include_router(judge_alignment.router) -api_router.include_router(metric_studio.router) -api_router.include_router(workspaces.router) -api_router.include_router(workspace_iam.router) -api_router.include_router(dashboard.router) -api_router.include_router(llm_gateway.router) -api_router.include_router(platform_admin.router) -api_router.include_router(org_usage.router) -api_router.include_router(usage_pricing.router) -api_router.include_router(synthetic_traces.router) +"""API v1 router aggregation.""" + +from fastapi import APIRouter +from app.api.v1.routes import ( + auth, + audio, + evaluations, + results, + agents, + personas, + scenarios, + knowledge_bases, + simulation_runs, + tool_connections, + iam, + profile, + integrations, + data_sources, + voicebundles, + aiproviders, + model_config, + manual_evaluations, + test_agents, + conversation_evaluations, + voice_agent, + evaluators, + evaluator_suites, + metrics, + scenario_metrics, + evaluator_results, + evaluator_result_metric_clusters, + chat, + chat_messaging_webhooks, + meta_whatsapp_webhooks, + twilio_sms_webhooks, + twilio_voice_webhooks, + telnyx_sms_webhooks, + telnyx_voice_webhooks, + playground, + settings, + observability, + alerts, + alert_settings, + cron_jobs, + voice_playground, + public_blind_test, + prompt_partials, + prompt_optimization, + telephony, + vobiz_telephony, + call_imports, + platform_call_imports, + call_import_schemas, + call_import_tags, + call_import_evaluations, + judge_alignment, + metric_studio, + workspaces, + workspace_iam, + dashboard, + llm_gateway, + platform_admin, + org_usage, + usage_pricing, + synthetic_traces, +) + +api_router = APIRouter() + +# Include all route routers +api_router.include_router(auth.router) +api_router.include_router(audio.router) +api_router.include_router(evaluations.router) +api_router.include_router(results.router) +api_router.include_router(agents.router) +api_router.include_router(personas.router) +api_router.include_router(scenarios.router) +api_router.include_router(knowledge_bases.router) +api_router.include_router(tool_connections.router) +api_router.include_router(simulation_runs.router) +api_router.include_router(iam.router) +api_router.include_router(profile.router) +api_router.include_router(integrations.router) +api_router.include_router(data_sources.router) +api_router.include_router(voicebundles.router) +api_router.include_router(aiproviders.router) +api_router.include_router(model_config.router) +api_router.include_router(manual_evaluations.router) +api_router.include_router(test_agents.router) +api_router.include_router(conversation_evaluations.router) +api_router.include_router(voice_agent.router) +api_router.include_router(evaluators.router) +api_router.include_router(evaluator_suites.router) +# Register before metrics.router so `/metrics/generate-from-scenarios` is not +# captured by `/metrics/{metric_id}` (which yields 405 for POST). +api_router.include_router(scenario_metrics.router) +api_router.include_router(metrics.router) +# Register before evaluator_results so `/evaluator-results/metric-clusters` +# is not captured by `/evaluator-results/{id}` with id="metric-clusters". +api_router.include_router(evaluator_result_metric_clusters.router) +api_router.include_router(evaluator_results.router) +api_router.include_router(chat.router) +api_router.include_router(chat_messaging_webhooks.router) +api_router.include_router(meta_whatsapp_webhooks.router) +api_router.include_router(twilio_sms_webhooks.router) +api_router.include_router(twilio_voice_webhooks.router) +api_router.include_router(telnyx_sms_webhooks.router) +api_router.include_router(telnyx_voice_webhooks.router) +api_router.include_router(playground.router) +api_router.include_router(settings.router) +api_router.include_router(observability.router) +api_router.include_router(alerts.router) +api_router.include_router(alert_settings.settings_router) +api_router.include_router(alert_settings.webhook_router) +api_router.include_router(cron_jobs.router) +api_router.include_router(voice_playground.router) +api_router.include_router(public_blind_test.router) +api_router.include_router(prompt_partials.router) +api_router.include_router(prompt_optimization.router) +api_router.include_router(telephony.router) +api_router.include_router(telephony.plivo_webhook_router) +api_router.include_router(vobiz_telephony.router) +api_router.include_router(call_imports.router) +api_router.include_router(platform_call_imports.router) +api_router.include_router(call_import_schemas.router) +api_router.include_router(call_import_tags.router) +api_router.include_router(call_import_evaluations.router) +api_router.include_router(judge_alignment.router) +api_router.include_router(metric_studio.router) +api_router.include_router(workspaces.router) +api_router.include_router(workspace_iam.router) +api_router.include_router(dashboard.router) +api_router.include_router(llm_gateway.router) +api_router.include_router(platform_admin.router) +api_router.include_router(org_usage.router) +api_router.include_router(usage_pricing.router) +api_router.include_router(synthetic_traces.router) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index b194f710..6e22d475 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -16,13 +16,14 @@ from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key from app.core.api_rate_limit import enforce_resource_create_rate_limit from app.core.oss_quotas import enforce_oss_quota -from app.core.auth import Principal +from app.core.auth import Principal, get_principal from app.services.billing.flexprice_service import record_agent_test_setup_generated from app.models.database import ( Agent, ConversationEvaluation, TestAgentConversation, VoiceBundle, AIProvider, Integration, IntegrationPlatform, CallMediumEnum, Evaluator, EvaluatorResult, EvaluatorSuite, CallRecording, Scenario, ) +from app.services.agents.favorites import apply_favorite, favorite_agent_ids, set_favorite from sqlalchemy import and_ from app.models.schemas import ( AgentCreate, @@ -31,6 +32,8 @@ AgentPhoneAssignmentCheckResponse, AgentPhoneAssignmentConflict, CallMediumEnum as CallMediumEnumSchema, + GenerateProductionPromptRequest, + GenerateProductionPromptResponse, GenerateTestPromptRequest, GenerateTestPromptResponse, GenerateScenariosFromPromptRequest, @@ -337,6 +340,73 @@ def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse] ] +@router.post("/generate-production-prompt", response_model=GenerateProductionPromptResponse) +async def generate_production_prompt( + data: GenerateProductionPromptRequest, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Generate a production system prompt from an agent brief (hosted agents).""" + from app.services.testing.agent_test_setup_generation import ( + generate_production_prompt_from_brief, + ) + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + + if not data.brief.strip(): + raise HTTPException(400, "Agent brief is required") + + provider_enum, model_str = _get_llm_provider_and_model( + organization_id, db, data.provider, data.model, data.credential_id + ) + + try: + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.AGENTS, + ) + ): + result = generate_production_prompt_from_brief( + data.brief, + agent_name=data.agent_name, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="production_prompt", + model=model_str, + ) + return GenerateProductionPromptResponse( + production_prompt=result.production_prompt, + provider=result.provider, + model=result.model, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as e: + logger.error(f"[Agents] Production prompt generation failed: {repr(e)}") + raise HTTPException(500, f"AI generation failed: {str(e)}") from e + + @router.post("/generate-test-prompt", response_model=GenerateTestPromptResponse) async def generate_test_prompt( data: GenerateTestPromptRequest, @@ -842,7 +912,50 @@ async def create_agent( except Exception as e: logger.warning(f"[Agents] Best-effort provider prompt sync failed on create: {e}") - return db_agent + if agent.knowledge_base_ids or agent.test_knowledge_base_ids: + from app.services.knowledge_bases.agent_links import ( + KB_ROLE_PRODUCTION, + KB_ROLE_TEST, + replace_agent_knowledge_bases, + ) + + try: + for role, kb_ids in ( + (KB_ROLE_PRODUCTION, agent.knowledge_base_ids), + (KB_ROLE_TEST, agent.test_knowledge_base_ids), + ): + if kb_ids: + replace_agent_knowledge_bases( + db, + agent=db_agent, + organization_id=organization_id, + workspace_id=workspace_id, + knowledge_base_ids=kb_ids, + role=role, + ) + db.commit() + db.refresh(db_agent) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + from app.services.agents.agent_serializers import agent_to_response + + return agent_to_response(db, db_agent) + + +class AgentFavoriteUpdate(BaseModel): + favorite: bool + + +def _with_favorites(db: Session, agents: list, user_id: Optional[UUID]) -> list: + from app.services.agents.agent_serializers import agents_to_responses + + responses = agents_to_responses(db, agents) + starred = favorite_agent_ids(db, user_id=user_id, agent_ids=[agent.id for agent in agents]) + return [ + apply_favorite(response, is_favorite=agent.id in starred) + for agent, response in zip(agents, responses) + ] @router.get("", response_model=List[AgentResponse]) @@ -851,6 +964,7 @@ async def list_agents( limit: int = 100, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), db: Session = Depends(get_db) ): """Get list of all agents for the active workspace. @@ -862,7 +976,7 @@ async def list_agents( Agent.organization_id == organization_id, Agent.workspace_id == workspace_id, ).offset(skip).limit(limit).all() - return agents + return _with_favorites(db, agents, principal.user_id) @router.get("/check-phone-assignment", response_model=AgentPhoneAssignmentCheckResponse) @@ -924,6 +1038,7 @@ async def get_agent( agent_id: str, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), db: Session = Depends(get_db) ): """Get a specific agent by ID (UUID) or agent_id (6-digit) within the active workspace.""" @@ -949,7 +1064,45 @@ async def get_agent( if not agent: raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - return agent + return _with_favorites(db, [agent], principal.user_id)[0] + + +@router.put("/{agent_id}/favorite", response_model=AgentResponse) +def update_agent_favorite( + agent_id: str, + body: AgentFavoriteUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +): + """Star or unstar an agent for the signed-in user only.""" + if principal.user_id is None: + raise HTTPException(status_code=400, detail="Sign in to save favorites") + try: + agent_uuid = UUID(agent_id) + agent = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + except ValueError: + agent = db.query(Agent).filter( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + if not agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + set_favorite( + db, + user_id=principal.user_id, + agent_id=agent.id, + organization_id=organization_id, + workspace_id=workspace_id, + favorite=body.favorite, + ) + return _with_favorites(db, [agent], principal.user_id)[0] @router.put("/{agent_id}", response_model=AgentResponse) @@ -1040,6 +1193,8 @@ async def update_agent( ) update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False) + kb_ids_update = update_data.pop("knowledge_base_ids", None) + test_kb_ids_update = update_data.pop("test_knowledge_base_ids", None) if "chat_connection_config" in update_data: conn_for_storage = update_data.get("chat_connection_type") or db_agent.chat_connection_type @@ -1151,6 +1306,30 @@ async def update_agent( if chat_err: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=chat_err) + if kb_ids_update is not None or test_kb_ids_update is not None: + from app.services.knowledge_bases.agent_links import ( + KB_ROLE_PRODUCTION, + KB_ROLE_TEST, + replace_agent_knowledge_bases, + ) + + try: + for role, kb_ids in ( + (KB_ROLE_PRODUCTION, kb_ids_update), + (KB_ROLE_TEST, test_kb_ids_update), + ): + if kb_ids is not None: + replace_agent_knowledge_bases( + db, + agent=db_agent, + organization_id=organization_id, + workspace_id=workspace_id, + knowledge_base_ids=kb_ids, + role=role, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + db.commit() db.refresh(db_agent) @@ -1173,7 +1352,9 @@ async def update_agent( except Exception as e: logger.warning(f"[Agents] Best-effort provider prompt sync failed on update: {e}") - return db_agent + from app.services.agents.agent_serializers import agent_to_response + + return agent_to_response(db, db_agent) @router.post("/{agent_id}/sync-provider-prompt") diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py deleted file mode 100644 index 17654b48..00000000 --- a/app/api/v1/routes/call_import_evaluations.py +++ /dev/null @@ -1,9937 +0,0 @@ -"""Evaluation routes scoped to a Call Import batch.""" - -from __future__ import annotations - -import asyncio -import base64 -import io -import json -import math -import re -import statistics -from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple -from uuid import UUID, uuid4 - -from datetime import date, datetime, timedelta, timezone - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status -from fastapi.responses import StreamingResponse -from loguru import logger -from pydantic import BaseModel, Field, field_validator -from sqlalchemy import Float, case, cast, desc, func, or_, text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -from app.core.auth import Principal, get_principal -from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message -from app.database import get_db -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.call_imports.audit import ( - actor_emails_for_evaluation, - emails_for_user_ids, - stamp_call_import_actor, - stamp_evaluation_actor, - user_ids_from_evaluations, -) -from app.services.workspace_rbac import resolve_workspace_capabilities -from app.models.database import ( - AIProvider, - CallImport, - CallImportEvaluation, - CallImportEvaluationReportSnapshot, - CallImportEvaluationPdfReport, - CallImportEvaluationRow, - CallImportRow, - Metric, - PromptPartial, - Workspace, -) -from app.models.enums import CallImportRowStatus, ModelProvider -from app.models.schemas import ( - CallImportEvaluationAggregateResponse, - CallImportEvaluationBulkDelete, - CallImportEvaluationBulkActionResponse, - CallImportEvaluationCreate, - CallImportEvaluationExportJobResponse, - CallImportEvaluationExportRequest, - CallImportEvaluationListResponse, - CallImportEvaluationResponse, - CallImportEvaluationRetryRequest, - CallImportEvaluationRetryResponse, - CallImportEvaluationRetrySkippedItem, - CallImportEvaluationRowListResponse, - CallImportEvaluationRowResponse, - CallImportEvaluationUpdate, - CallImportClassificationFacetCounts, - CallImportMetricAggregate, - CallImportMetricHistogramBucket, - CallImportMetricLabelPair, - CallImportMetricSummary, - CallImportMetricValueCount, - DiscoveredLabelDeleteRequest, - DiscoveredLabelItem, - DiscoveredLabelMergeRequest, - DiscoveredLabelsResponse, - DiscoveredMetricDeleteRequest, - DiscoveredMetricItem, - DiscoveredMetricMergeRequest, - DiscoveredMetricsResponse, - EvaluationInsightsRequest, - EvaluationTldrSummary, - EvaluationMetricClustersRequest, - EvaluationMetricClustersState, - EvaluationPromptImprovementsRequest, - EvaluationPromptImprovementsState, - PromptImprovementReviewUpdate, - PromptPushConfirm, - PromptPushPreviewResponse, - PromptPushResponse, - MetricFailurePoliciesResponse, - MetricFailurePoliciesSaveRequest, - MetricFailurePolicy, - MetricClusterEligibleRow, - MetricClusterEligibleRowsResponse, - EvaluationUserInsightsRequest, - EvaluationUserInsightsState, - MetricFlowEdge, - MetricPeriodDelta, - MetricFlowNode, - MetricFlowResponse, -) -from app.services.reporting.call_import_evaluation_pdf_report import ( - call_import_evaluation_pdf_report_service, -) -from app.services.reporting.call_import_pdf_report_storage import ( - build_pdf_report_s3_key, - compute_pdf_report_cache_fingerprint, - compute_pdf_report_config_fingerprint, - compute_pdf_report_content_fingerprint, - config_summary_from_report_config, - find_cached_pdf_report, - presigned_urls_for_pdf_report, -) -from app.services.call_import_metric_clusters import ( - METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - estimate_metric_clusters_llm_calls, - filter_completed_row_pairs, - list_eligible_cluster_rows, - metric_clusters_raw_is_cancelled, - metric_clusters_state_from_raw, - metric_clusters_state_to_db, -) -from app.services.classification_metric_scores import classification_facets_from_entry -from app.services.classification_metric_sql import ( - classification_choice_row_predicate, - classification_level_row_predicate, -) -from app.services.metric_classification_validation import is_classification_metric -from app.services.metric_failure_policy import ( - aggregate_primary_percent, - build_failure_policy_previews, - effective_policies, - failure_rate_percent_from_rows, - failure_policies_to_db, - has_clusterable_metrics, - merge_clustering_policies, - merge_failure_policies_into_raw, - policies_from_evaluation_raw, - validate_failure_policies_for_metrics, -) -from app.services.call_import_user_insights import ( - normalize_max_llm_calls, - total_llm_calls_for_rows, - user_insights_state_from_raw, -) - -router = APIRouter( - prefix="/call-imports/{call_import_id}/evaluations", - tags=["Call Import Evaluations"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -class CallImportEvaluationPdfReportRequest(BaseModel): - vendor_name: str = Field(..., min_length=1, max_length=120) - report_type: Literal["external", "internal"] = "external" - include_weekly_delta: bool = False - include_period_delta: bool = False - baseline_evaluation_id: Optional[str] = None - period_label: Optional[str] = Field(default=None, max_length=64) - use_case: Optional[str] = Field(default=None, max_length=120) - internal_brand_image_id: Optional[str] = None - external_brand_image_id: Optional[str] = None - report_config: Dict[str, Any] = Field(default_factory=dict) - platform_base_url: Optional[str] = Field( - default=None, - max_length=512, - description="Frontend origin for deep links to example calls in internal PDFs.", - ) - - @field_validator("vendor_name") - @classmethod - def _clean_vendor_name(cls, value: str) -> str: - cleaned = value.strip() - if not cleaned: - raise ValueError("Vendor name is required.") - return cleaned - - -class CallImportEvaluationPdfReportResponse(BaseModel): - id: str - filename: str - preview_url: Optional[str] = None - download_url: Optional[str] = None - created_at: datetime - created_by: Optional[str] = None - report_type: str - vendor_name: str - config_summary: Optional[str] = None - storage_available: bool = True - cache_hit: bool = False - - -class CallImportEvaluationPdfReportListItem(BaseModel): - id: str - filename: Optional[str] = None - vendor_name: str - report_type: str - created_by: Optional[str] = None - created_at: datetime - config_summary: Optional[str] = None - cache_fingerprint: Optional[str] = None - - -class CallImportEvaluationPdfReportListResponse(BaseModel): - items: List[CallImportEvaluationPdfReportListItem] - - -class CallImportEvaluationBaselineCandidate(BaseModel): - evaluation_id: str - name: str - dataset: str - period_label: Optional[str] = None - period_start: Optional[date] = None - period_end: Optional[date] = None - period_display: str - completed_rows: int - created_at: datetime - is_default: bool = False - - -class CallImportEvaluationBaselineCandidatesResponse(BaseModel): - items: List[CallImportEvaluationBaselineCandidate] - default_evaluation_id: Optional[str] = None - - -def _require_import( - db: Session, - call_import_id: UUID, - organization_id: UUID, -) -> CallImport: - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException(status_code=404, detail="Call import not found") - return call_import - - -def require_call_import_capability(capability: str): - """Ensure the caller has *capability* in the call import's workspace (not just the header).""" - - def _dep( - call_import_id: UUID, - principal: Principal = Depends(get_principal), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), - ) -> CallImport: - call_import = _require_import(db, call_import_id, organization_id) - caps, _, role = resolve_workspace_capabilities( - db, - principal=principal, - workspace_id=call_import.workspace_id, - organization_id=organization_id, - ) - if capability not in caps: - raise HTTPException( - status_code=403, - detail=capability_denied_message( - capability, - role_name=role.name if role else None, - workspace_label="the active workspace", - ), - ) - return call_import - - return _dep - - -def _pick_evaluation_row_transcript( - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> Optional[str]: - """Transcript shown in evaluation row detail for the run's source.""" - if source_row is None: - return None - source = ( - (evaluation.transcript_source or "diarised").strip().lower() - if evaluation is not None - else "diarised" - ) - if source == "production": - raw = (source_row.transcript or "").strip() - return raw or None - diarised = (source_row.diarised_transcript or "").strip() - if diarised: - return diarised - raw = (source_row.transcript or "").strip() - return raw or None - - -def _to_evaluation_row_response( - eval_row_obj: CallImportEvaluationRow, - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, - *, - include_text: bool = True, -) -> CallImportEvaluationRowResponse: - """Serialize one evaluation row plus joined source-row metadata. - - List pages pass ``include_text=False`` so transcript bodies stay off - the page. The single-row read passes ``True``. - """ - if include_text and source_row is not None: - transcript = _pick_evaluation_row_transcript(source_row, evaluation) - production_transcript = (source_row.transcript or "").strip() or None - diarised_transcript = (source_row.diarised_transcript or "").strip() or None - raw_columns = source_row.raw_columns - else: - transcript = None - production_transcript = None - diarised_transcript = None - raw_columns = None - return CallImportEvaluationRowResponse( - id=eval_row_obj.id, - evaluation_id=eval_row_obj.evaluation_id, - call_import_row_id=eval_row_obj.call_import_row_id, - row_index=source_row.row_index if source_row else None, - conversation_id=source_row.conversation_id if source_row else None, - transcript=transcript, - production_transcript=production_transcript, - diarised_transcript=diarised_transcript, - raw_columns=raw_columns, - recording_url=source_row.recording_url if source_row else None, - recording_date=source_row.recording_date if source_row else None, - recording_s3_key=source_row.recording_s3_key if source_row else None, - diarised_transcript_status=( - source_row.diarised_transcript_status if source_row else None - ), - diarised_transcript_error=( - source_row.diarised_transcript_error if source_row else None - ), - status=eval_row_obj.status, - metric_scores=eval_row_obj.metric_scores or {}, - error_message=eval_row_obj.error_message, - started_at=eval_row_obj.started_at, - finished_at=eval_row_obj.finished_at, - created_at=eval_row_obj.created_at, - updated_at=eval_row_obj.updated_at, - ) - - -def _serialize_selected_metric_ids(value) -> List[UUID]: - result: List[UUID] = [] - if not isinstance(value, list): - return result - for item in value: - try: - result.append(UUID(str(item))) - except (TypeError, ValueError): - continue - return result - - -def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: - if not ids: - return [] - rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(ids), - ) - .all() - ) - by_id = {row.id: row for row in rows} - return [by_id[mid] for mid in ids if mid in by_id] - - -def _expand_metric_selection( - db: Session, - org_id: UUID, - selected_ids: List[UUID], -) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: - """Resolve user-supplied metric ids into actual leaves + parent grouping. - - Rules: - * If a parent id is in ``selected_ids`` and no specific children of - that parent are also listed, include EVERY enabled child of that - parent. - * If a parent id AND some of its children are listed, include only - the listed children (treat the parent selection as the - "container" so users can deselect labels). - * Standalone metrics (no parent, no children) pass through - unchanged. - * Disabled metrics are filtered out at this layer so the caller - doesn't have to repeat the check. - - Returns: - (effective_metrics, parent_to_children) - - ``effective_metrics`` is the deduplicated list of metrics the - worker will actually score (children + standalone). Order is - preserved from ``selected_ids`` for display stability. - - ``parent_to_children`` maps each parent metric id (UUID) to the - list of its selected children. Useful for grouping in the LLM - prompt builder. - """ - if not selected_ids: - return [], {} - - requested = list(selected_ids) - initial_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(requested), - ) - .all() - ) - initial_by_id = {row.id: row for row in initial_rows} - - parent_ids_requested = { - m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id - } - # Map parent id -> children explicitly requested by the user. - explicit_children_by_parent: Dict[UUID, List[Metric]] = {} - for m in initial_rows: - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - explicit_children_by_parent.setdefault( - m.parent_metric_id, [] - ).append(m) - - # For parents without explicit children, hydrate every enabled child. - parents_needing_full_expansion = [ - pid - for pid in parent_ids_requested - if pid not in explicit_children_by_parent - ] - auto_expanded_children: Dict[UUID, List[Metric]] = {} - if parents_needing_full_expansion: - for pid in parents_needing_full_expansion: - child_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.parent_metric_id == pid, - Metric.enabled.is_(True), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - auto_expanded_children[pid] = child_rows - - parent_to_children: Dict[UUID, List[Metric]] = {} - for pid in parent_ids_requested: - children = explicit_children_by_parent.get( - pid - ) or auto_expanded_children.get(pid, []) - # Drop disabled children so the worker doesn't waste a slot on - # them. Empty parents (no enabled children) are still tracked - # because the UI may want to show "0 of 0" rather than swallow - # them silently. - parent_to_children[pid] = [c for c in children if c.enabled] - - effective: List[Metric] = [] - seen: set[UUID] = set() - for mid in requested: - m = initial_by_id.get(mid) - if m is None: - continue - if m.selection_mode and not m.parent_metric_id: - # Parent row itself is not scored — only its children. - for child in parent_to_children.get(m.id, []): - if child.id in seen or not child.enabled: - continue - seen.add(child.id) - effective.append(child) - continue - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - # Already accounted for via the parent expansion above. - continue - if not m.enabled: - continue - if m.id in seen: - continue - seen.add(m.id) - effective.append(m) - - return effective, parent_to_children - - -def _evaluation_bulk_operation_for_response( - evaluation_id: UUID, -) -> Optional[str]: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - return get_evaluation_bulk_operation(evaluation_id) - - -def _serialize_eval( - db: Session, - row: CallImportEvaluation, - *, - sibling_evaluation_ids: Optional[List[UUID]] = None, - user_emails: Optional[Dict[UUID, str]] = None, -) -> CallImportEvaluationResponse: - selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) - - # Pull every metric referenced anywhere in the run's grouping (leaves, - # standalone, AND parents from selected_metric_groups) so the UI can - # render parent labels even when only children were materialized into - # selected_metric_ids. - groups_raw: Dict[str, List[str]] = {} - if isinstance(row.selected_metric_groups, dict): - for parent_str, children in row.selected_metric_groups.items(): - if not isinstance(children, list): - continue - cleaned: List[str] = [] - for c in children: - try: - UUID(str(c)) - cleaned.append(str(c)) - except (TypeError, ValueError): - continue - try: - UUID(parent_str) - groups_raw[parent_str] = cleaned - except (TypeError, ValueError): - continue - - metric_ids_for_lookup: List[UUID] = list(selected_ids) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in metric_ids_for_lookup: - metric_ids_for_lookup.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids( - db, row.organization_id, metric_ids_for_lookup - ) - - from app.services.call_imports.progress_counters import merge_eval_counters_for_ui - - ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) - total = int(row.total_rows or 0) - ui_completed = ( - min(ui_completed_raw, total) if total else ui_completed_raw - ) - ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw - - if user_emails is None: - user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) - created_email, updated_email = actor_emails_for_evaluation(row, user_emails) - - from app.workers.tasks.evaluate_call_import_row_core import ( - count_distinct_llm_configs_for_metrics, - ) - - selected_set = set(selected_ids) - scoring_metrics = [m for m in metrics if m.id in selected_set] - expected_llm_calls_per_row = count_distinct_llm_configs_for_metrics( - scoring_metrics, - overrides=( - row.metric_llm_overrides - if isinstance(row.metric_llm_overrides, dict) - else {} - ), - run_provider=(row.llm_provider or "").strip() or None, - run_model=(row.llm_model or "").strip() or None, - run_llm_config=( - row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None - ), - run_credential_id=( - str(row.llm_credential_id) if row.llm_credential_id else None - ), - ) - - return CallImportEvaluationResponse( - id=row.id, - call_import_id=row.call_import_id, - organization_id=row.organization_id, - name=row.name, - selected_metric_ids=selected_ids, - selected_metric_groups=groups_raw or None, - metrics=[ - CallImportMetricSummary( - id=metric.id, - name=metric.name, - metric_type=metric.metric_type, - description=metric.description, - parent_metric_id=metric.parent_metric_id, - selection_mode=metric.selection_mode, - # Required by the Flow tab to know whether a parent - # opted into discovery; without it the - # DiscoveredLabelsPanel stays hidden even when the - # worker is actively producing discovered_labels. - allow_discovery=bool( - getattr(metric, "allow_discovery", False) - ), - ) - for metric in metrics - ], - status=row.status, - total_rows=row.total_rows, - completed_rows=ui_completed, - failed_rows=ui_failed, - error_message=row.error_message, - llm_provider=row.llm_provider, - llm_model=row.llm_model, - llm_credential_id=row.llm_credential_id, - llm_config=( - row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None - ), - metric_llm_overrides=( - row.metric_llm_overrides - if isinstance(row.metric_llm_overrides, dict) - else None - ), - stt_provider=row.stt_provider, - stt_model=row.stt_model, - stt_credential_id=row.stt_credential_id, - diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), - diarisation_llm_model=getattr(row, "diarisation_llm_model", None), - diarisation_llm_credential_id=getattr( - row, "diarisation_llm_credential_id", None - ), - diarisation_prompt=getattr(row, "diarisation_prompt", None), - transcribe_mode=( - (getattr(row, "transcribe_mode", None) or "stt_llm") - ), - transcript_source=(row.transcript_source or "diarised"), - sibling_evaluation_ids=list(sibling_evaluation_ids or []), - expected_llm_calls_per_row=expected_llm_calls_per_row, - started_at=row.started_at, - finished_at=row.finished_at, - created_at=row.created_at, - updated_at=row.updated_at, - created_by_email=created_email, - last_updated_by_email=updated_email, - tldr_summary=_tldr_summary_payload(row), - user_insights=_user_insights_payload(row), - metric_clusters=_metric_clusters_payload(row), - discover_new_metrics=bool( - getattr(row, "discover_new_metrics", False) - ), - bulk_operation=_evaluation_bulk_operation_for_response(row.id), - ) - - -def _normalize_name(value: Optional[str]) -> Optional[str]: - """Trim user-supplied name; empty string becomes ``NULL``.""" - if value is None: - return None - trimmed = value.strip() - return trimmed or None - - -def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: - """Recompute counters + terminal status after rows are added/removed. - - Uses a single aggregate query instead of loading every row status. - """ - from app.workers.tasks.evaluate_call_import_row_core import ( - _apply_parent_status_from_counters, - reconcile_evaluation_counters, - ) - - reconcile_evaluation_counters(db, evaluation) - _apply_parent_status_from_counters(evaluation) - db.flush() - - if evaluation.status in {"completed", "failed", "partial"}: - from app.models.database import CallImport - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - call_import = ( - db.query(CallImport) - .filter(CallImport.id == evaluation.call_import_id) - .first() - ) - if call_import is not None: - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "", - response_model=CallImportEvaluationResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="createCallImportEvaluation", -) -async def create_call_import_evaluation( - call_import_id: UUID, - payload: CallImportEvaluationCreate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - metric_ids = payload.metric_ids - if not metric_ids: - raise HTTPException( - status_code=400, - detail="Select at least one metric to run the evaluation against.", - ) - - org_metrics = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(metric_ids), - ) - .all() - ) - by_id = {metric.id: metric for metric in org_metrics} - unknown_ids = [mid for mid in metric_ids if mid not in by_id] - if unknown_ids: - raise HTTPException( - status_code=400, - detail=( - "These metric ids do not exist in your organization: " - f"{', '.join(str(mid) for mid in unknown_ids)}. " - "Refresh the metrics list and try again." - ), - ) - draft_metrics = [ - metric - for metric in org_metrics - if (getattr(metric, "lifecycle", None) or "active") == "draft" - ] - if draft_metrics: - names = ", ".join(metric.name for metric in draft_metrics) - raise HTTPException( - status_code=400, - detail=( - f"Draft metrics cannot be used in call import evaluations: {names}. " - "Promote them in Metrics Studio first." - ), - ) - # Parents themselves are containers, not scored rows, so a disabled - # parent shouldn't block the run as long as it has enabled children. - # We only reject disabled rows that the worker will actually try to - # evaluate (children + standalone leaves). - disabled_leaves = [ - metric - for metric in org_metrics - if not metric.enabled - and not (metric.selection_mode and not metric.parent_metric_id) - ] - if disabled_leaves: - names = ", ".join(metric.name for metric in disabled_leaves) - raise HTTPException( - status_code=400, - detail=( - f"These metrics are disabled and cannot be evaluated: {names}. " - "Enable them on the Metrics page (or pick different ones) and " - "try again." - ), - ) - - # Expand hierarchical selection: parents auto-include their enabled - # children, mixed parent+child selections respect the user's subset. - effective_metrics, parent_to_children = _expand_metric_selection( - db, organization_id, metric_ids - ) - if not effective_metrics: - raise HTTPException( - status_code=400, - detail=( - "None of the selected metrics yielded an enabled leaf to " - "evaluate. Check that parent categories have enabled " - "children, then try again." - ), - ) - - # The effective list (children + standalone leaves) is what gets - # persisted to ``selected_metric_ids`` and scored by the worker. - # The original parents are preserved in ``selected_metric_groups`` - # so the UI can rebuild the tree later. - leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] - selected_metric_groups: Dict[str, List[str]] = { - str(pid): [str(c.id) for c in children] - for pid, children in parent_to_children.items() - } - metric_rows = effective_metrics - valid_metric_id_strs = {str(m.id) for m in metric_rows} - - # ----- Validate run-level + per-metric LLM config ----- - llm_provider_norm: Optional[str] = None - llm_model_norm: Optional[str] = None - if payload.llm_provider or payload.llm_model: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail="Both llm_provider and llm_model are required when overriding the run LLM.", - ) - try: - llm_provider_norm = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - llm_model_norm = payload.llm_model.strip() or None - if not llm_model_norm: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in this " - "organization." - ), - ) - - # Per-metric overrides: keys can be either a leaf metric id (applies - # to that metric only) or a parent metric id (applies to every - # child of that parent). Parent keys are expanded to their - # children so the worker only sees concrete leaf ids. - metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None - if payload.metric_llm_overrides: - metric_overrides_payload = {} - for metric_id, override in payload.metric_llm_overrides.items(): - target_leaf_ids: List[str] = [] - if metric_id in valid_metric_id_strs: - target_leaf_ids = [metric_id] - else: - # Maybe it's a parent id — expand to the children that - # are part of THIS run. - try: - parent_uuid = UUID(metric_id) - except (TypeError, ValueError): - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a valid UUID." - ), - ) - children_for_parent = parent_to_children.get(parent_uuid) - if not children_for_parent: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not in metric_ids." - ), - ) - target_leaf_ids = [str(c.id) for c in children_for_parent] - - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a provider " - "but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses unknown " - f"provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - # Model without provider doesn't make sense — treat as 400 - # so the UI can fix it instead of silently falling back. - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model but " - "no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - for leaf_id in target_leaf_ids: - metric_overrides_payload[leaf_id] = override_dict - - # ----- Validate auto-transcribe settings ----- - # Diarised runs auto-diarise rows missing a diarised transcript and - # require STT + diariser LLM config. Production runs score the CSV - # transcript directly and skip diarisation entirely. - use_diarised = payload.transcript_sources[0] == "diarised" - auto_transcribe = use_diarised - - transcribe_mode_norm: Optional[str] = None - stt_provider_norm: Optional[str] = None - stt_model_norm: Optional[str] = None - diarisation_llm_provider_norm: Optional[str] = None - diarisation_llm_model_norm: Optional[str] = None - diarisation_prompt_norm: Optional[str] = None - - if use_diarised: - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) - - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: - raise HTTPException( - status_code=400, - detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." - ), - ) - if not payload.stt_model: - raise HTTPException( - status_code=400, - detail=( - "stt_model is required when transcribe_mode='stt_llm'." - ), - ) - try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): - raise HTTPException( - status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), - ) - - # --- Validate LLM diariser settings ----- - if not payload.diarization_llm_provider: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - if not payload.diarization_llm_model: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - try: - diarisation_llm_provider_norm = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - diarisation_llm_model_norm = ( - payload.diarization_llm_model.strip() or None - ) - if not diarisation_llm_model_norm: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - diarisation_prompt_norm = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - - from app.models.enums import CallImportParameterType, CallImportStatus - from app.services.call_imports.bulk_ops import ( - count_all_source_rows, - count_source_rows_with_production_transcript, - ) - - if use_diarised: - from app.api.v1.routes.call_imports import ( - _is_manual_audio_call_import, - _is_platform_call_import, - platform_import_hydration_complete, - ) - - if not _is_manual_audio_call_import(call_import) and not _is_platform_call_import( - call_import - ): - if call_import.schema_id: - from app.api.v1.routes.call_imports import ( - _resolve_schema, - _validate_diarised_eval_recording_ready, - ) - - diarised_schema = _resolve_schema( - db, organization_id, call_import.workspace_id, call_import.schema_id - ) - _validate_diarised_eval_recording_ready( - list(diarised_schema.parameters), - dict(call_import.parameter_mapping or {}), - ) - else: - legacy_recording_column = ( - (call_import.column_mapping or {}).get("recording_url") or "" - ).strip() - if not legacy_recording_column: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Diarize then evaluate requires a recording URL column " - "to be mapped." - ), - ) - - starting_from_mapped = False - from app.api.v1.routes.call_imports import ( - _is_platform_call_import, - platform_import_hydration_complete, - ) - - if _is_platform_call_import(call_import) and not platform_import_hydration_complete( - call_import - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Platform import is still discovering or fetching recordings. " - "Wait until the batch is ready before running evaluation." - ), - ) - - if call_import.status == CallImportStatus.MAPPED: - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file. Upload and map " - "a CSV/Excel file before running evaluation." - ), - ) - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot run evaluation without a mapped schema.", - ) - from app.api.v1.routes.call_imports import ( - _ensure_blob_storage_enabled, - _resolve_schema, - _resolve_telephony_integration, - _validate_direct_url_import_ready, - _validate_telephony_credentials_live, - ) - - workspace_id = call_import.workspace_id - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - if not use_diarised: - transcript_mapped = any( - param.type == CallImportParameterType.TRANSCRIPT - and (call_import.parameter_mapping or {}).get(param.name) - for param in parameters - ) - if not transcript_mapped: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No transcript column is mapped in this batch. " - "Map a schema transcript parameter to a CSV column, " - "or choose 'Diarize then evaluate'." - ), - ) - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - _validate_telephony_credentials_live(db, organization_id, integration) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - starting_from_mapped = True - - all_source_row_count = count_all_source_rows(db, call_import.id) - - if use_diarised: - # Diarized runs materialize every import row; recording fetch may - # still be pending when switching from a production eval-primary run. - total_row_count = all_source_row_count - else: - # Production runs score CSV text — rows need not wait for - # recording fetch to finish before they are evaluable. - total_row_count = count_source_rows_with_production_transcript( - db, call_import.id - ) - - if ( - use_diarised - and not starting_from_mapped - and all_source_row_count > 0 - and call_import.status != CallImportStatus.DELETING - ): - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - - requested_sources: List[str] = list(payload.transcript_sources) - - if ( - not use_diarised - and not starting_from_mapped - and count_all_source_rows(db, call_import.id) > 0 - and total_row_count == 0 - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No rows have a production transcript. " - "Choose 'Diarize then evaluate' or import rows with " - "a transcript column." - ), - ) - - base_name = _normalize_name(payload.name) - - def _name_for_source(source: str) -> Optional[str]: - # Single-source runs preserve the user's chosen name verbatim. - del source - return base_name - - created_evaluations: List[CallImportEvaluation] = [] - - for source in requested_sources: - evaluation = CallImportEvaluation( - call_import_id=call_import.id, - organization_id=organization_id, - # Mirror the parent CallImport's workspace so listings can - # filter on workspace_id directly without joining. - workspace_id=call_import.workspace_id, - name=_name_for_source(source), - selected_metric_ids=[ - str(metric_id) for metric_id in leaf_metric_ids - ], - selected_metric_groups=selected_metric_groups or None, - status="pending", - total_rows=total_row_count, - completed_rows=0, - failed_rows=0, - llm_provider=llm_provider_norm, - llm_model=llm_model_norm, - llm_credential_id=payload.llm_credential_id, - llm_config=payload.llm_config, - metric_llm_overrides=metric_overrides_payload, - stt_provider=stt_provider_norm, - stt_model=stt_model_norm, - stt_credential_id=( - payload.stt_credential_id if auto_transcribe else None - ), - diarisation_llm_provider=diarisation_llm_provider_norm, - diarisation_llm_model=diarisation_llm_model_norm, - diarisation_llm_credential_id=( - payload.diarization_llm_credential_id if auto_transcribe else None - ), - diarisation_prompt=diarisation_prompt_norm, - transcribe_mode=transcribe_mode_norm, - transcript_source=source, - discover_new_metrics=bool( - getattr(payload, "discover_new_metrics", False) - ), - ) - stamp_evaluation_actor(evaluation, principal, creating=True) - db.add(evaluation) - db.flush() - created_evaluations.append(evaluation) - - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - - primary_evaluation = created_evaluations[0] - sibling_ids = [e.id for e in created_evaluations[1:]] - - if not all_source_row_count and not starting_from_mapped: - for evaluation in created_evaluations: - evaluation.status = "completed" - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - if starting_from_mapped: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_mapped_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_mapped_call_import_evaluation_task.delay( - str(call_import.id), - str(organization_id), - str(call_import.workspace_id), - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - else: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_call_import_evaluation_task.delay( - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - - from app.services.billing.flexprice_service import ( - record_call_import_evaluation_started, - ) - - for evaluation in created_evaluations: - record_call_import_evaluation_started( - organization_id, - evaluation.id, - workspace_id=evaluation.workspace_id, - call_import_id=call_import.id, - total_rows=int(evaluation.total_rows or 0), - metric_count=len(leaf_metric_ids), - ) - - for evaluation in created_evaluations: - db.refresh(evaluation) - - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - -@router.get( - "", - response_model=CallImportEvaluationListResponse, - operation_id="listCallImportEvaluations", -) -async def list_call_import_evaluations( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .all() - ) - email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) - return CallImportEvaluationListResponse( - items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], - total=len(rows), - ) - - -@router.get( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="getCallImportEvaluation", -) -async def get_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - return _serialize_eval(db, row) - - -def _apply_classification_metric_filters( - query, - metric_id: UUID, - *, - yes_no: Optional[str] = None, - choice: Optional[str] = None, - level: Optional[str] = None, -): - """Restrict rows by Jev classification facets under ``metric_scores``.""" - mid = str(metric_id) - scores = CallImportEvaluationRow.metric_scores - - if yes_no and yes_no.strip(): - yn = yes_no.strip().lower() - if yn in ("yes", "no"): - noul_txt = func.json_extract_path_text( - scores, mid, "answers", "noul", "noul" - ) - noul_f = cast(noul_txt, Float) - derived = case( - (noul_f >= 0.5, "Yes"), - (noul_f.isnot(None), "No"), - else_=None, - ) - effective = func.coalesce( - func.json_extract_path_text(scores, mid, "classification_yes_no"), - derived, - ) - query = query.filter(func.lower(effective) == yn) - - if choice and choice.strip(): - query = query.filter(classification_choice_row_predicate(mid, choice)) - - if level and level.strip(): - query = query.filter(classification_level_row_predicate(mid, level)) - - return query - - -@router.get( - "/{eval_id}/rows", - response_model=CallImportEvaluationRowListResponse, - operation_id="listCallImportEvaluationRows", -) -async def list_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - page: int = Query(1, ge=1), - page_size: int = Query(100, ge=1, le=500), - q: Optional[str] = Query( - None, - description=( - "Free-text search across conversation_id and transcript " - "(case-insensitive substring match)." - ), - ), - metric_id: Optional[UUID] = Query( - None, - description=( - "If set, only return rows whose ``metric_scores[metric_id].value`` " - "exactly matches ``metric_value`` (string-compared). " - "Use together with ``metric_value``." - ), - ), - metric_value: Optional[str] = Query( - None, - description="Value to match against metric_id (string compare).", - ), - classification_yes_no: Optional[str] = Query( - None, - description=( - "With ``metric_id``, filter classification metrics by Yes/No " - "(case-insensitive)." - ), - ), - classification_choice: Optional[str] = Query( - None, - description=( - "With ``metric_id``, filter classification metrics by category " - "choice (case-insensitive)." - ), - ), - classification_level: Optional[str] = Query( - None, - description=( - "With ``metric_id``, filter classification metrics by level label " - "(case-insensitive)." - ), - ), - status_filter: Optional[str] = Query( - None, - alias="status", - description="Restrict to rows with this evaluation row status.", - ), - flow_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric whose ``sequence`` array should be " - "checked against ``flow_node`` and ``flow_edge_target``. Used " - "to drill into the calls behind a flow-chart node or edge." - ), - ), - flow_node: Optional[str] = Query( - None, - description=( - "If set together with ``flow_parent_id``, only return rows " - "whose sequence under that parent contains this step. Accepts " - "either a child metric UUID (resolved to slug(name)), a " - "``disc:`` discovered-label id, or a raw slug." - ), - ), - flow_edge_target: Optional[str] = Query( - None, - description=( - "Optional companion to ``flow_node``: when set, restrict to " - "rows whose sequence contains the directed transition " - "``flow_node -> flow_edge_target`` (immediately adjacent). " - "Same id format as ``flow_node``." - ), - ), - discovered_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric that defines the discovery scope " - "for ``discovered_label_key`` / ``has_discovered``." - ), - ), - discovered_label_key: Optional[str] = Query( - None, - description=( - "If set together with ``discovered_parent_id``, only return " - "rows whose ``metric_scores[parent].discovered_labels`` " - "list contains an entry with this slug (after applying " - "evaluation-level merge aliases)." - ), - ), - has_discovered: Optional[bool] = Query( - None, - description=( - "If true together with ``discovered_parent_id``, only return " - "rows that have at least one LLM-discovered label for the " - "parent. Useful to triage which calls produced novel labels." - ), - ), - sort_by: Optional[str] = Query( - None, - description=( - "Column to sort by. Accepted values: ``row_index`` (default " - "when omitted), ``conversation_id``, ``status`` (the " - "evaluation-row status), or ``metric:`` to sort " - "by ``metric_scores[].value``. Metric sorts compare " - "the extracted JSON text — adequate for booleans, enum " - "labels, and 0-1 ratings; large integer values may sort " - "lexicographically (10 before 2)." - ), - ), - sort_dir: Optional[str] = Query( - "asc", - description="Sort direction: ``asc`` (default) or ``desc``.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - eval_row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not eval_row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - query = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - ) - - # --- Filters ---------------------------------------------------------- - if q and q.strip(): - needle = f"%{q.strip()}%" - # Search across both transcript columns so a hit in either the - # production or the diarised version surfaces the row, - # independent of which source the evaluation actually scored. - query = query.filter( - or_( - CallImportRow.conversation_id.ilike(needle), - CallImportRow.transcript.ilike(needle), - CallImportRow.diarised_transcript.ilike(needle), - ) - ) - - if status_filter: - # The CallImportEvaluationRow.status column is a string in PG so a - # plain == filter works; we lowercase to match the stored values. - query = query.filter( - CallImportEvaluationRow.status == status_filter.strip().lower() - ) - - if metric_id is not None and metric_value is not None: - # ``metric_scores`` is a JSONB column shaped like - # ``{"": {"value": , "type": "boolean", ...}}``. We - # extract the nested ``value`` as text and compare to the user - # input as a string — that handles bool/int/enum without needing - # per-type casts. ``metric_value`` is matched case-insensitively - # so chart clicks on labels like "True" survive any casing drift - # between worker output and the chart label. - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_id), - "value", - ) - query = query.filter(func.lower(path_value) == metric_value.strip().lower()) - - if metric_id is not None and ( - (classification_yes_no and classification_yes_no.strip()) - or (classification_choice and classification_choice.strip()) - or (classification_level and classification_level.strip()) - ): - query = _apply_classification_metric_filters( - query, - metric_id, - yes_no=classification_yes_no, - choice=classification_choice, - level=classification_level, - ) - - # --- Flow chart drilldown filter ------------------------------------- - # Translates a clicked node (or edge) on the flow chart into a - # SQL filter against ``metric_scores[].sequence``. The - # frontend sends either a child UUID, a ``disc:`` discovered - # node id, or a raw slug — we normalize all three to the slug that - # actually appears in stored ``sequence`` arrays. - if flow_parent_id is not None and flow_node and flow_node.strip(): - parent_id_str_local = str(flow_parent_id) - alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) - - def _flow_node_to_slug(raw: str) -> Optional[str]: - raw_clean = raw.strip() - if not raw_clean: - return None - if raw_clean == _FLOW_START_NODE_ID: - # The synthetic START node isn't a real sequence entry; - # filtering on it is meaningless so we skip silently. - return None - if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): - return _resolve_alias( - alias_map_flow, - _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), - ) - # Try to interpret as a child metric UUID first; fall back - # to treating it as a slug. - try: - child_uuid = UUID(raw_clean) - except (TypeError, ValueError): - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - child = ( - db.query(Metric.name) - .filter( - Metric.id == child_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if child and child[0]: - return _resolve_alias(alias_map_flow, _slug_label(child[0])) - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - - from_slug = _flow_node_to_slug(flow_node) - target_slug: Optional[str] = None - if flow_edge_target and flow_edge_target.strip(): - target_slug = _flow_node_to_slug(flow_edge_target) - - if from_slug: - # The ``metric_scores`` column is declared as ``Column(JSON)`` - # in the model so on databases where the table was created - # from the model (rather than the migration) the physical - # type is ``json``, not ``jsonb``. The JSONB-only operators - # below (``jsonb_exists``, ``jsonb_array_elements_text``, - # ``@>``) require a JSONB input — we cast once up front so - # the same SQL works regardless of which path created the - # table. - scores_jsonb = ( - "(call_import_evaluation_rows.metric_scores)::jsonb" - ) - if target_slug: - # Edge filter: rows whose sequence under this parent - # contains ``from_slug`` immediately followed by - # ``target_slug``. Implemented as a correlated EXISTS - # over ``jsonb_array_elements_text`` with ORDINALITY, - # which is the portable way to express "next array - # index" against a JSONB array in Postgres. - edge_filter_sql = text( - f""" - EXISTS ( - SELECT 1 - FROM jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s1(elem, ord) - JOIN jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s2(elem, ord) - ON s2.ord = s1.ord + 1 - WHERE s1.elem = :from_slug - AND s2.elem = :to_slug - ) - """ - ).bindparams( - p_id=parent_id_str_local, - from_slug=from_slug, - to_slug=target_slug, - ) - query = query.filter(edge_filter_sql) - else: - # Node filter: rows whose ``metric_scores -> parent -> - # 'sequence'`` array contains ``from_slug``. We use the - # function form ``jsonb_exists`` rather than the ``?`` - # operator to avoid psycopg2 mistaking the question - # mark for a parameter placeholder. - node_filter_sql = text( - f""" - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - """ - ).bindparams(p_id=parent_id_str_local, slug=from_slug) - query = query.filter(node_filter_sql) - - # --- Discovered label filters --------------------------------------- - # Surfaces "which calls produced THIS LLM-discovered label" and the - # broader "which calls produced ANY LLM-discovered label". Both - # operate on ``metric_scores[].discovered_labels`` (a list - # of dicts) plus the same ``sequence`` array — covering both legacy - # rows where the slug only made it into ``sequence`` and newer - # rows where it landed in both. - if discovered_parent_id is not None and ( - discovered_label_key or has_discovered - ): - d_parent_str = str(discovered_parent_id) - alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) - # See note above: cast once so the JSONB operators don't reject - # the column when it's typed as ``json`` in the database. - scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" - if discovered_label_key and discovered_label_key.strip(): - target = _resolve_alias( - alias_map_disc, _slug_label(discovered_label_key) - ) - if target: - # Match rows whose discovered_labels list has an entry - # ``{"key": }`` OR whose sequence array still - # contains the slug. The latter covers older rows that - # were rewritten by a merge in the discovered_labels - # blob but whose sequence may have lagged. - contains_json = json.dumps( - {d_parent_str: {"discovered_labels": [{"key": target}]}} - ) - disc_filter_sql = text( - f""" - ( - {scores_jsonb} @> CAST(:contains AS JSONB) - OR - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - ) - """ - ).bindparams( - contains=contains_json, - p_id=d_parent_str, - slug=target, - ) - query = query.filter(disc_filter_sql) - elif has_discovered: - # No specific slug — just rows that surfaced any candidate - # under this parent. We coalesce missing paths to ``[]`` so - # ``jsonb_array_length`` always sees an array (it raises on - # non-array inputs, but our shape guarantees a list when - # the key is present). - has_disc_sql = text( - f""" - jsonb_array_length( - COALESCE( - {scores_jsonb} -> :p_id -> 'discovered_labels', - '[]'::jsonb - ) - ) > 0 - """ - ).bindparams(p_id=d_parent_str) - query = query.filter(has_disc_sql) - - # --- Sorting ---------------------------------------------------------- - # Column-click sorting from the UI. Falls back to ``row_index`` so - # paging stays stable when the user clears the sort. We always add a - # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. - # many rows with ``status = 'completed'``) keep a deterministic - # order across page boundaries — without this, pagination can - # double-show or skip rows when Postgres picks a different physical - # order on each query. - direction_desc = (sort_dir or "asc").strip().lower() == "desc" - - def _apply_direction(column_expr): - return column_expr.desc() if direction_desc else column_expr.asc() - - # Whether the caller's ``sort_by`` resolved to a known column. We - # use this flag to decide whether ``sort_dir`` is honoured on the - # fallback path: unrecognized columns (typos, stale UI state) fall - # back to the implicit ``row_index ASC`` default and intentionally - # ignore ``sort_dir`` so users don't get a surprise reverse order - # from a typo'd column name. - sort_recognized = False - sort_by_clean = (sort_by or "").strip() - primary_sort = None - metric_uuid: Optional[UUID] = None - if sort_by_clean == "row_index": - sort_recognized = True - # Falls through to the default ``order_by`` below with - # ``primary_sort`` still None — but ``sort_recognized=True`` - # tells the fallback branch to apply the requested direction. - elif sort_by_clean == "conversation_id": - sort_recognized = True - primary_sort = _apply_direction(CallImportRow.conversation_id) - elif sort_by_clean == "status": - sort_recognized = True - primary_sort = _apply_direction(CallImportEvaluationRow.status) - elif sort_by_clean.startswith("metric:"): - raw_metric_id = sort_by_clean.split(":", 1)[1].strip() - try: - metric_uuid = UUID(raw_metric_id) - except (TypeError, ValueError): - metric_uuid = None - if metric_uuid is not None: - sort_recognized = True - # ``metric_scores`` is JSON-typed but the helper functions - # for path extraction differ between Postgres (production) - # and SQLite (default test backend). Branch on the active - # dialect so we can use the right primitive: - # * Postgres → ``json_extract_path_text(col, key, "value")`` - # which returns the value as TEXT for both ``json`` and - # ``jsonb`` columns. - # * SQLite → ``json_extract(col, '$."".value')`` - # using JSONPath syntax. ``metric_uuid`` is already - # validated above (``UUID(raw_metric_id)``), so the - # interpolated path is safe from injection. - # NULL values (rows where the metric wasn't scored) sort - # to the END regardless of direction so un-scored rows - # don't crowd the top of an ascending sort. - dialect_name = ( - db.bind.dialect.name if db.bind is not None else "postgresql" - ) - if dialect_name == "sqlite": - json_path = f'$."{metric_uuid}".value' - path_value = func.json_extract( - CallImportEvaluationRow.metric_scores, - json_path, - ) - else: - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_uuid), - "value", - ) - primary_sort = ( - path_value.desc().nullslast() - if direction_desc - else path_value.asc().nullslast() - ) - - if primary_sort is not None: - query = query.order_by(primary_sort, CallImportRow.row_index.asc()) - elif sort_recognized: - # Explicit ``sort_by=row_index`` request — honour direction. - query = query.order_by(_apply_direction(CallImportRow.row_index)) - else: - # No sort requested OR unrecognized column — safe default of - # ``row_index ASC``. We deliberately ignore ``sort_dir`` here - # so a typo'd / stale ``sort_by`` doesn't quietly invert the - # default order. - query = query.order_by(CallImportRow.row_index.asc()) - from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page - from app.db_sharding.sessions import is_sharding_enabled - - def _pair_row_index( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> int: - return int(pair[1].row_index or 0) - - def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: - text = value or "" - if not desc: - return (0, *text.encode("utf-8")) - return (1, *(-byte for byte in text.encode("utf-8"))) - - if sort_by_clean == "conversation_id": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[1].conversation_id, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean == "status": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[0].status, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean.startswith("metric:") and metric_uuid is not None: - metric_id_str = str(metric_uuid) - - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - scores = pair[0].metric_scores or {} - entry = scores.get(metric_id_str, {}) - raw_value = entry.get("value") if isinstance(entry, dict) else None - null_rank = 1 if raw_value is None else 0 - return ( - null_rank, - _directed_string( - str(raw_value) if raw_value is not None else None, - direction_desc, - ), - _pair_row_index(pair), - ) - elif sort_recognized and sort_by_clean == "row_index": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - idx = _pair_row_index(pair) - return (-idx,) if direction_desc else (idx,) - else: - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - return (_pair_row_index(pair),) - - if is_sharding_enabled(): - def _build_query(session: Session): - return query.with_session(session) - - total, rows = fetch_evaluation_row_pairs_page( - db, - _build_query, - page=page, - page_size=page_size, - sort_key=_pair_sort_key, - bounded_shard_fetch=( - not sort_recognized or sort_by_clean == "row_index" - ), - row_index_desc=bool( - sort_recognized and sort_by_clean == "row_index" and direction_desc - ), - sort_reads_scores=sort_by_clean.startswith("metric:"), - ) - else: - from app.db_sharding.scatter_gather import defer_call_import_row_text - - total = query.count() - rows = ( - query.options(*defer_call_import_row_text()) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - - items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response( - eval_row_obj, - source_row, - eval_row, - include_text=False, - ) - for eval_row_obj, source_row in rows - ] - - return CallImportEvaluationRowListResponse( - items=items, - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/{eval_id}/rows/{row_id}", - response_model=CallImportEvaluationRowResponse, - operation_id="getCallImportEvaluationRow", -) -async def get_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """One evaluation row, including the transcript the list leaves off.""" - del api_key - _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if evaluation is None: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from app.db_sharding.sessions import is_sharding_enabled - - if not is_sharding_enabled(): - pair = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter( - CallImportEvaluationRow.id == row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - CallImportRow.call_import_id == call_import_id, - ) - .first() - ) - if pair is None: - raise HTTPException( - status_code=404, detail="Call import evaluation row not found" - ) - eval_row, source_row = pair - return _to_evaluation_row_response( - eval_row, - source_row, - evaluation, - include_text=True, - ) - - from app.db_sharding.row_ops import ( - close_row_sessions, - locate_call_import_evaluation_row, - ) - - try: - row_db, catalog_db, eval_row, source_row, _shard_id = ( - locate_call_import_evaluation_row(row_id) - ) - except LookupError: - raise HTTPException( - status_code=404, detail="Call import evaluation row not found" - ) from None - try: - if ( - eval_row.evaluation_id != eval_id - or source_row.call_import_id != call_import_id - ): - raise HTTPException( - status_code=404, detail="Call import evaluation row not found" - ) - return _to_evaluation_row_response( - eval_row, - source_row, - evaluation, - include_text=True, - ) - finally: - close_row_sessions(row_db, catalog_db) - - -@router.post( - "/{eval_id}/export", - response_model=CallImportEvaluationExportJobResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="startCallImportEvaluationExport", -) -async def start_call_import_evaluation_export( - call_import_id: UUID, - eval_id: UUID, - payload: Optional[CallImportEvaluationExportRequest] = Body(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationExportJobResponse: - """Queue a gzip CSV (or scores-only xlsx) export and return the job. - - The file is built off this request. Poll the job, then download each - part from blob storage. - """ - del api_key - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from sqlalchemy.exc import IntegrityError - - from app.models.database import CallImportEvaluationExport - from app.services.call_imports.evaluation_export import ( - XLSX_SHEET_ROW_CAP, - execute_evaluation_export, - export_job_is_stale, - ) - from app.services.storage.blob_storage_service import blob_storage_service - - body = payload or CallImportEvaluationExportRequest() - if body.format == "xlsx": - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - - row_count = count_evaluation_rows_for_run(db, eval_id) - if row_count > XLSX_SHEET_ROW_CAP: - raise HTTPException( - status_code=400, - detail=( - "This run has more rows than Excel allows on one sheet. " - "Download the CSV instead." - ), - ) - - active = _active_export_job(db, eval_id) - if active is not None and not _export_request_matches(active, body): - was_stale = export_job_is_stale(active) - if was_stale and _fail_stale_export_job(db, active): - active = None - elif was_stale: - active = _active_export_job(db, eval_id) - if active is not None and not _export_request_matches(active, body): - raise HTTPException( - status_code=409, - detail=( - "An export is already running for this evaluation. " - "Wait for it to finish before starting a different download." - ), - ) - if active is not None: - if blob_storage_service.is_enabled() and ( - active.status == "pending" or export_job_is_stale(active) - ): - try: - _enqueue_evaluation_export(str(active.id)) - except Exception as exc: - if active.status == "pending": - active.status = "failed" - active.error_message = f"Could not queue the export: {exc}" - active.finished_at = datetime.now(timezone.utc) - db.commit() - db.refresh(active) - return _export_job_response(active) - - job = CallImportEvaluationExport( - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - format=body.format, - include_transcripts=body.include_transcripts, - status="pending", - parts=[], - ) - db.add(job) - try: - db.commit() - except IntegrityError: - db.rollback() - active = _active_export_job(db, eval_id) - if active is not None and _export_request_matches(active, body): - return _export_job_response(active) - if active is not None: - raise HTTPException( - status_code=409, - detail=( - "An export is already running for this evaluation. " - "Wait for it to finish before starting a different download." - ), - ) - raise - db.refresh(job) - - if not blob_storage_service.is_enabled(): - execute_evaluation_export(db, job.id) - db.refresh(job) - return _export_job_response(job) - - try: - _enqueue_evaluation_export(str(job.id)) - except Exception as exc: - job.status = "failed" - job.error_message = f"Could not queue the export: {exc}" - job.finished_at = datetime.now(timezone.utc) - db.commit() - db.refresh(job) - return _export_job_response(job) - - -@router.get( - "/{eval_id}/export-jobs/{job_id}", - response_model=CallImportEvaluationExportJobResponse, - operation_id="getCallImportEvaluationExport", -) -async def get_call_import_evaluation_export( - call_import_id: UUID, - eval_id: UUID, - job_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationExportJobResponse: - del api_key - _require_import(db, call_import_id, organization_id) - from app.models.database import CallImportEvaluationExport - - job = ( - db.query(CallImportEvaluationExport) - .filter( - CallImportEvaluationExport.id == job_id, - CallImportEvaluationExport.evaluation_id == eval_id, - CallImportEvaluationExport.call_import_id == call_import_id, - CallImportEvaluationExport.organization_id == organization_id, - ) - .first() - ) - if job is None: - raise HTTPException(status_code=404, detail="Export job not found") - return _export_job_response(job, with_download_urls=True) - - -def _fail_stale_export_job(db: Session, job) -> bool: - """Fail a stale running export only while that same claim still holds it. - - The heartbeat and claim are the values already loaded on ``job``. A - worker that claimed the export after this read changes both, the - update matches nothing, and the caller re-reads the live job. - """ - from sqlalchemy import update - - from app.models.database import CallImportEvaluationExport - - observed_heartbeat = job.heartbeat_at - observed_claim = job.claim_token - heartbeat_still = ( - CallImportEvaluationExport.heartbeat_at.is_(None) - if observed_heartbeat is None - else CallImportEvaluationExport.heartbeat_at == observed_heartbeat - ) - claim_still = ( - CallImportEvaluationExport.claim_token.is_(None) - if observed_claim is None - else CallImportEvaluationExport.claim_token == observed_claim - ) - result = db.execute( - update(CallImportEvaluationExport) - .where( - CallImportEvaluationExport.id == job.id, - CallImportEvaluationExport.status == "running", - heartbeat_still, - claim_still, - ) - .values( - status="failed", - error_message="Stopped because a different export was requested.", - finished_at=datetime.now(timezone.utc), - ) - .execution_options(synchronize_session=False) - ) - db.commit() - return bool(result.rowcount) - - -def _export_request_matches(job, body) -> bool: - return (job.format or "csv") == body.format and bool(job.include_transcripts) == bool( - body.include_transcripts - ) - - -def _active_export_job(db: Session, evaluation_id: UUID): - from app.models.database import CallImportEvaluationExport - - return ( - db.query(CallImportEvaluationExport) - .filter( - CallImportEvaluationExport.evaluation_id == evaluation_id, - CallImportEvaluationExport.status.in_(("pending", "running")), - ) - .first() - ) - - -def _enqueue_evaluation_export(job_id: str) -> None: - from app.workers.tasks.call_import_bulk_ops import ( - export_call_import_evaluation_task, - ) - - export_call_import_evaluation_task.delay(job_id) - - -def _export_job_response(job, *, with_download_urls: bool = False): - from app.models.schemas import ( - CallImportEvaluationExportJobResponse, - CallImportEvaluationExportPart, - ) - from app.services.call_imports.evaluation_export import EXPORT_URL_TTL_SECONDS - from app.services.storage.blob_storage_service import blob_storage_service - - parts = [] - raw_parts = job.parts if isinstance(job.parts, list) else [] - for item in raw_parts: - if not isinstance(item, dict): - continue - download_url = None - key = item.get("s3_key") - filename = item.get("filename") or "export.csv.gz" - if ( - with_download_urls - and job.status == "completed" - and key - and blob_storage_service.is_enabled() - ): - download_url = blob_storage_service.generate_presigned_url_by_key( - key, - expiration=EXPORT_URL_TTL_SECONDS, - response_content_disposition=f'attachment; filename="{filename}"', - ) - parts.append( - CallImportEvaluationExportPart( - part=int(item.get("part") or 0), - filename=filename, - row_count=int(item.get("row_count") or 0), - byte_size=int(item.get("byte_size") or 0), - download_url=download_url, - ) - ) - return CallImportEvaluationExportJobResponse( - id=job.id, - evaluation_id=job.evaluation_id, - status=job.status, - format=job.format, - include_transcripts=bool(job.include_transcripts), - error_message=job.error_message, - parts=parts, - ) - - -def _report_filename_slug(value: str) -> str: - slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") - return slug or "client" - - -def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: - created_by = principal.email - if not created_by and principal.user_id: - created_by = str(principal.user_id) - return created_by, principal.user_id - - -def _pdf_report_response_from_row( - row: CallImportEvaluationPdfReport, - *, - cache_hit: bool = False, -) -> CallImportEvaluationPdfReportResponse: - filename = row.filename or "report.pdf" - preview_url, download_url = presigned_urls_for_pdf_report( - row.s3_key or "", - filename, - ) - return CallImportEvaluationPdfReportResponse( - id=str(row.id), - filename=filename, - preview_url=preview_url, - download_url=download_url, - created_at=row.created_at or datetime.now(timezone.utc), - created_by=row.created_by, - report_type=row.report_type, - vendor_name=row.vendor_name, - config_summary=config_summary_from_report_config( - row.report_config if isinstance(row.report_config, dict) else {} - ), - storage_available=bool(row.s3_key), - cache_hit=cache_hit, - ) - - -def _pdf_report_list_item_from_row( - row: CallImportEvaluationPdfReport, -) -> CallImportEvaluationPdfReportListItem: - return CallImportEvaluationPdfReportListItem( - id=str(row.id), - filename=row.filename, - vendor_name=row.vendor_name, - report_type=row.report_type, - created_by=row.created_by, - created_at=row.created_at or datetime.now(timezone.utc), - config_summary=config_summary_from_report_config( - row.report_config if isinstance(row.report_config, dict) else {} - ), - cache_fingerprint=row.cache_fingerprint, - ) - - -def _report_branding_for_import_workspace( - db: Session, - organization_id: UUID, - workspace_id: UUID, - *, - internal_brand_image_id: Optional[str] = None, - external_brand_image_id: Optional[str] = None, -) -> tuple[dict[str, str] | list[str], Optional[str]]: - workspace = ( - db.query(Workspace) - .filter( - Workspace.id == workspace_id, - Workspace.organization_id == organization_id, - ) - .first() - ) - raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} - images = raw.get("images") if isinstance(raw.get("images"), list) else [] - loaded_images: list[dict[str, str]] = [] - for item in images: - if not isinstance(item, dict) or not item.get("s3_key"): - continue - content_type = str(item.get("content_type") or "image/png") - try: - from app.services.storage.s3_service import s3_service - - image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Unable to load report branding image for workspace {}: {}", - workspace_id, - exc, - ) - continue - encoded = base64.b64encode(image_bytes).decode("ascii") - role = str(item.get("role") or "generic") - if role not in {"internal", "external", "generic"}: - role = "generic" - loaded_images.append( - { - "id": str(item.get("id") or ""), - "role": role, - "data_uri": f"data:{content_type};base64,{encoded}", - } - ) - - def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: - if selected_id: - for loaded in loaded_images: - if loaded["id"] == selected_id: - return loaded["data_uri"] - for loaded in loaded_images: - if loaded["role"] == role: - return loaded["data_uri"] - return None - - logo_data_uris: dict[str, str] = {} - internal_uri = _pick("internal", internal_brand_image_id) - external_uri = _pick("external", external_brand_image_id) - if internal_uri: - logo_data_uris["internal"] = internal_uri - if external_uri: - logo_data_uris["external"] = external_uri - if ( - not logo_data_uris - and not internal_brand_image_id - and not external_brand_image_id - ): - # Backward compatibility for workspaces that only had a generic logo - # library before the two-slot report header existed. - generic_uris = [ - loaded["data_uri"] - for loaded in loaded_images - if loaded.get("data_uri") - ] - if generic_uris: - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return generic_uris[:4], heading - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return logo_data_uris, heading - - -def _display_metrics_for_pdf_report( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, -) -> list[Metric]: - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - parent_id = UUID(parent_str) - except (TypeError, ValueError): - continue - if parent_id not in lookup_ids: - lookup_ids.append(parent_id) - - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - child_ids_in_groups: set[str] = set() - for child_strs in groups_raw.values(): - if not isinstance(child_strs, list): - continue - child_ids_in_groups.update(str(child_id) for child_id in child_strs) - - metrics_by_id = {str(metric.id): metric for metric in metrics} - display: list[Metric] = [] - seen: set[str] = set() - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(str(parent_str)) - if parent and str(parent.id) not in seen: - display.append(parent) - seen.add(str(parent.id)) - - for metric in metrics: - metric_id = str(metric.id) - if metric_id in seen or metric_id in child_ids_in_groups: - continue - if metric.selection_mode and not metric.parent_metric_id: - continue - display.append(metric) - seen.add(metric_id) - - return display - - -def _metrics_for_clustering( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[Metric]: - """All enabled quality metrics scored in this run, normalized for clustering. - - Hierarchical children are collapsed to their parent metric so cluster - groups render at the category level (e.g. ``AI reveal``) instead of the - child label level (e.g. ``Yes`` / ``No``). - """ - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - aggregate_metric_ids: List[UUID] = [] - for agg in aggregates: - if (agg.metric_category or "quality") == "user_insight": - continue - try: - aggregate_metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - if not aggregate_metric_ids: - return [] - - aggregate_metrics = _metrics_for_ids( - db, evaluation.organization_id, aggregate_metric_ids - ) - by_id = {metric.id: metric for metric in aggregate_metrics} - - normalized_ids: List[UUID] = [] - seen: set[UUID] = set() - for metric_id in aggregate_metric_ids: - metric = by_id.get(metric_id) - target_id = ( - metric.parent_metric_id - if metric is not None and metric.parent_metric_id - else metric_id - ) - if target_id in seen: - continue - seen.add(target_id) - normalized_ids.append(target_id) - - metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) - return [ - metric - for metric in metrics - if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) - ] - - -def _metric_is_user_insight(metric: Metric) -> bool: - if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": - return True - text_value = " ".join( - str(part or "").lower() - for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) - ) - normalized = text_value.replace("-", " ").replace("_", " ") - phrases = ( - "call context", - "caller context", - "product identification", - "out of scope", - "identity match", - "user identity", - "caller identity", - "frustration trigger", - "video call offer", - "video call reception", - ) - return any(phrase in normalized for phrase in phrases) - - -def _evaluation_rows_for_period( - db: Session, - evaluation_id: UUID, -) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: - return ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - -def _baseline_candidate_evaluations( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - *, - limit: int = 20, -) -> list[dict[str, Any]]: - candidates = ( - db.query(CallImportEvaluation, CallImport) - .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) - .filter( - CallImportEvaluation.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImportEvaluation.id != current_evaluation.id, - CallImportEvaluation.status == "completed", - CallImportEvaluation.completed_rows > 0, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .limit(limit * 3) - .all() - ) - items: list[dict[str, Any]] = [] - for candidate_eval, candidate_import in candidates: - rows = _evaluation_rows_for_period(db, candidate_eval.id) - period_start, period_end, period_label, period_display = _report_period_from_rows(rows) - if current_period_start and period_start and period_start >= current_period_start: - continue - dataset = ( - (candidate_import.dataset or "").strip() - or (candidate_import.original_filename or candidate_import.filename or "").strip() - or "Unknown dataset" - ) - evaluation_name = ( - (candidate_eval.name or "").strip() - or str(candidate_eval.id)[:8] - ) - items.append( - { - "evaluation_id": str(candidate_eval.id), - "name": evaluation_name, - "dataset": dataset, - "period_label": period_label, - "period_start": period_start, - "period_end": period_end, - "period_display": period_display, - "completed_rows": int(candidate_eval.completed_rows or 0), - "created_at": candidate_eval.created_at, - "is_default": False, - } - ) - if len(items) >= limit: - break - items.sort( - key=lambda item: ( - item["period_start"] or date.min, - item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), - ), - reverse=True, - ) - if items: - items[0]["is_default"] = True - return items - - -def _resolve_baseline_evaluation( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - baseline_evaluation_id: Optional[str], -) -> Optional[CallImportEvaluation]: - candidates = _baseline_candidate_evaluations( - db, - organization_id, - workspace_id, - current_evaluation, - current_period_start, - ) - allowed_ids = {item["evaluation_id"] for item in candidates} - if baseline_evaluation_id: - baseline_id = str(baseline_evaluation_id).strip() - if baseline_id not in allowed_ids: - raise HTTPException( - status_code=400, - detail="Selected baseline evaluation is not a valid prior run for this report.", - ) - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(baseline_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not candidates: - return None - default_id = candidates[0]["evaluation_id"] - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(default_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - - -def _benchmark_context_for_evaluation( - db: Session, - baseline_evaluation: Optional[CallImportEvaluation], -) -> Optional[dict[str, str]]: - if baseline_evaluation is None: - return None - baseline_import = ( - db.query(CallImport) - .filter(CallImport.id == baseline_evaluation.call_import_id) - .first() - ) - rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) - dataset = ( - (baseline_import.dataset or "").strip() - if baseline_import and baseline_import.dataset - else None - ) - filename = ( - (baseline_import.original_filename or baseline_import.filename or "").strip() - if baseline_import - else None - ) - evaluation_label = ( - (baseline_evaluation.name or "").strip() - if baseline_evaluation.name - else str(baseline_evaluation.id)[:8] - ) - period = period_label or ( - period_start.isoformat() if period_start else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(baseline_evaluation.id), - "period": period, - } - - -def _period_deltas_from_evaluation( - db: Session, - baseline_evaluation: CallImportEvaluation, - current_metric_aggregates: list[dict[str, Any]], - current_evaluation: CallImportEvaluation, - current_eval_rows: List[CallImportEvaluationRow], -) -> dict[str, dict[str, str]]: - baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] - baseline_aggregate_models = _compute_metric_aggregates( - db, - baseline_evaluation, - baseline_eval_rows, - ) - baseline_metric_aggregates = [ - _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models - ] - _metrics, _aggs, policies, _source, _child_map = _clustering_context( - db, current_evaluation, current_eval_rows - ) - metric_by_id = {str(m.id): m for m in _metrics} - current_by_id = { - str(item.get("metric_id")): item for item in current_metric_aggregates - } - previous_by_id = { - str(item.get("metric_id")): item - for item in baseline_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - metric = metric_by_id.get(metric_id) - policy = policies.get(metric_id) - previous_raw = previous_by_id.get(metric_id) - if metric is None or policy is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - current_pct = failure_rate_percent_from_rows( - current_eval_rows, metric, policy - ) - previous_pct = failure_rate_percent_from_rows( - baseline_eval_rows, metric, policy - ) - if current_pct is None or previous_pct is None: - current_pct = current_pct or _aggregate_primary_percent(current, policy) - previous_pct = ( - previous_pct or _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": ( - f"Current report {current_pct:.1f}% vs previous report " - f"{previous_pct:.1f}%" - ), - } - return deltas - - -_DELTA_EXPLANATION_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will receive " - "week-over-week metric failure-rate deltas plus reconciled failure " - "cluster context per metric.\n\n" - "Return STRICT JSON only:\n" - "{\n" - ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' - "}\n\n" - "Constraints:\n" - "- Only include metrics supplied in the prompt.\n" - "- Cluster labels are generated independently each run and are NOT stable " - "IDs. Never compare an unmatched current label to 0% baseline.\n" - "- Use matched_theme_shifts for label-aligned comparisons, " - "gap_label_shifts for structural shifts, and new_themes_current_period " - "for themes that emerged without a baseline match.\n" - "- If reconciliation is uncertain, explain using the numeric delta and " - "gap_label_shifts only.\n" - "- Keep each explanation to 1-2 short sentences (~220 chars).\n" - "- Vendor-safe, factual language; no markdown." -) - - -def _period_delta_explanation_cache_key( - baseline_evaluation_id: UUID, - *, - completed_rows: int, - baseline_completed_rows: int, -) -> str: - return ( - f"{baseline_evaluation_id}:{completed_rows}:" - f"{baseline_completed_rows}:reconciled-v2" - ) - - -def _normalize_cluster_label(label: str) -> str: - return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() - - -_CLUSTER_LABEL_STOPWORDS = frozenset( - { - "a", - "an", - "the", - "and", - "or", - "during", - "while", - "with", - "for", - "from", - "into", - "general", - "user", - "bot", - "agent", - } -) - - -def _cluster_label_tokens(label: str) -> set[str]: - return { - token - for token in _normalize_cluster_label(label).split() - if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 - } - - -def _cluster_label_similarity(left: str, right: str) -> float: - tokens_left = _cluster_label_tokens(left) - tokens_right = _cluster_label_tokens(right) - if not tokens_left or not tokens_right: - return 0.0 - intersection = tokens_left & tokens_right - if not intersection: - return 0.0 - union = tokens_left | tokens_right - jaccard = len(intersection) / len(union) - smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right - overlap_ratio = len(intersection) / len(smaller) - return max(jaccard, overlap_ratio * 0.85) - - -def _group_clusters_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, list[dict[str, Any]]]: - grouped: dict[str, list[dict[str, Any]]] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - grouped.setdefault(gap_label, []).append(cluster) - return grouped - - -def _append_matched_cluster_pair( - matched: list[dict[str, Any]], - current: dict[str, Any], - baseline: dict[str, Any], - *, - match_confidence: float, - match_method: str, -) -> None: - matched.append( - { - "current_label": current.get("label"), - "baseline_label": baseline.get("label"), - "gap_label": current.get("gap_label") or baseline.get("gap_label"), - "current_share_pct": current.get("share_pct"), - "baseline_share_pct": baseline.get("share_pct"), - "share_delta_pp": round( - float(current.get("share_pct") or 0.0) - - float(baseline.get("share_pct") or 0.0), - 1, - ), - "match_confidence": round(match_confidence, 2), - "match_method": match_method, - } - ) - - -def _aggregate_share_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, float]: - totals: dict[str, float] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - totals[gap_label] = totals.get(gap_label, 0.0) + float( - cluster.get("share_pct") or 0.0 - ) - return {gap: round(share, 1) for gap, share in totals.items()} - - -def _reconcile_cluster_periods( - current_clusters: list[dict[str, Any]], - baseline_clusters: list[dict[str, Any]], - *, - similarity_threshold: float = 0.35, -) -> dict[str, Any]: - """Align independently-generated cluster labels before delta explanation.""" - matched: list[dict[str, Any]] = [] - current_unmatched = list(current_clusters) - remaining_baseline = list(baseline_clusters) - - current_by_gap = _group_clusters_by_gap_label(current_unmatched) - baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) - for gap_label in list(current_by_gap): - current_group = current_by_gap.get(gap_label) or [] - baseline_group = baseline_by_gap.get(gap_label) or [] - if len(current_group) != 1 or len(baseline_group) != 1: - continue - current = current_group[0] - baseline = baseline_group[0] - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=0.75, - match_method="single_cluster_per_gap_label", - ) - current_unmatched.remove(current) - remaining_baseline.remove(baseline) - current_by_gap[gap_label] = [] - baseline_by_gap[gap_label] = [] - - for current in list(current_unmatched): - best_idx: Optional[int] = None - best_score = 0.0 - for idx, baseline in enumerate(remaining_baseline): - score = _cluster_label_similarity( - str(current.get("label") or ""), - str(baseline.get("label") or ""), - ) - if current.get("gap_label") == baseline.get("gap_label"): - score += 0.1 - if score > best_score: - best_score = score - best_idx = idx - - if best_idx is not None and best_score >= similarity_threshold: - baseline = remaining_baseline.pop(best_idx) - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=best_score, - match_method="label_similarity", - ) - - matched_current_labels = { - str(item.get("current_label") or "") for item in matched - } - matched_baseline_labels = { - str(item.get("baseline_label") or "") for item in matched - } - current_unmatched = [ - cluster - for cluster in current_clusters - if str(cluster.get("label") or "") not in matched_current_labels - ] - remaining_baseline = [ - cluster - for cluster in baseline_clusters - if str(cluster.get("label") or "") not in matched_baseline_labels - ] - - new_themes = [ - { - "label": cluster.get("label"), - "gap_label": cluster.get("gap_label"), - "share_pct": cluster.get("share_pct"), - "note": "New theme in current period (no close baseline match).", - } - for cluster in current_unmatched - ] - - retired_themes = [ - { - "label": baseline.get("label"), - "gap_label": baseline.get("gap_label"), - "share_pct": baseline.get("share_pct"), - "note": "Theme present in baseline only (retired or renamed).", - } - for baseline in remaining_baseline - ] - - current_gap = _aggregate_share_by_gap_label(current_clusters) - baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) - gap_label_shifts: dict[str, dict[str, float]] = {} - for gap_label in set(current_gap) | set(baseline_gap): - current_share = current_gap.get(gap_label, 0.0) - baseline_share = baseline_gap.get(gap_label, 0.0) - if abs(current_share - baseline_share) >= 0.5: - gap_label_shifts[gap_label] = { - "current_share_pct": current_share, - "baseline_share_pct": baseline_share, - "share_delta_pp": round(current_share - baseline_share, 1), - } - - return { - "matched_theme_shifts": matched, - "new_themes_current_period": new_themes, - "retired_themes_baseline_period": retired_themes, - "gap_label_shifts": gap_label_shifts, - "reconciliation_note": ( - "Cluster labels are generated independently each run and may " - "rename the same failure mode. Do not treat unmatched current " - "labels as 0% in the baseline period." - ), - } - - -def _load_period_delta_explanations_cache( - evaluation: CallImportEvaluation, - cache_key: str, -) -> Optional[dict[str, str]]: - raw = getattr(evaluation, "period_delta_explanations", None) - if not isinstance(raw, dict): - return None - entry = raw.get(cache_key) - if not isinstance(entry, dict): - return None - explanations_raw = entry.get("explanations") - if not isinstance(explanations_raw, dict): - return None - return { - str(metric_id): str(why).strip() - for metric_id, why in explanations_raw.items() - if str(metric_id).strip() and isinstance(why, str) and why.strip() - } - - -def _save_period_delta_explanations_cache( - db: Session, - evaluation: CallImportEvaluation, - cache_key: str, - explanations: dict[str, str], -) -> None: - raw = evaluation.period_delta_explanations - if not isinstance(raw, dict): - raw = {} - updated = dict(raw) - updated[cache_key] = { - "explanations": explanations, - "generated_at": datetime.now(timezone.utc).isoformat(), - } - evaluation.period_delta_explanations = updated - flag_modified(evaluation, "period_delta_explanations") - db.commit() - - -def _cluster_summary_for_metric( - state: Optional[EvaluationMetricClustersState], - metric_id: str, -) -> list[dict[str, Any]]: - if state is None or state.status != "completed": - return [] - for group in state.groups: - if str(group.metric_id) != metric_id: - continue - return [ - { - "label": cluster.label, - "gap_label": cluster.gap_label, - "share_pct": round(cluster.share_pct, 1), - "count": cluster.count, - } - for cluster in group.clusters[:5] - ] - return [] - - -def _merge_delta_why( - raw_deltas: dict[str, dict[str, str]], - explanations: dict[str, str], -) -> dict[str, dict[str, str]]: - if not explanations: - return raw_deltas - merged: dict[str, dict[str, str]] = {} - for metric_id, delta in raw_deltas.items(): - updated = dict(delta) - why = explanations.get(metric_id) - if why: - updated["why"] = why - merged[metric_id] = updated - return merged - - -def _explain_period_deltas( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], - *, - min_delta_pp: float = 0.5, -) -> dict[str, dict[str, str]]: - """Attach ``why`` explanations to period deltas using cached LLM output.""" - if not raw_deltas: - return raw_deltas - - cache_key = _period_delta_explanation_cache_key( - baseline_evaluation.id, - completed_rows=evaluation.completed_rows, - baseline_completed_rows=baseline_evaluation.completed_rows, - ) - cached = _load_period_delta_explanations_cache(evaluation, cache_key) - if cached is not None: - return _merge_delta_why(raw_deltas, cached) - - current_clusters = _metric_clusters_payload(evaluation) - baseline_clusters = _metric_clusters_payload(baseline_evaluation) - metrics_for_prompt: list[dict[str, Any]] = [] - for metric_id, delta in raw_deltas.items(): - label = delta.get("label") or "" - if "No previous-week baseline" in label: - continue - match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) - if match and abs(float(match.group(1))) < min_delta_pp: - continue - current_summary = _cluster_summary_for_metric(current_clusters, metric_id) - baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) - if not current_summary and not baseline_summary: - continue - cluster_reconciliation = _reconcile_cluster_periods( - current_summary, - baseline_summary, - ) - metrics_for_prompt.append( - { - "metric_id": metric_id, - "delta_label": label, - "delta_detail": delta.get("detail") or "", - "cluster_reconciliation": cluster_reconciliation, - } - ) - - if not metrics_for_prompt: - return raw_deltas - - provider_hint: Optional[str] = None - model_hint: Optional[str] = None - tldr_raw = evaluation.tldr_summary - if isinstance(tldr_raw, dict): - if isinstance(tldr_raw.get("provider"), str): - provider_hint = tldr_raw["provider"] - if isinstance(tldr_raw.get("model"), str): - model_hint = tldr_raw["model"] - - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - from app.services.call_import_user_insights import _call_llm, _parse_json_object - from app.services.usage.call_import_context import ( - call_import_evaluation_usage_context, - ) - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, provider_hint, model_hint - ) - usage_ctx = call_import_evaluation_usage_context( - organization_id=organization_id, - workspace_id=evaluation.workspace_id, - evaluation_id=evaluation.id, - call_import_id=evaluation.call_import_id, - ) - try: - text = _call_llm( - db, - organization_id, - provider_enum, - model_str, - [ - {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, - { - "role": "user", - "content": json.dumps( - {"metrics": metrics_for_prompt}, - ensure_ascii=False, - default=str, - ), - }, - ], - temperature=0.3, - max_tokens=900, - usage_ctx=usage_ctx, - ) - except Exception as exc: - logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) - return raw_deltas - - parsed = _parse_json_object(text) - explanations_raw = parsed.get("explanations") - explanations: dict[str, str] = {} - if isinstance(explanations_raw, dict): - for metric_id, why in explanations_raw.items(): - if isinstance(why, str) and why.strip(): - explanations[str(metric_id)] = why.strip() - - if explanations: - _save_period_delta_explanations_cache( - db, evaluation, cache_key, explanations - ) - return _merge_delta_why(raw_deltas, explanations) - - -def _period_deltas_with_explanations( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], -) -> dict[str, dict[str, str]]: - return _explain_period_deltas( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - - -def _benchmark_context_for_snapshot( - db: Session, - previous_snapshot: Optional[CallImportEvaluationReportSnapshot], -) -> Optional[dict[str, str]]: - if previous_snapshot is None: - return None - previous_import = ( - db.query(CallImport) - .filter(CallImport.id == previous_snapshot.call_import_id) - .first() - ) - previous_eval = ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) - .first() - ) - dataset = ( - (previous_import.dataset or "").strip() - if previous_import and previous_import.dataset - else None - ) - filename = ( - (previous_import.original_filename or previous_import.filename or "").strip() - if previous_import - else None - ) - evaluation_label = ( - (previous_eval.name or "").strip() - if previous_eval and previous_eval.name - else str(previous_snapshot.evaluation_id)[:8] - ) - period = previous_snapshot.period_label or ( - previous_snapshot.period_start.isoformat() - if previous_snapshot.period_start - else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(previous_snapshot.evaluation_id), - "period": period, - } - - -def _clamp_prose_to_sentences( - text: str, - *, - max_sentences: int = 3, - max_chars: int = 300, -) -> str: - """Keep concise audit/TLDR prose within sentence and character limits.""" - cleaned = (text or "").strip() - if not cleaned: - return cleaned - cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() - sentences = [ - sentence.strip() - for sentence in re.split(r"(?<=[.!?])\s+", cleaned) - if sentence.strip() - ] - if sentences: - result = " ".join(sentences[:max_sentences]).strip() - else: - result = cleaned - if len(result) > max_chars: - trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") - result = f"{trimmed}..." if trimmed else result[:max_chars] - return result - - -def _audit_summary_text_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> Optional[str]: - if summary is None: - return None - narrative = _clamp_prose_to_sentences(summary.narrative.strip()) - return narrative or None - - -def _metric_insights_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> dict[str, str]: - if summary is None: - return {} - return { - str(metric_id): insight.strip() - for metric_id, insight in summary.metric_insights.items() - if str(metric_id).strip() and insight.strip() - } - - -def _report_period_from_rows( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], -) -> tuple[Optional[date], Optional[date], Optional[str], str]: - dates = [ - source_row.recording_date - for eval_row, source_row in rows - if eval_row.status == "completed" and source_row.recording_date - ] - if not dates: - return None, None, None, "Not specified" - start = min(dates) - end = max(dates) - week_anchor = max(dates) - week_start = week_anchor - timedelta(days=week_anchor.weekday()) - week_end = week_start + timedelta(days=6) - iso_year, iso_week, _ = week_anchor.isocalendar() - label = f"{iso_year}-W{iso_week:02d}" - if week_start.year == week_end.year: - week_range = f"{week_start.strftime('%b %d')}ΓÇô{week_end.strftime('%b %d, %Y')}" - else: - week_range = ( - f"{week_start.strftime('%b %d, %Y')}ΓÇô{week_end.strftime('%b %d, %Y')}" - ) - display = f"W{iso_week:02d} · {week_range}" - return start, end, label, display - - -def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: - if hasattr(aggregate, "model_dump"): - return aggregate.model_dump(mode="json") - return aggregate.dict() - - -def _aggregate_primary_percent( - raw: dict[str, Any], - policy: Optional[MetricFailurePolicy] = None, -) -> Optional[float]: - return aggregate_primary_percent(raw, policy) - - -def _child_names_by_parent( - db: Session, - organization_id: UUID, - parent_metric_ids: Sequence[UUID], -) -> Dict[str, List[str]]: - if not parent_metric_ids: - return {} - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.in_(list(parent_metric_ids)), - ) - .all() - ) - out: Dict[str, List[str]] = {} - for child in children: - pid = str(child.parent_metric_id) - out.setdefault(pid, []).append(child.name) - return out - - -def _clustering_context( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[ - List[Metric], - List[CallImportMetricAggregate], - Dict[str, MetricFailurePolicy], - Literal["inferred", "user"], - Dict[str, List[str]], -]: - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, source = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - return metrics, aggregates, policies, source, child_names_by_parent - - -def _period_deltas_from_aggregates( - previous_metric_aggregates: list[dict[str, Any]], - current_metric_aggregates: list[dict[str, Any]], - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> dict[str, dict[str, str]]: - current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} - previous_by_id = { - str(item.get("metric_id")): item - for item in previous_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - previous_raw = previous_by_id.get(metric_id) - policy = (policies or {}).get(metric_id) - current_pct = _aggregate_primary_percent(current, policy) - previous_pct = ( - _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", - } - return deltas - - -def _period_deltas_from_snapshot( - previous: Optional[CallImportEvaluationReportSnapshot], - current_metric_aggregates: list[dict[str, Any]], -) -> dict[str, dict[str, str]]: - previous_items = ( - previous.metric_aggregates - if previous and isinstance(previous.metric_aggregates, list) - else [] - ) - return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) - - -def _sample_evidence_for_metrics( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], - metric_ids: set[str], -) -> dict[str, list[dict[str, str]]]: - samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} - for eval_row, source_row in rows: - scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - for metric_id in metric_ids: - if len(samples.get(metric_id, [])) >= 4: - continue - score = scores.get(metric_id) - if not isinstance(score, dict): - continue - rationale = score.get("rationale") - transcript = source_row.diarised_transcript or source_row.transcript or "" - quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] - if quote: - samples.setdefault(metric_id, []).append( - { - "conversation_id": source_row.conversation_id, - "quote": str(quote).strip()[:500], - } - ) - return samples - - -def _fallback_report_narrative( - insight_aggregates: list[dict[str, Any]], - evidence_samples: dict[str, list[dict[str, str]]], -) -> dict[str, Any]: - observations: dict[str, str] = {} - evidence: dict[str, dict[str, str]] = {} - design_notes: list[str] = [] - for aggregate in insight_aggregates: - metric_id = str(aggregate.get("metric_id") or "") - name = str(aggregate.get("metric_name") or "Insight") - counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] - if counts: - top = counts[0] - total = int(aggregate.get("count") or 0) or sum( - int(item.get("count") or 0) for item in counts if isinstance(item, dict) - ) - pct = (int(top.get("count") or 0) / total) * 100 if total else 0 - observations[metric_id] = ( - f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." - ) - design_notes.append( - f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." - ) - sample = (evidence_samples.get(metric_id) or [{}])[0] - if sample: - evidence[metric_id] = sample - return { - "observations": observations, - "evidence": evidence, - "design_notes": design_notes[:7], - "audit_summary": None, - } - - -def _generate_report_narrative( - db: Session, - organization_id: UUID, - *, - metric_aggregates: list[dict[str, Any]], - insight_aggregates: list[dict[str, Any]], - period_delta_by_metric: dict[str, dict[str, str]], - evidence_samples: dict[str, list[dict[str, str]]], - report_config: dict[str, Any], -) -> dict[str, Any]: - if not insight_aggregates: - return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} - try: - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - from app.services.ai.llm_service import llm_service - - llm_provider = (report_config.get("llm_provider") or "").strip() or None - llm_model = (report_config.get("llm_model") or "").strip() or None - llm_credential_id = report_config.get("llm_credential_id") - if not llm_provider and not llm_credential_id: - return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, - db, - llm_provider, - llm_model, - credential_id=llm_credential_id, - ) - prompt = ( - "You are writing a vendor-safe external call quality audit report. " - "Return strict JSON with keys observations (object keyed by metric_id), " - "evidence (object keyed by metric_id with conversation_id and quote), " - "design_notes (array of concise numbered-note strings), and audit_summary (string). " - "Use only the supplied aggregates and evidence samples.\n\n" - + json.dumps( - { - "metric_aggregates": metric_aggregates[:30], - "insight_aggregates": insight_aggregates, - "period_deltas": period_delta_by_metric, - "evidence_samples": evidence_samples, - "report_config": report_config, - }, - default=str, - ) - ) - llm_result = llm_service.generate_response( - messages=[ - {"role": "system", "content": "Return JSON only. No markdown."}, - {"role": "user", "content": prompt}, - ], - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.2, - max_tokens=1200, - ) - parsed = json.loads(str(llm_result.content or "{}")) - if isinstance(parsed, dict): - fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) - return { - "observations": parsed.get("observations") or fallback["observations"], - "evidence": parsed.get("evidence") or fallback["evidence"], - "design_notes": parsed.get("design_notes") or fallback["design_notes"], - "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], - } - except Exception as exc: # noqa: BLE001 - logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) - return _fallback_report_narrative(insight_aggregates, evidence_samples) - - -@router.get( - "/{eval_id}/baseline-candidates", - response_model=CallImportEvaluationBaselineCandidatesResponse, - operation_id="listCallImportEvaluationBaselineCandidates", -) -async def list_call_import_evaluation_baseline_candidates( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBaselineCandidatesResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = _evaluation_rows_for_period(db, evaluation.id) - period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( - rows - ) - candidates = _baseline_candidate_evaluations( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - ) - default_evaluation_id = next( - (item["evaluation_id"] for item in candidates if item.get("is_default")), - None, - ) - return CallImportEvaluationBaselineCandidatesResponse( - items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], - default_evaluation_id=default_evaluation_id, - ) - - -@router.post( - "/{eval_id}/pdf-report", - operation_id="generateCallImportEvaluationPdfReport", - dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], -) -async def generate_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationPdfReportRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -): - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - is_internal = payload.report_type == "internal" - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - report_config = payload.report_config if isinstance(payload.report_config, dict) else {} - metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) - configured_quality_ids = { - str(item) - for item in report_config.get("quality_metric_ids", []) - if item - } - configured_insight_ids = { - str(item.get("metric_id") or item) - for item in report_config.get("insights", []) - if item - } - if configured_quality_ids or configured_insight_ids: - allowed_ids = configured_quality_ids | configured_insight_ids - metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] - - eval_rows = [eval_row for eval_row, _source_row in rows] - aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) - selected_report_metric_ids = {str(metric.id) for metric in metrics} - aggregate_dicts = [ - _aggregate_to_dict(aggregate) - for aggregate in aggregate_models - if aggregate.metric_id in selected_report_metric_ids - ] - insight_metric_ids = { - str(metric.id) - for metric in metrics - if _metric_is_user_insight(metric) - } - metric_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids - ] - insight_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids - ] - period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) - period_label = (payload.period_label or derived_period_label or "").strip() or None - include_period_delta = ( - payload.include_period_delta or payload.include_weekly_delta - ) - previous_snapshot = None - period_delta_by_metric: dict[str, dict[str, str]] = {} - baseline_evaluation: Optional[CallImportEvaluation] = None - if include_period_delta and period_start: - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - payload.baseline_evaluation_id, - ) - if baseline_evaluation: - period_delta_by_metric = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates, - evaluation, - [eval_row for eval_row, _ in rows], - ) - period_delta_by_metric = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - period_delta_by_metric, - ) - benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) - evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) - cached_tldr_summary = _tldr_summary_payload(evaluation) - cached_user_insights = _user_insights_payload(evaluation) - cached_metric_clusters = _metric_clusters_payload(evaluation) - cached_prompt_improvements = _prompt_improvements_payload(evaluation) - generated_insights_for_pdf = _selected_generated_user_insights( - cached_user_insights, - report_config, - ) - metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( - cached_metric_clusters, - report_config, - ) - prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( - cached_prompt_improvements, - report_config, - ) - branding_images, custom_heading = _report_branding_for_import_workspace( - db, - organization_id, - call_import.workspace_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - ) - eval_row_list = [eval_row for eval_row, _ in rows] - pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) - pdf_parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - pdf_child_map = _child_names_by_parent( - db, evaluation.organization_id, pdf_parent_ids - ) - failure_policies_for_pdf, _fp_source = effective_policies( - evaluation, - metrics, - pdf_aggregates, - child_names_by_parent=pdf_child_map, - ) - - from app.services.storage.s3_service import s3_service - - config_fingerprint = compute_pdf_report_config_fingerprint( - report_type=payload.report_type, - include_period_delta=bool(payload.include_period_delta), - include_weekly_delta=bool(payload.include_weekly_delta), - baseline_evaluation_id=payload.baseline_evaluation_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - use_case=payload.use_case, - report_config=report_config, - report_heading=custom_heading, - vendor_name=payload.vendor_name, - platform_base_url=payload.platform_base_url, - period_label=period_label, - ) - content_fingerprint = compute_pdf_report_content_fingerprint( - evaluation_status=evaluation.status, - completed_rows=int(evaluation.completed_rows or 0), - total_rows=int(evaluation.total_rows or 0), - failed_rows=int(evaluation.failed_rows or 0), - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - period_delta_by_metric=period_delta_by_metric, - benchmark_context=benchmark_context, - metric_metadata=[ - { - "id": str(metric.id), - "name": metric.name, - "description": metric.description, - } - for metric in metrics - ], - failure_policies=failure_policies_for_pdf, - tldr_summary=cached_tldr_summary, - user_insights_for_pdf=generated_insights_for_pdf, - metric_clusters_for_pdf=metric_clusters_for_pdf, - prompt_improvements_for_pdf=prompt_improvements_for_pdf, - ) - cache_fingerprint = compute_pdf_report_cache_fingerprint( - config_fingerprint=config_fingerprint, - content_fingerprint=content_fingerprint, - ) - if s3_service.is_enabled(): - cached_pdf_report = find_cached_pdf_report( - db, - evaluation_id=evaluation.id, - organization_id=organization_id, - cache_fingerprint=cache_fingerprint, - ) - if cached_pdf_report is not None: - logger.info( - "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", - cached_pdf_report.id, - eval_id, - ) - return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) - - from app.services.usage.call_import_context import ( - call_import_evaluation_usage_context, - ) - from app.services.usage.context import llm_usage_context - - with llm_usage_context( - call_import_evaluation_usage_context( - organization_id=organization_id, - workspace_id=getattr(call_import, "workspace_id", None) - or evaluation.workspace_id, - evaluation_id=evaluation.id, - call_import_id=evaluation.call_import_id, - ) - ): - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, - ) - - generated_at = datetime.now(timezone.utc) - try: - pdf_started = datetime.now(timezone.utc) - pdf_bytes = await asyncio.to_thread( - call_import_evaluation_pdf_report_service.render_pdf, - vendor_name=payload.vendor_name, - call_import=call_import, - evaluation=evaluation, - metrics=metrics, - rows=rows, - failure_policies=failure_policies_for_pdf, - generated_at=generated_at, - internal=is_internal, - logo_data_uris=branding_images, - custom_heading=custom_heading, - include_weekly_delta=include_period_delta, - period_delta_by_metric=period_delta_by_metric, - use_case=payload.use_case, - period_display=period_display, - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - report_config=report_config, - narrative=narrative, - audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), - metric_insights=_metric_insights_from_tldr(cached_tldr_summary), - benchmark_context=benchmark_context, - generated_user_insights=generated_insights_for_pdf, - user_insights_overview=( - cached_user_insights.overview if cached_user_insights else None - ), - metric_clusters=metric_clusters_for_pdf, - metric_clusters_overview=( - cached_metric_clusters.overview if cached_metric_clusters else None - ), - prompt_improvements=prompt_improvements_for_pdf, - platform_base_url=payload.platform_base_url, - ) - logger.info( - "PDF report render finished in {:.1f}s for evaluation {}", - (datetime.now(timezone.utc) - pdf_started).total_seconds(), - eval_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to generate PDF report for call import {} evaluation {}", - call_import_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to generate PDF report: {exc}", - ) from exc - - snapshot = CallImportEvaluationReportSnapshot( - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - period_label=period_label, - period_start=period_start, - period_end=period_end, - report_config=report_config, - selected_metric_ids=[str(metric.id) for metric in metrics], - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - narrative=narrative, - total_calls=evaluation.total_rows, - selected_metric_count=len(metrics), - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - ) - db.add(snapshot) - db.flush() - - filename = ( - f"{_report_filename_slug(payload.vendor_name)}-" - f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" - ) - - if not s3_service.is_enabled(): - db.commit() - return StreamingResponse( - iter([pdf_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - report_id = uuid4() - s3_key = build_pdf_report_s3_key( - organization_id=organization_id, - call_import_id=call_import.id, - evaluation_id=evaluation.id, - report_id=report_id, - ) - try: - s3_service.upload_file_by_key( - file_content=pdf_bytes, - key=s3_key, - content_type="application/pdf", - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to upload PDF report for evaluation {} to object storage", - eval_id, - ) - db.rollback() - raise HTTPException( - status_code=500, - detail=f"Failed to store PDF report: {exc}", - ) from exc - - created_by, created_by_user_id = _pdf_report_actor(principal) - pdf_report = CallImportEvaluationPdfReport( - id=report_id, - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - snapshot_id=snapshot.id, - vendor_name=payload.vendor_name, - report_type=payload.report_type, - filename=filename, - s3_key=s3_key, - report_config=report_config, - cache_fingerprint=cache_fingerprint, - created_by=created_by, - created_by_user_id=created_by_user_id, - ) - db.add(pdf_report) - try: - db.commit() - except IntegrityError: - db.rollback() - try: - s3_service.delete_file_by_key(s3_key) - except Exception: # noqa: BLE001 - logger.warning( - "Failed to delete orphan PDF after cache race for evaluation {}", - eval_id, - ) - raced_winner = find_cached_pdf_report( - db, - evaluation_id=evaluation.id, - organization_id=organization_id, - cache_fingerprint=cache_fingerprint, - ) - if raced_winner is not None: - logger.info( - "PDF report cache race resolved for evaluation {} (winner {})", - eval_id, - raced_winner.id, - ) - return _pdf_report_response_from_row(raced_winner, cache_hit=True) - raise HTTPException( - status_code=500, - detail="Failed to store PDF report due to a concurrent duplicate request.", - ) from None - db.refresh(pdf_report) - from app.services.billing.flexprice_service import ( - record_call_import_pdf_report_generated, - ) - - record_call_import_pdf_report_generated( - organization_id, - pdf_report.id, - workspace_id=pdf_report.workspace_id, - evaluation_id=evaluation.id, - call_import_id=call_import_id, - report_type=pdf_report.report_type, - ) - return _pdf_report_response_from_row(pdf_report) - - -@router.get( - "/{eval_id}/pdf-reports", - response_model=CallImportEvaluationPdfReportListResponse, - operation_id="listCallImportEvaluationPdfReports", -) -async def list_call_import_evaluation_pdf_reports( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationPdfReportListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluationPdfReport.created_at)) - .all() - ) - return CallImportEvaluationPdfReportListResponse( - items=[_pdf_report_list_item_from_row(row) for row in rows], - ) - - -@router.get( - "/{eval_id}/pdf-reports/{report_id}", - response_model=CallImportEvaluationPdfReportResponse, - operation_id="getCallImportEvaluationPdfReport", -) -async def get_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - report_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationPdfReportResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.id == report_id, - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.call_import_id == call_import_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="PDF report not found") - if not row.s3_key: - raise HTTPException( - status_code=404, - detail="PDF report file is not available in object storage", - ) - return _pdf_report_response_from_row(row) - - -@router.get( - "/{eval_id}/pdf-reports/{report_id}/download", - operation_id="downloadCallImportEvaluationPdfReport", -) -async def download_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - report_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -): - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.id == report_id, - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.call_import_id == call_import_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="PDF report not found") - if not row.s3_key: - raise HTTPException( - status_code=404, - detail="PDF report file is not available in object storage", - ) - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - raise HTTPException( - status_code=503, - detail="Object storage is not enabled or not configured.", - ) - try: - file_bytes = s3_service.download_file_by_key(row.s3_key) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to download PDF report {} for evaluation {}", - report_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to download PDF report: {exc}", - ) from exc - filename = row.filename or "report.pdf" - safe_name = filename.replace('"', "'") - return StreamingResponse( - iter([file_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{safe_name}"'}, - ) - - -@router.patch( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="updateCallImportEvaluation", -) -async def update_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - """Edit metadata on an existing evaluation run (currently just ``name``).""" - - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - # Treat unset vs explicit ``None`` differently: unset = leave alone, - # explicit ``None`` or empty string = clear the name. - payload_data = payload.model_dump(exclude_unset=True) - if "name" in payload_data: - row.name = _normalize_name(payload_data["name"]) - - stamp_evaluation_actor(row, principal) - db.commit() - db.refresh(row) - return _serialize_eval(db, row) - - -def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: - """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" - - if not evaluation.celery_group_id and not any( - r.celery_task_id for r in evaluation.row_results - ): - return - try: - from app.workers.celery_app import celery_app - - pending_task_ids = [ - eval_row.celery_task_id - for eval_row in evaluation.row_results - if eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ] - if pending_task_ids: - celery_app.control.revoke(pending_task_ids, terminate=False) - except Exception: - # Best effort — DB delete remains the source of truth. - pass - - -# --------------------------------------------------------------------------- -# User-initiated cancel for in-flight evaluation rows -# --------------------------------------------------------------------------- -# -# Evaluation rows can sit in ``running`` for many minutes when the underlying -# LLM / audio metric call is slow or wedged (the worker carries an 8 min -# soft / 10 min hard time limit). Without a cancel affordance the operator's -# only recourse is to wait for Celery's time limit to fire — or to manually -# mutate the DB. These helpers + the two endpoints below give the UI a -# first-class "Abort" button mirroring the diarisation cancel pattern at -# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM/audio call so the in-flight HTTP request actually aborts. -# ``terminate=True`` routes the signal to the executing process; we spell -# ``signal="SIGTERM"`` out for clarity even though it's the default. - -# Sentinel error message stamped on cancelled rows. Read by the eval worker's -# ``_was_cancelled_externally`` guard (see -# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already -# past its slowest operation can't overwrite the cancelled state with its own -# terminal status. Touching either copy means touching both. -EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" - - -def _cancellable_eval_states() -> Tuple[str, ...]: - """States that an evaluation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: - """Best-effort revoke of a single eval row's Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). - """ - task_id = (eval_row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked evaluation task {} for eval row {}", - task_id, - eval_row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke evaluation task {} for eval row {}: {}", - task_id, - eval_row.id, - exc, - ) - - -def _cancel_eval_row_with_source_diarisation( - eval_row: CallImportEvaluationRow, - source_row: CallImportRow | None, - *, - cascade_diarisation: bool, - now: datetime | None = None, -) -> list[str]: - """Mark an eval row cancelled; optionally fail in-flight diarisation. - - Returns Celery task ids to revoke (deduped). The caller may batch-revoke - them; when ``cascade_diarisation`` is true, diarisation revoke also runs - inside :func:`_apply_diarisation_cancel` for the source row's task id. - """ - cancellable_states = _cancellable_eval_states() - if (eval_row.status or "").lower() not in cancellable_states: - return [] - - stamp = now or datetime.now(timezone.utc) - task_ids: list[str] = [] - eval_task_id = (eval_row.celery_task_id or "").strip() - if eval_task_id: - task_ids.append(eval_task_id) - - eval_row.status = "failed" - eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR - eval_row.finished_at = stamp - eval_row.celery_task_id = None - - if cascade_diarisation and source_row is not None: - from app.api.v1.routes.call_imports import _apply_diarisation_cancel - - source_task_id = (source_row.celery_task_id or "").strip() - if source_task_id and source_task_id not in task_ids: - task_ids.append(source_task_id) - _apply_diarisation_cancel([source_row]) - - return list(dict.fromkeys(task_ids)) - - -def _apply_evaluation_cancel( - eval_rows: List[CallImportEvaluationRow], - *, - source_rows: dict[UUID, CallImportRow] | None = None, - cascade_diarisation: bool = False, -) -> Tuple[int, int]: - """Cancel every cancellable row in ``eval_rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_eval_states() - cancelled = 0 - skipped = 0 - now = datetime.now(timezone.utc) - source_rows = source_rows or {} - for eval_row in eval_rows: - if (eval_row.status or "").lower() not in cancellable_states: - skipped += 1 - continue - source_row = source_rows.get(eval_row.call_import_row_id) - task_ids = _cancel_eval_row_with_source_diarisation( - eval_row, - source_row, - cascade_diarisation=cascade_diarisation, - now=now, - ) - for task_id in task_ids: - _revoke_eval_task_by_id(task_id, eval_row_id=eval_row.id) - cancelled += 1 - return cancelled, skipped - - -def _revoke_eval_task_by_id(task_id: str, *, eval_row_id: UUID) -> None: - """Best-effort revoke when the task id is known outside the ORM row.""" - cleaned = (task_id or "").strip() - if not cleaned: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - cleaned, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked evaluation task {} for eval row {}", - cleaned, - eval_row_id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke evaluation task {} for eval row {}: {}", - cleaned, - eval_row_id, - exc, - ) - - -def _claim_evaluation_bulk_operation( - evaluation_id: UUID, - operation: str, -) -> None: - """Reserve the run for a single bulk worker pass; 409 if one is active.""" - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - try_set_evaluation_bulk_operation, - ) - - if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] - return - existing = get_evaluation_bulk_operation(evaluation_id) or operation - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - existing = get_evaluation_bulk_operation(evaluation_id) - if existing: - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -@router.post( - "/{eval_id}/cancel", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="cancelCallImportEvaluation", -) -async def cancel_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Abort all in-flight (or queued) rows in a single evaluation run. - - Idempotent: calling on a run whose rows are already terminal returns - ``target_count=0`` with 202 so the UI can fire this from an - "Abort" button without having to pre-check the state. - - Heavy row resets and Celery revokes run in a background worker so - large batches do not block the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") - if target_count == 0: - from app.services.call_imports.bulk_ops import ( - count_inflight_diarisation_for_evaluation, - ) - - inflight_diarisation = count_inflight_diarisation_for_evaluation( - db, - eval_id, - evaluation.call_import_id, - ) - if inflight_diarisation == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - _claim_evaluation_bulk_operation(eval_id, "abort") - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "abort") - evaluation.status = "cancelled" - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/force-fail-pending", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="forceFailCallImportEvaluationPending", -) -async def force_fail_pending_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Force-fail only rows currently in ``pending`` for a single run. - - This is narrower than :func:`cancel_call_import_evaluation`: it leaves - ``running`` rows untouched so operators can clear permanently queued rows - without interrupting in-flight evaluations. - - Row updates run in a background worker so large batches do not block - the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets( - db, eval_id, mode="force_fail_pending" - ) - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay( - str(eval_id), mode="force_fail_pending" - ) - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/cancel", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportEvaluationRow", -) -async def cancel_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Abort an in-flight (or queued) evaluation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed``) returns the row unchanged with a 200 so the UI can - wire this to a "Stop" button without having to pre-check the - state. Updates the parent run's rollup so its counters reflect - the cancel immediately. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import evaluation_row_session - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - try: - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - if eval_row.evaluation_id != eval_id: - raise HTTPException( - status_code=404, - detail="Evaluation row not found in this run", - ) - _apply_evaluation_cancel( - [eval_row], - source_rows={source_row.id: source_row}, - cascade_diarisation=True, - ) - row_db.commit() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row, evaluation) - except LookupError as exc: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) from exc - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - source_row = ( - db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - source_rows = ( - {source_row.id: source_row} if source_row is not None else None - ) - _apply_evaluation_cancel( - [eval_row], - source_rows=source_rows, - cascade_diarisation=True, - ) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(eval_row) - - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -@router.delete( - "/{eval_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluation", -) -async def delete_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - _revoke_pending_tasks(row) - - from app.services.call_imports.discovered_running_cache import ( - clear_discovered_running_cache, - ) - - clear_discovered_running_cache(eval_id) - db.delete(row) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -@router.post( - "/bulk-delete", - status_code=status.HTTP_200_OK, - operation_id="bulkDeleteCallImportEvaluations", -) -async def bulk_delete_call_import_evaluations( - call_import_id: UUID, - payload: CallImportEvaluationBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Dict[str, int]: - """Delete multiple evaluation runs scoped to one call import. - - Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI - can clear out a multi-select. Unknown ids (already deleted, or - belonging to a different org/import) are silently skipped — the - response just reports how many actually went away. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - if not payload.evaluation_ids: - return {"deleted": 0} - - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id.in_(payload.evaluation_ids), - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .all() - ) - deleted = 0 - for row in rows: - _revoke_pending_tasks(row) - db.delete(row) - deleted += 1 - db.commit() - return {"deleted": deleted} - - -# --------------------------------------------------------------------------- -# Aggregation: turns per-row metric scores into histograms / value counts. -# -# Designed to be cheap enough to call on every page load: we read each -# evaluation row once, bucket numeric values into a fixed 10-bin -# histogram, and tally the top categorical values. Scaling concerns -# (millions of rows) are deferred — at that point we'd push this into a -# Postgres aggregate query, but for typical CSV imports (<10k rows) the -# Python pass is fast enough and dramatically simpler. -# --------------------------------------------------------------------------- - - -_HISTOGRAM_BUCKETS = 10 -_TOP_VALUE_COUNTS = 10 - - -def _coerce_numeric(value: Any) -> Optional[float]: - """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" - if isinstance(value, bool): - # Booleans are ints in Python; treat them as categorical so - # pass/fail metrics show up in value_counts instead of becoming - # a degenerate {0,1} histogram. - return None - if isinstance(value, (int, float)) and math.isfinite(value): - return float(value) - if isinstance(value, str): - try: - f = float(value) - if math.isfinite(f): - return f - except ValueError: - return None - return None - - -def _coerce_category(value: Any) -> Optional[str]: - """Render ``value`` as a label suitable for a value_counts bucket.""" - if value is None: - return None - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - text = value.strip() - return text or None - # Lists / dicts: stringify so they still group sensibly without - # exploding the cardinality (worst case: everything is "[…]" once). - return str(value) - - -def _build_histogram( - values: List[float], -) -> List[CallImportMetricHistogramBucket]: - """Fixed-bin histogram over ``values``; returns [] for <2 values.""" - if len(values) < 2: - return [] - lo = min(values) - hi = max(values) - if lo == hi: - # All values identical — render a single bucket so the UI shows a - # spike rather than empty space. - return [ - CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) - ] - width = (hi - lo) / _HISTOGRAM_BUCKETS - buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] - for v in values: - # Right-edge inclusive on the last bucket so ``hi`` doesn't fall - # off into a non-existent bucket index. - idx = int((v - lo) / width) - if idx >= _HISTOGRAM_BUCKETS: - idx = _HISTOGRAM_BUCKETS - 1 - buckets[idx].append(v) - return [ - CallImportMetricHistogramBucket( - x0=lo + i * width, - x1=lo + (i + 1) * width, - count=len(bucket), - ) - for i, bucket in enumerate(buckets) - ] - - -def _percentile(values: List[float], pct: float) -> Optional[float]: - """Linear-interpolated percentile compatible with NumPy default.""" - if not values: - return None - sorted_vals = sorted(values) - if len(sorted_vals) == 1: - return sorted_vals[0] - rank = (pct / 100.0) * (len(sorted_vals) - 1) - lo = int(math.floor(rank)) - hi = int(math.ceil(rank)) - if lo == hi: - return sorted_vals[lo] - frac = rank - lo - return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac - - -def _compute_metric_aggregates( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[CallImportMetricAggregate]: - """Collapse per-row ``metric_scores`` into one aggregate per metric. - - Selected metrics are read fresh from the DB so the response always - surfaces the current ``metric.name`` / ``metric_type`` even when a - metric was renamed after the run finished. - """ - - selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metrics from selected_metric_groups so they appear - # alongside their children in the aggregate response. Use ``getattr`` - # with a default so the helper still works for callers that pass - # lightweight objects (tests, in-memory shims) that don't carry the - # attribute at all. - groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) - groups_raw = ( - groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in selected_ids: - selected_ids.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - # Default to selected metrics, but also include any metric ids that - # surface in row scores even if missing from the metric registry — - # otherwise renaming/deleting a metric mid-run would silently drop - # results from the chart. - discovered_ids: List[str] = list(metric_meta.keys()) - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id_str in scores.keys(): - if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: - discovered_ids.append(metric_id_str) - - results: List[CallImportMetricAggregate] = [] - - for metric_id_str in discovered_ids: - meta = metric_meta.get(metric_id_str) - numeric_values: List[float] = [] - category_counts: Dict[str, int] = {} - classification_yes_no_counts: Dict[str, int] = {} - classification_choice_counts: Dict[str, int] = {} - classification_level_counts: Dict[str, int] = {} - classification_rows_scored = 0 - is_classification_metric_agg = bool( - meta and is_classification_metric(custom_data_type=getattr(meta, "custom_data_type", None)) - ) - # For multi-label parents we still need to know how many rows - # were scored (each row votes for >=1 label) so the n-badge in - # the UI shows "n=50" instead of the misleading "n=208" sum. - multi_label_rows_scored = 0 - # Unordered pair tally for the co-occurrence heatmap. Keys are - # ``(label_a, label_b)`` with ``a < b`` so we never double-count - # the same unordered pair. Only populated for multi-label - # parents — every other metric leaves this empty. - multi_label_pair_counts: Dict[Tuple[str, str], int] = {} - skipped = 0 - errored = 0 - observed_metric_type: Optional[str] = None - observed_name: Optional[str] = None - - # ``meta`` is a real ``Metric`` row in production, but tests - # frequently pass a lightweight stub. Pull the two attributes - # we need via ``getattr`` so a stub that only sets ``id`` / - # ``name`` / ``metric_type`` doesn't blow up here. - is_multi_label_parent = bool( - meta - and getattr(meta, "selection_mode", None) == "multi_label" - and not getattr(meta, "parent_metric_id", None) - ) - - for row in eval_rows: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else {} - ) - entry = scores.get(metric_id_str) - if not isinstance(entry, dict): - continue - if entry.get("metric_name"): - observed_name = entry.get("metric_name") - if entry.get("type"): - observed_metric_type = entry.get("type") - if entry.get("type") == "classification": - is_classification_metric_agg = True - if entry.get("skipped"): - skipped += 1 - continue - if entry.get("error"): - errored += 1 - continue - - # Multi-label parents store a comma-joined value that - # isn't useful as a single category; instead tally each - # selected child individually so the chart shows per-label - # counts that mirror the children's own boolean histograms. - if is_multi_label_parent: - selected = entry.get("selected_child_names") - if isinstance(selected, list) and selected: - multi_label_rows_scored += 1 - cleaned: List[str] = [] - for label in selected: - text_label = str(label).strip() or None - if text_label: - cleaned.append(text_label) - category_counts[text_label] = ( - category_counts.get(text_label, 0) + 1 - ) - # Emit one increment per unordered pair of distinct - # labels that fired together on this row. ``cleaned`` - # is deduplicated first because the LLM occasionally - # repeats a label inside ``selected_child_names``. - distinct = sorted(set(cleaned)) - for i in range(len(distinct)): - for j in range(i + 1, len(distinct)): - pair = (distinct[i], distinct[j]) - multi_label_pair_counts[pair] = ( - multi_label_pair_counts.get(pair, 0) + 1 - ) - continue - - if is_classification_metric_agg or entry.get("type") == "classification": - facets = classification_facets_from_entry(entry) - scored_classification = False - yn = facets.get("yes_no") - if yn: - classification_yes_no_counts[yn] = ( - classification_yes_no_counts.get(yn, 0) + 1 - ) - scored_classification = True - ch = facets.get("choice") - if ch: - classification_choice_counts[ch] = ( - classification_choice_counts.get(ch, 0) + 1 - ) - scored_classification = True - lv = facets.get("level") - if lv: - classification_level_counts[lv] = ( - classification_level_counts.get(lv, 0) + 1 - ) - scored_classification = True - if scored_classification: - classification_rows_scored += 1 - continue - - value = entry.get("value") - numeric = _coerce_numeric(value) - if numeric is not None: - numeric_values.append(numeric) - continue - category = _coerce_category(value) - if category is not None: - category_counts[category] = category_counts.get(category, 0) + 1 - - # ``count`` is "rows scored". For numeric / single-choice - # metrics that's the same as ``len(numeric) + sum(categories)`` - # because each scored row contributes exactly one observation. - # Multi-label parents however contribute one observation per - # selected child, so summing ``category_counts`` over-counts — - # we tracked rows-scored separately above and use it here. - legacy_rows_scored = len(numeric_values) + sum(category_counts.values()) - # Mixed migrations may score some rows via classification facets and - # others via legacy numeric/category values for the same metric id. - rows_scored = ( - multi_label_rows_scored - if is_multi_label_parent - else (classification_rows_scored + legacy_rows_scored) - if is_classification_metric_agg - else legacy_rows_scored - ) - - # Build numeric stats first, then categorical (both can coexist). - agg = CallImportMetricAggregate( - metric_id=metric_id_str, - metric_name=( - (meta.name if meta else observed_name) or "Unknown metric" - ), - metric_type=( - meta.metric_type if meta else observed_metric_type - ), - metric_category=( - "user_insight" - if meta is not None and _metric_is_user_insight(meta) - else "quality" - ) - or "quality", - is_multi_label_parent=is_multi_label_parent, - count=rows_scored, - skipped_count=skipped, - error_count=errored, - ) - if numeric_values: - agg.mean = float(statistics.fmean(numeric_values)) - agg.median = float(statistics.median(numeric_values)) - agg.min = min(numeric_values) - agg.max = max(numeric_values) - agg.stddev = ( - float(statistics.pstdev(numeric_values)) - if len(numeric_values) > 1 - else 0.0 - ) - agg.p25 = _percentile(numeric_values, 25) - agg.p75 = _percentile(numeric_values, 75) - agg.p95 = _percentile(numeric_values, 95) - agg.histogram_buckets = _build_histogram(numeric_values) - if ( - is_classification_metric_agg - and ( - classification_yes_no_counts - or classification_choice_counts - or classification_level_counts - ) - ): - def _facet_value_counts(counts: Dict[str, int]) -> List[CallImportMetricValueCount]: - sorted_counts = sorted( - counts.items(), key=lambda kv: kv[1], reverse=True - ) - return [ - CallImportMetricValueCount(label=label, count=count) - for label, count in sorted_counts[:_TOP_VALUE_COUNTS] - ] - - agg.classification_facets = CallImportClassificationFacetCounts( - yes_no=_facet_value_counts(classification_yes_no_counts), - choice=_facet_value_counts(classification_choice_counts), - level=_facet_value_counts(classification_level_counts), - ) - if category_counts: - sorted_counts = sorted( - category_counts.items(), key=lambda kv: kv[1], reverse=True - ) - agg.value_counts = [ - CallImportMetricValueCount(label=label, count=count) - for label, count in sorted_counts[:_TOP_VALUE_COUNTS] - ] - # Restrict the heatmap to pairs of labels we actually - # rendered above so the frontend never has to match - # against truncated/missing rows. Sorted desc by pair - # count to keep the most informative cells in the - # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. - if is_multi_label_parent and multi_label_pair_counts: - kept_labels = { - label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] - } - pair_items = [ - (a, b, count) - for (a, b), count in multi_label_pair_counts.items() - if a in kept_labels and b in kept_labels - ] - pair_items.sort(key=lambda t: t[2], reverse=True) - agg.co_occurrence = [ - CallImportMetricLabelPair(a=a, b=b, count=count) - for a, b, count in pair_items - ] - - results.append(agg) - - # Sort so each parent metric immediately precedes its children. - # The Visualizations grid renders metrics top-to-bottom in this - # order, so multi-label parents (the "summary" chart) sit above - # the per-child boolean histograms that drill into them. Metrics - # whose ``meta`` row was deleted mid-run (``meta is None``) sink - # to the bottom but keep their relative order. - enumerated = list(enumerate(results)) - - def _sort_key(item: Tuple[int, CallImportMetricAggregate]): - original_idx, agg = item - meta = metric_meta.get(agg.metric_id) - if meta is None: - return (1, "", 1, "", original_idx) - parent_id = getattr(meta, "parent_metric_id", None) - # Group key: a child shares its parent's UUID; a parent - # uses its own UUID. Within a group, depth=0 (parent) sorts - # before depth=1 (child); ties break alphabetically by name - # so children render in a stable order regardless of which - # row scored which label first. - if parent_id is None: - group_key = str(meta.id) - depth = 0 - else: - group_key = str(parent_id) - depth = 1 - return ( - 0, - group_key, - depth, - (getattr(meta, "name", "") or "").lower(), - original_idx, - ) - - enumerated.sort(key=_sort_key) - return [agg for _idx, agg in enumerated] - - -@router.get( - "/{eval_id}/aggregate", - response_model=CallImportEvaluationAggregateResponse, - operation_id="getCallImportEvaluationAggregate", -) -async def get_call_import_evaluation_aggregate( - call_import_id: UUID, - eval_id: UUID, - baseline_evaluation_id: Optional[UUID] = Query(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationAggregateResponse: - """Return per-metric distributions for the Visualizations tab. - - The shape is intentionally chart-friendly: histograms for numeric - metrics, top-N value counts for categorical/text metrics, plus - summary stats (mean/p50/p95) so the UI can render summary cards - without recomputing on the client. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - - metrics = _compute_metric_aggregates(db, evaluation, eval_rows) - - period_deltas: dict[str, MetricPeriodDelta] = {} - resolved_baseline_id: Optional[UUID] = None - if baseline_evaluation_id is not None: - call_import = _require_import(db, call_import_id, organization_id) - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - rows = load_evaluation_row_pairs(db, eval_id) - period_start, _, _, _ = _report_period_from_rows(rows) - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - str(baseline_evaluation_id), - ) - if baseline_evaluation: - resolved_baseline_id = baseline_evaluation.id - metric_aggregates_dicts = [ - _aggregate_to_dict(agg) for agg in metrics - ] - raw_deltas = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates_dicts, - evaluation, - eval_rows, - ) - raw_deltas = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - period_deltas = { - metric_id: MetricPeriodDelta( - label=delta.get("label") or "", - detail=delta.get("detail") or "", - why=(delta.get("why") or "").strip() or None, - ) - for metric_id, delta in raw_deltas.items() - } - - _fp_stored, failure_policies_source = policies_from_evaluation_raw( - evaluation.metric_clusters - ) - return CallImportEvaluationAggregateResponse( - evaluation_id=eval_id, - total_rows=evaluation.total_rows, - completed_rows=evaluation.completed_rows, - failed_rows=evaluation.failed_rows, - metrics=metrics, - period_deltas=period_deltas, - baseline_evaluation_id=resolved_baseline_id, - failure_policies_source=failure_policies_source, - ) - - -# --------------------------------------------------------------------------- -# TLDR insights: LLM-generated narrative + bullet patterns rendered above -# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` -# so the page never auto-burns LLM tokens; the user explicitly clicks -# "Generate summary" or "Regenerate" from the empty-state CTA. -# --------------------------------------------------------------------------- - - -_INSIGHTS_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will be " - "given aggregated metric statistics + a sample of rationales for " - "the rows of a single call-import evaluation. Identify the most " - "useful PATTERNS that hold ACROSS the calls -- not just per-metric " - "numbers. Look for combinations (e.g. `when X happens, Y also " - "tends to happen`), notable outliers, frequent failure modes, and " - "any signal that would change how a reviewer triages the run.\n\n" - "Return STRICT JSON only, with this shape and no extra keys:\n" - "{\n" - ' "narrative": "",\n' - ' "patterns": ["", "", ...],\n' - ' "metric_insights": {"": "<2-3 line business meaning>"}\n' - "}\n\n" - "Constraints:\n" - "- narrative is the ONLY text shown in the external audit summary and " - "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" - "- patterns are optional supporting notes and are NOT rendered in the " - "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" - "- metric_insights must include one entry for each top-level metric id supplied.\n" - "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" - "- Avoid restating raw counts unless they reveal a pattern.\n" - "- Use neutral, factual language ('frustration appeared in...') " - "rather than judgemental ('the agents failed to...')." -) - - -def _tldr_summary_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (with ``is_stale`` set) or ``None``. - - ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we - have to validate shape defensively -- a half-written or hand-edited - blob should not break the aggregate response. Returns ``None`` when - no cached summary exists. - """ - raw = evaluation.tldr_summary - if not isinstance(raw, dict): - return None - narrative = raw.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - return None - patterns_raw = raw.get("patterns") - patterns = ( - [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] - if isinstance(patterns_raw, list) - else [] - ) - metric_insights_raw = raw.get("metric_insights") - metric_insights = ( - { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - if isinstance(metric_insights_raw, dict) - else {} - ) - generated_at_raw = raw.get("generated_at") - try: - generated_at = ( - datetime.fromisoformat(generated_at_raw) - if isinstance(generated_at_raw, str) - else evaluation.updated_at or datetime.now(timezone.utc) - ) - except ValueError: - generated_at = evaluation.updated_at or datetime.now(timezone.utc) - snapshot = raw.get("generated_at_completed_rows") - snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=generated_at, - generated_at_completed_rows=snapshot_int, - provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, - model=raw.get("model") if isinstance(raw.get("model"), str) else None, - is_stale=evaluation.completed_rows > snapshot_int, - ) - - -def _sample_rationales_per_metric( - eval_rows: List[CallImportEvaluationRow], - *, - per_metric_cap: int = 3, - rationale_char_cap: int = 600, -) -> Dict[str, List[str]]: - """Collect up to ``per_metric_cap`` distinct rationales per metric. - - Distinctness is case- and whitespace-insensitive. We truncate each - rationale to ``rationale_char_cap`` so a few unusually verbose rows - can't dominate the prompt budget. Empty / non-string rationales are - skipped. - """ - out: Dict[str, List[str]] = {} - seen: Dict[str, set[str]] = {} - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id, entry in scores.items(): - if not isinstance(entry, dict): - continue - rationale = entry.get("rationale") - if not isinstance(rationale, str): - continue - text = rationale.strip() - if not text: - continue - bucket = out.setdefault(metric_id, []) - if len(bucket) >= per_metric_cap: - continue - key = " ".join(text.lower().split()) - seen_set = seen.setdefault(metric_id, set()) - if key in seen_set: - continue - seen_set.add(key) - bucket.append(text[:rationale_char_cap]) - return out - - -def _build_insights_messages( - evaluation: CallImportEvaluation, - aggregate: List[CallImportMetricAggregate], - rationale_samples: Dict[str, List[str]], - metric_meta: Dict[str, Metric], -) -> List[Dict[str, str]]: - """Render the user prompt fed to the LLM. - - The shape is plain markdown-ish text instead of JSON so the LLM can - skim it without us spending tokens on verbose schema delimiters. - Parent metrics surface their child metrics nested underneath so the - model sees the hierarchy and can talk about "X often co-occurred - with Y" rather than treating sub-labels as standalone metrics. - """ - name = evaluation.name or f"Run {str(evaluation.id)[:8]}" - lines: List[str] = [ - f"Evaluation: {name}", - ( - f"Rows: total={evaluation.total_rows} " - f"completed={evaluation.completed_rows} " - f"failed={evaluation.failed_rows}" - ), - "", - "## Per-metric aggregate", - ] - - # Group metrics by parent so the prompt mirrors the hierarchy. Any - # aggregate row whose ``metric_id`` is missing from ``metric_meta`` - # is rendered as a leaf at the top-level list (handles renamed / - # deleted parents). - children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} - top_level: List[CallImportMetricAggregate] = [] - for agg in aggregate: - meta = metric_meta.get(agg.metric_id) - parent_id = ( - str(meta.parent_metric_id) - if meta is not None and getattr(meta, "parent_metric_id", None) - else None - ) - if parent_id: - children_by_parent.setdefault(parent_id, []).append(agg) - else: - top_level.append(agg) - - def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: - prefix = " " * indent + "- " - bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] - if agg.skipped_count: - bits.append(f", skipped={agg.skipped_count}") - if agg.error_count: - bits.append(f", errors={agg.error_count}") - bits.append(")") - meta = metric_meta.get(agg.metric_id) - description = (meta.description or "").strip() if meta else "" - if description: - bits.append(f" | definition={description[:500]}") - if agg.mean is not None: - mean_s = f"{agg.mean:.2f}" - stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" - bits.append(f" | mean={mean_s} stddev={stddev_s}") - if agg.min is not None and agg.max is not None: - bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") - if agg.value_counts: - total = sum(v.count for v in agg.value_counts) or 1 - top = agg.value_counts[:3] - shares = ", ".join( - f'"{v.label}"={v.count}/{total}' for v in top - ) - bits.append(f" | top={shares}") - result = ["".join(bits)] - rationales = rationale_samples.get(agg.metric_id, []) - for r in rationales: - result.append(" " * (indent + 1) + f"- rationale: {r}") - return result - - for agg in top_level: - lines.extend(_format_metric_block(agg, indent=0)) - meta = metric_meta.get(agg.metric_id) - children = children_by_parent.get(str(meta.id), []) if meta else [] - for child in children: - lines.extend(_format_metric_block(child, indent=1)) - - lines.append("") - top_level_ids = [agg.metric_id for agg in top_level] - if top_level_ids: - lines.append( - "metric_insights keys must exactly use these top-level metric ids: " - + ", ".join(top_level_ids) - ) - lines.append("") - lines.append( - "Write the JSON object as instructed. Do not include " - "preamble, code fences, or trailing commentary." - ) - - return [ - {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, - {"role": "user", "content": "\n".join(lines)}, - ] - - -def _parse_insights_response(text: str) -> EvaluationTldrSummary: - """Coerce the LLM response into ``narrative`` + ``patterns``. - - Matches the JSON-with-fallback pattern used by - ``app.api.v1.routes.metrics._parse_metric_generation_response``: try - ``json.loads`` first, then fall back to regex extraction of the - first ``{...}`` block. Raises ``HTTPException`` with a 502 when the - response can't be parsed at all. - """ - cleaned = (text or "").strip() - if not cleaned: - raise HTTPException( - status_code=502, detail="LLM returned an empty insights response" - ) - try: - parsed = json.loads(cleaned) - except json.JSONDecodeError: - import re - - match = re.search(r"\{.*\}", cleaned, re.DOTALL) - if not match: - raise HTTPException( - status_code=502, - detail="Could not parse LLM insights response as JSON", - ) - try: - parsed = json.loads(match.group(0)) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Could not parse LLM insights response: {e}", - ) - - if not isinstance(parsed, dict): - raise HTTPException( - status_code=502, detail="LLM insights JSON was not an object" - ) - - narrative = parsed.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - raise HTTPException( - status_code=502, - detail="LLM insights JSON missing 'narrative' string", - ) - - patterns_raw = parsed.get("patterns") - if patterns_raw is None: - patterns: List[str] = [] - elif isinstance(patterns_raw, list): - patterns = [ - str(p).strip() - for p in patterns_raw - if isinstance(p, str) and p.strip() - ] - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'patterns' must be a list of strings", - ) - metric_insights_raw = parsed.get("metric_insights") - if metric_insights_raw is None: - metric_insights: Dict[str, str] = {} - elif isinstance(metric_insights_raw, dict): - metric_insights = { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'metric_insights' must be an object", - ) - - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=0, # filled in by caller - is_stale=False, - ) - - -def _generate_and_persist_tldr_summary( - db: Session, - evaluation: CallImportEvaluation, - *, - organization_id: UUID, - provider: Optional[str] = None, - model: Optional[str] = None, -) -> EvaluationTldrSummary: - """LLM TLDR generation used by the imports-queue Celery worker.""" - eval_id = evaluation.id - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - pairs = load_evaluation_row_pairs(db, eval_id) - eval_rows = [eval_row for eval_row, _ in pairs] - aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) - if not aggregate: - raise HTTPException( - status_code=400, - detail=( - "No metric data yet. Wait for at least one row to " - "finish scoring before generating a summary." - ), - ) - - metric_ids: List[UUID] = [] - for agg in aggregate: - try: - metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, metric_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - rationale_samples = _sample_rationales_per_metric(eval_rows) - messages = _build_insights_messages( - evaluation, aggregate, rationale_samples, metric_meta - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - from app.services.ai.llm_service import llm_service - from app.services.usage.call_import_context import ( - call_import_evaluation_usage_context, - ) - from app.services.usage.context import llm_usage_context - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, provider, model - ) - - try: - with llm_usage_context( - call_import_evaluation_usage_context( - organization_id=organization_id, - workspace_id=evaluation.workspace_id, - evaluation_id=evaluation.id, - call_import_id=evaluation.call_import_id, - ) - ): - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.4, - max_tokens=4096, - ) - if llm_result.get("truncated"): - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.2, - max_tokens=8192, - ) - except Exception as e: - logger.error(f"[CallImportInsights] LLM call failed: {e}") - raise HTTPException( - status_code=502, detail=f"LLM call failed: {e}" - ) from e - - if llm_result.get("truncated"): - raise HTTPException( - status_code=502, - detail="Insights response was cut off before it finished. Try again.", - ) - - total = int(evaluation.total_rows or 0) - summary = _parse_insights_response(llm_result.get("text", "")) - ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( - evaluation.completed_rows or 0 - ) - summary.generated_at_completed_rows = ui_completed - summary.provider = provider_enum.value - summary.model = model_str - summary.is_stale = False - - evaluation.tldr_summary = { - "narrative": summary.narrative, - "patterns": summary.patterns, - "metric_insights": summary.metric_insights, - "generated_at": summary.generated_at.isoformat(), - "generated_at_completed_rows": summary.generated_at_completed_rows, - "provider": summary.provider, - "model": summary.model, - } - flag_modified(evaluation, "tldr_summary") - db.commit() - db.refresh(evaluation) - return summary - - -@router.get( - "/{eval_id}/insights", - response_model=Optional[EvaluationTldrSummary], - operation_id="getCallImportEvaluationInsights", -) -async def get_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (or ``null``) without contacting the LLM. - - Used by the Visualizations tab on first paint so the empty-state - CTA can show up before the user opts into generation. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _tldr_summary_payload(evaluation) - - -@router.post( - "/{eval_id}/insights", - response_model=EvaluationTldrSummary, - operation_id="generateCallImportEvaluationInsights", -) -async def generate_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationTldrSummary: - """Generate (or return-cached) the LLM TLDR for an evaluation run. - - Behavior: - - * ``body.regenerate=False`` and a cached summary at the current - ``completed_rows`` watermark exists -> return it as-is. - * ``body.regenerate=False`` and a stale cached summary exists - (``generated_at_completed_rows < completed_rows``) -> return it - with ``is_stale=True``; the UI prompts the user to regenerate. - * Otherwise -> resolve provider+model (auto-detect when omitted), - call the LLM, persist the new summary, return it. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate: - cached = _tldr_summary_payload(evaluation) - if cached is not None: - return cached - - # Run the TLDR LLM on the imports worker (not the default worker or API). - from app.workers.tasks.generate_evaluation_tldr_insights import ( - generate_evaluation_tldr_insights_task, - ) - - try: - task_result = generate_evaluation_tldr_insights_task.apply_async( - kwargs={ - "evaluation_id": str(eval_id), - "call_import_id": str(call_import_id), - "organization_id": str(organization_id), - "provider": body.provider, - "model": body.model, - }, - ).get(timeout=25 * 60) - except Exception as exc: - logger.error( - "[CallImportInsights] TLDR task failed for evaluation {}: {}", - eval_id, - exc, - ) - raise HTTPException( - status_code=502, - detail=f"Summary generation failed: {exc}", - ) from exc - - if isinstance(task_result, dict) and task_result.get("error"): - status_code = int(task_result.get("status_code") or 502) - raise HTTPException( - status_code=status_code, - detail=str(task_result["error"]), - ) - - summary = EvaluationTldrSummary.model_validate(task_result) - db.refresh(evaluation) - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=summary.provider or provider_enum.value, - model=summary.model or model_str, - force=body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - principal=principal, - ) - - return summary - - -def _user_insights_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationUserInsightsState]: - raw = getattr(evaluation, "user_insights", None) - if raw is None: - return None - return user_insights_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_generated_user_insights( - state: Optional[EvaluationUserInsightsState], - report_config: dict[str, Any], -) -> list[dict[str, Any]]: - """Filter and order generated insights for PDF section 03.""" - if state is None or state.status != "completed" or not state.insights: - return [] - - selected_ids = report_config.get("user_insight_ids") - if isinstance(selected_ids, list) and selected_ids: - allowed = {str(item) for item in selected_ids if item} - items = [item for item in state.insights if item.id in allowed] - else: - items = list(state.insights) - - order_raw = report_config.get("order") - order_ids: list[str] = [] - if isinstance(order_raw, dict): - user_order = order_raw.get("user_insights") - if isinstance(user_order, list): - order_ids = [str(item) for item in user_order if item] - - if order_ids: - by_id = {item.id: item for item in items} - ordered = [by_id[iid] for iid in order_ids if iid in by_id] - seen = set(order_ids) - ordered.extend(item for item in items if item.id not in seen) - items = ordered - - return [item.model_dump(mode="json") for item in items] - - -def _enqueue_user_insights_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - """Enqueue background user-insights generation unless already running.""" - current = _user_insights_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - - completed_count = ( - _count_completed_eval_rows(db, evaluation.id) - if db is not None - else evaluation.completed_rows - ) - total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) - evaluation.user_insights = { - "status": "running", - "insights": ( - (evaluation.user_insights or {}).get("insights", []) - if isinstance(evaluation.user_insights, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "user_insights") - db.commit() - - from app.workers.tasks.generate_evaluation_user_insights import ( - generate_evaluation_user_insights_task, - ) - - generate_evaluation_user_insights_task.delay( - str(evaluation.id), - provider=provider, - model=model, - max_llm_calls=llm_budget, - ) - - -@router.get( - "/{eval_id}/user-insights", - response_model=Optional[EvaluationUserInsightsState], - operation_id="getCallImportEvaluationUserInsights", -) -async def get_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationUserInsightsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _user_insights_payload(evaluation) - - -@router.post( - "/{eval_id}/user-insights", - response_model=EvaluationUserInsightsState, - operation_id="generateCallImportEvaluationUserInsights", -) -async def generate_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationUserInsightsRequest = Body( - default_factory=EvaluationUserInsightsRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationUserInsightsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _user_insights_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating user insights." - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=provider_enum.value, - model=model_str, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _user_insights_payload(evaluation) or EvaluationUserInsightsState( - status="running" - ) - - -def _metric_clusters_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationMetricClustersState]: - raw = getattr(evaluation, "metric_clusters", None) - if raw is None: - return None - return metric_clusters_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_metric_clusters_for_pdf( - state: Optional[EvaluationMetricClustersState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: - return {} - payload: dict[str, Any] = { - "groups": [g.model_dump(mode="json") for g in state.groups], - "discovered_problems": [ - d.model_dump(mode="json") for d in state.discovered_problems - ], - } - if state.rca_summary is not None: - payload["rca_summary"] = state.rca_summary.model_dump(mode="json") - return payload - - -def _prompt_improvements_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationPromptImprovementsState]: - from app.services.call_import_prompt_improvements import ( - prompt_improvements_state_from_raw, - ) - - raw = getattr(evaluation, "prompt_improvements", None) - if raw is None: - return None - return prompt_improvements_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_prompt_improvements_for_pdf( - state: Optional[EvaluationPromptImprovementsState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("prompt_improvements") is False: - return {} - return { - "imported_agent_id": state.imported_agent_id, - "imported_agent_name": state.imported_agent_name, - "overview": state.overview, - "suggestions": [s.model_dump(mode="json") for s in state.suggestions], - } - - -def _enqueue_prompt_improvements_job( - evaluation: CallImportEvaluation, - *, - imported_agent_id: UUID, - imported_agent_name: str, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - current = _prompt_improvements_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - evaluation.prompt_improvements = { - "status": "running", - "imported_agent_id": str(imported_agent_id), - "imported_agent_name": imported_agent_name, - "suggestions": [], - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "provider": provider, - "model": model, - "error_message": None, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "prompt_improvements") - db.commit() - - from app.workers.tasks.generate_evaluation_prompt_improvements import ( - generate_evaluation_prompt_improvements_task, - ) - - async_result = generate_evaluation_prompt_improvements_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "imported_agent_id": str(imported_agent_id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.prompt_improvements, dict): - evaluation.prompt_improvements["celery_task_id"] = async_result.id - flag_modified(evaluation, "prompt_improvements") - db.commit() - - -def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: - from app.db_sharding.scatter_gather import load_evaluation_score_rows - - return load_evaluation_score_rows(db, evaluation_id) - - -def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - - return count_evaluation_rows_for_run( - db, evaluation_id, statuses=["completed"] - ) - - -def _completed_row_pairs_for_evaluation( - db: Session, - evaluation_id: UUID, -) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - row_pairs = load_evaluation_row_pairs(db, evaluation_id) - return [ - (eval_row, source_row) - for eval_row, source_row in row_pairs - if eval_row.status == "completed" - ] - - -def _resolve_metric_cluster_row_selection( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], - evaluation_row_ids: Optional[List[UUID]], - *, - row_limit: Optional[int] = None, - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: - """Return filtered completed row pairs and the selected row id strings.""" - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - if policies is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] - eligible_id_set = set(eligible_ordered_ids) - - if evaluation_row_ids is None and row_limit is not None: - selected_ids = eligible_ordered_ids[:row_limit] - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - if evaluation_row_ids is None: - selected_ids = eligible_ordered_ids - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - requested = {str(rid) for rid in evaluation_row_ids} - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - not_eligible = sorted(requested - eligible_id_set) - if not_eligible: - raise HTTPException( - status_code=400, - detail=( - "Each selected row must have at least one flagged quality metric. " - "Ineligible row(s): " - + ", ".join(not_eligible[:5]) - + ("…" if len(not_eligible) > 5 else "") - ), - ) - selected_ids = sorted(requested) - filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) - return filtered, selected_ids - - -def _enqueue_metric_clusters_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - evaluation_row_ids: Optional[List[UUID]] = None, - selected_evaluation_row_ids: Optional[List[str]] = None, - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - current = _metric_clusters_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - total_calls = 1 - row_ids_for_task: Optional[List[str]] = None - if db is not None: - eval_rows = _load_eval_rows(db, evaluation.id) - if selected_evaluation_row_ids is None: - _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - evaluation_row_ids, - ) - completed_pairs = filter_completed_row_pairs( - _completed_row_pairs_for_evaluation(db, evaluation.id), - [UUID(rid) for rid in selected_evaluation_row_ids], - ) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - policies_for_estimate = failure_policies - if policies_for_estimate is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies_for_estimate, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - _, total_calls = estimate_metric_clusters_llm_calls( - evaluation, - metrics, - completed_pairs, - policies_for_estimate, - max_llm_calls=llm_budget, - ) - row_ids_for_task = list(selected_evaluation_row_ids) - - prior_raw = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - policy_blob: Dict[str, Any] = {} - if failure_policies: - policy_blob = failure_policies_to_db(failure_policies, source="user") - - evaluation.metric_clusters = { - "status": "running", - "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], - "discovered_problems": ( - prior_raw.get("discovered_problems", []) - if isinstance(prior_raw, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - "selected_evaluation_row_ids": selected_evaluation_row_ids or [], - **policy_blob, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "metric_clusters") - db.commit() - - from app.workers.tasks.generate_evaluation_metric_clusters import ( - generate_evaluation_metric_clusters_task, - ) - - async_result = generate_evaluation_metric_clusters_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - "max_llm_calls": llm_budget, - "evaluation_row_ids": row_ids_for_task, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.metric_clusters, dict): - evaluation.metric_clusters["celery_task_id"] = async_result.id - flag_modified(evaluation, "metric_clusters") - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - db.commit() - - -def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: - """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return - task_id = str(raw.get("celery_task_id") or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") - logger.info( - "Revoked metric-clusters task {} for evaluation {}", - task_id, - evaluation.id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Failed to revoke metric-clusters task {} for evaluation {}: {}", - task_id, - evaluation.id, - exc, - ) - - -def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: - """Mark clustering as cancelled and revoke the worker task. - - Returns True if a running job was cancelled, False if already terminal. - """ - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return False - if (raw.get("status") or "").lower() != "running": - return False - - _revoke_metric_clusters_task(evaluation) - progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} - evaluation.metric_clusters = { - **raw, - "status": "cancelled", - "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - "progress": progress, - "celery_task_id": None, - } - return True - - -@router.get( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="getCallImportEvaluationMetricClusterFailurePolicies", -) -async def get_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source=source, - updated_at=updated_at, - ) - - -@router.put( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", -) -async def save_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - body: MetricFailurePoliciesSaveRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - try: - validate_failure_policies_for_metrics(body.policies, metrics) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - prior = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - evaluation.metric_clusters = merge_failure_policies_into_raw( - prior, - body.policies, - source="user", - ) - flag_modified(evaluation, "metric_clusters") - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) - if source != "user": - source = "user" - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source="user", - updated_at=updated_at, - ) - - -@router.get( - "/{eval_id}/metric-clusters/eligible-rows", - response_model=MetricClusterEligibleRowsResponse, - operation_id="listCallImportEvaluationMetricClusterEligibleRows", -) -async def list_call_import_evaluation_metric_cluster_eligible_rows( - call_import_id: UUID, - eval_id: UUID, - limit: Optional[int] = Query(default=None, ge=1), - count_only: bool = Query(default=False), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricClusterEligibleRowsResponse: - """Completed rows that have at least one flagged quality metric.""" - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) - metrics, _aggregates, policies, _source, _child_map = _clustering_context( - db, evaluation, eval_rows - ) - all_eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - total = len(all_eligible) - if count_only: - return MetricClusterEligibleRowsResponse(items=[], total=total) - raw_items = all_eligible if limit is None else all_eligible[:limit] - items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] - return MetricClusterEligibleRowsResponse(items=items, total=total) - - -@router.get( - "/{eval_id}/metric-clusters", - response_model=Optional[EvaluationMetricClustersState], - operation_id="getCallImportEvaluationMetricClusters", -) -async def get_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationMetricClustersState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _metric_clusters_payload(evaluation) - - -@router.post( - "/{eval_id}/metric-clusters", - response_model=EvaluationMetricClustersState, - operation_id="generateCallImportEvaluationMetricClusters", -) -async def generate_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationMetricClustersRequest = Body( - default_factory=EvaluationMetricClustersRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _metric_clusters_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating metric clusters." - ), - ) - - if body.evaluation_row_ids and body.row_limit is not None: - raise HTTPException( - status_code=400, - detail="Specify either evaluation_row_ids or row_limit, not both.", - ) - - if body.evaluation_row_ids: - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - requested = {str(rid) for rid in body.evaluation_row_ids} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, body.provider, body.model, body.credential_id - ) - - metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - merged_policies = merge_clustering_policies( - body.failure_policies, - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - try: - validate_failure_policies_for_metrics( - body.failure_policies or merged_policies, metrics - ) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - if not has_clusterable_metrics(metrics, merged_policies, eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No calls match any failure policy. Select failure values on " - "metrics that have matching rows, or leave metrics with no " - "failures unchecked — they are skipped automatically." - ), - ) - - filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - body.evaluation_row_ids, - row_limit=body.row_limit, - policies=merged_policies, - ) - if not selected_row_ids: - raise HTTPException( - status_code=400, - detail=( - "No eligible rows to cluster. Select completed calls that match " - "at least one configured failure policy." - ), - ) - if not filtered_pairs: - raise HTTPException( - status_code=400, - detail="No completed rows match the selected evaluation_row_ids.", - ) - - _enqueue_metric_clusters_job( - evaluation, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - evaluation_row_ids=body.evaluation_row_ids, - selected_evaluation_row_ids=selected_row_ids, - failure_policies=merged_policies, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="running" - ) - - -@router.post( - "/{eval_id}/metric-clusters/cancel", - response_model=EvaluationMetricClustersState, - operation_id="cancelCallImportEvaluationMetricClusters", -) -async def cancel_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - """Abort in-flight failure-diagnostics clustering. - - Idempotent: if clustering is not ``running``, returns the current state - unchanged. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _apply_metric_clusters_cancel(evaluation) - flag_modified(evaluation, "metric_clusters") - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="idle" - ) - - -@router.get( - "/{eval_id}/prompt-improvements", - response_model=Optional[EvaluationPromptImprovementsState], - operation_id="getCallImportEvaluationPromptImprovements", -) -async def get_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationPromptImprovementsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _prompt_improvements_payload(evaluation) - - -@router.post( - "/{eval_id}/prompt-improvements", - response_model=EvaluationPromptImprovementsState, - operation_id="generateCallImportEvaluationPromptImprovements", -) -async def generate_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationPromptImprovementsRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationPromptImprovementsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - clusters = _metric_clusters_payload(evaluation) - if clusters is None or clusters.status != "completed": - raise HTTPException( - status_code=400, - detail=( - "Metric clusters must be completed before generating prompt " - "improvements. Run failure diagnostics first." - ), - ) - - from app.services.call_import_prompt_improvements import is_imported_agent - from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request - from app.services.platform_import_imported_agent import resolve_imported_agent_for_call_import - - call_import = _require_import(db, call_import_id, organization_id) - imported_agent_id = body.imported_agent_id - if imported_agent_id is None: - linked = resolve_imported_agent_for_call_import(db, call_import, principal) - if linked is None: - raise HTTPException( - status_code=400, - detail=( - "No imported agent is linked to this import. Select an imported agent " - "or use a single-agent platform import." - ), - ) - imported_agent_id = linked.id - - imported_agent = ( - db.query(PromptPartial) - .filter( - PromptPartial.id == imported_agent_id, - PromptPartial.organization_id == organization_id, - PromptPartial.workspace_id == workspace_id, - ) - .first() - ) - if imported_agent is None or not is_imported_agent(imported_agent): - from app.services.platform_import_imported_agent import ( - relink_evaluation_imported_agent, - resolve_imported_agent_for_call_import, - ) - - recovered = resolve_imported_agent_for_call_import(db, call_import, principal) - if recovered is None or not is_imported_agent(recovered): - raise HTTPException( - status_code=404, - detail="Imported agent not found in the active workspace", - ) - imported_agent = recovered - imported_agent_id = recovered.id - if isinstance(evaluation.prompt_improvements, dict): - relink_evaluation_imported_agent(evaluation, recovered) - stamp_evaluation_actor(evaluation, principal) - db.commit() - - if not body.regenerate and not body.force: - cached = _prompt_improvements_payload(evaluation) - if ( - cached is not None - and cached.status in {"running", "completed"} - and cached.imported_agent_id == str(imported_agent_id) - ): - return cached - - provider_enum, model_str = get_llm_provider_and_model_for_request( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_prompt_improvements_job( - evaluation, - imported_agent_id=imported_agent_id, - imported_agent_name=imported_agent.name, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( - status="running", - imported_agent_id=str(imported_agent_id), - imported_agent_name=imported_agent.name, - ) - - -def _load_evaluation_for_prompt_push( - db: Session, - call_import_id: UUID, - eval_id: UUID, - organization_id: UUID, -) -> tuple[CallImport, CallImportEvaluation]: - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - return call_import, evaluation - - -@router.patch( - "/{eval_id}/prompt-improvements/review", - response_model=EvaluationPromptImprovementsState, - operation_id="reviewCallImportEvaluationPromptImprovement", -) -async def review_call_import_evaluation_prompt_improvement( - call_import_id: UUID, - eval_id: UUID, - body: PromptImprovementReviewUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationPromptImprovementsState: - del api_key - from app.services.call_import_prompt_push import PromptPushError, set_suggestion_review - - _, evaluation = _load_evaluation_for_prompt_push(db, call_import_id, eval_id, organization_id) - try: - set_suggestion_review(evaluation, body.suggestion_id, body.review_status, body.edited_text) - except PromptPushError as exc: - raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState(status="idle") - - -@router.get( - "/{eval_id}/prompt-improvements/push-preview", - response_model=PromptPushPreviewResponse, - operation_id="previewCallImportEvaluationPromptPush", -) -async def preview_call_import_evaluation_prompt_push( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> PromptPushPreviewResponse: - del api_key - from app.services.call_import_prompt_push import PromptPushError, preview_evaluation_push - - call_import, evaluation = _load_evaluation_for_prompt_push( - db, call_import_id, eval_id, organization_id - ) - try: - payload = preview_evaluation_push( - db, evaluation, call_import, organization_id, workspace_id - ) - except PromptPushError as exc: - raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc - return PromptPushPreviewResponse(**payload) - - -@router.post( - "/{eval_id}/prompt-improvements/push", - response_model=PromptPushResponse, - operation_id="pushCallImportEvaluationPrompt", -) -async def push_call_import_evaluation_prompt( - call_import_id: UUID, - eval_id: UUID, - body: PromptPushConfirm, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> PromptPushResponse: - del api_key - from app.services.call_import_prompt_push import PromptPushError, push_evaluation_prompt - - call_import, evaluation = _load_evaluation_for_prompt_push( - db, call_import_id, eval_id, organization_id - ) - try: - row = push_evaluation_prompt( - db, - evaluation, - call_import, - organization_id, - workspace_id, - body.confirm, - principal, - push_to_live=body.push_to_live, - ) - except PromptPushError as exc: - raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc - return row - - -# --------------------------------------------------------------------------- -# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a -# directed graph of (label -> label) transitions across the whole run. -# Powers the aggregate Sankey-style React Flow chart on the evaluation -# overview; per-call flow charts are built client-side from the same -# ``sequence`` field on a single row's metric_scores entry. -# --------------------------------------------------------------------------- - - -_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. -_FLOW_START_NODE_ID = "__START__" -_DISCOVERED_NODE_PREFIX = "disc:" - - -def _slug_label(value: Any) -> str: - """Lowercase + whitespace-collapse + underscore-join. - - Used everywhere we need a stable key for a metric/label name — - matching the same convention the worker uses when emitting - ``sequence`` entries and discovered keys. - """ - if value is None: - return "" - return "_".join(str(value).strip().lower().split()) - - -def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: - """Walk the alias map until we hit a slug that doesn't redirect. - - The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete - endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark - a slug as tombstoned. Chains can accumulate when the user merges - A→B and later merges B→C; this helper collapses them so callers - always land on the final canonical slug. - - Returns: - * the canonical slug if it still resolves to a real label, - * an empty string if the slug has been tombstoned (callers MUST - treat an empty result as "drop this entry entirely"), - * the input ``key`` if it isn't aliased. - - Cycles are guarded by a hard step limit since the alias map is - user-driven. - """ - if not key: - return "" - if not alias_map: - return key - current = key - seen: set[str] = set() - for _ in range(16): - if current in seen: - return current - seen.add(current) - if current not in alias_map: - return current - nxt = alias_map[current] - if nxt == current: - return current - if nxt == "": - # Deletion sentinel — the user has explicitly retired this - # slug. Propagate the empty string up so callers drop it. - return "" - current = nxt - return current - - -# Reserved JSON key under which the worker stores top-level metric -# discoveries on each row's ``metric_scores`` dict. Mirrors the constant -# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to -# avoid a worker import cycle from the routes module. -DISCOVERED_METRICS_KEY = "__discovered_metrics__" -METRIC_SCORES_META_KEYS = frozenset({DISCOVERED_METRICS_KEY, "_billing"}) - - -def _is_metric_scores_meta_key(key: str) -> bool: - normalized = str(key or "").strip().lower() - if not normalized: - return True - if normalized in {item.lower() for item in METRIC_SCORES_META_KEYS}: - return True - return normalized.endswith("__discovered") - -# Allowed values for an LLM-suggested top-level metric type. Kept in -# sync with ``DiscoveredMetricSuggestedType`` in -# ``app/models/schemas.py``. -_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") - - -def normalize_scores_with_aliases( - metric_scores: Dict[str, Any], - evaluation: CallImportEvaluation, - db: Session, - organization_id: UUID, -) -> Dict[str, Any]: - """Rewrite per-row ``metric_scores`` to honor merges + promotions. - - Called by the worker right after ``evaluate_with_llm`` returns so - every row that finishes AFTER a user has merged or promoted a - discovered label persists data already reflecting that decision. - Without this hook, a worker holding a stale prompt could re-emit a - ``from_key`` slug long after the user merged it away. - - For every parent entry (``selection_mode != null`` and a - ``discovered_labels`` / ``sequence`` field) we: - - * resolve discovered slugs through the evaluation's - ``discovered_label_aliases`` map (transitively), - * drop any discovered_labels entry whose canonical slug now - matches a real promoted child of the parent (merging them out - of the panel for free), and - * collapse adjacent duplicate sequence entries that result. - - Returns ``metric_scores`` (mutated in place) for chaining. - """ - if not isinstance(metric_scores, dict): - return metric_scores - - aliases_top = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - - # Identify the parent entries inside metric_scores. They're the - # dicts that carry a ``selection_mode`` key (set by the LLM - # hierarchy parser) and either a ``sequence`` or a - # ``discovered_labels`` list. - for key, entry in list(metric_scores.items()): - if not isinstance(entry, dict): - continue - if entry.get("type") != "category" and not entry.get("selection_mode"): - continue - try: - parent_uuid = UUID(str(key)) - except (TypeError, ValueError): - continue - - alias_map = {} - sub = aliases_top.get(str(parent_uuid)) - if isinstance(sub, dict): - alias_map = { - str(k): str(v) - for k, v in sub.items() - if isinstance(k, str) and isinstance(v, str) - } - promoted = _promoted_child_slugs(db, parent_uuid, organization_id) - - # Rewrite discovered_labels: alias-resolve keys, drop duplicates - # post-resolution, and drop entries that have been promoted. - discovered = entry.get("discovered_labels") - if isinstance(discovered, list): - kept_disc: List[Dict[str, Any]] = [] - seen: set[str] = set() - for d in discovered: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(alias_map, slug) - if not slug or slug in promoted or slug in seen: - continue - seen.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_disc.append(new_entry) - entry["discovered_labels"] = kept_disc - - # Rewrite sequence: alias-resolve every entry; collapse adjacent - # duplicates that result. We DON'T drop slugs that match - # promoted children — the promoted child slug is still a valid - # sequence entry; the flow chart will resolve it to the real - # child node. - seq = entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - last: Optional[str] = None - for item in seq: - if not isinstance(item, str): - continue - slug = _resolve_alias(alias_map, _slug_label(item)) - if not slug or slug == last: - continue - new_seq.append(slug) - last = slug - entry["sequence"] = new_seq - - # Top-level metric discoveries live alongside the parent entries - # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the - # flat evaluation-level alias/tombstone map + suppress slugs that - # already correspond to a real top-level Metric so workers that - # finish AFTER the user has merged / deleted / promoted can't - # resurrect a retired candidate. - discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) - if isinstance(discovered_metrics_payload, list): - flat_alias_map = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - promoted_metric_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - kept_metrics: List[Dict[str, Any]] = [] - seen_metrics: set[str] = set() - for d in discovered_metrics_payload: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(flat_alias_map, slug) - if ( - not slug - or slug in promoted_metric_slugs - or slug in seen_metrics - ): - continue - seen_metrics.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_metrics.append(new_entry) - if kept_metrics: - metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics - else: - # No survivors — drop the empty array so empty-discovery rows - # keep their pre-feature payload shape. - metric_scores.pop(DISCOVERED_METRICS_KEY, None) - - return metric_scores - - -def _alias_map_for_parent( - evaluation: CallImportEvaluation, parent_metric_id: UUID -) -> Dict[str, str]: - """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. - - Stored shape on the evaluation row is - ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty - dict for parents that have never had a merge applied. - """ - raw = getattr(evaluation, "discovered_label_aliases", None) - if not isinstance(raw, dict): - return {} - submap = raw.get(str(parent_metric_id)) - if not isinstance(submap, dict): - return {} - return { - str(k): str(v) - for k, v in submap.items() - if isinstance(k, str) and isinstance(v, str) - } - - -def _promoted_child_slugs( - db: Session, parent_metric_id: UUID, organization_id: UUID -) -> set[str]: - """Slugs of every real child currently sitting under the parent. - - The Discovered Labels panel hides any candidate whose slug already - matches a real child — that covers both freshly-promoted candidates - and legacy children the LLM happened to re-discover. We pull from - the live ``metrics`` table rather than the eval's - ``selected_metric_groups`` snapshot so newly-promoted children take - effect immediately, even on evaluations that ran before the - promotion. - """ - children = ( - db.query(Metric.name) - .filter( - Metric.parent_metric_id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .all() - ) - out: set[str] = set() - for (name,) in children: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _promoted_top_level_metric_slugs( - db: Session, organization_id: UUID -) -> set[str]: - """Slugs of every top-level (non-child) Metric in the organization. - - Used to suppress discovered-metric candidates whose slug already - matches a real standalone metric. We intentionally include both - standalone metrics AND parent category metrics — a top-level - discovery that collides with either name is a duplicate by - definition. - """ - rows = ( - db.query(Metric.name) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.is_(None), - ) - .all() - ) - out: set[str] = set() - for (name,) in rows: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _get_running_discovered_labels( - db: Session, - eval_id: UUID, - parent_metric_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, - *, - from_cache: bool = False, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered label seen in this eval so far. - - Walks each ``call_import_evaluation_rows`` row's - ``metric_scores[parent_id]["discovered_labels"]`` and folds entries - that share the same slug. Returns a list ordered by descending - count and stable on label key, shaped like:: - - [{"key": "customer_on_hold", "name": "Customer put on hold", - "description": "...", "sample_rationale": "...", "count": 12}] - - Powers two callers: - * The worker prompt builder ("REUSE the existing key if it fits") - — invoked just before each row's LLM call to feed the model the - running list of previously-discovered labels in this evaluation. - * The ``/discovered-labels`` API surface used by the frontend - Discovered Labels panel to render candidates with counts + - sample rationales. - - Non-completed rows are skipped: an in-flight row's discoveries are - not yet reliable (the row could fail and never produce final - metric_scores). We accept the tradeoff that rows running - concurrently won't see each other's labels — slug-collision dedup - catches identical re-inventions, and near-paraphrases surface in - the UI panel where the user can manually merge. - """ - - parent_id_str = str(parent_metric_id) - aliases = alias_map or {} - if from_cache: - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_child_slugs( - db, parent_metric_id, organization_id - ) - from app.services.call_imports.discovered_running_cache import ( - load_running_label_entries, - project_discovered_entries, - ) - - return project_discovered_entries( - load_running_label_entries(db, eval_id, parent_metric_id), - resolve_key=lambda raw: _resolve_alias(aliases, _slug_label(raw)), - promoted_slugs=promoted_slugs, - ) - - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - # Suppress slugs that have either: - # * been promoted to a real child of the parent (so the panel doesn't - # keep nagging the user about a candidate they've already - # accepted), or - # * been merged INTO another slug (the "from" side of a merge) — - # those occurrences fold into the canonical target instead. - promoted_slugs = set() - if organization_id is not None: - promoted_slugs = _promoted_child_slugs( - db, parent_metric_id, organization_id - ) - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - discovered = parent_entry.get("discovered_labels") - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on a real child slug. Order matters: a - # candidate that was merged into a slug which has since - # been promoted should disappear, not show up at the - # canonical slug. An empty resolved key means the slug was - # tombstoned via the delete endpoint. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace("_", " ") - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - - existing = by_key.get(key) - if existing is None: - # Track up to N=3 distinct rationales per candidate so - # the Promote-to-child flow can pre-fill the new - # sub-metric's rubric with concrete LLM examples - # without the user copy-pasting from the row table. - # ``sample_rationale`` is preserved for back-compat - # with older clients; ``examples`` is the new field. - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Append distinct rationales (case-insensitive trim) up - # to a small cap. Headroom is intentionally one above - # what the UI surfaces (2) so we have a backup when the - # first rationale is unhelpful. - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _get_running_discovered_metrics( - db: Session, - eval_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, - *, - from_cache: bool = False, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered top-level metric in this eval. - - Mirrors :func:`_get_running_discovered_labels` but is keyed at the - evaluation level (no ``parent_metric_id``). Walks each completed - row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries - that share the same slug (post-alias resolution), and suppresses - slugs that already correspond to a real top-level :class:`Metric` - in the organization. - - Each returned entry is shaped:: - - {"key": "customer_satisfaction", - "name": "Customer Satisfaction", - "description": "...", - "suggested_type": "boolean" | "rating" | "category", - "sample_rationale": "...", - "examples": ["..."], - "count": 12} - """ - - aliases = alias_map or {} - if from_cache: - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - from app.services.call_imports.discovered_running_cache import ( - load_running_metric_entries, - project_discovered_entries, - ) - - return project_discovered_entries( - load_running_metric_entries(db, eval_id), - resolve_key=lambda raw: _resolve_alias(aliases, _slug_label(raw)), - promoted_slugs=promoted_slugs, - ) - - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - promoted_slugs = set() - if organization_id is not None: - promoted_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on an already-existing top-level metric - # slug. Empty resolved key = tombstoned. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace( - "_", " " - ) - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - raw_type = str(entry.get("suggested_type") or "").strip().lower() - if raw_type not in _DISCOVERED_METRIC_TYPES: - raw_type = "boolean" - - existing = by_key.get(key) - if existing is None: - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "suggested_type": raw_type, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Keep the most-frequently-suggested type. We don't track - # per-type frequency yet; defer to the first non-default - # type encountered when the existing entry has the default. - if existing.get("suggested_type") == "boolean" and raw_type != "boolean": - existing["suggested_type"] = raw_type - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _build_flow_graph( - eval_rows: List[CallImportEvaluationRow], - parent_metric: Metric, - children: List[Metric], - alias_map: Optional[Dict[str, str]] = None, - extra_children: Optional[List[Metric]] = None, -) -> MetricFlowResponse: - """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. - - A synthetic ``START`` node is prepended to every sequence so the - diagram has a single origin. Children that never appear in any - sequence are still emitted as nodes (count=0) so the UI can render - them in the legend. - - ``alias_map`` lets callers fold merged-out discovered slugs into - their canonical target before building the graph; ``extra_children`` - are children of the parent that aren't in the legend list (e.g. - children promoted *after* the evaluation was created and therefore - missing from ``selected_metric_groups``) but should still resolve in - sequences so the slug doesn't get redrawn as a discovered candidate. - """ - parent_id_str = str(parent_metric.id) - aliases = alias_map or {} - # Build a fast lookup keyed by both the lower_snake child key (what the - # LLM emits in ``sequence``) and the child UUID (what some clients may - # store) so legacy / drifted payloads still resolve. - child_lookup: Dict[str, Metric] = {} - for child in children: - slug = _slug_label(child.name) - child_lookup[slug] = child - child_lookup[str(child.id)] = child - # ``extra_children`` are resolved-only — they shouldn't add legend - # nodes (those come from the explicit ``children`` argument), but - # they need to be in ``child_lookup`` so a sequence step that - # matches a freshly-promoted child resolves to the real child UUID - # instead of falling through to ``discovered_lookup`` and rendering - # as a "discovered" node. - if extra_children: - for child in extra_children: - slug = _slug_label(child.name) - if slug and slug not in child_lookup: - child_lookup[slug] = child - cid = str(child.id) - child_lookup.setdefault(cid, child) - - # Discovered labels: walk every row's discovered_labels first so we - # know which discovered slugs are valid before resolving sequences. - # Discovered nodes get a ``disc:`` prefixed id so they can't collide - # with real child UUIDs in the node/edge graph. We apply - # ``alias_map`` first so merged-out source slugs fold into their - # canonical target — preserving the user's "merge" intent on still- - # in-flight rows whose JSON wasn't rewritten by the merge endpoint. - discovered_lookup: Dict[str, Dict[str, Any]] = {} - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_discovered = parent_entry.get("discovered_labels") - if not isinstance(raw_discovered, list): - continue - for entry in raw_discovered: - if not isinstance(entry, dict): - continue - slug = _slug_label(entry.get("key") or entry.get("name")) - slug = _resolve_alias(aliases, slug) - if not slug or slug in child_lookup: - continue - name = (entry.get("name") or "").strip() or slug.replace("_", " ") - existing = discovered_lookup.get(slug) - if existing is None: - discovered_lookup[slug] = { - "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", - "name": name, - } - - node_counts: Dict[str, int] = {} - edge_counts: Dict[tuple[str, str], int] = {} - terminal_counts: Dict[str, int] = {} - - total_rows = len(eval_rows) - rows_with_sequence = 0 - - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_sequence = parent_entry.get("sequence") - if not isinstance(raw_sequence, list): - continue - - resolved_ids: List[str] = [] - last_resolved: Optional[str] = None - for item in raw_sequence: - if not isinstance(item, str): - continue - normalized = _resolve_alias(aliases, _slug_label(item)) - child = child_lookup.get(normalized) or child_lookup.get(item) - if child is not None: - cid = str(child.id) - # Adjacent dedupe AFTER alias resolution so two - # different raw slugs that fold to the same target - # don't draw a self-edge through the chart. - if cid == last_resolved: - continue - resolved_ids.append(cid) - last_resolved = cid - continue - disc = discovered_lookup.get(normalized) - if disc is not None: - if disc["id"] == last_resolved: - continue - resolved_ids.append(disc["id"]) - last_resolved = disc["id"] - - if not resolved_ids: - continue - - rows_with_sequence += 1 - for nid in resolved_ids: - node_counts[nid] = node_counts.get(nid, 0) + 1 - - edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( - edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 - ) - for src, tgt in zip(resolved_ids, resolved_ids[1:]): - if src == tgt: - continue - edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 - - terminal_id = resolved_ids[-1] - terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 - - nodes: List[MetricFlowNode] = [] - # Always include a START node so the UI has a stable entry point. - nodes.append( - MetricFlowNode( - id=_FLOW_START_NODE_ID, - label="Start", - count=rows_with_sequence, - is_terminal=False, - ) - ) - - def _emit_child_node(child: Metric) -> None: - cid = str(child.id) - count = node_counts.get(cid, 0) - terminal_count = terminal_counts.get(cid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=cid, - label=child.name, - count=count, - is_terminal=is_terminal, - ) - ) - - emitted_child_ids: set[str] = set() - for child in children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Extra children (promoted after the eval was created) only get - # legend nodes if they actually appear in the data — otherwise we'd - # pollute the diagram with every standalone promotion the user has - # ever made under this parent. - if extra_children: - for child in extra_children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - if node_counts.get(cid, 0) == 0: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Append discovered nodes after the real children so legend ordering - # keeps user-defined labels first. - for slug, info in discovered_lookup.items(): - nid = info["id"] - count = node_counts.get(nid, 0) - terminal_count = terminal_counts.get(nid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=nid, - label=info["name"], - count=count, - is_terminal=is_terminal, - is_discovered=True, - ) - ) - - edges: List[MetricFlowEdge] = [ - MetricFlowEdge(source=src, target=tgt, count=count) - for (src, tgt), count in sorted( - edge_counts.items(), key=lambda kv: kv[1], reverse=True - ) - ] - - return MetricFlowResponse( - parent_metric_id=parent_id_str, - parent_metric_name=parent_metric.name, - selection_mode=parent_metric.selection_mode, - nodes=nodes, - edges=edges, - total_rows=total_rows, - rows_with_sequence=rows_with_sequence, - ) - - -@router.get( - "/{eval_id}/flow", - response_model=MetricFlowResponse, - operation_id="getCallImportEvaluationFlow", -) -async def get_call_import_evaluation_flow( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose children's sequences should be " - "aggregated into a flow graph." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFlowResponse: - """Aggregate the LLM-inferred per-row sequences into one flow graph. - - Returns ``nodes`` (one per child of the parent metric, plus a - synthetic ``START`` node) and ``edges`` (counts of consecutive - label transitions across every row that produced a sequence). The - frontend feeds this directly into a React Flow / xyflow canvas; - edge thickness should scale with ``count / total_rows`` and - ``is_terminal`` nodes should be styled as outcomes. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - if not parent.selection_mode: - raise HTTPException( - status_code=400, - detail=( - "Flow charts are only meaningful for parent metrics " - "(selection_mode set). This metric is standalone." - ), - ) - - # Children are taken from selected_metric_groups when present so the - # flow chart reflects exactly the subset that ran in this - # evaluation; otherwise fall back to every enabled child of the - # parent. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_id_str = str(parent.id) - children: List[Metric] = [] - if parent_id_str in groups_raw and isinstance( - groups_raw[parent_id_str], list - ): - child_ids: List[UUID] = [] - for c in groups_raw[parent_id_str]: - try: - child_ids.append(UUID(str(c))) - except (TypeError, ValueError): - continue - if child_ids: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(child_ids), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - if not children: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .order_by(Metric.created_at.asc()) - .all() - ) - - # Children promoted AFTER this evaluation was created aren't in - # ``selected_metric_groups`` but their slugs still appear in already- - # scored rows' sequences. Pass them as ``extra_children`` so those - # sequence entries resolve against the real (now promoted) child - # instead of being redrawn as discovered candidates. - extra_children: List[Metric] = [] - if children: - existing_ids = {child.id for child in children} - all_children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .all() - ) - extra_children = [c for c in all_children if c.id not in existing_ids] - - eval_rows = _load_eval_rows(db, eval_id) - - alias_map = _alias_map_for_parent(evaluation, parent.id) - return _build_flow_graph( - eval_rows, - parent, - children, - alias_map=alias_map, - extra_children=extra_children, - ) - - -@router.get( - "/{eval_id}/discovered-labels", - response_model=DiscoveredLabelsResponse, - operation_id="getCallImportEvaluationDiscoveredLabels", -) -async def get_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose LLM-discovered candidate " - "sub-labels should be aggregated across rows." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Aggregate candidate sub-labels the LLM discovered during this eval. - - Only meaningful for parents with ``allow_discovery=true``; for other - parents we just return an empty ``items`` list rather than 400-ing - so the frontend can call the endpoint unconditionally for every - parent on the Flow tab without branching. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - alias_map = _alias_map_for_parent(evaluation, parent_metric_id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - parent_metric_id, - organization_id=organization_id, - alias_map=alias_map, - ) - items = [DiscoveredLabelItem(**item) for item in items_raw] - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), items=items - ) - - -@router.post( - "/{eval_id}/discovered-labels/merge", - response_model=DiscoveredLabelsResponse, - operation_id="mergeCallImportEvaluationDiscoveredLabels", -) -async def merge_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. - - Idempotent — re-merging the same pair is a no-op. Discovered slugs - inside per-row ``sequence`` arrays are also rewritten so the flow - chart stays consistent with the panel. When a row already has - ``to_key`` and we're merging ``from_key`` into it, we drop the - ``from_key`` entry instead of producing two entries with the same - slug. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - # No-op; just return the current aggregate so the client can - # refresh its view. - alias_map_existing = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_existing, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept: List[Dict[str, Any]] = [] - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - parent_entry["discovered_labels"] = kept - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == from_key: - seq_changed = True - if last_added == to_key: - continue - new_seq.append(to_key) - last_added = to_key - else: - new_seq.append(item) - last_added = ( - _slug_label(item) if isinstance(item, str) else None - ) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) - - # Persist the merge at the evaluation level too. This is what makes - # the merge survive future scoring: rows that finish AFTER this - # call (e.g. retries, in-flight workers) will go through the - # alias map in the API surface even if the per-row JSON they - # write still mentions ``from_key``. We chain through any existing - # alias so merging A→B and then B→C resolves A→C in the panel. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - # Resolve transitively: if to_key itself was previously merged into - # something else, point from_key at the canonical end-of-chain. - canonical_to = _resolve_alias(parent_aliases, to_key) - parent_aliases[from_key] = canonical_to - # Re-target any earlier aliases that pointed AT from_key — without - # this, A→B and then B→C would leave A still pointing to B (now a - # broken pointer because B is gone). Rewriting them keeps the - # alias map self-consistent. - for k, v in list(parent_aliases.items()): - if v == from_key: - parent_aliases[k] = canonical_to - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-labels/delete", - response_model=DiscoveredLabelsResponse, - operation_id="deleteCallImportEvaluationDiscoveredLabel", -) -async def delete_call_import_evaluation_discovered_label( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Tombstone a single LLM-discovered candidate for this evaluation. - - Symmetric with the merge endpoint, but instead of redirecting the - slug at another candidate we mark it as deleted. After this call: - - * the slug is stripped from every row's - ``metric_scores[parent].discovered_labels`` list, and from - every row's ``sequence`` array (so the flow chart no longer - draws a node for it); - * the slug is recorded in - ``evaluation.discovered_label_aliases[parent][slug] = ""`` - so any worker that finishes a row AFTER this call (e.g. a row - still in flight when the user clicked Delete) silently drops - the slug instead of resurrecting it. - - Idempotent: deleting an already-deleted slug is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) != len(discovered): - parent_entry["discovered_labels"] = kept - mutated = True - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == target_key: - seq_changed = True - continue - if isinstance(item, str): - norm = _slug_label(item) - if norm == last_added: - seq_changed = True - continue - last_added = norm - new_seq.append(item) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) - - # 3. Persist the tombstone on the evaluation so workers that finish - # later don't re-surface the deleted slug. We also retarget any - # existing aliases whose ``to_key`` was the deleted slug — without - # this, a previous merge that pointed at this slug would leave a - # dangling pointer. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - parent_aliases[target_key] = "" # deletion sentinel - for k, v in list(parent_aliases.items()): - if v == target_key: - parent_aliases[k] = "" - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -# --------------------------------------------------------------------------- -# Discovered TOP-LEVEL METRICS (per-evaluation discovery) -# -# These endpoints are the parallel of the discovered-labels trio above but -# scoped to the evaluation as a whole instead of to a parent metric. They -# all live under ``/{eval_id}/discovered-metrics`` and operate on the -# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row -# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` -# map (no parent-id nesting). -# --------------------------------------------------------------------------- - - -def _flat_metric_aliases( - evaluation: CallImportEvaluation, -) -> Dict[str, str]: - """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" - raw = getattr(evaluation, "discovered_metric_aliases", None) - if not isinstance(raw, dict): - return {} - return { - str(k): str(v) - for k, v in raw.items() - if isinstance(k, str) and isinstance(v, str) - } - - -@router.get( - "/{eval_id}/discovered-metrics", - response_model=DiscoveredMetricsResponse, - operation_id="getCallImportEvaluationDiscoveredMetrics", -) -async def get_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Aggregate top-level metric candidates the LLM discovered during this eval. - - Returns an empty ``items`` list when the evaluation did not opt - into top-level metric discovery; this keeps the frontend able to - call the endpoint unconditionally without branching on the - evaluation's ``discover_new_metrics`` flag. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not bool(getattr(evaluation, "discover_new_metrics", False)): - return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/merge", - response_model=DiscoveredMetricsResponse, - operation_id="mergeCallImportEvaluationDiscoveredMetrics", -) -async def merge_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Rewrite every row's ``__discovered_metrics__`` entry from→to. - - Mirrors the discovered-labels merge endpoint but operates on the - flat top-level metric list. Idempotent — re-merging is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - - kept: List[Dict[str, Any]] = [] - mutated = False - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - scores[DISCOVERED_METRICS_KEY] = kept - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - canonical_to = _resolve_alias(aliases, to_key) - aliases[from_key] = canonical_to - for k, v in list(aliases.items()): - if v == from_key: - aliases[k] = canonical_to - evaluation.discovered_metric_aliases = aliases - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/delete", - response_model=DiscoveredMetricsResponse, - operation_id="deleteCallImportEvaluationDiscoveredMetric", -) -async def delete_call_import_evaluation_discovered_metric( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Tombstone a single LLM-discovered top-level metric candidate.""" - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) == len(discovered): - return False - if kept: - scores[DISCOVERED_METRICS_KEY] = kept - else: - scores.pop(DISCOVERED_METRICS_KEY, None) - row.metric_scores = dict(scores) - return True - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - aliases[target_key] = "" # tombstone - for k, v in list(aliases.items()): - if v == target_key: - aliases[k] = "" - evaluation.discovered_metric_aliases = aliases - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.delete( - "/{eval_id}/rows/{eval_row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluationRow", -) -async def delete_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - """Delete a single per-row scoring entry within an evaluation run. - - Useful when the user wants to drop a noisy row before re-exporting - the CSV — e.g. a row whose audio was corrupt and skewed the - aggregate. Counters on the parent are recomputed so the rolled-up - status stays accurate. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import delete_evaluation_row_on_shards - - if not delete_evaluation_row_on_shards(eval_row_id, eval_id): - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - # If the row was still in flight, best-effort revoke the worker task - # so it doesn't try to write into a deleted DB row mid-execution. - if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: - pass - - db.delete(eval_row) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -# --------------------------------------------------------------------------- -# Retry endpoints -# --------------------------------------------------------------------------- -# -# The create endpoint enqueues every row of a fresh run; these endpoints -# let the user re-enqueue a *subset* of rows in an existing run — most -# commonly the ones that failed. We keep the worker contract identical -# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path -# only has to reset row state and re-fan-out. When a row is missing its -# diarised transcript and the run was configured for diarised -# transcripts, we chain through ``transcribe_call_import_row_task`` the -# same way the create endpoint does — that's what makes "retry" feel -# like "just fix it" instead of "fail again immediately". - - -def source_retry_assignments( - source_row: CallImportRow, - *, - transcribe_overwrite: bool, -) -> dict: - """Columns a retry should write. Empty when the source row is already reusable.""" - from app.api.v1.routes.call_imports import _is_stuck_pending_import_row - from app.db_sharding.scatter_gather import row_text_present - - changes: dict = {} - if (getattr(source_row, "celery_task_id", None) or "").strip(): - changes["celery_task_id"] = None - - no_audio = not (getattr(source_row, "recording_s3_key", None) or "").strip() - status = getattr(source_row, "status", None) - requeue = no_audio and status in ( - CallImportRowStatus.FAILED, - CallImportRowStatus.PROCESSING, - ) - if no_audio and status == CallImportRowStatus.PENDING: - requeue = _is_stuck_pending_import_row(source_row) - if requeue: - if status != CallImportRowStatus.PENDING: - changes["status"] = CallImportRowStatus.PENDING - if getattr(source_row, "error_message", None) is not None: - changes["error_message"] = None - if (getattr(source_row, "attempts", 0) or 0) != 0: - changes["attempts"] = 0 - - has_dia = row_text_present( - source_row, "diarised_transcript", "diarised_transcript_char_length" - ) - if transcribe_overwrite and has_dia: - changes["diarised_transcript"] = None - has_dia = False - - dia_status = ( - getattr(source_row, "diarised_transcript_status", None) or "" - ).strip().lower() - if has_dia and not transcribe_overwrite: - if dia_status != "completed": - changes["diarised_transcript_status"] = "completed" - if getattr(source_row, "diarised_transcript_error", None) is not None: - changes["diarised_transcript_error"] = None - return changes - - if dia_status in {"failed", "pending", "running", "idle"}: - if dia_status != "idle": - changes["diarised_transcript_status"] = "idle" - if getattr(source_row, "diarised_transcript_error", None) is not None: - changes["diarised_transcript_error"] = None - return changes - - -def _prepare_source_row_for_retry( - source_row: CallImportRow, - *, - transcribe_overwrite: bool, -) -> None: - """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" - for column, value in source_retry_assignments( - source_row, - transcribe_overwrite=transcribe_overwrite, - ).items(): - setattr(source_row, column, value) - - -def _metric_scores_for_retry(existing, metric_ids: Optional[List[UUID]]) -> dict: - """Full retry clears every score. A metric subset removes only those keys.""" - if not metric_ids: - return {} - scores = existing if isinstance(existing, dict) else {} - target_keys = {str(mid).lower() for mid in metric_ids} - return { - key: value - for key, value in scores.items() - if _is_metric_scores_meta_key(str(key)) - or str(key).lower() not in target_keys - } - - -def _reset_eval_row_for_retry( - eval_row: CallImportEvaluationRow, - *, - metric_ids: Optional[List[UUID]] = None, - skip_revoke: bool = False, - skip_chain: bool = False, -) -> None: - """Wipe per-row state so the worker can re-run it cleanly. - - Mirrors the initial state used by ``create_call_import_evaluation`` - when it first inserts a row, with the addition of revoking any - lingering Celery task id. - - When ``metric_ids`` is provided, this is a **metric-subset retry**: - only the scores for those metrics are removed from - ``metric_scores`` (other metrics' previously-computed values are - preserved so the worker's partial-merge write keeps them intact). - Otherwise the entire ``metric_scores`` dict is reset, matching the - legacy behaviour. - """ - if ( - not skip_revoke - and eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ): - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: # noqa: BLE001 — revoke is best-effort - pass - eval_row.status = "pending" - eval_row.error_message = None - eval_row.metric_scores = _metric_scores_for_retry( - eval_row.metric_scores, - metric_ids, - ) - eval_row.started_at = None - eval_row.finished_at = None - task_id = (eval_row.celery_task_id or "").strip() - eval_row.celery_task_id = None - if skip_chain or not task_id: - return - from app.workers.tasks.evaluate_call_import_row_audio import ( - clear_eval_audio_chain, - ) - - clear_eval_audio_chain(str(eval_row.id)) - - -def _enqueue_eval_rows_with_optional_transcribe( - db: Session, - evaluation: CallImportEvaluation, - eval_rows_with_source: List[ - Tuple[CallImportEvaluationRow, CallImportRow] - ], - *, - transcribe_overwrite: bool = False, - restricted_metric_ids: Optional[List[UUID]] = None, -) -> Tuple[int, int]: - """Schedule throttled evaluation dispatch for pending eval rows. - - Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for - logging/UI compatibility. Actual Celery fan-out is handled by - :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. - """ - from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval - from app.workers.concurrency.fair_dispatch import ( - schedule_fair_dispatch, - store_evaluation_transcribe_overwrite, - store_row_restricted_metrics, - ) - - eval_only_count = 0 - transcribe_count = 0 - if eval_rows_with_source: - for eval_row, source_row in eval_rows_with_source: - if _needs_transcribe_for_eval( - evaluation, - source_row, - transcribe_overwrite=transcribe_overwrite, - ): - transcribe_count += 1 - else: - eval_only_count += 1 - - restricted_metric_ids_str: Optional[List[str]] = ( - [str(mid) for mid in restricted_metric_ids] - if restricted_metric_ids - else None - ) - if restricted_metric_ids_str: - for eval_row, _ in eval_rows_with_source: - store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) - else: - restricted_metric_ids_str = ( - [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None - ) - store_evaluation_transcribe_overwrite( - evaluation.id, - overwrite=transcribe_overwrite, - ) - schedule_fair_dispatch(max_workspace_turns=999) - return eval_only_count, transcribe_count - - -def _apply_telephony_retry_overrides( - db: Session, - *, - call_import: CallImport, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Pin or clear telephony credentials on the batch for this retry pass.""" - fields_set = payload.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return - - from app.api.v1.routes.call_imports import ( - _resolve_telephony_integration, - _validate_telephony_credentials_live, - ) - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - _validate_telephony_credentials_live(db, organization_id, integration) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - -def _apply_retry_overrides( - db: Session, - evaluation: CallImportEvaluation, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Validate + persist the LLM/STT override fields on the run. - - Mirrors the validation in ``create_call_import_evaluation`` but - only touches the fields the caller actually sent — leaving any - field ``None`` preserves the run's existing value. Raises - ``HTTPException(400)`` on bad input so the route handler can let - FastAPI turn it into a clean 400 response. - """ - # --- LLM provider + model (must be sent together) --- - if payload.llm_provider is not None or payload.llm_model is not None: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail=( - "Both llm_provider and llm_model are required when " - "overriding the run LLM on retry." - ), - ) - try: - evaluation.llm_provider = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - new_model = payload.llm_model.strip() or None - if not new_model: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - evaluation.llm_model = new_model - - # --- LLM credential pin --- - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in " - "this organization." - ), - ) - evaluation.llm_credential_id = payload.llm_credential_id - - if payload.llm_config is not None: - evaluation.llm_config = payload.llm_config - - # --- Per-metric LLM overrides --- - # We accept the same dict shape as the create endpoint but - # constrain keys to leaf metrics that are actually in this run. - # Passing an empty dict explicitly clears existing overrides. - if payload.metric_llm_overrides is not None: - valid_leaf_ids = { - str(mid) for mid in (evaluation.selected_metric_ids or []) - } - overrides_payload: Dict[str, Dict[str, Any]] = {} - for metric_id, override in payload.metric_llm_overrides.items(): - if metric_id not in valid_leaf_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a leaf metric in " - "this run." - ), - ) - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a " - "provider but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses " - f"unknown provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model " - "but no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - overrides_payload[metric_id] = override_dict - evaluation.metric_llm_overrides = overrides_payload or None - - # --- STT provider + model (must be sent together) --- - if payload.stt_provider is not None or payload.stt_model is not None: - if not (payload.stt_provider and payload.stt_model): - raise HTTPException( - status_code=400, - detail=( - "Both stt_provider and stt_model are required " - "when overriding the run STT on retry." - ), - ) - try: - evaluation.stt_provider = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - new_stt_model = payload.stt_model.strip() or None - if not new_stt_model: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - evaluation.stt_model = new_stt_model - - # --- STT credential pin --- - if payload.stt_credential_id is not None: - evaluation.stt_credential_id = payload.stt_credential_id - - # --- LLM diariser provider + model (must be sent together) --- - if ( - payload.diarization_llm_provider is not None - or payload.diarization_llm_model is not None - ): - if not ( - payload.diarization_llm_provider - and payload.diarization_llm_model - ): - raise HTTPException( - status_code=400, - detail=( - "Both diarization_llm_provider and " - "diarization_llm_model are required when overriding " - "the run diariser on retry." - ), - ) - try: - evaluation.diarisation_llm_provider = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - "Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - new_diariser_model = ( - payload.diarization_llm_model.strip() or None - ) - if not new_diariser_model: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - evaluation.diarisation_llm_model = new_diariser_model - - if payload.diarization_llm_credential_id is not None: - evaluation.diarisation_llm_credential_id = ( - payload.diarization_llm_credential_id - ) - - # ``diarization_prompt`` semantics: None = leave untouched; - # empty string = clear (fall back to the canonical default at - # worker time); anything else = persist verbatim. - if payload.diarization_prompt is not None: - cleaned = payload.diarization_prompt.strip() - evaluation.diarisation_prompt = cleaned or None - - if payload.transcribe_mode is not None: - mode = payload.transcribe_mode.strip().lower() - if mode not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Valid values are 'stt_llm' and 'llm_only'." - ), - ) - evaluation.transcribe_mode = mode - - -def _gather_retry_targets( - db: Session, - evaluation: CallImportEvaluation, - requested_ids: Optional[List[UUID]], - *, - include_completed: bool = False, -) -> Tuple[ - List[Tuple[CallImportEvaluationRow, CallImportRow]], - List[CallImportEvaluationRetrySkippedItem], -]: - """Resolve which rows to retry + reasons for any we refuse. - - When ``requested_ids`` is None we retry every row whose status is - ``failed`` (or every row when ``include_completed`` is also set — - used by the metric-subset retry path which legitimately wants to - recompute a metric on already-successful rows). When the caller - passes ids explicitly we still filter out rows that are currently - in flight; ``include_completed`` controls whether previously- - successful rows are eligible. - """ - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import gather_retry_targets_sharded - - return gather_retry_targets_sharded( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - - from app.db_sharding.eval_rows import _collect_retry_targets_on_session - - targets, skipped, found = _collect_retry_targets_on_session( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - if requested_ids is not None: - for missing in set(requested_ids) - found: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=missing, - reason="unknown", - ) - ) - return targets, skipped - - -@router.post( - "/{eval_id}/retry", - response_model=CallImportEvaluationRetryResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluation", -) -async def retry_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRetryResponse: - """Re-enqueue failed rows in an evaluation run. - - Default behavior (no body) is "retry every row that failed". Pass - ``eval_row_ids`` to scope the retry to a specific subset (e.g. the - single row a user clicked in the UI). Rows that are still - in-flight or already completed are returned in ``skipped`` rather - than re-enqueued, so this endpoint is always safe to call. - - When ``metric_ids`` is set in the payload, this is a **metric- - subset retry**: only the listed metrics are recomputed (and merged - into the row's existing ``metric_scores`` — other metrics' values - are preserved). The route auto-flips ``include_completed=True`` in - that case so previously-successful rows are eligible for re- - scoring; without it the call would no-op because every row would - be skipped as ``completed``. - - The worker contract is the same as the create endpoint: - ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. - When the run is configured for diarised transcripts and the row's - diarised transcript is missing, we chain through - ``transcribe_call_import_row_task`` first — matching the - auto-transcribe behavior of POST ``/evaluations``. - """ - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - requested_ids = payload.eval_row_ids if payload else None - # Metric-subset retry: validate that every metric is something this - # run actually scored. Empty list is rejected too — callers that - # want a full re-run should omit the field entirely. - # - # ``selected_metric_ids`` holds the LEAVES only (children for - # hierarchical / category metrics, standalone metrics otherwise) — - # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. - # Parent IDs for hierarchical metrics live separately in - # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the - # UI can reconstruct the tree without round-tripping through the - # metric table. - # - # The Re-run-metrics modal surfaces PARENTS for hierarchical - # metrics (it suppresses individual children via - # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a - # naive ``metric_ids Γèå selected_metric_ids`` check rejects every - # parent-ID request with a misleading "unknown ids" 400. We accept - # both shapes here and then EXPAND any parent IDs into - # ``{parent_id, *child_ids}`` so the downstream helpers see the - # full set of keys that need clearing + the full set of leaves - # that need re-scoring. - metric_ids: Optional[List[UUID]] = ( - payload.metric_ids if payload else None - ) - if metric_ids is not None: - metric_ids = [ - mid for mid in metric_ids if not _is_metric_scores_meta_key(str(mid)) - ] - if not metric_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a non-empty list of metric UUIDs. " - "Omit the field to re-run all metrics." - ), - ) - - leaf_set: Set[str] = { - str(item).lower() - for item in (evaluation.selected_metric_ids or []) - } - # ``selected_metric_groups`` is a dict ``{parent_id_str: - # [child_id_str, ...]}`` (see line ~487 in - # ``create_call_import_evaluation``). We tolerate stale data - # (string / UUID / non-dict) without crashing the retry path — - # if it's malformed we just treat it as "no parents" and fall - # back to the leaf-only check. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_to_children_str: Dict[str, List[str]] = {} - for parent_key, children_raw in groups_raw.items(): - if not isinstance(children_raw, (list, tuple)): - continue - children_norm = [ - str(c).lower() for c in children_raw if c is not None - ] - parent_to_children_str[str(parent_key).lower()] = children_norm - parent_set = set(parent_to_children_str.keys()) - - unknown = [ - mid for mid in metric_ids - if str(mid).lower() not in leaf_set - and str(mid).lower() not in parent_set - ] - if unknown: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a subset of this evaluation's " - f"selected metrics; unknown ids: {[str(u) for u in unknown]}." - ), - ) - - # Expand parent IDs into ``{parent, *children}`` so: - # * ``_reset_eval_row_for_retry`` strips BOTH the parent - # entry (with ``chosen_child_id`` / rationale) AND every - # per-child boolean entry that the LLM evaluator wrote - # under each child's ID (see - # ``app/workers/tasks/helpers/llm_evaluation.py`` lines - # 1584 and 1649). - # * ``_enqueue_eval_rows_with_optional_transcribe`` → - # ``evaluate_call_import_row_task`` filters the work-list - # off ``selected_metric_ids`` (leaves), so we MUST hand it - # the child IDs for the parent to actually get re-scored. - # Leaves pass through unchanged. - expanded: List[UUID] = [] - seen: Set[str] = set() - for mid in metric_ids: - mid_norm = str(mid).lower() - children_str = parent_to_children_str.get(mid_norm) - if children_str is not None: - # Parent: include the parent ID itself (so the parent - # entry in ``metric_scores`` is also cleared) and all - # of its children. - candidates = [mid_norm, *children_str] - else: - candidates = [mid_norm] - for candidate in candidates: - if candidate in seen: - continue - try: - expanded.append(UUID(candidate)) - except (TypeError, ValueError): - # Defensive: skip junk values rather than 500. - continue - seen.add(candidate) - metric_ids = expanded - - # ``include_completed`` is auto-enabled when the caller asked for a - # metric subset (otherwise the metric-subset retry would always - # no-op on a green run, which is the whole reason this feature - # exists). The explicit payload flag wins for full-row retries. - include_completed = bool( - (payload.include_completed if payload else False) - or (metric_ids is not None) - ) - - transcribe_overwrite = bool( - payload.transcribe_overwrite if payload else False - ) - - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - if requested_ids is None: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - statuses = ( - ["failed", "completed"] if include_completed else ["failed"] - ) - target_count = count_evaluation_rows_for_run( - db, eval_id, statuses=statuses - ) - else: - from sqlalchemy import func - - count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( - CallImportEvaluationRow.evaluation_id == eval_id - ) - if include_completed: - count_query = count_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ) - else: - count_query = count_query.filter( - CallImportEvaluationRow.status == "failed" - ) - target_count = int(count_query.scalar() or 0) - if target_count == 0: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - else: - targets, skipped = _gather_retry_targets( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - if not targets: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - target_count = len(targets) - - # Apply LLM / STT overrides BEFORE enqueueing so the persisted run - # config is correct by the time the worker reads it. - if payload is not None: - _apply_retry_overrides(db, evaluation, organization_id, payload) - _apply_telephony_retry_overrides( - db, - call_import=call_import, - organization_id=organization_id, - payload=payload, - ) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - - _claim_evaluation_bulk_operation(eval_id, "retry") - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - retry_call_import_evaluation_task, - ) - - retry_call_import_evaluation_task.delay( - str(eval_id), - { - "eval_row_ids": [str(rid) for rid in requested_ids] - if requested_ids - else None, - "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, - "include_completed": include_completed, - "transcribe_overwrite": transcribe_overwrite, - }, - ) - - return CallImportEvaluationRetryResponse( - requeued=target_count, - transcribe_requeued=0, - skipped=skipped, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/retry", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluationRow", -) -async def retry_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Re-enqueue a single failed evaluation row. - - Convenience wrapper around ``retry_call_import_evaluation`` for the - "Retry this row" affordance in the row table. Returns the - refreshed row so the UI can update its badge immediately, without - waiting for the next polling tick. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import ( - evaluation_row_session, - find_evaluation_row_in_run, - ) - from app.db_sharding.sessions import is_sharding_enabled - - eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) - if eval_row is None: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - if eval_row.status in {"pending", "running"}: - raise HTTPException( - status_code=409, - detail=( - "This row is still in progress — wait for it to finish " - "before retrying." - ), - ) - - targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) - if not targets: - raise HTTPException( - status_code=409, - detail=( - "This row cannot be retried in its current state " - f"(status={eval_row.status})." - ), - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(eval_row) - row_db.commit() - targets = [(eval_row, source_row)] - else: - for er, source_row in targets: - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(er) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - - try: - _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to re-enqueue retry for evaluation row {}", eval_row_id - ) - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - _source_row, - _shard_id, - ): - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - row_db.commit() - else: - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - _rollup_evaluation_status(evaluation, db) - db.commit() - raise HTTPException( - status_code=500, - detail=f"Failed to re-enqueue retry: {exc}", - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - _row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - db.refresh(eval_row) - source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=EVALS_VIEW, - manage_capability=EVALS_RUN, - run_capability=EVALS_RUN, - report_capability=REPORTS_GENERATE, -) diff --git a/app/api/v1/routes/call_import_evaluations/__init__.py b/app/api/v1/routes/call_import_evaluations/__init__.py new file mode 100644 index 00000000..48019ce7 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/__init__.py @@ -0,0 +1,79 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +from app.api.v1.routes.call_import_evaluations import ( # noqa: F401 + aggregate, + clusters, + discovered, + evaluation_cancel, + evaluation_create, + evaluation_read, + evaluation_retry, + exports, + flow, + insights, + prompts, + rows, +) +from app.api.v1.routes.call_import_evaluations._router import router +from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=EVALS_VIEW, + manage_capability=EVALS_RUN, + run_capability=EVALS_RUN, + report_capability=REPORTS_GENERATE, +) + +from app.api.v1.routes.call_import_evaluations.aggregate import ( # noqa: E402 + _aggregate_primary_percent, + _aggregate_to_dict, + _child_names_by_parent, + _compute_metric_aggregates, + _explain_period_deltas, + _period_delta_explanation_cache_key, + _reconcile_cluster_periods, +) +from app.api.v1.routes.call_import_evaluations.discovered import ( # noqa: E402 + DISCOVERED_METRICS_KEY, + _alias_map_for_parent, + _get_running_discovered_labels, + _get_running_discovered_metrics, + _is_metric_scores_meta_key, + _resolve_alias, + _slug_label, + normalize_scores_with_aliases, +) +from app.api.v1.routes.call_import_evaluations.exports import ( # noqa: E402 + _fail_stale_export_job, +) +from app.api.v1.routes.call_import_evaluations.flow import ( # noqa: E402 + _FLOW_START_NODE_ID, + _build_flow_graph, +) +from app.api.v1.routes.call_import_evaluations.insights import ( # noqa: E402 + _generate_and_persist_tldr_summary, + _tldr_summary_payload, +) +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _count_completed_eval_rows, + _expand_metric_selection, + _metrics_for_ids, + _rollup_evaluation_status, + _serialize_eval, + _serialize_selected_metric_ids, + require_call_import_capability, +) +from app.api.v1.routes.call_import_evaluations.evaluation_cancel import ( # noqa: E402 + EVAL_CANCELLED_BY_USER_ERROR, +) +from app.api.v1.routes.call_import_evaluations.evaluation_retry import ( # noqa: E402 + _enqueue_eval_rows_with_optional_transcribe, + _metric_scores_for_retry, + _prepare_source_row_for_retry, + _reset_eval_row_for_retry, + source_retry_assignments, +) diff --git a/app/api/v1/routes/call_import_evaluations/_router.py b/app/api/v1/routes/call_import_evaluations/_router.py new file mode 100644 index 00000000..7c8c9842 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/_router.py @@ -0,0 +1,11 @@ +"""Shared APIRouter for call import evaluation endpoints.""" + +from fastapi import APIRouter, Depends + +from app.dependencies import require_enterprise_feature + +router = APIRouter( + prefix="/call-imports/{call_import_id}/evaluations", + tags=["Call Import Evaluations"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) diff --git a/app/api/v1/routes/call_import_evaluations/_shared.py b/app/api/v1/routes/call_import_evaluations/_shared.py new file mode 100644 index 00000000..687fb786 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/_shared.py @@ -0,0 +1,656 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +def _require_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImport: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + return call_import + + +def require_call_import_capability(capability: str): + """Ensure the caller has *capability* in the call import's workspace (not just the header).""" + + def _dep( + call_import_id: UUID, + principal: Principal = Depends(get_principal), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), + ) -> CallImport: + call_import = _require_import(db, call_import_id, organization_id) + caps, _, role = resolve_workspace_capabilities( + db, + principal=principal, + workspace_id=call_import.workspace_id, + organization_id=organization_id, + ) + if capability not in caps: + raise HTTPException( + status_code=403, + detail=capability_denied_message( + capability, + role_name=role.name if role else None, + workspace_label="the active workspace", + ), + ) + return call_import + + return _dep + + +def _pick_evaluation_row_transcript( + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> Optional[str]: + """Transcript shown in evaluation row detail for the run's source.""" + if source_row is None: + return None + source = ( + (evaluation.transcript_source or "diarised").strip().lower() + if evaluation is not None + else "diarised" + ) + if source == "production": + raw = (source_row.transcript or "").strip() + return raw or None + diarised = (source_row.diarised_transcript or "").strip() + if diarised: + return diarised + raw = (source_row.transcript or "").strip() + return raw or None + + +def _to_evaluation_row_response( + eval_row_obj: CallImportEvaluationRow, + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, + *, + include_text: bool = True, +) -> CallImportEvaluationRowResponse: + """Serialize one evaluation row plus joined source-row metadata. + + List pages pass ``include_text=False`` so transcript bodies stay off + the page. The single-row read passes ``True``. + """ + if include_text and source_row is not None: + transcript = _pick_evaluation_row_transcript(source_row, evaluation) + production_transcript = (source_row.transcript or "").strip() or None + diarised_transcript = (source_row.diarised_transcript or "").strip() or None + raw_columns = source_row.raw_columns + else: + transcript = None + production_transcript = None + diarised_transcript = None + raw_columns = None + return CallImportEvaluationRowResponse( + id=eval_row_obj.id, + evaluation_id=eval_row_obj.evaluation_id, + call_import_row_id=eval_row_obj.call_import_row_id, + row_index=source_row.row_index if source_row else None, + conversation_id=source_row.conversation_id if source_row else None, + transcript=transcript, + production_transcript=production_transcript, + diarised_transcript=diarised_transcript, + raw_columns=raw_columns, + recording_url=source_row.recording_url if source_row else None, + recording_date=source_row.recording_date if source_row else None, + recording_s3_key=source_row.recording_s3_key if source_row else None, + diarised_transcript_status=( + source_row.diarised_transcript_status if source_row else None + ), + diarised_transcript_error=( + source_row.diarised_transcript_error if source_row else None + ), + status=eval_row_obj.status, + metric_scores=eval_row_obj.metric_scores or {}, + error_message=eval_row_obj.error_message, + started_at=eval_row_obj.started_at, + finished_at=eval_row_obj.finished_at, + created_at=eval_row_obj.created_at, + updated_at=eval_row_obj.updated_at, + ) + + +def _serialize_selected_metric_ids(value) -> List[UUID]: + result: List[UUID] = [] + if not isinstance(value, list): + return result + for item in value: + try: + result.append(UUID(str(item))) + except (TypeError, ValueError): + continue + return result + + +def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: + if not ids: + return [] + rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(ids), + ) + .all() + ) + by_id = {row.id: row for row in rows} + return [by_id[mid] for mid in ids if mid in by_id] + + +def _expand_metric_selection( + db: Session, + org_id: UUID, + selected_ids: List[UUID], +) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: + """Resolve user-supplied metric ids into actual leaves + parent grouping. + + Rules: + * If a parent id is in ``selected_ids`` and no specific children of + that parent are also listed, include EVERY enabled child of that + parent. + * If a parent id AND some of its children are listed, include only + the listed children (treat the parent selection as the + "container" so users can deselect labels). + * Standalone metrics (no parent, no children) pass through + unchanged. + * Disabled metrics are filtered out at this layer so the caller + doesn't have to repeat the check. + + Returns: + (effective_metrics, parent_to_children) + + ``effective_metrics`` is the deduplicated list of metrics the + worker will actually score (children + standalone). Order is + preserved from ``selected_ids`` for display stability. + + ``parent_to_children`` maps each parent metric id (UUID) to the + list of its selected children. Useful for grouping in the LLM + prompt builder. + """ + if not selected_ids: + return [], {} + + requested = list(selected_ids) + initial_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(requested), + ) + .all() + ) + initial_by_id = {row.id: row for row in initial_rows} + + parent_ids_requested = { + m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id + } + # Map parent id -> children explicitly requested by the user. + explicit_children_by_parent: Dict[UUID, List[Metric]] = {} + for m in initial_rows: + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + explicit_children_by_parent.setdefault( + m.parent_metric_id, [] + ).append(m) + + # For parents without explicit children, hydrate every enabled child. + parents_needing_full_expansion = [ + pid + for pid in parent_ids_requested + if pid not in explicit_children_by_parent + ] + auto_expanded_children: Dict[UUID, List[Metric]] = {} + if parents_needing_full_expansion: + for pid in parents_needing_full_expansion: + child_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.parent_metric_id == pid, + Metric.enabled.is_(True), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + auto_expanded_children[pid] = child_rows + + parent_to_children: Dict[UUID, List[Metric]] = {} + for pid in parent_ids_requested: + children = explicit_children_by_parent.get( + pid + ) or auto_expanded_children.get(pid, []) + # Drop disabled children so the worker doesn't waste a slot on + # them. Empty parents (no enabled children) are still tracked + # because the UI may want to show "0 of 0" rather than swallow + # them silently. + parent_to_children[pid] = [c for c in children if c.enabled] + + effective: List[Metric] = [] + seen: set[UUID] = set() + for mid in requested: + m = initial_by_id.get(mid) + if m is None: + continue + if m.selection_mode and not m.parent_metric_id: + # Parent row itself is not scored — only its children. + for child in parent_to_children.get(m.id, []): + if child.id in seen or not child.enabled: + continue + seen.add(child.id) + effective.append(child) + continue + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + # Already accounted for via the parent expansion above. + continue + if not m.enabled: + continue + if m.id in seen: + continue + seen.add(m.id) + effective.append(m) + + return effective, parent_to_children + + +def _evaluation_bulk_operation_for_response( + evaluation_id: UUID, +) -> Optional[str]: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + return get_evaluation_bulk_operation(evaluation_id) + + +def _serialize_eval( + db: Session, + row: CallImportEvaluation, + *, + sibling_evaluation_ids: Optional[List[UUID]] = None, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportEvaluationResponse: + from app.api.v1.routes.call_import_evaluations.clusters import ( + _metric_clusters_payload, + ) + from app.api.v1.routes.call_import_evaluations.insights import ( + _tldr_summary_payload, + _user_insights_payload, + ) + + selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) + + # Pull every metric referenced anywhere in the run's grouping (leaves, + # standalone, AND parents from selected_metric_groups) so the UI can + # render parent labels even when only children were materialized into + # selected_metric_ids. + groups_raw: Dict[str, List[str]] = {} + if isinstance(row.selected_metric_groups, dict): + for parent_str, children in row.selected_metric_groups.items(): + if not isinstance(children, list): + continue + cleaned: List[str] = [] + for c in children: + try: + UUID(str(c)) + cleaned.append(str(c)) + except (TypeError, ValueError): + continue + try: + UUID(parent_str) + groups_raw[parent_str] = cleaned + except (TypeError, ValueError): + continue + + metric_ids_for_lookup: List[UUID] = list(selected_ids) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in metric_ids_for_lookup: + metric_ids_for_lookup.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids( + db, row.organization_id, metric_ids_for_lookup + ) + + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) + total = int(row.total_rows or 0) + ui_completed = ( + min(ui_completed_raw, total) if total else ui_completed_raw + ) + ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw + + if user_emails is None: + user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) + created_email, updated_email = actor_emails_for_evaluation(row, user_emails) + + from app.workers.tasks.evaluate_call_import_row_core import ( + count_distinct_llm_configs_for_metrics, + ) + + selected_set = set(selected_ids) + scoring_metrics = [m for m in metrics if m.id in selected_set] + expected_llm_calls_per_row = count_distinct_llm_configs_for_metrics( + scoring_metrics, + overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else {} + ), + run_provider=(row.llm_provider or "").strip() or None, + run_model=(row.llm_model or "").strip() or None, + run_llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + run_credential_id=( + str(row.llm_credential_id) if row.llm_credential_id else None + ), + ) + + return CallImportEvaluationResponse( + id=row.id, + call_import_id=row.call_import_id, + organization_id=row.organization_id, + name=row.name, + selected_metric_ids=selected_ids, + selected_metric_groups=groups_raw or None, + metrics=[ + CallImportMetricSummary( + id=metric.id, + name=metric.name, + metric_type=metric.metric_type, + description=metric.description, + parent_metric_id=metric.parent_metric_id, + selection_mode=metric.selection_mode, + # Required by the Flow tab to know whether a parent + # opted into discovery; without it the + # DiscoveredLabelsPanel stays hidden even when the + # worker is actively producing discovered_labels. + allow_discovery=bool( + getattr(metric, "allow_discovery", False) + ), + ) + for metric in metrics + ], + status=row.status, + total_rows=row.total_rows, + completed_rows=ui_completed, + failed_rows=ui_failed, + error_message=row.error_message, + llm_provider=row.llm_provider, + llm_model=row.llm_model, + llm_credential_id=row.llm_credential_id, + llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + metric_llm_overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else None + ), + stt_provider=row.stt_provider, + stt_model=row.stt_model, + stt_credential_id=row.stt_credential_id, + diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), + diarisation_llm_model=getattr(row, "diarisation_llm_model", None), + diarisation_llm_credential_id=getattr( + row, "diarisation_llm_credential_id", None + ), + diarisation_prompt=getattr(row, "diarisation_prompt", None), + transcribe_mode=( + (getattr(row, "transcribe_mode", None) or "stt_llm") + ), + transcript_source=(row.transcript_source or "diarised"), + sibling_evaluation_ids=list(sibling_evaluation_ids or []), + expected_llm_calls_per_row=expected_llm_calls_per_row, + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + updated_at=row.updated_at, + created_by_email=created_email, + last_updated_by_email=updated_email, + tldr_summary=_tldr_summary_payload(row), + user_insights=_user_insights_payload(row), + metric_clusters=_metric_clusters_payload(row), + discover_new_metrics=bool( + getattr(row, "discover_new_metrics", False) + ), + bulk_operation=_evaluation_bulk_operation_for_response(row.id), + ) + + +def _normalize_name(value: Optional[str]) -> Optional[str]: + """Trim user-supplied name; empty string becomes ``NULL``.""" + if value is None: + return None + trimmed = value.strip() + return trimmed or None + + +def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: + """Recompute counters + terminal status after rows are added/removed. + + Uses a single aggregate query instead of loading every row status. The + counter reconcile is the fenced recount, so buffered Redis deltas are + discarded rather than re-applied. + """ + from app.workers.tasks.evaluate_call_import_row_core import ( + _apply_parent_status_from_counters, + reconcile_evaluation_counters, + ) + + reconcile_evaluation_counters(db, evaluation) + _apply_parent_status_from_counters(evaluation) + db.flush() + + from app.services.call_imports.eval_flexprice_billing import ( + maybe_finalize_evaluation_flexprice_billing, + ) + + maybe_finalize_evaluation_flexprice_billing(db, evaluation) + + if evaluation.status in {"completed", "failed", "partial"}: + from app.models.database import CallImport + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + from app.services.call_imports.eval_progress_buffer import ( + delete as delete_eval_counters, + ) + + delete_eval_counters(evaluation.id) + + call_import = ( + db.query(CallImport) + .filter(CallImport.id == evaluation.call_import_id) + .first() + ) + if call_import is not None: + rollup_call_import_batch_status(db, call_import) + +def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: + from app.db_sharding.scatter_gather import load_evaluation_score_rows + + return load_evaluation_score_rows(db, evaluation_id) + + +def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + + return count_evaluation_rows_for_run( + db, evaluation_id, statuses=["completed"] + ) + + +def _completed_row_pairs_for_evaluation( + db: Session, + evaluation_id: UUID, +) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + row_pairs = load_evaluation_row_pairs(db, evaluation_id) + return [ + (eval_row, source_row) + for eval_row, source_row in row_pairs + if eval_row.status == "completed" + ] diff --git a/app/api/v1/routes/call_import_evaluations/aggregate.py b/app/api/v1/routes/call_import_evaluations/aggregate.py new file mode 100644 index 00000000..28cf7933 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/aggregate.py @@ -0,0 +1,1778 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _require_import, + _serialize_selected_metric_ids, +) + + +def _evaluation_rows_for_period( + db: Session, + evaluation_id: UUID, +) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + return sorted( + load_evaluation_row_pairs(db, evaluation_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + return ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + +def _metric_is_user_insight(metric: Metric) -> bool: + if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": + return True + text_value = " ".join( + str(part or "").lower() + for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) + ) + normalized = text_value.replace("-", " ").replace("_", " ") + phrases = ( + "call context", + "caller context", + "product identification", + "out of scope", + "identity match", + "user identity", + "caller identity", + "frustration trigger", + "video call offer", + "video call reception", + ) + return any(phrase in normalized for phrase in phrases) + + +def _baseline_candidate_evaluations( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + *, + limit: int = 20, +) -> list[dict[str, Any]]: + from app.services.call_imports.evaluation_report_period import ( + ensure_evaluation_report_period, + evaluation_report_period_is_current, + report_period_tuple, + ) + + ensure_evaluation_report_period(db, current_evaluation) + if current_period_start is None: + current_period_start, _, _, _ = report_period_tuple(current_evaluation) + + candidates = ( + db.query(CallImportEvaluation, CallImport) + .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImportEvaluation.id != current_evaluation.id, + CallImportEvaluation.status == "completed", + CallImportEvaluation.completed_rows > 0, + ) + ) + if current_period_start is not None: + candidates = candidates.filter( + or_( + CallImportEvaluation.report_period_start.is_(None), + CallImportEvaluation.report_period_start < current_period_start, + ) + ) + candidates = candidates.order_by(desc(CallImportEvaluation.created_at)).limit( + limit * 4 + ).all() + items: list[dict[str, Any]] = [] + for candidate_eval, candidate_import in candidates: + if not evaluation_report_period_is_current(candidate_eval): + ensure_evaluation_report_period(db, candidate_eval) + period_start = candidate_eval.report_period_start + period_end = candidate_eval.report_period_end + period_label = candidate_eval.report_period_label + period_display = candidate_eval.report_period_display or "Not specified" + if period_start is None: + continue + if current_period_start and period_start >= current_period_start: + continue + dataset = ( + (candidate_import.dataset or "").strip() + or (candidate_import.original_filename or candidate_import.filename or "").strip() + or "Unknown dataset" + ) + evaluation_name = ( + (candidate_eval.name or "").strip() + or str(candidate_eval.id)[:8] + ) + items.append( + { + "evaluation_id": str(candidate_eval.id), + "name": evaluation_name, + "dataset": dataset, + "period_label": period_label, + "period_start": period_start, + "period_end": period_end, + "period_display": period_display, + "completed_rows": int(candidate_eval.completed_rows or 0), + "created_at": candidate_eval.created_at, + "is_default": False, + } + ) + if len(items) >= limit: + break + items.sort( + key=lambda item: ( + item["period_start"] or date.min, + item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), + ), + reverse=True, + ) + if items: + items[0]["is_default"] = True + return items + + +def _resolve_baseline_evaluation( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + baseline_evaluation_id: Optional[str], +) -> Optional[CallImportEvaluation]: + candidates = _baseline_candidate_evaluations( + db, + organization_id, + workspace_id, + current_evaluation, + current_period_start, + ) + allowed_ids = {item["evaluation_id"] for item in candidates} + if baseline_evaluation_id: + baseline_id = str(baseline_evaluation_id).strip() + if baseline_id not in allowed_ids: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(baseline_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not candidates: + return None + default_id = candidates[0]["evaluation_id"] + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(default_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + + +def _load_baseline_evaluation_for_aggregate( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + baseline_evaluation_id: UUID, +) -> CallImportEvaluation: + """Validate a client-supplied baseline without scanning candidate runs.""" + if baseline_evaluation_id == current_evaluation.id: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + baseline = ( + db.query(CallImportEvaluation) + .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) + .filter( + CallImportEvaluation.id == baseline_evaluation_id, + CallImportEvaluation.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImportEvaluation.status == "completed", + CallImportEvaluation.completed_rows > 0, + ) + .first() + ) + if baseline is None: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + return baseline + + +def _benchmark_context_for_evaluation( + db: Session, + baseline_evaluation: Optional[CallImportEvaluation], +) -> Optional[dict[str, str]]: + if baseline_evaluation is None: + return None + baseline_import = ( + db.query(CallImport) + .filter(CallImport.id == baseline_evaluation.call_import_id) + .first() + ) + rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) + dataset = ( + (baseline_import.dataset or "").strip() + if baseline_import and baseline_import.dataset + else None + ) + filename = ( + (baseline_import.original_filename or baseline_import.filename or "").strip() + if baseline_import + else None + ) + evaluation_label = ( + (baseline_evaluation.name or "").strip() + if baseline_evaluation.name + else str(baseline_evaluation.id)[:8] + ) + period = period_label or ( + period_start.isoformat() if period_start else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(baseline_evaluation.id), + "period": period, + } + + +def _period_deltas_from_evaluation( + db: Session, + baseline_evaluation: CallImportEvaluation, + current_metric_aggregates: list[dict[str, Any]], + current_evaluation: CallImportEvaluation, + current_eval_rows: List[CallImportEvaluationRow], +) -> dict[str, dict[str, str]]: + from app.db_sharding.scatter_gather import load_evaluation_score_rows_for_aggregate + from app.services.call_imports.metric_aggregate_cache import ( + metrics_for_aggregate_endpoint, + ) + + baseline_eval_rows = load_evaluation_score_rows_for_aggregate( + db, baseline_evaluation.id + ) + baseline_aggregate_models = metrics_for_aggregate_endpoint( + db, baseline_evaluation + ) + baseline_metric_aggregates = [ + _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models + ] + _metrics, _aggs, policies, _source, _child_map = _clustering_context( + db, current_evaluation, current_eval_rows + ) + metric_by_id = {str(m.id): m for m in _metrics} + current_by_id = { + str(item.get("metric_id")): item for item in current_metric_aggregates + } + previous_by_id = { + str(item.get("metric_id")): item + for item in baseline_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + metric = metric_by_id.get(metric_id) + policy = policies.get(metric_id) + previous_raw = previous_by_id.get(metric_id) + if metric is None or policy is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + current_pct = failure_rate_percent_from_rows( + current_eval_rows, metric, policy + ) + previous_pct = failure_rate_percent_from_rows( + baseline_eval_rows, metric, policy + ) + if current_pct is None or previous_pct is None: + current_pct = current_pct or _aggregate_primary_percent(current, policy) + previous_pct = ( + previous_pct or _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": ( + f"Current report {current_pct:.1f}% vs previous report " + f"{previous_pct:.1f}%" + ), + } + return deltas + + +_DELTA_EXPLANATION_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will receive " + "week-over-week metric failure-rate deltas plus reconciled failure " + "cluster context per metric.\n\n" + "Return STRICT JSON only:\n" + "{\n" + ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' + "}\n\n" + "Constraints:\n" + "- Only include metrics supplied in the prompt.\n" + "- Cluster labels are generated independently each run and are NOT stable " + "IDs. Never compare an unmatched current label to 0% baseline.\n" + "- Use matched_theme_shifts for label-aligned comparisons, " + "gap_label_shifts for structural shifts, and new_themes_current_period " + "for themes that emerged without a baseline match.\n" + "- If reconciliation is uncertain, explain using the numeric delta and " + "gap_label_shifts only.\n" + "- Keep each explanation to 1-2 short sentences (~220 chars).\n" + "- Vendor-safe, factual language; no markdown." +) + + +def _period_delta_explanation_cache_key( + baseline_evaluation_id: UUID, + *, + completed_rows: int, + baseline_completed_rows: int, + metric_clusters: Any = None, +) -> str: + from app.services.call_imports.evaluation_period_delta_cache import ( + period_delta_numeric_cache_key, + ) + + return period_delta_numeric_cache_key( + baseline_evaluation_id, + completed_rows=completed_rows, + baseline_completed_rows=baseline_completed_rows, + metric_clusters=metric_clusters, + ) + + +def _normalize_cluster_label(label: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() + + +_CLUSTER_LABEL_STOPWORDS = frozenset( + { + "a", + "an", + "the", + "and", + "or", + "during", + "while", + "with", + "for", + "from", + "into", + "general", + "user", + "bot", + "agent", + } +) + + +def _cluster_label_tokens(label: str) -> set[str]: + return { + token + for token in _normalize_cluster_label(label).split() + if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 + } + + +def _cluster_label_similarity(left: str, right: str) -> float: + tokens_left = _cluster_label_tokens(left) + tokens_right = _cluster_label_tokens(right) + if not tokens_left or not tokens_right: + return 0.0 + intersection = tokens_left & tokens_right + if not intersection: + return 0.0 + union = tokens_left | tokens_right + jaccard = len(intersection) / len(union) + smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right + overlap_ratio = len(intersection) / len(smaller) + return max(jaccard, overlap_ratio * 0.85) + + +def _group_clusters_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + grouped.setdefault(gap_label, []).append(cluster) + return grouped + + +def _append_matched_cluster_pair( + matched: list[dict[str, Any]], + current: dict[str, Any], + baseline: dict[str, Any], + *, + match_confidence: float, + match_method: str, +) -> None: + matched.append( + { + "current_label": current.get("label"), + "baseline_label": baseline.get("label"), + "gap_label": current.get("gap_label") or baseline.get("gap_label"), + "current_share_pct": current.get("share_pct"), + "baseline_share_pct": baseline.get("share_pct"), + "share_delta_pp": round( + float(current.get("share_pct") or 0.0) + - float(baseline.get("share_pct") or 0.0), + 1, + ), + "match_confidence": round(match_confidence, 2), + "match_method": match_method, + } + ) + + +def _aggregate_share_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, float]: + totals: dict[str, float] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + totals[gap_label] = totals.get(gap_label, 0.0) + float( + cluster.get("share_pct") or 0.0 + ) + return {gap: round(share, 1) for gap, share in totals.items()} + + +def _reconcile_cluster_periods( + current_clusters: list[dict[str, Any]], + baseline_clusters: list[dict[str, Any]], + *, + similarity_threshold: float = 0.35, +) -> dict[str, Any]: + """Align independently-generated cluster labels before delta explanation.""" + matched: list[dict[str, Any]] = [] + current_unmatched = list(current_clusters) + remaining_baseline = list(baseline_clusters) + + current_by_gap = _group_clusters_by_gap_label(current_unmatched) + baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) + for gap_label in list(current_by_gap): + current_group = current_by_gap.get(gap_label) or [] + baseline_group = baseline_by_gap.get(gap_label) or [] + if len(current_group) != 1 or len(baseline_group) != 1: + continue + current = current_group[0] + baseline = baseline_group[0] + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=0.75, + match_method="single_cluster_per_gap_label", + ) + current_unmatched.remove(current) + remaining_baseline.remove(baseline) + current_by_gap[gap_label] = [] + baseline_by_gap[gap_label] = [] + + for current in list(current_unmatched): + best_idx: Optional[int] = None + best_score = 0.0 + for idx, baseline in enumerate(remaining_baseline): + score = _cluster_label_similarity( + str(current.get("label") or ""), + str(baseline.get("label") or ""), + ) + if current.get("gap_label") == baseline.get("gap_label"): + score += 0.1 + if score > best_score: + best_score = score + best_idx = idx + + if best_idx is not None and best_score >= similarity_threshold: + baseline = remaining_baseline.pop(best_idx) + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=best_score, + match_method="label_similarity", + ) + + matched_current_labels = { + str(item.get("current_label") or "") for item in matched + } + matched_baseline_labels = { + str(item.get("baseline_label") or "") for item in matched + } + current_unmatched = [ + cluster + for cluster in current_clusters + if str(cluster.get("label") or "") not in matched_current_labels + ] + remaining_baseline = [ + cluster + for cluster in baseline_clusters + if str(cluster.get("label") or "") not in matched_baseline_labels + ] + + new_themes = [ + { + "label": cluster.get("label"), + "gap_label": cluster.get("gap_label"), + "share_pct": cluster.get("share_pct"), + "note": "New theme in current period (no close baseline match).", + } + for cluster in current_unmatched + ] + + retired_themes = [ + { + "label": baseline.get("label"), + "gap_label": baseline.get("gap_label"), + "share_pct": baseline.get("share_pct"), + "note": "Theme present in baseline only (retired or renamed).", + } + for baseline in remaining_baseline + ] + + current_gap = _aggregate_share_by_gap_label(current_clusters) + baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) + gap_label_shifts: dict[str, dict[str, float]] = {} + for gap_label in set(current_gap) | set(baseline_gap): + current_share = current_gap.get(gap_label, 0.0) + baseline_share = baseline_gap.get(gap_label, 0.0) + if abs(current_share - baseline_share) >= 0.5: + gap_label_shifts[gap_label] = { + "current_share_pct": current_share, + "baseline_share_pct": baseline_share, + "share_delta_pp": round(current_share - baseline_share, 1), + } + + return { + "matched_theme_shifts": matched, + "new_themes_current_period": new_themes, + "retired_themes_baseline_period": retired_themes, + "gap_label_shifts": gap_label_shifts, + "reconciliation_note": ( + "Cluster labels are generated independently each run and may " + "rename the same failure mode. Do not treat unmatched current " + "labels as 0% in the baseline period." + ), + } + + +def _load_period_delta_explanations_cache( + evaluation: CallImportEvaluation, + cache_key: str, +) -> Optional[dict[str, str]]: + raw = getattr(evaluation, "period_delta_explanations", None) + if not isinstance(raw, dict): + return None + entry = raw.get(cache_key) + if not isinstance(entry, dict): + return None + explanations_raw = entry.get("explanations") + if not isinstance(explanations_raw, dict): + return None + return { + str(metric_id): str(why).strip() + for metric_id, why in explanations_raw.items() + if str(metric_id).strip() and isinstance(why, str) and why.strip() + } + + +def _save_period_delta_explanations_cache( + db: Session, + evaluation: CallImportEvaluation, + cache_key: str, + explanations: dict[str, str], +) -> None: + raw = evaluation.period_delta_explanations + if not isinstance(raw, dict): + raw = {} + updated = dict(raw) + updated[cache_key] = { + "explanations": explanations, + "generated_at": datetime.now(timezone.utc).isoformat(), + } + evaluation.period_delta_explanations = updated + flag_modified(evaluation, "period_delta_explanations") + db.commit() + + +def _cluster_summary_for_metric( + state: Optional[EvaluationMetricClustersState], + metric_id: str, +) -> list[dict[str, Any]]: + if state is None or state.status != "completed": + return [] + for group in state.groups: + if str(group.metric_id) != metric_id: + continue + return [ + { + "label": cluster.label, + "gap_label": cluster.gap_label, + "share_pct": round(cluster.share_pct, 1), + "count": cluster.count, + } + for cluster in group.clusters[:5] + ] + return [] + + +def _merge_delta_why( + raw_deltas: dict[str, dict[str, str]], + explanations: dict[str, str], +) -> dict[str, dict[str, str]]: + if not explanations: + return raw_deltas + merged: dict[str, dict[str, str]] = {} + for metric_id, delta in raw_deltas.items(): + updated = dict(delta) + why = explanations.get(metric_id) + if why: + updated["why"] = why + merged[metric_id] = updated + return merged + + +def _explain_period_deltas( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], + *, + min_delta_pp: float = 0.5, +) -> dict[str, dict[str, str]]: + """Attach ``why`` explanations to period deltas using cached LLM output.""" + if not raw_deltas: + return raw_deltas + + cache_key = _period_delta_explanation_cache_key( + baseline_evaluation.id, + completed_rows=evaluation.completed_rows, + baseline_completed_rows=baseline_evaluation.completed_rows, + metric_clusters=evaluation.metric_clusters, + ) + cached = _load_period_delta_explanations_cache(evaluation, cache_key) + if cached is not None: + return _merge_delta_why(raw_deltas, cached) + + from app.api.v1.routes.call_import_evaluations.clusters import ( + _metric_clusters_payload, + ) + + current_clusters = _metric_clusters_payload(evaluation) + baseline_clusters = _metric_clusters_payload(baseline_evaluation) + metrics_for_prompt: list[dict[str, Any]] = [] + for metric_id, delta in raw_deltas.items(): + label = delta.get("label") or "" + if "No previous-week baseline" in label: + continue + match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) + if match and abs(float(match.group(1))) < min_delta_pp: + continue + current_summary = _cluster_summary_for_metric(current_clusters, metric_id) + baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) + if not current_summary and not baseline_summary: + continue + cluster_reconciliation = _reconcile_cluster_periods( + current_summary, + baseline_summary, + ) + metrics_for_prompt.append( + { + "metric_id": metric_id, + "delta_label": label, + "delta_detail": delta.get("detail") or "", + "cluster_reconciliation": cluster_reconciliation, + } + ) + + if not metrics_for_prompt: + return raw_deltas + + provider_hint: Optional[str] = None + model_hint: Optional[str] = None + tldr_raw = evaluation.tldr_summary + if isinstance(tldr_raw, dict): + if isinstance(tldr_raw.get("provider"), str): + provider_hint = tldr_raw["provider"] + if isinstance(tldr_raw.get("model"), str): + model_hint = tldr_raw["model"] + + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + from app.services.call_import_user_insights import _call_llm, _parse_json_object + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + ) + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, provider_hint, model_hint + ) + usage_ctx = call_import_evaluation_usage_context( + organization_id=organization_id, + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) + try: + text = _call_llm( + db, + organization_id, + provider_enum, + model_str, + [ + {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps( + {"metrics": metrics_for_prompt}, + ensure_ascii=False, + default=str, + ), + }, + ], + temperature=0.3, + max_tokens=900, + usage_ctx=usage_ctx, + ) + except Exception as exc: + logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) + return raw_deltas + + parsed = _parse_json_object(text) + explanations_raw = parsed.get("explanations") + explanations: dict[str, str] = {} + if isinstance(explanations_raw, dict): + for metric_id, why in explanations_raw.items(): + if isinstance(why, str) and why.strip(): + explanations[str(metric_id)] = why.strip() + + if explanations: + _save_period_delta_explanations_cache( + db, evaluation, cache_key, explanations + ) + return _merge_delta_why(raw_deltas, explanations) + + +def _period_deltas_with_explanations( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], +) -> dict[str, dict[str, str]]: + return _explain_period_deltas( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + + +def _benchmark_context_for_snapshot( + db: Session, + previous_snapshot: Optional[CallImportEvaluationReportSnapshot], +) -> Optional[dict[str, str]]: + if previous_snapshot is None: + return None + previous_import = ( + db.query(CallImport) + .filter(CallImport.id == previous_snapshot.call_import_id) + .first() + ) + previous_eval = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) + .first() + ) + dataset = ( + (previous_import.dataset or "").strip() + if previous_import and previous_import.dataset + else None + ) + filename = ( + (previous_import.original_filename or previous_import.filename or "").strip() + if previous_import + else None + ) + evaluation_label = ( + (previous_eval.name or "").strip() + if previous_eval and previous_eval.name + else str(previous_snapshot.evaluation_id)[:8] + ) + period = previous_snapshot.period_label or ( + previous_snapshot.period_start.isoformat() + if previous_snapshot.period_start + else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(previous_snapshot.evaluation_id), + "period": period, + } + + +def _clamp_prose_to_sentences( + text: str, + *, + max_sentences: int = 3, + max_chars: int = 300, +) -> str: + """Keep concise audit/TLDR prose within sentence and character limits.""" + cleaned = (text or "").strip() + if not cleaned: + return cleaned + cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() + sentences = [ + sentence.strip() + for sentence in re.split(r"(?<=[.!?])\s+", cleaned) + if sentence.strip() + ] + if sentences: + result = " ".join(sentences[:max_sentences]).strip() + else: + result = cleaned + if len(result) > max_chars: + trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") + result = f"{trimmed}..." if trimmed else result[:max_chars] + return result + + +def _audit_summary_text_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> Optional[str]: + if summary is None: + return None + narrative = _clamp_prose_to_sentences(summary.narrative.strip()) + return narrative or None + + +def _metric_insights_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> dict[str, str]: + if summary is None: + return {} + return { + str(metric_id): insight.strip() + for metric_id, insight in summary.metric_insights.items() + if str(metric_id).strip() and insight.strip() + } + + +def _report_period_from_rows( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], +) -> tuple[Optional[date], Optional[date], Optional[str], str]: + dates = [ + source_row.recording_date + for eval_row, source_row in rows + if eval_row.status == "completed" and source_row.recording_date + ] + if not dates: + return None, None, None, "Not specified" + start = min(dates) + end = max(dates) + week_anchor = max(dates) + week_start = week_anchor - timedelta(days=week_anchor.weekday()) + week_end = week_start + timedelta(days=6) + iso_year, iso_week, _ = week_anchor.isocalendar() + label = f"{iso_year}-W{iso_week:02d}" + if week_start.year == week_end.year: + week_range = f"{week_start.strftime('%b %d')}ΓÇô{week_end.strftime('%b %d, %Y')}" + else: + week_range = ( + f"{week_start.strftime('%b %d, %Y')}ΓÇô{week_end.strftime('%b %d, %Y')}" + ) + display = f"W{iso_week:02d} · {week_range}" + return start, end, label, display + + +def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: + if hasattr(aggregate, "model_dump"): + return aggregate.model_dump(mode="json") + return aggregate.dict() + + +def _aggregate_primary_percent( + raw: dict[str, Any], + policy: Optional[MetricFailurePolicy] = None, +) -> Optional[float]: + return aggregate_primary_percent(raw, policy) + + +def _child_names_by_parent( + db: Session, + organization_id: UUID, + parent_metric_ids: Sequence[UUID], +) -> Dict[str, List[str]]: + if not parent_metric_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(list(parent_metric_ids)), + ) + .all() + ) + out: Dict[str, List[str]] = {} + for child in children: + pid = str(child.parent_metric_id) + out.setdefault(pid, []).append(child.name) + return out + + +def _clustering_context( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[ + List[Metric], + List[CallImportMetricAggregate], + Dict[str, MetricFailurePolicy], + Literal["inferred", "user"], + Dict[str, List[str]], +]: + from app.api.v1.routes.call_import_evaluations.clusters import ( + _metrics_for_clustering, + ) + + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, source = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + return metrics, aggregates, policies, source, child_names_by_parent + + +def _period_deltas_from_aggregates( + previous_metric_aggregates: list[dict[str, Any]], + current_metric_aggregates: list[dict[str, Any]], + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> dict[str, dict[str, str]]: + current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} + previous_by_id = { + str(item.get("metric_id")): item + for item in previous_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + previous_raw = previous_by_id.get(metric_id) + policy = (policies or {}).get(metric_id) + current_pct = _aggregate_primary_percent(current, policy) + previous_pct = ( + _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", + } + return deltas + + +def _period_deltas_from_snapshot( + previous: Optional[CallImportEvaluationReportSnapshot], + current_metric_aggregates: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + previous_items = ( + previous.metric_aggregates + if previous and isinstance(previous.metric_aggregates, list) + else [] + ) + return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) + + +# --------------------------------------------------------------------------- +# Aggregation: turns per-row metric scores into histograms / value counts. +# +# Designed to be cheap enough to call on every page load: we read each +# evaluation row once, bucket numeric values into a fixed 10-bin +# histogram, and tally the top categorical values. Scaling concerns +# (millions of rows) are deferred — at that point we'd push this into a +# Postgres aggregate query, but for typical CSV imports (<10k rows) the +# Python pass is fast enough and dramatically simpler. +# --------------------------------------------------------------------------- + + +_HISTOGRAM_BUCKETS = 10 +_TOP_VALUE_COUNTS = 10 + + +def _coerce_numeric(value: Any) -> Optional[float]: + """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" + if isinstance(value, bool): + # Booleans are ints in Python; treat them as categorical so + # pass/fail metrics show up in value_counts instead of becoming + # a degenerate {0,1} histogram. + return None + if isinstance(value, (int, float)) and math.isfinite(value): + return float(value) + if isinstance(value, str): + try: + f = float(value) + if math.isfinite(f): + return f + except ValueError: + return None + return None + + +def _coerce_category(value: Any) -> Optional[str]: + """Render ``value`` as a label suitable for a value_counts bucket.""" + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + text = value.strip() + return text or None + # Lists / dicts: stringify so they still group sensibly without + # exploding the cardinality (worst case: everything is "[…]" once). + return str(value) + + +def _build_histogram( + values: List[float], +) -> List[CallImportMetricHistogramBucket]: + """Fixed-bin histogram over ``values``; returns [] for <2 values.""" + if len(values) < 2: + return [] + lo = min(values) + hi = max(values) + if lo == hi: + # All values identical — render a single bucket so the UI shows a + # spike rather than empty space. + return [ + CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) + ] + width = (hi - lo) / _HISTOGRAM_BUCKETS + buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] + for v in values: + # Right-edge inclusive on the last bucket so ``hi`` doesn't fall + # off into a non-existent bucket index. + idx = int((v - lo) / width) + if idx >= _HISTOGRAM_BUCKETS: + idx = _HISTOGRAM_BUCKETS - 1 + buckets[idx].append(v) + return [ + CallImportMetricHistogramBucket( + x0=lo + i * width, + x1=lo + (i + 1) * width, + count=len(bucket), + ) + for i, bucket in enumerate(buckets) + ] + + +def _percentile(values: List[float], pct: float) -> Optional[float]: + """Linear-interpolated percentile compatible with NumPy default.""" + if not values: + return None + sorted_vals = sorted(values) + if len(sorted_vals) == 1: + return sorted_vals[0] + rank = (pct / 100.0) * (len(sorted_vals) - 1) + lo = int(math.floor(rank)) + hi = int(math.ceil(rank)) + if lo == hi: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac + + +def _compute_metric_aggregates( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[CallImportMetricAggregate]: + """Collapse per-row ``metric_scores`` into one aggregate per metric. + + Selected metrics are read fresh from the DB so the response always + surfaces the current ``metric.name`` / ``metric_type`` even when a + metric was renamed after the run finished. + """ + + selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metrics from selected_metric_groups so they appear + # alongside their children in the aggregate response. Use ``getattr`` + # with a default so the helper still works for callers that pass + # lightweight objects (tests, in-memory shims) that don't carry the + # attribute at all. + groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) + groups_raw = ( + groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in selected_ids: + selected_ids.append(pid) + except (TypeError, ValueError): + continue + + from app.api.v1.routes import call_import_evaluations as eval_routes + + metrics = eval_routes._metrics_for_ids(db, evaluation.organization_id, selected_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + # Default to selected metrics, but also include any metric ids that + # surface in row scores even if missing from the metric registry — + # otherwise renaming/deleting a metric mid-run would silently drop + # results from the chart. + discovered_ids: List[str] = list(metric_meta.keys()) + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id_str in scores.keys(): + if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: + discovered_ids.append(metric_id_str) + + results: List[CallImportMetricAggregate] = [] + + for metric_id_str in discovered_ids: + meta = metric_meta.get(metric_id_str) + numeric_values: List[float] = [] + category_counts: Dict[str, int] = {} + classification_yes_no_counts: Dict[str, int] = {} + classification_choice_counts: Dict[str, int] = {} + classification_level_counts: Dict[str, int] = {} + classification_rows_scored = 0 + is_classification_metric_agg = bool( + meta and is_classification_metric(custom_data_type=getattr(meta, "custom_data_type", None)) + ) + # For multi-label parents we still need to know how many rows + # were scored (each row votes for >=1 label) so the n-badge in + # the UI shows "n=50" instead of the misleading "n=208" sum. + multi_label_rows_scored = 0 + # Unordered pair tally for the co-occurrence heatmap. Keys are + # ``(label_a, label_b)`` with ``a < b`` so we never double-count + # the same unordered pair. Only populated for multi-label + # parents — every other metric leaves this empty. + multi_label_pair_counts: Dict[Tuple[str, str], int] = {} + skipped = 0 + errored = 0 + observed_metric_type: Optional[str] = None + observed_name: Optional[str] = None + corpus_error_words = 0 + corpus_ref_words = 0 + corpus_char_distance = 0 + corpus_ref_chars = 0 + + # ``meta`` is a real ``Metric`` row in production, but tests + # frequently pass a lightweight stub. Pull the two attributes + # we need via ``getattr`` so a stub that only sets ``id`` / + # ``name`` / ``metric_type`` doesn't blow up here. + is_multi_label_parent = bool( + meta + and getattr(meta, "selection_mode", None) == "multi_label" + and not getattr(meta, "parent_metric_id", None) + ) + + for row in eval_rows: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else {} + ) + entry = scores.get(metric_id_str) + if not isinstance(entry, dict): + continue + if entry.get("metric_name"): + observed_name = entry.get("metric_name") + if entry.get("type"): + observed_metric_type = entry.get("type") + if entry.get("type") == "classification": + is_classification_metric_agg = True + if entry.get("skipped"): + skipped += 1 + continue + if entry.get("error"): + errored += 1 + continue + + # Multi-label parents store a comma-joined value that + # isn't useful as a single category; instead tally each + # selected child individually so the chart shows per-label + # counts that mirror the children's own boolean histograms. + if is_multi_label_parent: + selected = entry.get("selected_child_names") + if isinstance(selected, list) and selected: + multi_label_rows_scored += 1 + cleaned: List[str] = [] + for label in selected: + text_label = str(label).strip() or None + if text_label: + cleaned.append(text_label) + category_counts[text_label] = ( + category_counts.get(text_label, 0) + 1 + ) + # Emit one increment per unordered pair of distinct + # labels that fired together on this row. ``cleaned`` + # is deduplicated first because the LLM occasionally + # repeats a label inside ``selected_child_names``. + distinct = sorted(set(cleaned)) + for i in range(len(distinct)): + for j in range(i + 1, len(distinct)): + pair = (distinct[i], distinct[j]) + multi_label_pair_counts[pair] = ( + multi_label_pair_counts.get(pair, 0) + 1 + ) + continue + + if is_classification_metric_agg or entry.get("type") == "classification": + facets = classification_facets_from_entry(entry) + scored_classification = False + yn = facets.get("yes_no") + if yn: + classification_yes_no_counts[yn] = ( + classification_yes_no_counts.get(yn, 0) + 1 + ) + scored_classification = True + ch = facets.get("choice") + if ch: + classification_choice_counts[ch] = ( + classification_choice_counts.get(ch, 0) + 1 + ) + scored_classification = True + lv = facets.get("level") + if lv: + classification_level_counts[lv] = ( + classification_level_counts.get(lv, 0) + 1 + ) + scored_classification = True + if scored_classification: + classification_rows_scored += 1 + continue + + value = entry.get("value") + numeric = _coerce_numeric(value) + if numeric is not None: + numeric_values.append(numeric) + details = entry.get("details") + if isinstance(details, dict): + ref_words = details.get("ref_words") + if ref_words is None: + ref_words = details.get("ref_word_count") + if ref_words: + try: + rw = int(ref_words) + except (TypeError, ValueError): + rw = 0 + if rw > 0: + word_distance = details.get("word_distance") + if word_distance is None: + word_distance = ( + int(details.get("substitutions") or 0) + + int(details.get("deletions") or 0) + + int(details.get("insertions") or 0) + ) + corpus_error_words += int(word_distance) + corpus_ref_words += rw + ref_chars = details.get("ref_chars") + char_distance = details.get("char_distance") + if ref_chars and char_distance is not None: + try: + rc = int(ref_chars) + cd = int(char_distance) + except (TypeError, ValueError): + rc = 0 + cd = 0 + if rc > 0: + corpus_char_distance += cd + corpus_ref_chars += rc + continue + category = _coerce_category(value) + if category is not None: + category_counts[category] = category_counts.get(category, 0) + 1 + + # ``count`` is "rows scored". For numeric / single-choice + # metrics that's the same as ``len(numeric) + sum(categories)`` + # because each scored row contributes exactly one observation. + # Multi-label parents however contribute one observation per + # selected child, so summing ``category_counts`` over-counts — + # we tracked rows-scored separately above and use it here. + legacy_rows_scored = len(numeric_values) + sum(category_counts.values()) + # Mixed migrations may score some rows via classification facets and + # others via legacy numeric/category values for the same metric id. + rows_scored = ( + multi_label_rows_scored + if is_multi_label_parent + else (classification_rows_scored + legacy_rows_scored) + if is_classification_metric_agg + else legacy_rows_scored + ) + + # Build numeric stats first, then categorical (both can coexist). + agg = CallImportMetricAggregate( + metric_id=metric_id_str, + metric_name=( + (meta.name if meta else observed_name) or "Unknown metric" + ), + metric_type=( + meta.metric_type if meta else observed_metric_type + ), + metric_category=( + "user_insight" + if meta is not None and _metric_is_user_insight(meta) + else "quality" + ) + or "quality", + is_multi_label_parent=is_multi_label_parent, + count=rows_scored, + skipped_count=skipped, + error_count=errored, + ) + if numeric_values: + agg.mean = float(statistics.fmean(numeric_values)) + agg.median = float(statistics.median(numeric_values)) + agg.min = min(numeric_values) + agg.max = max(numeric_values) + agg.stddev = ( + float(statistics.pstdev(numeric_values)) + if len(numeric_values) > 1 + else 0.0 + ) + agg.p25 = _percentile(numeric_values, 25) + agg.p75 = _percentile(numeric_values, 75) + agg.p95 = _percentile(numeric_values, 95) + agg.histogram_buckets = _build_histogram(numeric_values) + from app.services.metrics.computed_metrics import ( + COMPUTED_TRANSCRIPT_SCORING_METHODS, + metric_scoring_method_key, + ) + + if ( + meta + and metric_scoring_method_key(meta) in COMPUTED_TRANSCRIPT_SCORING_METHODS + and corpus_ref_words > 0 + ): + agg.corpus_wer = float(corpus_error_words) / float(corpus_ref_words) + if ( + meta + and metric_scoring_method_key(meta) in COMPUTED_TRANSCRIPT_SCORING_METHODS + and corpus_ref_chars > 0 + ): + agg.corpus_cer = float(corpus_char_distance) / float(corpus_ref_chars) + if ( + is_classification_metric_agg + and ( + classification_yes_no_counts + or classification_choice_counts + or classification_level_counts + ) + ): + def _facet_value_counts(counts: Dict[str, int]) -> List[CallImportMetricValueCount]: + sorted_counts = sorted( + counts.items(), key=lambda kv: kv[1], reverse=True + ) + return [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted_counts[:_TOP_VALUE_COUNTS] + ] + + agg.classification_facets = CallImportClassificationFacetCounts( + yes_no=_facet_value_counts(classification_yes_no_counts), + choice=_facet_value_counts(classification_choice_counts), + level=_facet_value_counts(classification_level_counts), + ) + if category_counts: + sorted_counts = sorted( + category_counts.items(), key=lambda kv: kv[1], reverse=True + ) + agg.value_counts = [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted_counts[:_TOP_VALUE_COUNTS] + ] + # Restrict the heatmap to pairs of labels we actually + # rendered above so the frontend never has to match + # against truncated/missing rows. Sorted desc by pair + # count to keep the most informative cells in the + # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. + if is_multi_label_parent and multi_label_pair_counts: + kept_labels = { + label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] + } + pair_items = [ + (a, b, count) + for (a, b), count in multi_label_pair_counts.items() + if a in kept_labels and b in kept_labels + ] + pair_items.sort(key=lambda t: t[2], reverse=True) + agg.co_occurrence = [ + CallImportMetricLabelPair(a=a, b=b, count=count) + for a, b, count in pair_items + ] + + results.append(agg) + + # Sort so each parent metric immediately precedes its children. + # The Visualizations grid renders metrics top-to-bottom in this + # order, so multi-label parents (the "summary" chart) sit above + # the per-child boolean histograms that drill into them. Metrics + # whose ``meta`` row was deleted mid-run (``meta is None``) sink + # to the bottom but keep their relative order. + enumerated = list(enumerate(results)) + + def _sort_key(item: Tuple[int, CallImportMetricAggregate]): + original_idx, agg = item + meta = metric_meta.get(agg.metric_id) + if meta is None: + return (1, "", 1, "", original_idx) + parent_id = getattr(meta, "parent_metric_id", None) + # Group key: a child shares its parent's UUID; a parent + # uses its own UUID. Within a group, depth=0 (parent) sorts + # before depth=1 (child); ties break alphabetically by name + # so children render in a stable order regardless of which + # row scored which label first. + if parent_id is None: + group_key = str(meta.id) + depth = 0 + else: + group_key = str(parent_id) + depth = 1 + return ( + 0, + group_key, + depth, + (getattr(meta, "name", "") or "").lower(), + original_idx, + ) + + enumerated.sort(key=_sort_key) + return [agg for _idx, agg in enumerated] + + +@router.get( + "/{eval_id}/aggregate", + response_model=CallImportEvaluationAggregateResponse, + operation_id="getCallImportEvaluationAggregate", +) +async def get_call_import_evaluation_aggregate( + call_import_id: UUID, + eval_id: UUID, + baseline_evaluation_id: Optional[UUID] = Query(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationAggregateResponse: + """Return per-metric distributions for the Visualizations tab. + + The shape is intentionally chart-friendly: histograms for numeric + metrics, top-N value counts for categorical/text metrics, plus + summary stats (mean/p50/p95) so the UI can render summary cards + without recomputing on the client. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.metric_aggregate_cache import ( + metrics_for_aggregate_endpoint, + ) + + metrics = metrics_for_aggregate_endpoint(db, evaluation) + + period_deltas: dict[str, MetricPeriodDelta] = {} + resolved_baseline_id: Optional[UUID] = None + if baseline_evaluation_id is not None: + call_import = _require_import(db, call_import_id, organization_id) + from app.db_sharding.scatter_gather import ( + load_evaluation_score_rows_for_aggregate, + ) + from app.services.call_imports.evaluation_period_delta_cache import ( + load_cached_period_deltas, + period_delta_numeric_cache_key, + save_cached_period_deltas, + ) + + baseline_evaluation = _load_baseline_evaluation_for_aggregate( + db, + organization_id, + call_import.workspace_id, + evaluation, + baseline_evaluation_id, + ) + resolved_baseline_id = baseline_evaluation.id + cache_key = period_delta_numeric_cache_key( + baseline_evaluation.id, + completed_rows=int(evaluation.completed_rows or 0), + baseline_completed_rows=int(baseline_evaluation.completed_rows or 0), + metric_clusters=evaluation.metric_clusters, + ) + raw_deltas = load_cached_period_deltas(evaluation, cache_key) + if raw_deltas is None: + current_eval_rows = load_evaluation_score_rows_for_aggregate( + db, eval_id + ) + metric_aggregates_dicts = [ + _aggregate_to_dict(agg) for agg in metrics + ] + raw_deltas = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates_dicts, + evaluation, + current_eval_rows, + ) + try: + save_cached_period_deltas( + db, evaluation, cache_key, raw_deltas + ) + db.commit() + except Exception as exc: + logger.warning( + "period delta cache save failed for evaluation {}: {}", + evaluation.id, + exc, + ) + db.rollback() + cached_why = _load_period_delta_explanations_cache(evaluation, cache_key) + if cached_why: + raw_deltas = _merge_delta_why(raw_deltas, cached_why) + period_deltas = { + metric_id: MetricPeriodDelta( + label=delta.get("label") or "", + detail=delta.get("detail") or "", + why=(delta.get("why") or "").strip() or None, + ) + for metric_id, delta in raw_deltas.items() + } + + _fp_stored, failure_policies_source = policies_from_evaluation_raw( + evaluation.metric_clusters + ) + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + ui_completed, ui_failed = merge_eval_counters_for_ui(evaluation) + return CallImportEvaluationAggregateResponse( + evaluation_id=eval_id, + total_rows=evaluation.total_rows, + completed_rows=ui_completed, + failed_rows=ui_failed, + metrics=metrics, + period_deltas=period_deltas, + baseline_evaluation_id=resolved_baseline_id, + failure_policies_source=failure_policies_source, + ) + diff --git a/app/api/v1/routes/call_import_evaluations/clusters.py b/app/api/v1/routes/call_import_evaluations/clusters.py new file mode 100644 index 00000000..fe8db95a --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/clusters.py @@ -0,0 +1,918 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _completed_row_pairs_for_evaluation, + _load_eval_rows, + _metrics_for_ids, + _require_import, +) +from app.api.v1.routes.call_import_evaluations.aggregate import ( # noqa: E402 + _child_names_by_parent, + _clustering_context, + _compute_metric_aggregates, + _metric_is_user_insight, +) + +def _metrics_for_clustering( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[Metric]: + """All enabled quality metrics scored in this run, normalized for clustering. + + Hierarchical children are collapsed to their parent metric so cluster + groups render at the category level (e.g. ``AI reveal``) instead of the + child label level (e.g. ``Yes`` / ``No``). + """ + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + aggregate_metric_ids: List[UUID] = [] + for agg in aggregates: + if (agg.metric_category or "quality") == "user_insight": + continue + try: + aggregate_metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + if not aggregate_metric_ids: + return [] + + aggregate_metrics = _metrics_for_ids( + db, evaluation.organization_id, aggregate_metric_ids + ) + by_id = {metric.id: metric for metric in aggregate_metrics} + + normalized_ids: List[UUID] = [] + seen: set[UUID] = set() + for metric_id in aggregate_metric_ids: + metric = by_id.get(metric_id) + target_id = ( + metric.parent_metric_id + if metric is not None and metric.parent_metric_id + else metric_id + ) + if target_id in seen: + continue + seen.add(target_id) + normalized_ids.append(target_id) + + metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) + return [ + metric + for metric in metrics + if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) + ] +def _metric_clusters_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationMetricClustersState]: + raw = getattr(evaluation, "metric_clusters", None) + if raw is None: + return None + return metric_clusters_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_metric_clusters_for_pdf( + state: Optional[EvaluationMetricClustersState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: + return {} + payload: dict[str, Any] = { + "groups": [g.model_dump(mode="json") for g in state.groups], + "discovered_problems": [ + d.model_dump(mode="json") for d in state.discovered_problems + ], + } + if state.rca_summary is not None: + payload["rca_summary"] = state.rca_summary.model_dump(mode="json") + return payload +def _resolve_metric_cluster_row_selection( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], + evaluation_row_ids: Optional[List[UUID]], + *, + row_limit: Optional[int] = None, + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: + """Return filtered completed row pairs and the selected row id strings.""" + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + if policies is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] + eligible_id_set = set(eligible_ordered_ids) + + if evaluation_row_ids is None and row_limit is not None: + selected_ids = eligible_ordered_ids[:row_limit] + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + if evaluation_row_ids is None: + selected_ids = eligible_ordered_ids + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + requested = {str(rid) for rid in evaluation_row_ids} + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + not_eligible = sorted(requested - eligible_id_set) + if not_eligible: + raise HTTPException( + status_code=400, + detail=( + "Each selected row must have at least one flagged quality metric. " + "Ineligible row(s): " + + ", ".join(not_eligible[:5]) + + ("…" if len(not_eligible) > 5 else "") + ), + ) + selected_ids = sorted(requested) + filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) + return filtered, selected_ids + + +def _enqueue_metric_clusters_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + evaluation_row_ids: Optional[List[UUID]] = None, + selected_evaluation_row_ids: Optional[List[str]] = None, + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, + transcript_source: Optional[str] = None, + resume: bool = False, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _metric_clusters_payload(evaluation) + if current is not None and current.status == "running" and not force and not resume: + return + + del max_llm_calls + if db is not None: + eval_rows = _load_eval_rows(db, evaluation.id) + if selected_evaluation_row_ids is None: + _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + evaluation_row_ids, + ) + + prior_raw = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + policy_blob: Dict[str, Any] = {} + if failure_policies: + policy_blob = failure_policies_to_db(failure_policies, source="user") + + from app.services.call_import_metric_clusters_wave import ( + resume_metric_clusters_job_header, + start_metric_clusters_job_header, + ) + from app.services.evaluation_analysis_checkpoints import new_job_run_id, normalize_transcript_source + + source = normalize_transcript_source( + transcript_source or prior_raw.get("transcript_source") or evaluation.transcript_source + ) + run_id = ( + UUID(str(prior_raw["job_run_id"])) + if resume and prior_raw.get("job_run_id") + else new_job_run_id() + ) + merged_raw = { + **prior_raw, + **policy_blob, + "selected_evaluation_row_ids": selected_evaluation_row_ids or [], + } + if db is not None: + if resume and prior_raw.get("job_run_id"): + evaluation.metric_clusters = resume_metric_clusters_job_header( + db, + evaluation, + merged_raw, + transcript_source=source, + provider=provider, + model=model, + ) + else: + evaluation.metric_clusters = start_metric_clusters_job_header( + evaluation, + db, + job_run_id=run_id, + transcript_source=source, + provider=provider, + model=model, + metric_clusters_raw=merged_raw, + ) + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "metric_clusters") + db.commit() + + from app.workers.tasks.generate_evaluation_metric_clusters import ( + generate_evaluation_metric_clusters_task, + ) + + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + # Kept for observability; the worker reads the authoritative + # selection from the job header (selected_evaluation_row_ids). + "evaluation_row_ids": list(selected_evaluation_row_ids or []), + "transcript_source": source, + "resume": resume, + "job_run_id": str(run_id), + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["celery_task_id"] = async_result.id + flag_modified(evaluation, "metric_clusters") + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + db.commit() + + +def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: + """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([task_id]) + logger.info( + "Revoked metric-clusters task {} for evaluation {}", + task_id, + evaluation.id, + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke metric-clusters task {} for evaluation {}: {}", + task_id, + evaluation.id, + exc, + ) + + +def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: + """Mark clustering as cancelled and revoke the worker task. + + Returns True if a running job was cancelled, False if already terminal. + """ + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + + _revoke_metric_clusters_task(evaluation) + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + evaluation.metric_clusters = { + **raw, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + "cancelled_at": datetime.now(timezone.utc).isoformat(), + } + return True + + +@router.get( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="getCallImportEvaluationMetricClusterFailurePolicies", +) +async def get_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source=source, + updated_at=updated_at, + ) + + +@router.put( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", +) +async def save_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + body: MetricFailurePoliciesSaveRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + try: + validate_failure_policies_for_metrics(body.policies, metrics) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + evaluation.metric_clusters = merge_failure_policies_into_raw( + prior, + body.policies, + source="user", + ) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) + if source != "user": + source = "user" + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source="user", + updated_at=updated_at, + ) + + +@router.get( + "/{eval_id}/metric-clusters/eligible-rows", + response_model=MetricClusterEligibleRowsResponse, + operation_id="listCallImportEvaluationMetricClusterEligibleRows", +) +async def list_call_import_evaluation_metric_cluster_eligible_rows( + call_import_id: UUID, + eval_id: UUID, + limit: Optional[int] = Query(default=None, ge=1), + count_only: bool = Query(default=False), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricClusterEligibleRowsResponse: + """Completed rows that have at least one flagged quality metric.""" + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) + metrics, _aggregates, policies, _source, _child_map = _clustering_context( + db, evaluation, eval_rows + ) + all_eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + total = len(all_eligible) + if count_only: + return MetricClusterEligibleRowsResponse(items=[], total=total) + raw_items = all_eligible if limit is None else all_eligible[:limit] + items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] + return MetricClusterEligibleRowsResponse(items=items, total=total) + + +@router.get( + "/{eval_id}/metric-clusters", + response_model=Optional[EvaluationMetricClustersState], + operation_id="getCallImportEvaluationMetricClusters", +) +async def get_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationMetricClustersState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _metric_clusters_payload(evaluation) + + +@router.post( + "/{eval_id}/metric-clusters", + response_model=EvaluationMetricClustersState, + operation_id="generateCallImportEvaluationMetricClusters", +) +async def generate_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationMetricClustersRequest = Body( + default_factory=EvaluationMetricClustersRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force and not body.resume: + cached = _metric_clusters_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating metric clusters." + ), + ) + + if body.evaluation_row_ids and body.row_limit is not None: + raise HTTPException( + status_code=400, + detail="Specify either evaluation_row_ids or row_limit, not both.", + ) + + if body.evaluation_row_ids: + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + requested = {str(rid) for rid in body.evaluation_row_ids} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + + prior_clusters_raw = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + prior_provider = ( + prior_clusters_raw.get("provider") + if isinstance(prior_clusters_raw.get("provider"), str) + else None + ) + prior_model = ( + prior_clusters_raw.get("model") + if isinstance(prior_clusters_raw.get("model"), str) + else None + ) + resolve_provider = (body.provider or "").strip() or ( + prior_provider if body.resume else None + ) + resolve_model = (body.model or "").strip() or ( + prior_model if body.resume else None + ) + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, resolve_provider, resolve_model, body.credential_id + ) + + metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + merged_policies = merge_clustering_policies( + body.failure_policies, + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + try: + validate_failure_policies_for_metrics( + body.failure_policies or merged_policies, metrics + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not has_clusterable_metrics(metrics, merged_policies, eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No calls match any failure policy. Select failure values on " + "metrics that have matching rows, or leave metrics with no " + "failures unchecked — they are skipped automatically." + ), + ) + + selection_row_ids = body.evaluation_row_ids + selection_row_limit = body.row_limit + if body.resume and selection_row_ids is None and selection_row_limit is None: + saved_selection = prior_clusters_raw.get("selected_evaluation_row_ids") + if isinstance(saved_selection, list) and saved_selection: + selection_row_ids = [UUID(str(item)) for item in saved_selection] + filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + selection_row_ids, + row_limit=selection_row_limit, + policies=merged_policies, + ) + if not selected_row_ids: + raise HTTPException( + status_code=400, + detail=( + "No eligible rows to cluster. Select completed calls that match " + "at least one configured failure policy." + ), + ) + if not filtered_pairs: + raise HTTPException( + status_code=400, + detail="No completed rows match the selected evaluation_row_ids.", + ) + + _enqueue_metric_clusters_job( + evaluation, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + evaluation_row_ids=body.evaluation_row_ids, + selected_evaluation_row_ids=selected_row_ids, + failure_policies=merged_policies, + transcript_source=body.transcript_source, + resume=body.resume, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="running" + ) + + +@router.post( + "/{eval_id}/metric-clusters/cancel", + response_model=EvaluationMetricClustersState, + operation_id="cancelCallImportEvaluationMetricClusters", +) +async def cancel_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + """Abort in-flight failure-diagnostics clustering. + + Idempotent: if clustering is not ``running``, returns the current state + unchanged. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_metric_clusters_cancel(evaluation) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="idle" + ) diff --git a/app/api/v1/routes/call_import_evaluations/discovered.py b/app/api/v1/routes/call_import_evaluations/discovered.py new file mode 100644 index 00000000..db87e751 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/discovered.py @@ -0,0 +1,1462 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _require_import, +) + +def _slug_label(value: Any) -> str: + """Lowercase + whitespace-collapse + underscore-join. + + Used everywhere we need a stable key for a metric/label name — + matching the same convention the worker uses when emitting + ``sequence`` entries and discovered keys. + """ + if value is None: + return "" + return "_".join(str(value).strip().lower().split()) + + +def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: + """Walk the alias map until we hit a slug that doesn't redirect. + + The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete + endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark + a slug as tombstoned. Chains can accumulate when the user merges + A→B and later merges B→C; this helper collapses them so callers + always land on the final canonical slug. + + Returns: + * the canonical slug if it still resolves to a real label, + * an empty string if the slug has been tombstoned (callers MUST + treat an empty result as "drop this entry entirely"), + * the input ``key`` if it isn't aliased. + + Cycles are guarded by a hard step limit since the alias map is + user-driven. + """ + if not key: + return "" + if not alias_map: + return key + current = key + seen: set[str] = set() + for _ in range(16): + if current in seen: + return current + seen.add(current) + if current not in alias_map: + return current + nxt = alias_map[current] + if nxt == current: + return current + if nxt == "": + # Deletion sentinel — the user has explicitly retired this + # slug. Propagate the empty string up so callers drop it. + return "" + current = nxt + return current + + +# Reserved JSON key under which the worker stores top-level metric +# discoveries on each row's ``metric_scores`` dict. Mirrors the constant +# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to +# avoid a worker import cycle from the routes module. +DISCOVERED_METRICS_KEY = "__discovered_metrics__" +METRIC_SCORES_META_KEYS = frozenset({DISCOVERED_METRICS_KEY, "_billing"}) + + +def _is_metric_scores_meta_key(key: str) -> bool: + normalized = str(key or "").strip().lower() + if not normalized: + return True + if normalized in {item.lower() for item in METRIC_SCORES_META_KEYS}: + return True + return normalized.endswith("__discovered") + +# Allowed values for an LLM-suggested top-level metric type. Kept in +# sync with ``DiscoveredMetricSuggestedType`` in +# ``app/models/schemas.py``. +_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") + + +def normalize_scores_with_aliases( + metric_scores: Dict[str, Any], + evaluation: CallImportEvaluation, + db: Session, + organization_id: UUID, +) -> Dict[str, Any]: + """Rewrite per-row ``metric_scores`` to honor merges + promotions. + + Called by the worker right after ``evaluate_with_llm`` returns so + every row that finishes AFTER a user has merged or promoted a + discovered label persists data already reflecting that decision. + Without this hook, a worker holding a stale prompt could re-emit a + ``from_key`` slug long after the user merged it away. + + For every parent entry (``selection_mode != null`` and a + ``discovered_labels`` / ``sequence`` field) we: + + * resolve discovered slugs through the evaluation's + ``discovered_label_aliases`` map (transitively), + * drop any discovered_labels entry whose canonical slug now + matches a real promoted child of the parent (merging them out + of the panel for free), and + * collapse adjacent duplicate sequence entries that result. + + Returns ``metric_scores`` (mutated in place) for chaining. + """ + if not isinstance(metric_scores, dict): + return metric_scores + + aliases_top = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + + # Identify the parent entries inside metric_scores. They're the + # dicts that carry a ``selection_mode`` key (set by the LLM + # hierarchy parser) and either a ``sequence`` or a + # ``discovered_labels`` list. + for key, entry in list(metric_scores.items()): + if not isinstance(entry, dict): + continue + if entry.get("type") != "category" and not entry.get("selection_mode"): + continue + try: + parent_uuid = UUID(str(key)) + except (TypeError, ValueError): + continue + + alias_map = {} + sub = aliases_top.get(str(parent_uuid)) + if isinstance(sub, dict): + alias_map = { + str(k): str(v) + for k, v in sub.items() + if isinstance(k, str) and isinstance(v, str) + } + promoted = _promoted_child_slugs(db, parent_uuid, organization_id) + + # Rewrite discovered_labels: alias-resolve keys, drop duplicates + # post-resolution, and drop entries that have been promoted. + discovered = entry.get("discovered_labels") + if isinstance(discovered, list): + kept_disc: List[Dict[str, Any]] = [] + seen: set[str] = set() + for d in discovered: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(alias_map, slug) + if not slug or slug in promoted or slug in seen: + continue + seen.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_disc.append(new_entry) + entry["discovered_labels"] = kept_disc + + # Rewrite sequence: alias-resolve every entry; collapse adjacent + # duplicates that result. We DON'T drop slugs that match + # promoted children — the promoted child slug is still a valid + # sequence entry; the flow chart will resolve it to the real + # child node. + seq = entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + last: Optional[str] = None + for item in seq: + if not isinstance(item, str): + continue + slug = _resolve_alias(alias_map, _slug_label(item)) + if not slug or slug == last: + continue + new_seq.append(slug) + last = slug + entry["sequence"] = new_seq + + # Top-level metric discoveries live alongside the parent entries + # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the + # flat evaluation-level alias/tombstone map + suppress slugs that + # already correspond to a real top-level Metric so workers that + # finish AFTER the user has merged / deleted / promoted can't + # resurrect a retired candidate. + discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) + if isinstance(discovered_metrics_payload, list): + flat_alias_map = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + promoted_metric_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + kept_metrics: List[Dict[str, Any]] = [] + seen_metrics: set[str] = set() + for d in discovered_metrics_payload: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(flat_alias_map, slug) + if ( + not slug + or slug in promoted_metric_slugs + or slug in seen_metrics + ): + continue + seen_metrics.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_metrics.append(new_entry) + if kept_metrics: + metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics + else: + # No survivors — drop the empty array so empty-discovery rows + # keep their pre-feature payload shape. + metric_scores.pop(DISCOVERED_METRICS_KEY, None) + + return metric_scores + + +def _alias_map_for_parent( + evaluation: CallImportEvaluation, parent_metric_id: UUID +) -> Dict[str, str]: + """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. + + Stored shape on the evaluation row is + ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty + dict for parents that have never had a merge applied. + """ + raw = getattr(evaluation, "discovered_label_aliases", None) + if not isinstance(raw, dict): + return {} + submap = raw.get(str(parent_metric_id)) + if not isinstance(submap, dict): + return {} + return { + str(k): str(v) + for k, v in submap.items() + if isinstance(k, str) and isinstance(v, str) + } + + +def _promoted_child_slugs( + db: Session, parent_metric_id: UUID, organization_id: UUID +) -> set[str]: + """Slugs of every real child currently sitting under the parent. + + The Discovered Labels panel hides any candidate whose slug already + matches a real child — that covers both freshly-promoted candidates + and legacy children the LLM happened to re-discover. We pull from + the live ``metrics`` table rather than the eval's + ``selected_metric_groups`` snapshot so newly-promoted children take + effect immediately, even on evaluations that ran before the + promotion. + """ + children = ( + db.query(Metric.name) + .filter( + Metric.parent_metric_id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .all() + ) + out: set[str] = set() + for (name,) in children: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _promoted_top_level_metric_slugs( + db: Session, organization_id: UUID +) -> set[str]: + """Slugs of every top-level (non-child) Metric in the organization. + + Used to suppress discovered-metric candidates whose slug already + matches a real standalone metric. We intentionally include both + standalone metrics AND parent category metrics — a top-level + discovery that collides with either name is a duplicate by + definition. + """ + rows = ( + db.query(Metric.name) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.is_(None), + ) + .all() + ) + out: set[str] = set() + for (name,) in rows: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _get_running_discovered_labels( + db: Session, + eval_id: UUID, + parent_metric_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, + *, + from_cache: bool = False, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered label seen in this eval so far. + + Walks each ``call_import_evaluation_rows`` row's + ``metric_scores[parent_id]["discovered_labels"]`` and folds entries + that share the same slug. Returns a list ordered by descending + count and stable on label key, shaped like:: + + [{"key": "customer_on_hold", "name": "Customer put on hold", + "description": "...", "sample_rationale": "...", "count": 12}] + + Powers two callers: + * The worker prompt builder ("REUSE the existing key if it fits") + — invoked just before each row's LLM call to feed the model the + running list of previously-discovered labels in this evaluation. + * The ``/discovered-labels`` API surface used by the frontend + Discovered Labels panel to render candidates with counts + + sample rationales. + + Non-completed rows are skipped: an in-flight row's discoveries are + not yet reliable (the row could fail and never produce final + metric_scores). We accept the tradeoff that rows running + concurrently won't see each other's labels — slug-collision dedup + catches identical re-inventions, and near-paraphrases surface in + the UI panel where the user can manually merge. + """ + + parent_id_str = str(parent_metric_id) + aliases = alias_map or {} + if from_cache: + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_child_slugs( + db, parent_metric_id, organization_id + ) + from app.services.call_imports.discovered_running_cache import ( + load_running_label_entries, + project_discovered_entries, + ) + + return project_discovered_entries( + load_running_label_entries(db, eval_id, parent_metric_id), + resolve_key=lambda raw: _resolve_alias(aliases, _slug_label(raw)), + promoted_slugs=promoted_slugs, + ) + + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + # Suppress slugs that have either: + # * been promoted to a real child of the parent (so the panel doesn't + # keep nagging the user about a candidate they've already + # accepted), or + # * been merged INTO another slug (the "from" side of a merge) — + # those occurrences fold into the canonical target instead. + promoted_slugs = set() + if organization_id is not None: + promoted_slugs = _promoted_child_slugs( + db, parent_metric_id, organization_id + ) + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + discovered = parent_entry.get("discovered_labels") + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on a real child slug. Order matters: a + # candidate that was merged into a slug which has since + # been promoted should disappear, not show up at the + # canonical slug. An empty resolved key means the slug was + # tombstoned via the delete endpoint. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace("_", " ") + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + + existing = by_key.get(key) + if existing is None: + # Track up to N=3 distinct rationales per candidate so + # the Promote-to-child flow can pre-fill the new + # sub-metric's rubric with concrete LLM examples + # without the user copy-pasting from the row table. + # ``sample_rationale`` is preserved for back-compat + # with older clients; ``examples`` is the new field. + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Append distinct rationales (case-insensitive trim) up + # to a small cap. Headroom is intentionally one above + # what the UI surfaces (2) so we have a backup when the + # first rationale is unhelpful. + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _get_running_discovered_metrics( + db: Session, + eval_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, + *, + from_cache: bool = False, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered top-level metric in this eval. + + Mirrors :func:`_get_running_discovered_labels` but is keyed at the + evaluation level (no ``parent_metric_id``). Walks each completed + row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries + that share the same slug (post-alias resolution), and suppresses + slugs that already correspond to a real top-level :class:`Metric` + in the organization. + + Each returned entry is shaped:: + + {"key": "customer_satisfaction", + "name": "Customer Satisfaction", + "description": "...", + "suggested_type": "boolean" | "rating" | "category", + "sample_rationale": "...", + "examples": ["..."], + "count": 12} + """ + + aliases = alias_map or {} + if from_cache: + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + from app.services.call_imports.discovered_running_cache import ( + load_running_metric_entries, + project_discovered_entries, + ) + + return project_discovered_entries( + load_running_metric_entries(db, eval_id), + resolve_key=lambda raw: _resolve_alias(aliases, _slug_label(raw)), + promoted_slugs=promoted_slugs, + ) + + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + promoted_slugs = set() + if organization_id is not None: + promoted_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on an already-existing top-level metric + # slug. Empty resolved key = tombstoned. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace( + "_", " " + ) + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + raw_type = str(entry.get("suggested_type") or "").strip().lower() + if raw_type not in _DISCOVERED_METRIC_TYPES: + raw_type = "boolean" + + existing = by_key.get(key) + if existing is None: + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "suggested_type": raw_type, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Keep the most-frequently-suggested type. We don't track + # per-type frequency yet; defer to the first non-default + # type encountered when the existing entry has the default. + if existing.get("suggested_type") == "boolean" and raw_type != "boolean": + existing["suggested_type"] = raw_type + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +@router.get( + "/{eval_id}/discovered-labels", + response_model=DiscoveredLabelsResponse, + operation_id="getCallImportEvaluationDiscoveredLabels", +) +async def get_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose LLM-discovered candidate " + "sub-labels should be aggregated across rows." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Aggregate candidate sub-labels the LLM discovered during this eval. + + Only meaningful for parents with ``allow_discovery=true``; for other + parents we just return an empty ``items`` list rather than 400-ing + so the frontend can call the endpoint unconditionally for every + parent on the Flow tab without branching. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + alias_map = _alias_map_for_parent(evaluation, parent_metric_id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + parent_metric_id, + organization_id=organization_id, + alias_map=alias_map, + ) + items = [DiscoveredLabelItem(**item) for item in items_raw] + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), items=items + ) + + +@router.post( + "/{eval_id}/discovered-labels/merge", + response_model=DiscoveredLabelsResponse, + operation_id="mergeCallImportEvaluationDiscoveredLabels", +) +async def merge_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. + + Idempotent — re-merging the same pair is a no-op. Discovered slugs + inside per-row ``sequence`` arrays are also rewritten so the flow + chart stays consistent with the panel. When a row already has + ``to_key`` and we're merging ``from_key`` into it, we drop the + ``from_key`` entry instead of producing two entries with the same + slug. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + # No-op; just return the current aggregate so the client can + # refresh its view. + alias_map_existing = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_existing, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept: List[Dict[str, Any]] = [] + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + parent_entry["discovered_labels"] = kept + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == from_key: + seq_changed = True + if last_added == to_key: + continue + new_seq.append(to_key) + last_added = to_key + else: + new_seq.append(item) + last_added = ( + _slug_label(item) if isinstance(item, str) else None + ) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) + + # Persist the merge at the evaluation level too. This is what makes + # the merge survive future scoring: rows that finish AFTER this + # call (e.g. retries, in-flight workers) will go through the + # alias map in the API surface even if the per-row JSON they + # write still mentions ``from_key``. We chain through any existing + # alias so merging A→B and then B→C resolves A→C in the panel. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + # Resolve transitively: if to_key itself was previously merged into + # something else, point from_key at the canonical end-of-chain. + canonical_to = _resolve_alias(parent_aliases, to_key) + parent_aliases[from_key] = canonical_to + # Re-target any earlier aliases that pointed AT from_key — without + # this, A→B and then B→C would leave A still pointing to B (now a + # broken pointer because B is gone). Rewriting them keeps the + # alias map self-consistent. + for k, v in list(parent_aliases.items()): + if v == from_key: + parent_aliases[k] = canonical_to + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-labels/delete", + response_model=DiscoveredLabelsResponse, + operation_id="deleteCallImportEvaluationDiscoveredLabel", +) +async def delete_call_import_evaluation_discovered_label( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Tombstone a single LLM-discovered candidate for this evaluation. + + Symmetric with the merge endpoint, but instead of redirecting the + slug at another candidate we mark it as deleted. After this call: + + * the slug is stripped from every row's + ``metric_scores[parent].discovered_labels`` list, and from + every row's ``sequence`` array (so the flow chart no longer + draws a node for it); + * the slug is recorded in + ``evaluation.discovered_label_aliases[parent][slug] = ""`` + so any worker that finishes a row AFTER this call (e.g. a row + still in flight when the user clicked Delete) silently drops + the slug instead of resurrecting it. + + Idempotent: deleting an already-deleted slug is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) != len(discovered): + parent_entry["discovered_labels"] = kept + mutated = True + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == target_key: + seq_changed = True + continue + if isinstance(item, str): + norm = _slug_label(item) + if norm == last_added: + seq_changed = True + continue + last_added = norm + new_seq.append(item) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) + + # 3. Persist the tombstone on the evaluation so workers that finish + # later don't re-surface the deleted slug. We also retarget any + # existing aliases whose ``to_key`` was the deleted slug — without + # this, a previous merge that pointed at this slug would leave a + # dangling pointer. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + parent_aliases[target_key] = "" # deletion sentinel + for k, v in list(parent_aliases.items()): + if v == target_key: + parent_aliases[k] = "" + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +# --------------------------------------------------------------------------- +# Discovered TOP-LEVEL METRICS (per-evaluation discovery) +# +# These endpoints are the parallel of the discovered-labels trio above but +# scoped to the evaluation as a whole instead of to a parent metric. They +# all live under ``/{eval_id}/discovered-metrics`` and operate on the +# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row +# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` +# map (no parent-id nesting). +# --------------------------------------------------------------------------- + + +def _flat_metric_aliases( + evaluation: CallImportEvaluation, +) -> Dict[str, str]: + """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" + raw = getattr(evaluation, "discovered_metric_aliases", None) + if not isinstance(raw, dict): + return {} + return { + str(k): str(v) + for k, v in raw.items() + if isinstance(k, str) and isinstance(v, str) + } + + +@router.get( + "/{eval_id}/discovered-metrics", + response_model=DiscoveredMetricsResponse, + operation_id="getCallImportEvaluationDiscoveredMetrics", +) +async def get_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Aggregate top-level metric candidates the LLM discovered during this eval. + + Returns an empty ``items`` list when the evaluation did not opt + into top-level metric discovery; this keeps the frontend able to + call the endpoint unconditionally without branching on the + evaluation's ``discover_new_metrics`` flag. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not bool(getattr(evaluation, "discover_new_metrics", False)): + return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/merge", + response_model=DiscoveredMetricsResponse, + operation_id="mergeCallImportEvaluationDiscoveredMetrics", +) +async def merge_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Rewrite every row's ``__discovered_metrics__`` entry from→to. + + Mirrors the discovered-labels merge endpoint but operates on the + flat top-level metric list. Idempotent — re-merging is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + + kept: List[Dict[str, Any]] = [] + mutated = False + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + scores[DISCOVERED_METRICS_KEY] = kept + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + canonical_to = _resolve_alias(aliases, to_key) + aliases[from_key] = canonical_to + for k, v in list(aliases.items()): + if v == from_key: + aliases[k] = canonical_to + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/delete", + response_model=DiscoveredMetricsResponse, + operation_id="deleteCallImportEvaluationDiscoveredMetric", +) +async def delete_call_import_evaluation_discovered_metric( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Tombstone a single LLM-discovered top-level metric candidate.""" + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) == len(discovered): + return False + if kept: + scores[DISCOVERED_METRICS_KEY] = kept + else: + scores.pop(DISCOVERED_METRICS_KEY, None) + row.metric_scores = dict(scores) + return True + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + aliases[target_key] = "" # tombstone + for k, v in list(aliases.items()): + if v == target_key: + aliases[k] = "" + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + diff --git a/app/api/v1/routes/call_import_evaluations/evaluation_cancel.py b/app/api/v1/routes/call_import_evaluations/evaluation_cancel.py new file mode 100644 index 00000000..ab52b1e4 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/evaluation_cancel.py @@ -0,0 +1,739 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _require_import, + _rollup_evaluation_status, + _to_evaluation_row_response, +) +def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: + """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" + + if not evaluation.celery_group_id and not any( + r.celery_task_id for r in evaluation.row_results + ): + return + try: + from app.workers.celery_app import celery_app + + pending_task_ids = [ + eval_row.celery_task_id + for eval_row in evaluation.row_results + if eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ] + if pending_task_ids: + celery_app.control.revoke(pending_task_ids, terminate=False) + except Exception: + # Best effort — DB delete remains the source of truth. + pass + + +# --------------------------------------------------------------------------- +# User-initiated cancel for in-flight evaluation rows +# --------------------------------------------------------------------------- +# +# Evaluation rows can sit in ``running`` for many minutes when the underlying +# LLM / audio metric call is slow or wedged (the worker carries an 8 min +# soft / 10 min hard time limit). Without a cancel affordance the operator's +# only recourse is to wait for Celery's time limit to fire — or to manually +# mutate the DB. These helpers + the two endpoints below give the UI a +# first-class "Abort" button mirroring the diarisation cancel pattern at +# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). +# +# Revoke uses :mod:`app.workers.celery_revoke` — ``terminate`` is off by default +# because ``worker-imports`` runs ``--pool=threads`` (Celery cannot kill threads). +# Rows are marked cancelled in the DB first; eval workers cooperatively exit via +# ``was_cancelled_externally``. Set ``celery.revoke_use_terminate: true`` only +# when every targeted worker uses prefork and you need SIGTERM mid-flight. + +# Sentinel error message stamped on cancelled rows. Read by the eval worker's +# ``_was_cancelled_externally`` guard (see +# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already +# past its slowest operation can't overwrite the cancelled state with its own +# terminal status. Touching either copy means touching both. +EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" + + +def _cancellable_eval_states() -> Tuple[str, ...]: + """States that an evaluation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: + """Best-effort revoke of a single eval row's Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). + """ + task_id = (eval_row.celery_task_id or "").strip() + if not task_id: + return + from app.workers.celery_revoke import revoke_celery_tasks + + if revoke_celery_tasks([task_id]): + logger.info( + "Revoked evaluation task {} for eval row {}", + task_id, + eval_row.id, + ) + + +def _cancel_eval_row_with_source_diarisation( + eval_row: CallImportEvaluationRow, + source_row: CallImportRow | None, + *, + cascade_diarisation: bool, + now: datetime | None = None, +) -> list[str]: + """Mark an eval row cancelled; optionally fail in-flight diarisation. + + Returns Celery task ids to revoke (deduped). The caller may batch-revoke + them; when ``cascade_diarisation`` is true, diarisation revoke also runs + inside :func:`_apply_diarisation_cancel` for the source row's task id. + """ + cancellable_states = _cancellable_eval_states() + if (eval_row.status or "").lower() not in cancellable_states: + return [] + + stamp = now or datetime.now(timezone.utc) + task_ids: list[str] = [] + eval_task_id = (eval_row.celery_task_id or "").strip() + if eval_task_id: + task_ids.append(eval_task_id) + + eval_row.status = "failed" + eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR + eval_row.finished_at = stamp + eval_row.celery_task_id = None + + if cascade_diarisation and source_row is not None: + from app.api.v1.routes.call_imports import _apply_diarisation_cancel + + source_task_id = (source_row.celery_task_id or "").strip() + if source_task_id and source_task_id not in task_ids: + task_ids.append(source_task_id) + _apply_diarisation_cancel([source_row]) + + return list(dict.fromkeys(task_ids)) + + +def _apply_evaluation_cancel( + eval_rows: List[CallImportEvaluationRow], + *, + source_rows: dict[UUID, CallImportRow] | None = None, + cascade_diarisation: bool = False, +) -> Tuple[int, int]: + """Cancel every cancellable row in ``eval_rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_eval_states() + cancelled = 0 + skipped = 0 + now = datetime.now(timezone.utc) + source_rows = source_rows or {} + for eval_row in eval_rows: + if (eval_row.status or "").lower() not in cancellable_states: + skipped += 1 + continue + source_row = source_rows.get(eval_row.call_import_row_id) + task_ids = _cancel_eval_row_with_source_diarisation( + eval_row, + source_row, + cascade_diarisation=cascade_diarisation, + now=now, + ) + for task_id in task_ids: + _revoke_eval_task_by_id(task_id, eval_row_id=eval_row.id) + cancelled += 1 + return cancelled, skipped + + +def _revoke_eval_task_by_id(task_id: str, *, eval_row_id: UUID) -> None: + """Best-effort revoke when the task id is known outside the ORM row.""" + cleaned = (task_id or "").strip() + if not cleaned: + return + from app.workers.celery_revoke import revoke_celery_tasks + + if revoke_celery_tasks([cleaned]): + logger.info( + "Revoked evaluation task {} for eval row {}", + cleaned, + eval_row_id, + ) + + +def _claim_evaluation_bulk_operation( + evaluation_id: UUID, + operation: str, +) -> None: + """Reserve the run for a single bulk worker pass; 409 if one is active.""" + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + try_set_evaluation_bulk_operation, + ) + + if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] + return + existing = get_evaluation_bulk_operation(evaluation_id) or operation + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + existing = get_evaluation_bulk_operation(evaluation_id) + if existing: + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +@router.post( + "/{eval_id}/cancel", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="cancelCallImportEvaluation", +) +async def cancel_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Abort all in-flight (or queued) rows in a single evaluation run. + + Idempotent: calling on a run whose rows are already terminal returns + ``target_count=0`` with 202 so the UI can fire this from an + "Abort" button without having to pre-check the state. + + Heavy row resets and Celery revokes run in a background worker so + large batches do not block the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") + if target_count == 0: + from app.services.call_imports.bulk_ops import ( + count_inflight_diarisation_for_evaluation, + ) + + inflight_diarisation = count_inflight_diarisation_for_evaluation( + db, + eval_id, + evaluation.call_import_id, + ) + if inflight_diarisation == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + worker_enqueued=False, + evaluation_id=eval_id, + ) + _claim_evaluation_bulk_operation(eval_id, "abort") + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + worker_enqueued=True, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "abort") + evaluation.status = "cancelled" + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + worker_enqueued=True, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/force-fail-pending", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="forceFailCallImportEvaluationPending", +) +async def force_fail_pending_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Force-fail only rows currently in ``pending`` for a single run. + + This is narrower than :func:`cancel_call_import_evaluation`: it leaves + ``running`` rows untouched so operators can clear permanently queued rows + without interrupting in-flight evaluations. + + Row updates run in a background worker so large batches do not block + the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets( + db, eval_id, mode="force_fail_pending" + ) + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + worker_enqueued=False, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay( + str(eval_id), mode="force_fail_pending" + ) + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + worker_enqueued=True, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/cancel", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportEvaluationRow", +) +async def cancel_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Abort an in-flight (or queued) evaluation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed``) returns the row unchanged with a 200 so the UI can + wire this to a "Stop" button without having to pre-check the + state. Updates the parent run's rollup so its counters reflect + the cancel immediately. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import evaluation_row_session + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + try: + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + if eval_row.evaluation_id != eval_id: + raise HTTPException( + status_code=404, + detail="Evaluation row not found in this run", + ) + _apply_evaluation_cancel( + [eval_row], + source_rows={source_row.id: source_row}, + cascade_diarisation=True, + ) + row_db.commit() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + row_db.refresh(eval_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) + except LookupError as exc: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) from exc + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + source_rows = ( + {source_row.id: source_row} if source_row is not None else None + ) + _apply_evaluation_cancel( + [eval_row], + source_rows=source_rows, + cascade_diarisation=True, + ) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(eval_row) + + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +@router.delete( + "/{eval_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluation", +) +async def delete_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + _revoke_pending_tasks(row) + + from app.services.call_imports.discovered_running_cache import ( + clear_discovered_running_cache, + ) + + clear_discovered_running_cache(eval_id) + db.delete(row) + db.commit() + from app.services.call_imports.eval_progress_buffer import ( + delete as delete_eval_counters, + ) + + delete_eval_counters(eval_id) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post( + "/bulk-delete", + status_code=status.HTTP_200_OK, + operation_id="bulkDeleteCallImportEvaluations", +) +async def bulk_delete_call_import_evaluations( + call_import_id: UUID, + payload: CallImportEvaluationBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Dict[str, int]: + """Delete multiple evaluation runs scoped to one call import. + + Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI + can clear out a multi-select. Unknown ids (already deleted, or + belonging to a different org/import) are silently skipped — the + response just reports how many actually went away. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + if not payload.evaluation_ids: + return {"deleted": 0} + + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id.in_(payload.evaluation_ids), + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .all() + ) + deleted = 0 + deleted_ids = [] + for row in rows: + _revoke_pending_tasks(row) + deleted_ids.append(row.id) + db.delete(row) + deleted += 1 + db.commit() + from app.services.call_imports.eval_progress_buffer import ( + delete as delete_eval_counters, + ) + + for deleted_id in deleted_ids: + delete_eval_counters(deleted_id) + return {"deleted": deleted} + diff --git a/app/api/v1/routes/call_import_evaluations/evaluation_create.py b/app/api/v1/routes/call_import_evaluations/evaluation_create.py new file mode 100644 index 00000000..599d26e7 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/evaluation_create.py @@ -0,0 +1,40 @@ +"""POST handler to create a call-import evaluation run.""" + +from __future__ import annotations + +from uuid import UUID + +from fastapi import Depends, status +from sqlalchemy.orm import Session + +from app.core.auth import Principal, get_principal +from app.database import get_db +from app.dependencies import get_api_key, get_organization_id +from app.models.schemas import CallImportEvaluationCreate, CallImportEvaluationResponse +from app.services.call_imports.evaluation_create import create_call_import_evaluations + +from app.api.v1.routes.call_import_evaluations._router import router + + +@router.post( + "", + response_model=CallImportEvaluationResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="createCallImportEvaluation", +) +async def create_call_import_evaluation( + call_import_id: UUID, + payload: CallImportEvaluationCreate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + return create_call_import_evaluations( + db, + organization_id=organization_id, + principal=principal, + call_import_id=call_import_id, + payload=payload, + ) diff --git a/app/api/v1/routes/call_import_evaluations/evaluation_read.py b/app/api/v1/routes/call_import_evaluations/evaluation_read.py new file mode 100644 index 00000000..93294e80 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/evaluation_read.py @@ -0,0 +1,258 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _normalize_name, + _require_import, + _serialize_eval, +) +@router.get( + "", + response_model=CallImportEvaluationListResponse, + operation_id="listCallImportEvaluations", +) +async def list_call_import_evaluations( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .all() + ) + email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) + return CallImportEvaluationListResponse( + items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], + total=len(rows), + ) + + +@router.get( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="getCallImportEvaluation", +) +async def get_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return _serialize_eval(db, row) + + +@router.patch( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="updateCallImportEvaluation", +) +async def update_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + """Edit metadata on an existing evaluation run (currently just ``name``).""" + + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + # Treat unset vs explicit ``None`` differently: unset = leave alone, + # explicit ``None`` or empty string = clear the name. + payload_data = payload.model_dump(exclude_unset=True) + if "name" in payload_data: + row.name = _normalize_name(payload_data["name"]) + + stamp_evaluation_actor(row, principal) + db.commit() + db.refresh(row) + return _serialize_eval(db, row) + diff --git a/app/api/v1/routes/call_import_evaluations/evaluation_retry.py b/app/api/v1/routes/call_import_evaluations/evaluation_retry.py new file mode 100644 index 00000000..82ebf5cb --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/evaluation_retry.py @@ -0,0 +1,1116 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _require_import, + _rollup_evaluation_status, + _to_evaluation_row_response, +) +from app.api.v1.routes.call_import_evaluations.discovered import ( # noqa: E402 + _is_metric_scores_meta_key, +) +from app.api.v1.routes.call_import_evaluations.evaluation_cancel import ( # noqa: E402 + _claim_evaluation_bulk_operation, + _require_no_evaluation_bulk_operation, +) +def source_retry_assignments( + source_row: CallImportRow, + *, + transcribe_overwrite: bool, +) -> dict: + """Columns a retry should write. Empty when the source row is already reusable.""" + from app.api.v1.routes.call_imports import _is_stuck_pending_import_row + from app.db_sharding.scatter_gather import row_text_present + + changes: dict = {} + if (getattr(source_row, "celery_task_id", None) or "").strip(): + changes["celery_task_id"] = None + no_audio = not (getattr(source_row, "recording_s3_key", None) or "").strip() + status = getattr(source_row, "status", None) + requeue = no_audio and status in ( + CallImportRowStatus.FAILED, + CallImportRowStatus.PROCESSING, + ) + if no_audio and status == CallImportRowStatus.PENDING: + requeue = _is_stuck_pending_import_row(source_row) + if requeue: + if status != CallImportRowStatus.PENDING: + changes["status"] = CallImportRowStatus.PENDING + if getattr(source_row, "error_message", None) is not None: + changes["error_message"] = None + if (getattr(source_row, "attempts", 0) or 0) != 0: + changes["attempts"] = 0 + + has_dia = row_text_present( + source_row, "diarised_transcript", "diarised_transcript_char_length" + ) + if transcribe_overwrite: + # Retry overwrite re-diarises *this* row only; the eval run's own + # overwrite marker is stored separately so `_needs_transcribe_for_eval` + # knows the fresh transcript belongs to the current pass. + changes["diarised_transcript"] = None + changes["diarised_transcript_status"] = "idle" + changes["diarised_transcript_error"] = None + changes["diarised_at"] = None + return changes + + dia_status = ( + getattr(source_row, "diarised_transcript_status", None) or "" + ).strip().lower() + if has_dia and not transcribe_overwrite: + if dia_status != "completed": + changes["diarised_transcript_status"] = "completed" + if getattr(source_row, "diarised_transcript_error", None) is not None: + changes["diarised_transcript_error"] = None + return changes + + if dia_status in {"failed", "pending", "running", "idle"}: + if dia_status != "idle": + changes["diarised_transcript_status"] = "idle" + if getattr(source_row, "diarised_transcript_error", None) is not None: + changes["diarised_transcript_error"] = None + return changes + + +def _prepare_source_row_for_retry( + source_row: CallImportRow, + *, + transcribe_overwrite: bool, +) -> None: + """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" + for column, value in source_retry_assignments( + source_row, + transcribe_overwrite=transcribe_overwrite, + ).items(): + setattr(source_row, column, value) + + +def _metric_scores_for_retry(existing, metric_ids: Optional[List[UUID]]) -> dict: + """Full retry clears every score. A metric subset removes only those keys.""" + if not metric_ids: + return {} + scores = existing if isinstance(existing, dict) else {} + target_keys = {str(mid).lower() for mid in metric_ids} + return { + key: value + for key, value in scores.items() + if _is_metric_scores_meta_key(str(key)) + or str(key).lower() not in target_keys + } + + +def _mark_evaluation_running_for_retry(evaluation: CallImportEvaluation) -> None: + """Open a new retry pass. + + The stale-run sweep ages a parent from ``started_at``. Keeping the + original timestamp makes a retry of a run older than 24 hours eligible + for cancellation on the next sweep. + """ + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + evaluation.started_at = datetime.now(timezone.utc) + + +def _reset_eval_row_for_retry( + eval_row: CallImportEvaluationRow, + *, + metric_ids: Optional[List[UUID]] = None, + skip_revoke: bool = False, + skip_chain: bool = False, +) -> None: + """Wipe per-row state so the worker can re-run it cleanly. + + Mirrors the initial state used by ``create_call_import_evaluation`` + when it first inserts a row, with the addition of revoking any + lingering Celery task id. + + When ``metric_ids`` is provided, this is a **metric-subset retry**: + only the scores for those metrics are removed from + ``metric_scores`` (other metrics' previously-computed values are + preserved so the worker's partial-merge write keeps them intact). + Otherwise the entire ``metric_scores`` dict is reset, matching the + legacy behaviour. + """ + if ( + not skip_revoke + and eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ): + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: # noqa: BLE001 — revoke is best-effort + pass + eval_row.status = "pending" + eval_row.error_message = None + eval_row.metric_scores = _metric_scores_for_retry( + eval_row.metric_scores, + metric_ids, + ) + eval_row.started_at = None + eval_row.finished_at = None + task_id = (eval_row.celery_task_id or "").strip() + eval_row.celery_task_id = None + if skip_chain or not task_id: + return + from app.workers.tasks.evaluate_call_import_row_audio import ( + clear_eval_audio_chain, + ) + + clear_eval_audio_chain(str(eval_row.id)) + + +def _enqueue_eval_rows_with_optional_transcribe( + db: Session, + evaluation: CallImportEvaluation, + eval_rows_with_source: List[ + Tuple[CallImportEvaluationRow, CallImportRow] + ], + *, + transcribe_overwrite: bool = False, + restricted_metric_ids: Optional[List[UUID]] = None, +) -> Tuple[int, int]: + """Schedule throttled evaluation dispatch for pending eval rows. + + Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for + logging/UI compatibility. Actual Celery fan-out is handled by + :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. + """ + from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval + from app.workers.concurrency.fair_dispatch import ( + schedule_fair_dispatch, + store_evaluation_transcribe_overwrite, + store_row_restricted_metrics, + ) + + eval_only_count = 0 + transcribe_count = 0 + if eval_rows_with_source: + for eval_row, source_row in eval_rows_with_source: + if _needs_transcribe_for_eval( + evaluation, + source_row, + transcribe_overwrite=transcribe_overwrite, + ): + transcribe_count += 1 + else: + eval_only_count += 1 + + restricted_metric_ids_str: Optional[List[str]] = ( + [str(mid) for mid in restricted_metric_ids] + if restricted_metric_ids + else None + ) + if restricted_metric_ids_str: + for eval_row, _ in eval_rows_with_source: + store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) + else: + restricted_metric_ids_str = ( + [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None + ) + store_evaluation_transcribe_overwrite( + evaluation.id, + overwrite=transcribe_overwrite, + ) + schedule_fair_dispatch(max_workspace_turns=999) + return eval_only_count, transcribe_count + + +def _apply_telephony_retry_overrides( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Pin or clear telephony credentials on the batch for this retry pass.""" + fields_set = payload.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return + + from app.api.v1.routes.call_imports import ( + _resolve_telephony_integration, + _validate_telephony_credentials_live, + ) + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + +def _apply_retry_overrides( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Validate + persist the LLM/STT override fields on the run. + + Mirrors the validation in ``create_call_import_evaluation`` but + only touches the fields the caller actually sent — leaving any + field ``None`` preserves the run's existing value. Raises + ``HTTPException(400)`` on bad input so the route handler can let + FastAPI turn it into a clean 400 response. + """ + # --- LLM provider + model (must be sent together) --- + if payload.llm_provider is not None or payload.llm_model is not None: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail=( + "Both llm_provider and llm_model are required when " + "overriding the run LLM on retry." + ), + ) + try: + evaluation.llm_provider = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + new_model = payload.llm_model.strip() or None + if not new_model: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + evaluation.llm_model = new_model + + # --- LLM credential pin --- + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in " + "this organization." + ), + ) + evaluation.llm_credential_id = payload.llm_credential_id + + if payload.llm_config is not None: + evaluation.llm_config = payload.llm_config + + # --- Per-metric LLM overrides --- + # We accept the same dict shape as the create endpoint but + # constrain keys to leaf metrics that are actually in this run. + # Passing an empty dict explicitly clears existing overrides. + if payload.metric_llm_overrides is not None: + valid_leaf_ids = { + str(mid) for mid in (evaluation.selected_metric_ids or []) + } + overrides_payload: Dict[str, Dict[str, Any]] = {} + for metric_id, override in payload.metric_llm_overrides.items(): + if metric_id not in valid_leaf_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a leaf metric in " + "this run." + ), + ) + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a " + "provider but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses " + f"unknown provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model " + "but no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + overrides_payload[metric_id] = override_dict + evaluation.metric_llm_overrides = overrides_payload or None + + # --- STT provider + model (must be sent together) --- + if payload.stt_provider is not None or payload.stt_model is not None: + if not (payload.stt_provider and payload.stt_model): + raise HTTPException( + status_code=400, + detail=( + "Both stt_provider and stt_model are required " + "when overriding the run STT on retry." + ), + ) + try: + evaluation.stt_provider = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + new_stt_model = payload.stt_model.strip() or None + if not new_stt_model: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + evaluation.stt_model = new_stt_model + + # --- STT credential pin --- + if payload.stt_credential_id is not None: + evaluation.stt_credential_id = payload.stt_credential_id + + # --- LLM diariser provider + model (must be sent together) --- + if ( + payload.diarization_llm_provider is not None + or payload.diarization_llm_model is not None + ): + if not ( + payload.diarization_llm_provider + and payload.diarization_llm_model + ): + raise HTTPException( + status_code=400, + detail=( + "Both diarization_llm_provider and " + "diarization_llm_model are required when overriding " + "the run diariser on retry." + ), + ) + try: + evaluation.diarisation_llm_provider = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + "Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + new_diariser_model = ( + payload.diarization_llm_model.strip() or None + ) + if not new_diariser_model: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + evaluation.diarisation_llm_model = new_diariser_model + + if payload.diarization_llm_credential_id is not None: + evaluation.diarisation_llm_credential_id = ( + payload.diarization_llm_credential_id + ) + + # ``diarization_prompt`` semantics: None = leave untouched; + # empty string = clear (fall back to the canonical default at + # worker time); anything else = persist verbatim. + if payload.diarization_prompt is not None: + cleaned = payload.diarization_prompt.strip() + evaluation.diarisation_prompt = cleaned or None + + if payload.transcribe_mode is not None: + mode = payload.transcribe_mode.strip().lower() + if mode not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Valid values are 'stt_llm' and 'llm_only'." + ), + ) + evaluation.transcribe_mode = mode + + +def _gather_retry_targets( + db: Session, + evaluation: CallImportEvaluation, + requested_ids: Optional[List[UUID]], + *, + include_completed: bool = False, +) -> Tuple[ + List[Tuple[CallImportEvaluationRow, CallImportRow]], + List[CallImportEvaluationRetrySkippedItem], +]: + """Resolve which rows to retry + reasons for any we refuse. + + When ``requested_ids`` is None we retry every row whose status is + ``failed`` (or every row when ``include_completed`` is also set — + used by the metric-subset retry path which legitimately wants to + recompute a metric on already-successful rows). When the caller + passes ids explicitly we still filter out rows that are currently + in flight; ``include_completed`` controls whether previously- + successful rows are eligible. + """ + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import gather_retry_targets_sharded + + return gather_retry_targets_sharded( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + + from app.db_sharding.eval_rows import _collect_retry_targets_on_session + + targets, skipped, found = _collect_retry_targets_on_session( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + if requested_ids is not None: + for missing in set(requested_ids) - found: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=missing, + reason="unknown", + ) + ) + return targets, skipped + + +@router.post( + "/{eval_id}/retry", + response_model=CallImportEvaluationRetryResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluation", +) +async def retry_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRetryResponse: + """Re-enqueue failed rows in an evaluation run. + + Default behavior (no body) is "retry every row that failed". Pass + ``eval_row_ids`` to scope the retry to a specific subset (e.g. the + single row a user clicked in the UI). Rows that are still + in-flight or already completed are returned in ``skipped`` rather + than re-enqueued, so this endpoint is always safe to call. + + When ``metric_ids`` is set in the payload, this is a **metric- + subset retry**: only the listed metrics are recomputed (and merged + into the row's existing ``metric_scores`` — other metrics' values + are preserved). The route auto-flips ``include_completed=True`` in + that case so previously-successful rows are eligible for re- + scoring; without it the call would no-op because every row would + be skipped as ``completed``. + + The worker contract is the same as the create endpoint: + ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. + When the run is configured for diarised transcripts and the row's + diarised transcript is missing, we chain through + ``transcribe_call_import_row_task`` first — matching the + auto-transcribe behavior of POST ``/evaluations``. + """ + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + requested_ids = payload.eval_row_ids if payload else None + retry_filter = payload.row_filter if payload else None + _MAX_EXPLICIT_EVAL_RETRY_IDS = 500 + if requested_ids is not None and len(requested_ids) > _MAX_EXPLICIT_EVAL_RETRY_IDS: + raise HTTPException( + status_code=400, + detail=( + f"At most {_MAX_EXPLICIT_EVAL_RETRY_IDS} eval_row_ids per request; " + "use row_filter for bulk retry." + ), + ) + # Metric-subset retry: validate that every metric is something this + # run actually scored. Empty list is rejected too — callers that + # want a full re-run should omit the field entirely. + # + # ``selected_metric_ids`` holds the LEAVES only (children for + # hierarchical / category metrics, standalone metrics otherwise) — + # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. + # Parent IDs for hierarchical metrics live separately in + # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the + # UI can reconstruct the tree without round-tripping through the + # metric table. + # + # The Re-run-metrics modal surfaces PARENTS for hierarchical + # metrics (it suppresses individual children via + # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a + # naive ``metric_ids Γèå selected_metric_ids`` check rejects every + # parent-ID request with a misleading "unknown ids" 400. We accept + # both shapes here and then EXPAND any parent IDs into + # ``{parent_id, *child_ids}`` so the downstream helpers see the + # full set of keys that need clearing + the full set of leaves + # that need re-scoring. + metric_ids: Optional[List[UUID]] = ( + payload.metric_ids if payload else None + ) + if metric_ids is not None: + metric_ids = [ + mid for mid in metric_ids if not _is_metric_scores_meta_key(str(mid)) + ] + if not metric_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a non-empty list of metric UUIDs. " + "Omit the field to re-run all metrics." + ), + ) + + leaf_set: Set[str] = { + str(item).lower() + for item in (evaluation.selected_metric_ids or []) + } + # ``selected_metric_groups`` is a dict ``{parent_id_str: + # [child_id_str, ...]}`` (see line ~487 in + # ``create_call_import_evaluation``). We tolerate stale data + # (string / UUID / non-dict) without crashing the retry path — + # if it's malformed we just treat it as "no parents" and fall + # back to the leaf-only check. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_to_children_str: Dict[str, List[str]] = {} + for parent_key, children_raw in groups_raw.items(): + if not isinstance(children_raw, (list, tuple)): + continue + children_norm = [ + str(c).lower() for c in children_raw if c is not None + ] + parent_to_children_str[str(parent_key).lower()] = children_norm + parent_set = set(parent_to_children_str.keys()) + + unknown = [ + mid for mid in metric_ids + if str(mid).lower() not in leaf_set + and str(mid).lower() not in parent_set + ] + if unknown: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a subset of this evaluation's " + f"selected metrics; unknown ids: {[str(u) for u in unknown]}." + ), + ) + + # Expand parent IDs into ``{parent, *children}`` so: + # * ``_reset_eval_row_for_retry`` strips BOTH the parent + # entry (with ``chosen_child_id`` / rationale) AND every + # per-child boolean entry that the LLM evaluator wrote + # under each child's ID (see + # ``app/workers/tasks/helpers/llm_evaluation.py`` lines + # 1584 and 1649). + # * ``_enqueue_eval_rows_with_optional_transcribe`` → + # ``evaluate_call_import_row_task`` filters the work-list + # off ``selected_metric_ids`` (leaves), so we MUST hand it + # the child IDs for the parent to actually get re-scored. + # Leaves pass through unchanged. + expanded: List[UUID] = [] + seen: Set[str] = set() + for mid in metric_ids: + mid_norm = str(mid).lower() + children_str = parent_to_children_str.get(mid_norm) + if children_str is not None: + # Parent: include the parent ID itself (so the parent + # entry in ``metric_scores`` is also cleared) and all + # of its children. + candidates = [mid_norm, *children_str] + else: + candidates = [mid_norm] + for candidate in candidates: + if candidate in seen: + continue + try: + expanded.append(UUID(candidate)) + except (TypeError, ValueError): + # Defensive: skip junk values rather than 500. + continue + seen.add(candidate) + metric_ids = expanded + + # ``include_completed`` is auto-enabled when the caller asked for a + # metric subset (otherwise the metric-subset retry would always + # no-op on a green run, which is the whole reason this feature + # exists). The explicit payload flag wins for full-row retries. + include_completed = bool( + (payload.include_completed if payload else False) + or (metric_ids is not None) + ) + + transcribe_overwrite = bool( + payload.transcribe_overwrite if payload else False + ) + + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + retry_search = (retry_filter.q or "").strip() if retry_filter else "" + retry_status = ( + (retry_filter.status or "").strip().lower() if retry_filter and retry_filter.status else None + ) + if requested_ids is None: + from app.db_sharding.eval_rows import ( + count_evaluation_retry_targets, + count_evaluation_rows_for_run, + ) + from app.db_sharding.sessions import is_sharding_enabled + + if retry_filter is not None or retry_search: + if retry_status and include_completed and retry_status not in { + "failed", + "completed", + }: + statuses = [retry_status] + elif retry_status: + statuses = [retry_status] + else: + statuses = ( + ["failed", "completed"] if include_completed else ["failed"] + ) + target_count = count_evaluation_retry_targets( + db, + eval_id, + statuses=statuses, + search_term=retry_search, + ) + elif is_sharding_enabled(): + statuses = ( + ["failed", "completed"] if include_completed else ["failed"] + ) + target_count = count_evaluation_rows_for_run( + db, eval_id, statuses=statuses + ) + else: + from sqlalchemy import func + + count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( + CallImportEvaluationRow.evaluation_id == eval_id + ) + if include_completed: + count_query = count_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ) + else: + count_query = count_query.filter( + CallImportEvaluationRow.status == "failed" + ) + target_count = int(count_query.scalar() or 0) + if target_count == 0: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + else: + targets, skipped = _gather_retry_targets( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + if not targets: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + target_count = len(targets) + + # Apply LLM / STT overrides BEFORE enqueueing so the persisted run + # config is correct by the time the worker reads it. + if payload is not None: + _apply_retry_overrides(db, evaluation, organization_id, payload) + _apply_telephony_retry_overrides( + db, + call_import=call_import, + organization_id=organization_id, + payload=payload, + ) + + _mark_evaluation_running_for_retry(evaluation) + + _claim_evaluation_bulk_operation(eval_id, "retry") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + retry_call_import_evaluation_task, + ) + + retry_call_import_evaluation_task.delay( + str(eval_id), + { + "eval_row_ids": [str(rid) for rid in requested_ids] + if requested_ids + else None, + "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, + "include_completed": include_completed, + "transcribe_overwrite": transcribe_overwrite, + "retry_search_term": retry_search or None, + "retry_status_filter": retry_status, + }, + ) + + return CallImportEvaluationRetryResponse( + requeued=target_count, + transcribe_requeued=0, + skipped=skipped, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/retry", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluationRow", +) +async def retry_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Re-enqueue a single failed evaluation row. + + Convenience wrapper around ``retry_call_import_evaluation`` for the + "Retry this row" affordance in the row table. Returns the + refreshed row so the UI can update its badge immediately, without + waiting for the next polling tick. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import ( + evaluation_row_session, + find_evaluation_row_in_run, + ) + from app.db_sharding.sessions import is_sharding_enabled + + eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) + if eval_row is None: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + if eval_row.status in {"pending", "running"}: + raise HTTPException( + status_code=409, + detail=( + "This row is still in progress — wait for it to finish " + "before retrying." + ), + ) + + targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) + if not targets: + raise HTTPException( + status_code=409, + detail=( + "This row cannot be retried in its current state " + f"(status={eval_row.status})." + ), + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(eval_row) + row_db.commit() + targets = [(eval_row, source_row)] + else: + for er, source_row in targets: + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(er) + + _mark_evaluation_running_for_retry(evaluation) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + + try: + _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to re-enqueue retry for evaluation row {}", eval_row_id + ) + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + _source_row, + _shard_id, + ): + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + row_db.commit() + else: + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + _rollup_evaluation_status(evaluation, db) + db.commit() + raise HTTPException( + status_code=500, + detail=f"Failed to re-enqueue retry: {exc}", + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + _row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + db.refresh(eval_row) + source_row = targets[0][1] + return _to_evaluation_row_response(eval_row, source_row, evaluation) diff --git a/app/api/v1/routes/call_import_evaluations/exports.py b/app/api/v1/routes/call_import_evaluations/exports.py new file mode 100644 index 00000000..b9fa6122 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/exports.py @@ -0,0 +1,487 @@ +"""Evaluation export jobs and PDF report route wrappers.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional +from uuid import UUID + +from fastapi import Body, Depends, HTTPException, status +from sqlalchemy import desc +from sqlalchemy.orm import Session + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE +from app.database import get_db +from app.dependencies import get_api_key, get_organization_id +from app.models.database import CallImportEvaluation, CallImportEvaluationPdfReport +from app.models.schemas import ( + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, +) +from app.services.reporting.call_import_evaluation_pdf_api import ( + CallImportEvaluationBaselineCandidatesResponse, + CallImportEvaluationPdfReportListResponse, + CallImportEvaluationPdfReportRequest, + CallImportEvaluationPdfReportResponse, + baseline_candidates_response, + generate_call_import_evaluation_pdf_report, + pdf_report_download_response, + pdf_report_list_item_from_row, + pdf_report_response_from_row, +) + +from app.api.v1.routes.call_import_evaluations._router import router +from app.api.v1.routes.call_import_evaluations._shared import ( + _require_import, + require_call_import_capability, +) + +@router.post( + "/{eval_id}/export", + response_model=CallImportEvaluationExportJobResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="startCallImportEvaluationExport", +) +async def start_call_import_evaluation_export( + call_import_id: UUID, + eval_id: UUID, + payload: Optional[CallImportEvaluationExportRequest] = Body(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationExportJobResponse: + """Queue a gzip CSV (or scores-only xlsx) export and return the job. + + The file is built off this request. Poll the job, then download each + part from blob storage. + """ + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from sqlalchemy.exc import IntegrityError + + from app.models.database import CallImportEvaluationExport + from app.services.call_imports.evaluation_export import ( + XLSX_SHEET_ROW_CAP, + execute_evaluation_export, + export_job_is_stale, + ) + from app.services.storage.blob_storage_service import blob_storage_service + + body = payload or CallImportEvaluationExportRequest() + if body.format == "xlsx": + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + + row_count = count_evaluation_rows_for_run(db, eval_id) + if row_count > XLSX_SHEET_ROW_CAP: + raise HTTPException( + status_code=400, + detail=( + "This run has more rows than Excel allows on one sheet. " + "Download the CSV instead." + ), + ) + + active = _active_export_job(db, eval_id) + if active is not None and not _export_request_matches(active, body): + was_stale = export_job_is_stale(active) + if was_stale and _fail_stale_export_job(db, active): + active = None + elif was_stale: + active = _active_export_job(db, eval_id) + if active is not None and not _export_request_matches(active, body): + raise HTTPException( + status_code=409, + detail=( + "An export is already running for this evaluation. " + "Wait for it to finish before starting a different download." + ), + ) + if active is not None: + if blob_storage_service.is_enabled() and ( + active.status == "pending" or export_job_is_stale(active) + ): + try: + _enqueue_evaluation_export(str(active.id)) + except Exception as exc: + if active.status == "pending": + active.status = "failed" + active.error_message = f"Could not queue the export: {exc}" + active.finished_at = datetime.now(timezone.utc) + db.commit() + db.refresh(active) + return _export_job_response(active) + + job = CallImportEvaluationExport( + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + format=body.format, + include_transcripts=body.include_transcripts, + status="pending", + parts=[], + ) + db.add(job) + try: + db.commit() + except IntegrityError: + db.rollback() + active = _active_export_job(db, eval_id) + if active is not None and _export_request_matches(active, body): + return _export_job_response(active) + if active is not None: + raise HTTPException( + status_code=409, + detail=( + "An export is already running for this evaluation. " + "Wait for it to finish before starting a different download." + ), + ) + raise + db.refresh(job) + + if not blob_storage_service.is_enabled(): + execute_evaluation_export(db, job.id) + db.refresh(job) + return _export_job_response(job) + + try: + _enqueue_evaluation_export(str(job.id)) + except Exception as exc: + job.status = "failed" + job.error_message = f"Could not queue the export: {exc}" + job.finished_at = datetime.now(timezone.utc) + db.commit() + db.refresh(job) + return _export_job_response(job) + + +@router.get( + "/{eval_id}/export-jobs/{job_id}", + response_model=CallImportEvaluationExportJobResponse, + operation_id="getCallImportEvaluationExport", +) +async def get_call_import_evaluation_export( + call_import_id: UUID, + eval_id: UUID, + job_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationExportJobResponse: + del api_key + _require_import(db, call_import_id, organization_id) + from app.models.database import CallImportEvaluationExport + + job = ( + db.query(CallImportEvaluationExport) + .filter( + CallImportEvaluationExport.id == job_id, + CallImportEvaluationExport.evaluation_id == eval_id, + CallImportEvaluationExport.call_import_id == call_import_id, + CallImportEvaluationExport.organization_id == organization_id, + ) + .first() + ) + if job is None: + raise HTTPException(status_code=404, detail="Export job not found") + return _export_job_response(job, with_download_urls=True) + + +def _fail_stale_export_job(db: Session, job) -> bool: + """Fail a stale running export only while that same claim still holds it. + + The heartbeat and claim are the values already loaded on ``job``. A + worker that claimed the export after this read changes both, the + update matches nothing, and the caller re-reads the live job. + """ + from sqlalchemy import update + + from app.models.database import CallImportEvaluationExport + + observed_heartbeat = job.heartbeat_at + observed_claim = job.claim_token + heartbeat_still = ( + CallImportEvaluationExport.heartbeat_at.is_(None) + if observed_heartbeat is None + else CallImportEvaluationExport.heartbeat_at == observed_heartbeat + ) + claim_still = ( + CallImportEvaluationExport.claim_token.is_(None) + if observed_claim is None + else CallImportEvaluationExport.claim_token == observed_claim + ) + result = db.execute( + update(CallImportEvaluationExport) + .where( + CallImportEvaluationExport.id == job.id, + CallImportEvaluationExport.status == "running", + heartbeat_still, + claim_still, + ) + .values( + status="failed", + error_message="Stopped because a different export was requested.", + finished_at=datetime.now(timezone.utc), + ) + .execution_options(synchronize_session=False) + ) + db.commit() + return bool(result.rowcount) + + +def _export_request_matches(job, body) -> bool: + return (job.format or "csv") == body.format and bool(job.include_transcripts) == bool( + body.include_transcripts + ) + + +def _active_export_job(db: Session, evaluation_id: UUID): + from app.models.database import CallImportEvaluationExport + + return ( + db.query(CallImportEvaluationExport) + .filter( + CallImportEvaluationExport.evaluation_id == evaluation_id, + CallImportEvaluationExport.status.in_(("pending", "running")), + ) + .first() + ) + + +def _enqueue_evaluation_export(job_id: str) -> None: + from app.workers.tasks.call_import_bulk_ops import ( + export_call_import_evaluation_task, + ) + + export_call_import_evaluation_task.delay(job_id) + + +def _export_job_response(job, *, with_download_urls: bool = False): + from app.models.schemas import ( + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportPart, + ) + from app.services.call_imports.evaluation_export import EXPORT_URL_TTL_SECONDS + from app.services.storage.blob_storage_service import blob_storage_service + + parts = [] + raw_parts = job.parts if isinstance(job.parts, list) else [] + for item in raw_parts: + if not isinstance(item, dict): + continue + download_url = None + key = item.get("s3_key") + filename = item.get("filename") or "export.csv.gz" + if ( + with_download_urls + and job.status == "completed" + and key + and blob_storage_service.is_enabled() + ): + download_url = blob_storage_service.generate_presigned_url_by_key( + key, + expiration=EXPORT_URL_TTL_SECONDS, + response_content_disposition=f'attachment; filename="{filename}"', + ) + parts.append( + CallImportEvaluationExportPart( + part=int(item.get("part") or 0), + filename=filename, + row_count=int(item.get("row_count") or 0), + byte_size=int(item.get("byte_size") or 0), + download_url=download_url, + ) + ) + return CallImportEvaluationExportJobResponse( + id=job.id, + evaluation_id=job.evaluation_id, + status=job.status, + format=job.format, + include_transcripts=bool(job.include_transcripts), + error_message=job.error_message, + parts=parts, + ) + +@router.get( + "/{eval_id}/baseline-candidates", + response_model=CallImportEvaluationBaselineCandidatesResponse, + operation_id="listCallImportEvaluationBaselineCandidates", +) +async def list_call_import_evaluation_baseline_candidates( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationBaselineCandidatesResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return baseline_candidates_response( + db, + call_import=call_import, + evaluation=evaluation, + organization_id=organization_id, + ) + + +@router.post( + "/{eval_id}/pdf-report", + operation_id="generateCallImportEvaluationPdfReport", + dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], +) +async def generate_call_import_evaluation_pdf_report_route( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationPdfReportRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +): + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return await generate_call_import_evaluation_pdf_report( + db, + call_import=call_import, + evaluation=evaluation, + organization_id=organization_id, + eval_id=eval_id, + call_import_id=call_import_id, + payload=payload, + principal=principal, + ) + + +@router.get( + "/{eval_id}/pdf-reports", + response_model=CallImportEvaluationPdfReportListResponse, + operation_id="listCallImportEvaluationPdfReports", +) +async def list_call_import_evaluation_pdf_reports( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .all() + ) + return CallImportEvaluationPdfReportListResponse( + items=[pdf_report_list_item_from_row(row) for row in rows], + ) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}", + response_model=CallImportEvaluationPdfReportResponse, + operation_id="getCallImportEvaluationPdfReport", +) +async def get_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + return pdf_report_response_from_row(row) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}/download", + operation_id="downloadCallImportEvaluationPdfReport", +) +async def download_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + del api_key + _require_import(db, call_import_id, organization_id) + return pdf_report_download_response( + db, + call_import_id=call_import_id, + eval_id=eval_id, + report_id=report_id, + organization_id=organization_id, + ) + diff --git a/app/api/v1/routes/call_import_evaluations/flow.py b/app/api/v1/routes/call_import_evaluations/flow.py new file mode 100644 index 00000000..fd00e8ad --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/flow.py @@ -0,0 +1,531 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _load_eval_rows, + _metrics_for_ids, + _require_import, +) +from app.api.v1.routes.call_import_evaluations.discovered import ( # noqa: E402 + _alias_map_for_parent, + _promoted_child_slugs, + _resolve_alias, + _slug_label, +) + +_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. +_FLOW_START_NODE_ID = "__START__" +_DISCOVERED_NODE_PREFIX = "disc:" +def _build_flow_graph( + eval_rows: List[CallImportEvaluationRow], + parent_metric: Metric, + children: List[Metric], + alias_map: Optional[Dict[str, str]] = None, + extra_children: Optional[List[Metric]] = None, +) -> MetricFlowResponse: + """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. + + A synthetic ``START`` node is prepended to every sequence so the + diagram has a single origin. Children that never appear in any + sequence are still emitted as nodes (count=0) so the UI can render + them in the legend. + + ``alias_map`` lets callers fold merged-out discovered slugs into + their canonical target before building the graph; ``extra_children`` + are children of the parent that aren't in the legend list (e.g. + children promoted *after* the evaluation was created and therefore + missing from ``selected_metric_groups``) but should still resolve in + sequences so the slug doesn't get redrawn as a discovered candidate. + """ + parent_id_str = str(parent_metric.id) + aliases = alias_map or {} + # Build a fast lookup keyed by both the lower_snake child key (what the + # LLM emits in ``sequence``) and the child UUID (what some clients may + # store) so legacy / drifted payloads still resolve. + child_lookup: Dict[str, Metric] = {} + for child in children: + slug = _slug_label(child.name) + child_lookup[slug] = child + child_lookup[str(child.id)] = child + # ``extra_children`` are resolved-only — they shouldn't add legend + # nodes (those come from the explicit ``children`` argument), but + # they need to be in ``child_lookup`` so a sequence step that + # matches a freshly-promoted child resolves to the real child UUID + # instead of falling through to ``discovered_lookup`` and rendering + # as a "discovered" node. + if extra_children: + for child in extra_children: + slug = _slug_label(child.name) + if slug and slug not in child_lookup: + child_lookup[slug] = child + cid = str(child.id) + child_lookup.setdefault(cid, child) + + # Discovered labels: walk every row's discovered_labels first so we + # know which discovered slugs are valid before resolving sequences. + # Discovered nodes get a ``disc:`` prefixed id so they can't collide + # with real child UUIDs in the node/edge graph. We apply + # ``alias_map`` first so merged-out source slugs fold into their + # canonical target — preserving the user's "merge" intent on still- + # in-flight rows whose JSON wasn't rewritten by the merge endpoint. + discovered_lookup: Dict[str, Dict[str, Any]] = {} + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_discovered = parent_entry.get("discovered_labels") + if not isinstance(raw_discovered, list): + continue + for entry in raw_discovered: + if not isinstance(entry, dict): + continue + slug = _slug_label(entry.get("key") or entry.get("name")) + slug = _resolve_alias(aliases, slug) + if not slug or slug in child_lookup: + continue + name = (entry.get("name") or "").strip() or slug.replace("_", " ") + existing = discovered_lookup.get(slug) + if existing is None: + discovered_lookup[slug] = { + "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", + "name": name, + } + + node_counts: Dict[str, int] = {} + edge_counts: Dict[tuple[str, str], int] = {} + terminal_counts: Dict[str, int] = {} + + total_rows = len(eval_rows) + rows_with_sequence = 0 + + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_sequence = parent_entry.get("sequence") + if not isinstance(raw_sequence, list): + continue + + resolved_ids: List[str] = [] + last_resolved: Optional[str] = None + for item in raw_sequence: + if not isinstance(item, str): + continue + normalized = _resolve_alias(aliases, _slug_label(item)) + child = child_lookup.get(normalized) or child_lookup.get(item) + if child is not None: + cid = str(child.id) + # Adjacent dedupe AFTER alias resolution so two + # different raw slugs that fold to the same target + # don't draw a self-edge through the chart. + if cid == last_resolved: + continue + resolved_ids.append(cid) + last_resolved = cid + continue + disc = discovered_lookup.get(normalized) + if disc is not None: + if disc["id"] == last_resolved: + continue + resolved_ids.append(disc["id"]) + last_resolved = disc["id"] + + if not resolved_ids: + continue + + rows_with_sequence += 1 + for nid in resolved_ids: + node_counts[nid] = node_counts.get(nid, 0) + 1 + + edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( + edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 + ) + for src, tgt in zip(resolved_ids, resolved_ids[1:]): + if src == tgt: + continue + edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 + + terminal_id = resolved_ids[-1] + terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 + + nodes: List[MetricFlowNode] = [] + # Always include a START node so the UI has a stable entry point. + nodes.append( + MetricFlowNode( + id=_FLOW_START_NODE_ID, + label="Start", + count=rows_with_sequence, + is_terminal=False, + ) + ) + + def _emit_child_node(child: Metric) -> None: + cid = str(child.id) + count = node_counts.get(cid, 0) + terminal_count = terminal_counts.get(cid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=cid, + label=child.name, + count=count, + is_terminal=is_terminal, + ) + ) + + emitted_child_ids: set[str] = set() + for child in children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Extra children (promoted after the eval was created) only get + # legend nodes if they actually appear in the data — otherwise we'd + # pollute the diagram with every standalone promotion the user has + # ever made under this parent. + if extra_children: + for child in extra_children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + if node_counts.get(cid, 0) == 0: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Append discovered nodes after the real children so legend ordering + # keeps user-defined labels first. + for slug, info in discovered_lookup.items(): + nid = info["id"] + count = node_counts.get(nid, 0) + terminal_count = terminal_counts.get(nid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=nid, + label=info["name"], + count=count, + is_terminal=is_terminal, + is_discovered=True, + ) + ) + + edges: List[MetricFlowEdge] = [ + MetricFlowEdge(source=src, target=tgt, count=count) + for (src, tgt), count in sorted( + edge_counts.items(), key=lambda kv: kv[1], reverse=True + ) + ] + + return MetricFlowResponse( + parent_metric_id=parent_id_str, + parent_metric_name=parent_metric.name, + selection_mode=parent_metric.selection_mode, + nodes=nodes, + edges=edges, + total_rows=total_rows, + rows_with_sequence=rows_with_sequence, + ) + + +@router.get( + "/{eval_id}/flow", + response_model=MetricFlowResponse, + operation_id="getCallImportEvaluationFlow", +) +async def get_call_import_evaluation_flow( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose children's sequences should be " + "aggregated into a flow graph." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFlowResponse: + """Aggregate the LLM-inferred per-row sequences into one flow graph. + + Returns ``nodes`` (one per child of the parent metric, plus a + synthetic ``START`` node) and ``edges`` (counts of consecutive + label transitions across every row that produced a sequence). The + frontend feeds this directly into a React Flow / xyflow canvas; + edge thickness should scale with ``count / total_rows`` and + ``is_terminal`` nodes should be styled as outcomes. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + if not parent.selection_mode: + raise HTTPException( + status_code=400, + detail=( + "Flow charts are only meaningful for parent metrics " + "(selection_mode set). This metric is standalone." + ), + ) + + # Children are taken from selected_metric_groups when present so the + # flow chart reflects exactly the subset that ran in this + # evaluation; otherwise fall back to every enabled child of the + # parent. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_id_str = str(parent.id) + children: List[Metric] = [] + if parent_id_str in groups_raw and isinstance( + groups_raw[parent_id_str], list + ): + child_ids: List[UUID] = [] + for c in groups_raw[parent_id_str]: + try: + child_ids.append(UUID(str(c))) + except (TypeError, ValueError): + continue + if child_ids: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(child_ids), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + if not children: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .order_by(Metric.created_at.asc()) + .all() + ) + + # Children promoted AFTER this evaluation was created aren't in + # ``selected_metric_groups`` but their slugs still appear in already- + # scored rows' sequences. Pass them as ``extra_children`` so those + # sequence entries resolve against the real (now promoted) child + # instead of being redrawn as discovered candidates. + extra_children: List[Metric] = [] + if children: + existing_ids = {child.id for child in children} + all_children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .all() + ) + extra_children = [c for c in all_children if c.id not in existing_ids] + + eval_rows = _load_eval_rows(db, eval_id) + + alias_map = _alias_map_for_parent(evaluation, parent.id) + return _build_flow_graph( + eval_rows, + parent, + children, + alias_map=alias_map, + extra_children=extra_children, + ) diff --git a/app/api/v1/routes/call_import_evaluations/insights.py b/app/api/v1/routes/call_import_evaluations/insights.py new file mode 100644 index 00000000..aff4a7b8 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/insights.py @@ -0,0 +1,1138 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) +from app.services.import_eval_llm_output_budgets import ( + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _count_completed_eval_rows, + _load_eval_rows, + _metrics_for_ids, + _require_import, + _serialize_selected_metric_ids, +) +from app.api.v1.routes.call_import_evaluations.aggregate import ( # noqa: E402 + _clamp_prose_to_sentences, + _compute_metric_aggregates, +) + +# TLDR insights: LLM-generated narrative + bullet patterns rendered above +# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` +# so the page never auto-burns LLM tokens; the user explicitly clicks +# "Generate summary" or "Regenerate" from the empty-state CTA. +# --------------------------------------------------------------------------- + + +_INSIGHTS_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will be " + "given aggregated metric statistics + a sample of rationales for " + "the rows of a single call-import evaluation. Identify the most " + "useful PATTERNS that hold ACROSS the calls -- not just per-metric " + "numbers. Look for combinations (e.g. `when X happens, Y also " + "tends to happen`), notable outliers, frequent failure modes, and " + "any signal that would change how a reviewer triages the run.\n\n" + "Return STRICT JSON only, with this shape and no extra keys:\n" + "{\n" + ' "narrative": "",\n' + ' "patterns": ["", "", ...],\n' + ' "metric_insights": {"": "<2-3 line business meaning>"}\n' + "}\n\n" + "Constraints:\n" + "- narrative is the ONLY text shown in the external audit summary and " + "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" + "- patterns are optional supporting notes and are NOT rendered in the " + "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" + "- metric_insights must include one entry for each top-level metric id supplied.\n" + "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" + "- Avoid restating raw counts unless they reveal a pattern.\n" + "- Use neutral, factual language ('frustration appeared in...') " + "rather than judgemental ('the agents failed to...')." +) + + +def _tldr_summary_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (with ``is_stale`` set) or ``None``. + + ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we + have to validate shape defensively -- a half-written or hand-edited + blob should not break the aggregate response. Returns ``None`` when + no cached summary exists. + """ + raw = evaluation.tldr_summary + if not isinstance(raw, dict): + return None + narrative = raw.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + return None + patterns_raw = raw.get("patterns") + patterns = ( + [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] + if isinstance(patterns_raw, list) + else [] + ) + metric_insights_raw = raw.get("metric_insights") + metric_insights = ( + { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + if isinstance(metric_insights_raw, dict) + else {} + ) + generated_at_raw = raw.get("generated_at") + try: + generated_at = ( + datetime.fromisoformat(generated_at_raw) + if isinstance(generated_at_raw, str) + else evaluation.updated_at or datetime.now(timezone.utc) + ) + except ValueError: + generated_at = evaluation.updated_at or datetime.now(timezone.utc) + snapshot = raw.get("generated_at_completed_rows") + snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + # Same live count the snapshot was stamped with (catalog + unflushed + # Redis), so new completions mark the summary stale without waiting for + # the next progress flush. + merged_completed, _merged_failed = merge_eval_counters_for_ui(evaluation) + current_completed = max(int(evaluation.completed_rows or 0), merged_completed) + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=generated_at, + generated_at_completed_rows=snapshot_int, + provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, + model=raw.get("model") if isinstance(raw.get("model"), str) else None, + is_stale=current_completed > snapshot_int, + ) + + +def _sample_rationales_per_metric( + eval_rows: List[CallImportEvaluationRow], + *, + per_metric_cap: int = 3, + rationale_char_cap: int = 600, +) -> Dict[str, List[str]]: + """Collect up to ``per_metric_cap`` distinct rationales per metric. + + Distinctness is case- and whitespace-insensitive. We truncate each + rationale to ``rationale_char_cap`` so a few unusually verbose rows + can't dominate the prompt budget. Empty / non-string rationales are + skipped. + """ + out: Dict[str, List[str]] = {} + seen: Dict[str, set[str]] = {} + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id, entry in scores.items(): + if not isinstance(entry, dict): + continue + rationale = entry.get("rationale") + if not isinstance(rationale, str): + continue + text = rationale.strip() + if not text: + continue + bucket = out.setdefault(metric_id, []) + if len(bucket) >= per_metric_cap: + continue + key = " ".join(text.lower().split()) + seen_set = seen.setdefault(metric_id, set()) + if key in seen_set: + continue + seen_set.add(key) + bucket.append(text[:rationale_char_cap]) + return out + + +def _build_insights_messages( + evaluation: CallImportEvaluation, + aggregate: List[CallImportMetricAggregate], + rationale_samples: Dict[str, List[str]], + metric_meta: Dict[str, Metric], +) -> List[Dict[str, str]]: + """Render the user prompt fed to the LLM. + + The shape is plain markdown-ish text instead of JSON so the LLM can + skim it without us spending tokens on verbose schema delimiters. + Parent metrics surface their child metrics nested underneath so the + model sees the hierarchy and can talk about "X often co-occurred + with Y" rather than treating sub-labels as standalone metrics. + """ + name = evaluation.name or f"Run {str(evaluation.id)[:8]}" + lines: List[str] = [ + f"Evaluation: {name}", + ( + f"Rows: total={evaluation.total_rows} " + f"completed={evaluation.completed_rows} " + f"failed={evaluation.failed_rows}" + ), + "", + "## Per-metric aggregate", + ] + + # Group metrics by parent so the prompt mirrors the hierarchy. Any + # aggregate row whose ``metric_id`` is missing from ``metric_meta`` + # is rendered as a leaf at the top-level list (handles renamed / + # deleted parents). + children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} + top_level: List[CallImportMetricAggregate] = [] + for agg in aggregate: + meta = metric_meta.get(agg.metric_id) + parent_id = ( + str(meta.parent_metric_id) + if meta is not None and getattr(meta, "parent_metric_id", None) + else None + ) + if parent_id: + children_by_parent.setdefault(parent_id, []).append(agg) + else: + top_level.append(agg) + + def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: + prefix = " " * indent + "- " + bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] + if agg.skipped_count: + bits.append(f", skipped={agg.skipped_count}") + if agg.error_count: + bits.append(f", errors={agg.error_count}") + bits.append(")") + meta = metric_meta.get(agg.metric_id) + description = (meta.description or "").strip() if meta else "" + if description: + bits.append(f" | definition={description[:500]}") + if agg.mean is not None: + mean_s = f"{agg.mean:.2f}" + stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" + bits.append(f" | mean={mean_s} stddev={stddev_s}") + if agg.min is not None and agg.max is not None: + bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") + if agg.value_counts: + total = sum(v.count for v in agg.value_counts) or 1 + top = agg.value_counts[:3] + shares = ", ".join( + f'"{v.label}"={v.count}/{total}' for v in top + ) + bits.append(f" | top={shares}") + result = ["".join(bits)] + rationales = rationale_samples.get(agg.metric_id, []) + for r in rationales: + result.append(" " * (indent + 1) + f"- rationale: {r}") + return result + + for agg in top_level: + lines.extend(_format_metric_block(agg, indent=0)) + meta = metric_meta.get(agg.metric_id) + children = children_by_parent.get(str(meta.id), []) if meta else [] + for child in children: + lines.extend(_format_metric_block(child, indent=1)) + + lines.append("") + top_level_ids = [agg.metric_id for agg in top_level] + if top_level_ids: + lines.append( + "metric_insights keys must exactly use these top-level metric ids: " + + ", ".join(top_level_ids) + ) + lines.append("") + lines.append( + "Write the JSON object as instructed. Do not include " + "preamble, code fences, or trailing commentary." + ) + + return [ + {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, + {"role": "user", "content": "\n".join(lines)}, + ] + + +def _parse_insights_response(text: str) -> EvaluationTldrSummary: + """Coerce the LLM response into ``narrative`` + ``patterns``. + + Matches the JSON-with-fallback pattern used by + ``app.api.v1.routes.metrics._parse_metric_generation_response``: try + ``json.loads`` first, then fall back to regex extraction of the + first ``{...}`` block. Raises ``HTTPException`` with a 502 when the + response can't be parsed at all. + """ + cleaned = (text or "").strip() + if not cleaned: + raise HTTPException( + status_code=502, detail="LLM returned an empty insights response" + ) + try: + parsed = json.loads(cleaned) + except json.JSONDecodeError: + import re + + match = re.search(r"\{.*\}", cleaned, re.DOTALL) + if not match: + raise HTTPException( + status_code=502, + detail="Could not parse LLM insights response as JSON", + ) + try: + parsed = json.loads(match.group(0)) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=502, + detail=f"Could not parse LLM insights response: {e}", + ) + + if not isinstance(parsed, dict): + raise HTTPException( + status_code=502, detail="LLM insights JSON was not an object" + ) + + narrative = parsed.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + raise HTTPException( + status_code=502, + detail="LLM insights JSON missing 'narrative' string", + ) + + patterns_raw = parsed.get("patterns") + if patterns_raw is None: + patterns: List[str] = [] + elif isinstance(patterns_raw, list): + patterns = [ + str(p).strip() + for p in patterns_raw + if isinstance(p, str) and p.strip() + ] + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'patterns' must be a list of strings", + ) + metric_insights_raw = parsed.get("metric_insights") + if metric_insights_raw is None: + metric_insights: Dict[str, str] = {} + elif isinstance(metric_insights_raw, dict): + metric_insights = { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'metric_insights' must be an object", + ) + + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=0, # filled in by caller + is_stale=False, + ) + + +def _generate_and_persist_tldr_summary( + db: Session, + evaluation: CallImportEvaluation, + *, + organization_id: UUID, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> EvaluationTldrSummary: + """LLM TLDR generation used by the imports-queue Celery worker.""" + eval_id = evaluation.id + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + # Stamp the summary with the completion count at the moment its input is + # collected. Rows finishing while the LLM runs are not in the summary, so + # they must still make it stale. + input_completed_rows, _input_failed_rows = merge_eval_counters_for_ui(evaluation) + from app.db_sharding.scatter_gather import ( + INSIGHTS_TLDR_ROW_SAMPLE, + load_completed_evaluation_row_pairs_sample, + ) + from app.services.call_imports.metric_aggregate_cache import ( + evaluation_metric_aggregates_for_analysis, + ) + + aggregate = evaluation_metric_aggregates_for_analysis(db, evaluation) + if not aggregate: + raise HTTPException( + status_code=400, + detail=( + "No metric data yet. Wait for at least one row to " + "finish scoring before generating a summary." + ), + ) + + metric_ids: List[UUID] = [] + for agg in aggregate: + try: + metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, metric_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + rationale_samples = _sample_rationales_per_metric( + [eval_row for eval_row, _ in load_completed_evaluation_row_pairs_sample( + db, eval_id, max_pairs=INSIGHTS_TLDR_ROW_SAMPLE + )] + ) + messages = _build_insights_messages( + evaluation, aggregate, rationale_samples, metric_meta + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + from app.services.ai.llm_service import llm_service + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + ) + from app.services.usage.context import llm_usage_context + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, provider, model + ) + + try: + with llm_usage_context( + call_import_evaluation_usage_context( + organization_id=organization_id, + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) + ): + budgets = [ + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, + ] + summary: Optional[EvaluationTldrSummary] = None + for budget_idx, max_tok in enumerate(budgets): + temperature = 0.4 if budget_idx == 0 else 0.2 + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=temperature, + max_tokens=max_tok, + ) + if llm_result.get("truncated"): + logger.warning( + "[CallImportInsights] TLDR truncated at max_tokens={} (model={})", + max_tok, + model_str, + ) + if budget_idx >= len(budgets) - 1: + raise HTTPException( + status_code=502, + detail=( + "Insights response was cut off before it finished. Try again." + ), + ) + continue + try: + summary = _parse_insights_response(llm_result.get("text", "")) + break + except HTTPException as parse_exc: + logger.warning( + "[CallImportInsights] TLDR parse failed at max_tokens={} (model={}): {}", + max_tok, + model_str, + parse_exc.detail, + ) + if budget_idx >= len(budgets) - 1: + raise parse_exc + except Exception as e: + logger.error(f"[CallImportInsights] LLM call failed: {e}") + raise HTTPException( + status_code=502, detail=f"LLM call failed: {e}" + ) from e + + if summary is None: + raise HTTPException( + status_code=502, + detail="Insights response was cut off before it finished. Try again.", + ) + + summary.generated_at_completed_rows = input_completed_rows + summary.provider = provider_enum.value + summary.model = model_str + summary.is_stale = False + + evaluation.tldr_summary = { + "narrative": summary.narrative, + "patterns": summary.patterns, + "metric_insights": summary.metric_insights, + "generated_at": summary.generated_at.isoformat(), + "generated_at_completed_rows": summary.generated_at_completed_rows, + "provider": summary.provider, + "model": summary.model, + } + flag_modified(evaluation, "tldr_summary") + db.commit() + db.refresh(evaluation) + return summary + + +@router.get( + "/{eval_id}/insights", + response_model=Optional[EvaluationTldrSummary], + operation_id="getCallImportEvaluationInsights", +) +async def get_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (or ``null``) without contacting the LLM. + + Used by the Visualizations tab on first paint so the empty-state + CTA can show up before the user opts into generation. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _tldr_summary_payload(evaluation) + + +@router.post( + "/{eval_id}/insights", + response_model=EvaluationTldrSummary, + operation_id="generateCallImportEvaluationInsights", +) +async def generate_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationTldrSummary: + """Generate (or return-cached) the LLM TLDR for an evaluation run. + + Behavior: + + * ``body.regenerate=False`` and a cached summary at the current + ``completed_rows`` watermark exists -> return it as-is. + * ``body.regenerate=False`` and a stale cached summary exists + (``generated_at_completed_rows < completed_rows``) -> return it + with ``is_stale=True``; the UI prompts the user to regenerate. + * Otherwise -> resolve provider+model (auto-detect when omitted), + call the LLM, persist the new summary, return it. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate: + cached = _tldr_summary_payload(evaluation) + if cached is not None: + return cached + + # Run the TLDR LLM on the imports worker (not the default worker or API). + from app.workers.tasks.generate_evaluation_tldr_insights import ( + generate_evaluation_tldr_insights_task, + ) + + try: + task_result = generate_evaluation_tldr_insights_task.apply_async( + kwargs={ + "evaluation_id": str(eval_id), + "call_import_id": str(call_import_id), + "organization_id": str(organization_id), + "provider": body.provider, + "model": body.model, + }, + ).get(timeout=25 * 60) + except Exception as exc: + logger.error( + "[CallImportInsights] TLDR task failed for evaluation {}: {}", + eval_id, + exc, + ) + raise HTTPException( + status_code=502, + detail=f"Summary generation failed: {exc}", + ) from exc + + if isinstance(task_result, dict) and task_result.get("error"): + status_code = int(task_result.get("status_code") or 502) + raise HTTPException( + status_code=status_code, + detail=str(task_result["error"]), + ) + + summary = EvaluationTldrSummary.model_validate(task_result) + db.refresh(evaluation) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=summary.provider or provider_enum.value, + model=summary.model or model_str, + force=body.regenerate, + max_llm_calls=body.max_llm_calls, + transcript_source=body.transcript_source, + db=db, + principal=principal, + ) + + return summary + + +def _user_insights_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationUserInsightsState]: + raw = getattr(evaluation, "user_insights", None) + if raw is None: + return None + return user_insights_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_generated_user_insights( + state: Optional[EvaluationUserInsightsState], + report_config: dict[str, Any], +) -> list[dict[str, Any]]: + """Filter and order generated insights for PDF section 03.""" + if state is None or state.status != "completed" or not state.insights: + return [] + + selected_ids = report_config.get("user_insight_ids") + if isinstance(selected_ids, list) and selected_ids: + allowed = {str(item) for item in selected_ids if item} + items = [item for item in state.insights if item.id in allowed] + else: + items = list(state.insights) + + order_raw = report_config.get("order") + order_ids: list[str] = [] + if isinstance(order_raw, dict): + user_order = order_raw.get("user_insights") + if isinstance(user_order, list): + order_ids = [str(item) for item in user_order if item] + + if order_ids: + by_id = {item.id: item for item in items} + ordered = [by_id[iid] for iid in order_ids if iid in by_id] + seen = set(order_ids) + ordered.extend(item for item in items if item.id not in seen) + items = ordered + + return [item.model_dump(mode="json") for item in items] + + +def _stored_user_insights_llm( + evaluation: CallImportEvaluation, + *, + provider: Optional[str], + model: Optional[str], +) -> tuple[Optional[str], Optional[str]]: + """Prefer explicit request fields; otherwise keep the model from the last job.""" + prov = (provider or "").strip() or None + mod = (model or "").strip() or None + if prov and mod: + return prov, mod + prior = evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + if not prov and isinstance(prior.get("provider"), str) and prior["provider"].strip(): + prov = prior["provider"].strip() + if not mod and isinstance(prior.get("model"), str) and prior["model"].strip(): + mod = prior["model"].strip() + return prov, mod + + +def _resolve_user_insights_llm( + organization_id: UUID, + db: Session, + evaluation: CallImportEvaluation, + body: EvaluationUserInsightsRequest, +) -> tuple[ModelProvider, str]: + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + + provider, model = _stored_user_insights_llm( + evaluation, + provider=body.provider, + model=body.model, + ) + if body.resume and not (body.provider or "").strip() and not (body.model or "").strip(): + if not provider or not model: + raise HTTPException( + status_code=400, + detail=( + "Cannot resume user insights without a saved provider and model. " + "Start a new run from the summary tab and pick a model." + ), + ) + return get_llm_provider_and_model_for_request( + organization_id, db, provider, model, body.credential_id + ) + + +def _enqueue_user_insights_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + transcript_source: Optional[str] = None, + resume: bool = False, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + """Enqueue background user-insights generation unless already running.""" + del max_llm_calls + current = _user_insights_payload(evaluation) + if current is not None and current.status == "running" and not force and not resume: + return + + from app.services.call_import_user_insights_wave import ( + resume_user_insights_job_header, + start_user_insights_job_header, + ) + from app.services.evaluation_analysis_checkpoints import new_job_run_id, normalize_transcript_source + + prior = evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + source = normalize_transcript_source( + transcript_source or prior.get("transcript_source") or evaluation.transcript_source + ) + prior_insights = prior.get("insights") if isinstance(prior.get("insights"), list) else [] + run_id = ( + UUID(str(prior["job_run_id"])) + if resume and prior.get("job_run_id") + else new_job_run_id() + ) + if db is not None: + if resume and prior.get("job_run_id"): + evaluation.user_insights = resume_user_insights_job_header( + db, + evaluation, + prior, + transcript_source=source, + provider=provider, + model=model, + ) + else: + evaluation.user_insights = start_user_insights_job_header( + evaluation, + db, + job_run_id=run_id, + transcript_source=source, + provider=provider, + model=model, + prior_insights=prior_insights if prior.get("status") == "completed" else [], + ) + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "user_insights") + db.commit() + + task_header = ( + evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + ) + task_provider = task_header.get("provider") if isinstance(task_header.get("provider"), str) else provider + task_model = task_header.get("model") if isinstance(task_header.get("model"), str) else model + + from app.workers.tasks.generate_evaluation_user_insights import ( + generate_evaluation_user_insights_task, + ) + + async_result = generate_evaluation_user_insights_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "provider": task_provider, + "model": task_model, + "job_run_id": str(run_id), + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.user_insights, dict): + evaluation.user_insights["celery_task_id"] = async_result.id + flag_modified(evaluation, "user_insights") + db.commit() + + +@router.get( + "/{eval_id}/user-insights", + response_model=Optional[EvaluationUserInsightsState], + operation_id="getCallImportEvaluationUserInsights", +) +async def get_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationUserInsightsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _user_insights_payload(evaluation) + + +@router.post( + "/{eval_id}/user-insights", + response_model=EvaluationUserInsightsState, + operation_id="generateCallImportEvaluationUserInsights", +) +async def generate_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationUserInsightsRequest = Body( + default_factory=EvaluationUserInsightsRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationUserInsightsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force and not body.resume: + cached = _user_insights_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating user insights." + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + + provider_enum, model_str = _resolve_user_insights_llm( + organization_id, db, evaluation, body + ) + + _enqueue_user_insights_job( + evaluation, + provider=provider_enum.value, + model=model_str, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + transcript_source=body.transcript_source, + resume=body.resume, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _user_insights_payload(evaluation) or EvaluationUserInsightsState( + status="running" + ) + + +def _revoke_user_insights_task(evaluation: CallImportEvaluation) -> None: + raw = evaluation.user_insights + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([task_id]) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke user-insights task for evaluation {}: {}", + evaluation.id, + exc, + ) + + +def _apply_user_insights_cancel(evaluation: CallImportEvaluation) -> bool: + from app.services.evaluation_analysis_checkpoints import USER_INSIGHTS_CANCELLED_BY_USER_ERROR + + raw = evaluation.user_insights + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + _revoke_user_insights_task(evaluation) + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + evaluation.user_insights = { + **raw, + "status": "cancelled", + "error_message": USER_INSIGHTS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + "cancelled_at": datetime.now(timezone.utc).isoformat(), + } + return True + + +@router.post( + "/{eval_id}/user-insights/cancel", + response_model=EvaluationUserInsightsState, + operation_id="cancelCallImportEvaluationUserInsights", +) +async def cancel_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationUserInsightsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_user_insights_cancel(evaluation) + flag_modified(evaluation, "user_insights") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _user_insights_payload(evaluation) or EvaluationUserInsightsState( + status="idle" + ) + diff --git a/app/api/v1/routes/call_import_evaluations/prompts.py b/app/api/v1/routes/call_import_evaluations/prompts.py new file mode 100644 index 00000000..f0a3a185 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/prompts.py @@ -0,0 +1,657 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _require_import, +) +from app.api.v1.routes.call_import_evaluations.clusters import ( # noqa: E402 + _metric_clusters_payload, +) +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + require_call_import_capability, +) + +def _prompt_improvements_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationPromptImprovementsState]: + from app.services.call_import_prompt_improvements import ( + prompt_improvements_state_from_raw, + ) + + raw = getattr(evaluation, "prompt_improvements", None) + if raw is None: + return None + return prompt_improvements_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_prompt_improvements_for_pdf( + state: Optional[EvaluationPromptImprovementsState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("prompt_improvements") is False: + return {} + return { + "imported_agent_id": state.imported_agent_id, + "imported_agent_name": state.imported_agent_name, + "overview": state.overview, + "suggestions": [s.model_dump(mode="json") for s in state.suggestions], + } + + +def _enqueue_prompt_improvements_job( + evaluation: CallImportEvaluation, + *, + imported_agent_id: UUID, + imported_agent_name: str, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + resume: bool = False, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _prompt_improvements_payload(evaluation) + if current is not None and current.status == "running" and not force and not resume: + return + + prior = evaluation.prompt_improvements if isinstance(evaluation.prompt_improvements, dict) else {} + prior_suggestions = prior.get("suggestions") if isinstance(prior.get("suggestions"), list) else [] + prior_provider = ( + prior.get("provider") if isinstance(prior.get("provider"), str) else None + ) + prior_model = prior.get("model") if isinstance(prior.get("model"), str) else None + resolved_provider = (provider or "").strip() or (prior_provider if resume else None) + resolved_model = (model or "").strip() or (prior_model if resume else None) + + # Resume must keep the original job_run_id so checkpoint batches stay + # readable; a regenerate/new run gets a fresh id. + resumed_run_id = prior.get("job_run_id") if resume else None + if resume and not resumed_run_id: + resume = False + assigned_run_id = str(resumed_run_id) if resumed_run_id else str(uuid4()) + + evaluation.prompt_improvements = { + **prior, + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent_name, + "suggestions": prior_suggestions if prior.get("status") == "completed" else [], + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "provider": resolved_provider, + "model": resolved_model, + "error_message": None, + "celery_task_id": None, + "completed_group_metric_ids": (prior.get("completed_group_metric_ids") or []) + if resume + else [], + "completed_group_count": int(prior.get("completed_group_count") or 0) + if resume + else 0, + "job_run_id": assigned_run_id, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "prompt_improvements") + db.commit() + + from app.workers.tasks.generate_evaluation_prompt_improvements import ( + generate_evaluation_prompt_improvements_task, + ) + + async_result = generate_evaluation_prompt_improvements_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "imported_agent_id": str(imported_agent_id), + "provider": resolved_provider, + "model": resolved_model, + "credential_id": str(credential_id) if credential_id else None, + "resume": resume, + "job_run_id": assigned_run_id, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.prompt_improvements, dict): + evaluation.prompt_improvements["celery_task_id"] = async_result.id + flag_modified(evaluation, "prompt_improvements") + db.commit() + + +@router.get( + "/{eval_id}/prompt-improvements", + response_model=Optional[EvaluationPromptImprovementsState], + operation_id="getCallImportEvaluationPromptImprovements", +) +async def get_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationPromptImprovementsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _prompt_improvements_payload(evaluation) + + +@router.post( + "/{eval_id}/prompt-improvements", + response_model=EvaluationPromptImprovementsState, + operation_id="generateCallImportEvaluationPromptImprovements", +) +async def generate_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationPromptImprovementsRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + clusters = _metric_clusters_payload(evaluation) + if clusters is None or clusters.status != "completed": + raise HTTPException( + status_code=400, + detail=( + "Metric clusters must be completed before generating prompt " + "improvements. Run failure diagnostics first." + ), + ) + + from app.services.call_import_prompt_improvements import is_imported_agent + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + from app.services.platform_import_imported_agent import resolve_imported_agent_for_call_import + + call_import = _require_import(db, call_import_id, organization_id) + imported_agent_id = body.imported_agent_id + if imported_agent_id is None and body.resume: + saved_agent = ( + evaluation.prompt_improvements.get("imported_agent_id") + if isinstance(evaluation.prompt_improvements, dict) + else None + ) + if saved_agent: + try: + imported_agent_id = UUID(str(saved_agent)) + except ValueError: + imported_agent_id = None + if imported_agent_id is None: + linked = resolve_imported_agent_for_call_import(db, call_import, principal) + if linked is None: + raise HTTPException( + status_code=400, + detail=( + "No imported agent is linked to this import. Select an imported agent " + "or use a single-agent platform import." + ), + ) + imported_agent_id = linked.id + + imported_agent = ( + db.query(PromptPartial) + .filter( + PromptPartial.id == imported_agent_id, + PromptPartial.organization_id == organization_id, + PromptPartial.workspace_id == workspace_id, + ) + .first() + ) + if imported_agent is None or not is_imported_agent(imported_agent): + from app.services.platform_import_imported_agent import ( + relink_evaluation_imported_agent, + resolve_imported_agent_for_call_import, + ) + + recovered = resolve_imported_agent_for_call_import(db, call_import, principal) + if recovered is None or not is_imported_agent(recovered): + raise HTTPException( + status_code=404, + detail="Imported agent not found in the active workspace", + ) + imported_agent = recovered + imported_agent_id = recovered.id + if isinstance(evaluation.prompt_improvements, dict): + relink_evaluation_imported_agent(evaluation, recovered) + stamp_evaluation_actor(evaluation, principal) + db.commit() + + if not body.regenerate and not body.force and not body.resume: + cached = _prompt_improvements_payload(evaluation) + if ( + cached is not None + and cached.status in {"running", "completed"} + and cached.imported_agent_id == str(imported_agent_id) + ): + return cached + + prior_prompt_raw = ( + evaluation.prompt_improvements + if isinstance(evaluation.prompt_improvements, dict) + else {} + ) + prior_prompt_provider = ( + prior_prompt_raw.get("provider") + if isinstance(prior_prompt_raw.get("provider"), str) + else None + ) + prior_prompt_model = ( + prior_prompt_raw.get("model") + if isinstance(prior_prompt_raw.get("model"), str) + else None + ) + resolve_provider = (body.provider or "").strip() or ( + prior_prompt_provider if body.resume else None + ) + resolve_model = (body.model or "").strip() or ( + prior_prompt_model if body.resume else None + ) + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, resolve_provider, resolve_model, body.credential_id + ) + + _enqueue_prompt_improvements_job( + evaluation, + imported_agent_id=imported_agent_id, + imported_agent_name=imported_agent.name, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + resume=body.resume, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( + status="running", + imported_agent_id=str(imported_agent_id), + imported_agent_name=imported_agent.name, + ) + + +def _revoke_prompt_improvements_task(evaluation: CallImportEvaluation) -> None: + raw = evaluation.prompt_improvements + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([task_id]) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke prompt-improvements task for evaluation {}: {}", + evaluation.id, + exc, + ) + + +def _apply_prompt_improvements_cancel(evaluation: CallImportEvaluation) -> bool: + from app.services.evaluation_analysis_checkpoints import ( + PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR, + ) + + raw = evaluation.prompt_improvements + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + _revoke_prompt_improvements_task(evaluation) + evaluation.prompt_improvements = { + **raw, + "status": "cancelled", + "error_message": PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR, + "celery_task_id": None, + "cancelled_at": datetime.now(timezone.utc).isoformat(), + } + return True + + +@router.post( + "/{eval_id}/prompt-improvements/cancel", + response_model=EvaluationPromptImprovementsState, + operation_id="cancelCallImportEvaluationPromptImprovements", +) +async def cancel_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_prompt_improvements_cancel(evaluation) + flag_modified(evaluation, "prompt_improvements") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( + status="idle" + ) + + +def _load_evaluation_for_prompt_push( + db: Session, + call_import_id: UUID, + eval_id: UUID, + organization_id: UUID, +) -> tuple[CallImport, CallImportEvaluation]: + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return call_import, evaluation + + +@router.patch( + "/{eval_id}/prompt-improvements/review", + response_model=EvaluationPromptImprovementsState, + operation_id="reviewCallImportEvaluationPromptImprovement", +) +async def review_call_import_evaluation_prompt_improvement( + call_import_id: UUID, + eval_id: UUID, + body: PromptImprovementReviewUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + from app.services.call_import_prompt_push import PromptPushError, set_suggestion_review + + _, evaluation = _load_evaluation_for_prompt_push(db, call_import_id, eval_id, organization_id) + try: + set_suggestion_review(evaluation, body.suggestion_id, body.review_status, body.edited_text) + except PromptPushError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState(status="idle") + + +@router.get( + "/{eval_id}/prompt-improvements/push-preview", + response_model=PromptPushPreviewResponse, + operation_id="previewCallImportEvaluationPromptPush", +) +async def preview_call_import_evaluation_prompt_push( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> PromptPushPreviewResponse: + del api_key + from app.services.call_import_prompt_push import PromptPushError, preview_evaluation_push + + call_import, evaluation = _load_evaluation_for_prompt_push( + db, call_import_id, eval_id, organization_id + ) + try: + payload = preview_evaluation_push( + db, evaluation, call_import, organization_id, workspace_id + ) + except PromptPushError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + return PromptPushPreviewResponse(**payload) + + +@router.post( + "/{eval_id}/prompt-improvements/push", + response_model=PromptPushResponse, + operation_id="pushCallImportEvaluationPrompt", +) +async def push_call_import_evaluation_prompt( + call_import_id: UUID, + eval_id: UUID, + body: PromptPushConfirm, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> PromptPushResponse: + del api_key + from app.services.call_import_prompt_push import PromptPushError, push_evaluation_prompt + + call_import, evaluation = _load_evaluation_for_prompt_push( + db, call_import_id, eval_id, organization_id + ) + try: + row = push_evaluation_prompt( + db, + evaluation, + call_import, + organization_id, + workspace_id, + body.confirm, + principal, + push_to_live=body.push_to_live, + ) + except PromptPushError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + return row + diff --git a/app/api/v1/routes/call_import_evaluations/rows.py b/app/api/v1/routes/call_import_evaluations/rows.py new file mode 100644 index 00000000..5bb88ec0 --- /dev/null +++ b/app/api/v1/routes/call_import_evaluations/rows.py @@ -0,0 +1,1076 @@ +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import Float, case, cast, desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationExportJobResponse, + CallImportEvaluationExportRequest, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportClassificationFacetCounts, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + PromptPushConfirm, + PromptPushPreviewResponse, + PromptPushResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.classification_metric_sql import ( + classification_choice_row_predicate, + classification_level_row_predicate, +) +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +from app.api.v1.routes.call_import_evaluations._router import router + +from app.api.v1.routes.call_import_evaluations._shared import ( # noqa: E402 + _pick_evaluation_row_transcript, + _require_import, + _rollup_evaluation_status, + _to_evaluation_row_response, +) +from app.api.v1.routes.call_import_evaluations.evaluation_cancel import ( # noqa: E402 + _apply_evaluation_cancel, + _cancel_eval_row_with_source_diarisation, + _require_no_evaluation_bulk_operation, + _revoke_eval_task, +) +from app.api.v1.routes.call_import_evaluations.discovered import ( # noqa: E402 + _alias_map_for_parent, + _resolve_alias, + _slug_label, +) +from app.api.v1.routes.call_import_evaluations.flow import ( # noqa: E402 + _DISCOVERED_NODE_PREFIX, + _FLOW_START_NODE_ID, +) + +def _apply_classification_metric_filters( + query, + metric_id: UUID, + *, + yes_no: Optional[str] = None, + choice: Optional[str] = None, + level: Optional[str] = None, +): + """Restrict rows by Jev classification facets under ``metric_scores``.""" + mid = str(metric_id) + scores = CallImportEvaluationRow.metric_scores + + if yes_no and yes_no.strip(): + yn = yes_no.strip().lower() + if yn in ("yes", "no"): + noul_txt = func.json_extract_path_text( + scores, mid, "answers", "noul", "noul" + ) + noul_f = cast(noul_txt, Float) + derived = case( + (noul_f >= 0.5, "Yes"), + (noul_f.isnot(None), "No"), + else_=None, + ) + effective = func.coalesce( + func.json_extract_path_text(scores, mid, "classification_yes_no"), + derived, + ) + query = query.filter(func.lower(effective) == yn) + + if choice and choice.strip(): + query = query.filter(classification_choice_row_predicate(mid, choice)) + + if level and level.strip(): + query = query.filter(classification_level_row_predicate(mid, level)) + + return query + + +@router.get( + "/{eval_id}/rows", + response_model=CallImportEvaluationRowListResponse, + operation_id="listCallImportEvaluationRows", +) +async def list_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + page: int = Query(1, ge=1), + page_size: int = Query(100, ge=1, le=500), + q: Optional[str] = Query( + None, + description=( + "Free-text search across conversation_id, transcript, and " + "diarised_transcript (case-insensitive substring match)." + ), + ), + metric_id: Optional[UUID] = Query( + None, + description=( + "If set, only return rows whose ``metric_scores[metric_id].value`` " + "exactly matches ``metric_value`` (string-compared). " + "Use together with ``metric_value``." + ), + ), + metric_value: Optional[str] = Query( + None, + description="Value to match against metric_id (string compare).", + ), + classification_yes_no: Optional[str] = Query( + None, + description=( + "With ``metric_id``, filter classification metrics by Yes/No " + "(case-insensitive)." + ), + ), + classification_choice: Optional[str] = Query( + None, + description=( + "With ``metric_id``, filter classification metrics by category " + "choice (case-insensitive)." + ), + ), + classification_level: Optional[str] = Query( + None, + description=( + "With ``metric_id``, filter classification metrics by level label " + "(case-insensitive)." + ), + ), + status_filter: Optional[str] = Query( + None, + alias="status", + description="Restrict to rows with this evaluation row status.", + ), + flow_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric whose ``sequence`` array should be " + "checked against ``flow_node`` and ``flow_edge_target``. Used " + "to drill into the calls behind a flow-chart node or edge." + ), + ), + flow_node: Optional[str] = Query( + None, + description=( + "If set together with ``flow_parent_id``, only return rows " + "whose sequence under that parent contains this step. Accepts " + "either a child metric UUID (resolved to slug(name)), a " + "``disc:`` discovered-label id, or a raw slug." + ), + ), + flow_edge_target: Optional[str] = Query( + None, + description=( + "Optional companion to ``flow_node``: when set, restrict to " + "rows whose sequence contains the directed transition " + "``flow_node -> flow_edge_target`` (immediately adjacent). " + "Same id format as ``flow_node``." + ), + ), + discovered_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric that defines the discovery scope " + "for ``discovered_label_key`` / ``has_discovered``." + ), + ), + discovered_label_key: Optional[str] = Query( + None, + description=( + "If set together with ``discovered_parent_id``, only return " + "rows whose ``metric_scores[parent].discovered_labels`` " + "list contains an entry with this slug (after applying " + "evaluation-level merge aliases)." + ), + ), + has_discovered: Optional[bool] = Query( + None, + description=( + "If true together with ``discovered_parent_id``, only return " + "rows that have at least one LLM-discovered label for the " + "parent. Useful to triage which calls produced novel labels." + ), + ), + sort_by: Optional[str] = Query( + None, + description=( + "Column to sort by. Accepted values: ``row_index`` (default " + "when omitted), ``conversation_id``, ``status`` (the " + "evaluation-row status), or ``metric:`` to sort " + "by ``metric_scores[].value``. Metric sorts compare " + "the extracted JSON text — adequate for booleans, enum " + "labels, and 0-1 ratings; large integer values may sort " + "lexicographically (10 before 2)." + ), + ), + sort_dir: Optional[str] = Query( + "asc", + description="Sort direction: ``asc`` (default) or ``desc``.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + eval_row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not eval_row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + query = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + ) + + # --- Filters ---------------------------------------------------------- + from app.services.call_imports.row_text_search import apply_evaluation_row_search + + query = apply_evaluation_row_search(query, q) + + if status_filter: + # The CallImportEvaluationRow.status column is a string in PG so a + # plain == filter works; we lowercase to match the stored values. + query = query.filter( + CallImportEvaluationRow.status == status_filter.strip().lower() + ) + + if metric_id is not None and metric_value is not None: + # ``metric_scores`` is a JSONB column shaped like + # ``{"": {"value": , "type": "boolean", ...}}``. We + # extract the nested ``value`` as text and compare to the user + # input as a string — that handles bool/int/enum without needing + # per-type casts. ``metric_value`` is matched case-insensitively + # so chart clicks on labels like "True" survive any casing drift + # between worker output and the chart label. + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_id), + "value", + ) + query = query.filter(func.lower(path_value) == metric_value.strip().lower()) + + if metric_id is not None and ( + (classification_yes_no and classification_yes_no.strip()) + or (classification_choice and classification_choice.strip()) + or (classification_level and classification_level.strip()) + ): + query = _apply_classification_metric_filters( + query, + metric_id, + yes_no=classification_yes_no, + choice=classification_choice, + level=classification_level, + ) + + # --- Flow chart drilldown filter ------------------------------------- + # Translates a clicked node (or edge) on the flow chart into a + # SQL filter against ``metric_scores[].sequence``. The + # frontend sends either a child UUID, a ``disc:`` discovered + # node id, or a raw slug — we normalize all three to the slug that + # actually appears in stored ``sequence`` arrays. + if flow_parent_id is not None and flow_node and flow_node.strip(): + parent_id_str_local = str(flow_parent_id) + alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) + + def _flow_node_to_slug(raw: str) -> Optional[str]: + raw_clean = raw.strip() + if not raw_clean: + return None + if raw_clean == _FLOW_START_NODE_ID: + # The synthetic START node isn't a real sequence entry; + # filtering on it is meaningless so we skip silently. + return None + if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): + return _resolve_alias( + alias_map_flow, + _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), + ) + # Try to interpret as a child metric UUID first; fall back + # to treating it as a slug. + try: + child_uuid = UUID(raw_clean) + except (TypeError, ValueError): + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + child = ( + db.query(Metric.name) + .filter( + Metric.id == child_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if child and child[0]: + return _resolve_alias(alias_map_flow, _slug_label(child[0])) + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + + from_slug = _flow_node_to_slug(flow_node) + target_slug: Optional[str] = None + if flow_edge_target and flow_edge_target.strip(): + target_slug = _flow_node_to_slug(flow_edge_target) + + if from_slug: + # The ``metric_scores`` column is declared as ``Column(JSON)`` + # in the model so on databases where the table was created + # from the model (rather than the migration) the physical + # type is ``json``, not ``jsonb``. The JSONB-only operators + # below (``jsonb_exists``, ``jsonb_array_elements_text``, + # ``@>``) require a JSONB input — we cast once up front so + # the same SQL works regardless of which path created the + # table. + scores_jsonb = ( + "(call_import_evaluation_rows.metric_scores)::jsonb" + ) + if target_slug: + # Edge filter: rows whose sequence under this parent + # contains ``from_slug`` immediately followed by + # ``target_slug``. Implemented as a correlated EXISTS + # over ``jsonb_array_elements_text`` with ORDINALITY, + # which is the portable way to express "next array + # index" against a JSONB array in Postgres. + edge_filter_sql = text( + f""" + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s1(elem, ord) + JOIN jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s2(elem, ord) + ON s2.ord = s1.ord + 1 + WHERE s1.elem = :from_slug + AND s2.elem = :to_slug + ) + """ + ).bindparams( + p_id=parent_id_str_local, + from_slug=from_slug, + to_slug=target_slug, + ) + query = query.filter(edge_filter_sql) + else: + # Node filter: rows whose ``metric_scores -> parent -> + # 'sequence'`` array contains ``from_slug``. We use the + # function form ``jsonb_exists`` rather than the ``?`` + # operator to avoid psycopg2 mistaking the question + # mark for a parameter placeholder. + node_filter_sql = text( + f""" + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + """ + ).bindparams(p_id=parent_id_str_local, slug=from_slug) + query = query.filter(node_filter_sql) + + # --- Discovered label filters --------------------------------------- + # Surfaces "which calls produced THIS LLM-discovered label" and the + # broader "which calls produced ANY LLM-discovered label". Both + # operate on ``metric_scores[].discovered_labels`` (a list + # of dicts) plus the same ``sequence`` array — covering both legacy + # rows where the slug only made it into ``sequence`` and newer + # rows where it landed in both. + if discovered_parent_id is not None and ( + discovered_label_key or has_discovered + ): + d_parent_str = str(discovered_parent_id) + alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) + # See note above: cast once so the JSONB operators don't reject + # the column when it's typed as ``json`` in the database. + scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" + if discovered_label_key and discovered_label_key.strip(): + target = _resolve_alias( + alias_map_disc, _slug_label(discovered_label_key) + ) + if target: + # Match rows whose discovered_labels list has an entry + # ``{"key": }`` OR whose sequence array still + # contains the slug. The latter covers older rows that + # were rewritten by a merge in the discovered_labels + # blob but whose sequence may have lagged. + contains_json = json.dumps( + {d_parent_str: {"discovered_labels": [{"key": target}]}} + ) + disc_filter_sql = text( + f""" + ( + {scores_jsonb} @> CAST(:contains AS JSONB) + OR + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + ) + """ + ).bindparams( + contains=contains_json, + p_id=d_parent_str, + slug=target, + ) + query = query.filter(disc_filter_sql) + elif has_discovered: + # No specific slug — just rows that surfaced any candidate + # under this parent. We coalesce missing paths to ``[]`` so + # ``jsonb_array_length`` always sees an array (it raises on + # non-array inputs, but our shape guarantees a list when + # the key is present). + has_disc_sql = text( + f""" + jsonb_array_length( + COALESCE( + {scores_jsonb} -> :p_id -> 'discovered_labels', + '[]'::jsonb + ) + ) > 0 + """ + ).bindparams(p_id=d_parent_str) + query = query.filter(has_disc_sql) + + # --- Sorting ---------------------------------------------------------- + # Column-click sorting from the UI. Falls back to ``row_index`` so + # paging stays stable when the user clears the sort. We always add a + # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. + # many rows with ``status = 'completed'``) keep a deterministic + # order across page boundaries — without this, pagination can + # double-show or skip rows when Postgres picks a different physical + # order on each query. + direction_desc = (sort_dir or "asc").strip().lower() == "desc" + + def _apply_direction(column_expr): + return column_expr.desc() if direction_desc else column_expr.asc() + + # Whether the caller's ``sort_by`` resolved to a known column. We + # use this flag to decide whether ``sort_dir`` is honoured on the + # fallback path: unrecognized columns (typos, stale UI state) fall + # back to the implicit ``row_index ASC`` default and intentionally + # ignore ``sort_dir`` so users don't get a surprise reverse order + # from a typo'd column name. + sort_recognized = False + sort_by_clean = (sort_by or "").strip() + primary_sort = None + metric_uuid: Optional[UUID] = None + if sort_by_clean == "row_index": + sort_recognized = True + # Falls through to the default ``order_by`` below with + # ``primary_sort`` still None — but ``sort_recognized=True`` + # tells the fallback branch to apply the requested direction. + elif sort_by_clean == "conversation_id": + sort_recognized = True + primary_sort = _apply_direction(CallImportRow.conversation_id) + elif sort_by_clean == "status": + sort_recognized = True + primary_sort = _apply_direction(CallImportEvaluationRow.status) + elif sort_by_clean.startswith("metric:"): + raw_metric_id = sort_by_clean.split(":", 1)[1].strip() + try: + metric_uuid = UUID(raw_metric_id) + except (TypeError, ValueError): + metric_uuid = None + if metric_uuid is not None: + sort_recognized = True + # ``metric_scores`` is JSON-typed but the helper functions + # for path extraction differ between Postgres (production) + # and SQLite (default test backend). Branch on the active + # dialect so we can use the right primitive: + # * Postgres → ``json_extract_path_text(col, key, "value")`` + # which returns the value as TEXT for both ``json`` and + # ``jsonb`` columns. + # * SQLite → ``json_extract(col, '$."".value')`` + # using JSONPath syntax. ``metric_uuid`` is already + # validated above (``UUID(raw_metric_id)``), so the + # interpolated path is safe from injection. + # NULL values (rows where the metric wasn't scored) sort + # to the END regardless of direction so un-scored rows + # don't crowd the top of an ascending sort. + dialect_name = ( + db.bind.dialect.name if db.bind is not None else "postgresql" + ) + if dialect_name == "sqlite": + json_path = f'$."{metric_uuid}".value' + path_value = func.json_extract( + CallImportEvaluationRow.metric_scores, + json_path, + ) + else: + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_uuid), + "value", + ) + primary_sort = ( + path_value.desc().nullslast() + if direction_desc + else path_value.asc().nullslast() + ) + + if primary_sort is not None: + query = query.order_by(primary_sort, CallImportRow.row_index.asc()) + elif sort_recognized: + # Explicit ``sort_by=row_index`` request — honour direction. + query = query.order_by(_apply_direction(CallImportRow.row_index)) + else: + # No sort requested OR unrecognized column — safe default of + # ``row_index ASC``. We deliberately ignore ``sort_dir`` here + # so a typo'd / stale ``sort_by`` doesn't quietly invert the + # default order. + query = query.order_by(CallImportRow.row_index.asc()) + from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page + from app.db_sharding.sessions import is_sharding_enabled + + def _pair_row_index( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> int: + return int(pair[1].row_index or 0) + + def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: + text = value or "" + if not desc: + return (0, *text.encode("utf-8")) + return (1, *(-byte for byte in text.encode("utf-8"))) + + if sort_by_clean == "conversation_id": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[1].conversation_id, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean == "status": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[0].status, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean.startswith("metric:") and metric_uuid is not None: + metric_id_str = str(metric_uuid) + + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + scores = pair[0].metric_scores or {} + entry = scores.get(metric_id_str, {}) + raw_value = entry.get("value") if isinstance(entry, dict) else None + null_rank = 1 if raw_value is None else 0 + return ( + null_rank, + _directed_string( + str(raw_value) if raw_value is not None else None, + direction_desc, + ), + _pair_row_index(pair), + ) + elif sort_recognized and sort_by_clean == "row_index": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + idx = _pair_row_index(pair) + return (-idx,) if direction_desc else (idx,) + else: + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + return (_pair_row_index(pair),) + + if is_sharding_enabled(): + def _build_query(session: Session): + return query.with_session(session) + + total, rows = fetch_evaluation_row_pairs_page( + db, + _build_query, + page=page, + page_size=page_size, + sort_key=_pair_sort_key, + bounded_shard_fetch=( + not sort_recognized or sort_by_clean == "row_index" + ), + row_index_desc=bool( + sort_recognized and sort_by_clean == "row_index" and direction_desc + ), + sort_reads_scores=sort_by_clean.startswith("metric:"), + ) + else: + from app.db_sharding.scatter_gather import defer_call_import_row_text + + total = query.count() + rows = ( + query.options(*defer_call_import_row_text()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + items: List[CallImportEvaluationRowResponse] = [ + _to_evaluation_row_response( + eval_row_obj, + source_row, + eval_row, + include_text=False, + ) + for eval_row_obj, source_row in rows + ] + + return CallImportEvaluationRowListResponse( + items=items, + total=total, + page=page, + page_size=page_size, + ) + + +class WerAlignmentResponse(BaseModel): + reference: List[str] = Field(default_factory=list) + hypothesis: List[str] = Field(default_factory=list) + chunks: List[Dict[str, Any]] = Field(default_factory=list) + substitutions: int = 0 + deletions: int = 0 + insertions: int = 0 + hits: int = 0 + wer: float = 0.0 + reference_text: str = "" + hypothesis_text: str = "" + + +@router.get( + "/{eval_id}/rows/{row_id}/wer-alignment", + response_model=WerAlignmentResponse, + operation_id="getCallImportEvaluationRowWerAlignment", +) +async def get_call_import_evaluation_row_wer_alignment( + call_import_id: UUID, + eval_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> WerAlignmentResponse: + """Recompute word alignment for production user turns vs diarised user turns.""" + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if evaluation is None: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + source_row: CallImportRow | None = None + if not is_sharding_enabled(): + pair = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter( + CallImportEvaluationRow.id == row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + CallImportRow.call_import_id == call_import_id, + ) + .first() + ) + if pair is None: + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) + _eval_row, source_row = pair + else: + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + ) + + row_db = catalog_db = None + try: + row_db, catalog_db, eval_row, source_row, _shard_id = ( + locate_call_import_evaluation_row(row_id) + ) + if ( + eval_row.evaluation_id != eval_id + or source_row.call_import_id != call_import_id + ): + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) + except LookupError: + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) from None + finally: + if row_db is not None: + close_row_sessions(row_db, catalog_db) + + from app.services.evaluation.transcript_error_rates import ( + build_word_alignment, + select_comparison_texts, + ) + + segments = ( + source_row.diarised_segments + if isinstance(source_row.diarised_segments, list) + else None + ) + raw_aliases = getattr(call_import, "speaker_aliases", None) + selected = select_comparison_texts( + production_transcript=(source_row.transcript or "").strip(), + diarised_transcript=(source_row.diarised_transcript or "").strip(), + diarised_segments=segments, + diarised_native_transcript=( + getattr(source_row, "diarised_native_transcript", None) or "" + ).strip(), + diarised_speaker_swap=bool( + getattr(source_row, "diarised_speaker_swap", False) + ), + speaker_aliases=raw_aliases if isinstance(raw_aliases, dict) else None, + ) + if not (selected["reference"] or "").strip(): + raise HTTPException( + status_code=422, + detail="Production user transcript is empty", + ) + alignment = build_word_alignment( + selected["reference"], + selected["hypothesis"], + romanised=selected["mode"] == "romanised_fallback", + ) + return WerAlignmentResponse( + reference=alignment.get("reference") or [], + hypothesis=alignment.get("hypothesis") or [], + chunks=alignment.get("chunks") or [], + substitutions=int(alignment.get("substitutions") or 0), + deletions=int(alignment.get("deletions") or 0), + insertions=int(alignment.get("insertions") or 0), + hits=int(alignment.get("hits") or 0), + wer=float(alignment.get("wer") or 0.0), + reference_text=selected["reference"], + hypothesis_text=selected["hypothesis"], + ) + + +@router.get( + "/{eval_id}/rows/{row_id}", + response_model=CallImportEvaluationRowResponse, + operation_id="getCallImportEvaluationRow", +) +async def get_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """One evaluation row, including the transcript the list leaves off.""" + del api_key + _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if evaluation is None: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + if not is_sharding_enabled(): + pair = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter( + CallImportEvaluationRow.id == row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + CallImportRow.call_import_id == call_import_id, + ) + .first() + ) + if pair is None: + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) + eval_row, source_row = pair + return _to_evaluation_row_response( + eval_row, + source_row, + evaluation, + include_text=True, + ) + + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + ) + + try: + row_db, catalog_db, eval_row, source_row, _shard_id = ( + locate_call_import_evaluation_row(row_id) + ) + except LookupError: + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) from None + try: + if ( + eval_row.evaluation_id != eval_id + or source_row.call_import_id != call_import_id + ): + raise HTTPException( + status_code=404, detail="Call import evaluation row not found" + ) + return _to_evaluation_row_response( + eval_row, + source_row, + evaluation, + include_text=True, + ) + finally: + close_row_sessions(row_db, catalog_db) + + +@router.delete( + "/{eval_id}/rows/{eval_row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluationRow", +) +async def delete_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single per-row scoring entry within an evaluation run. + + Useful when the user wants to drop a noisy row before re-exporting + the CSV — e.g. a row whose audio was corrupt and skewed the + aggregate. Counters on the parent are recomputed so the rolled-up + status stays accurate. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import delete_evaluation_row_on_shards + + if not delete_evaluation_row_on_shards(eval_row_id, eval_id): + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + # If the row was still in flight, best-effort revoke the worker task + # so it doesn't try to write into a deleted DB row mid-execution. + if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: + pass + + db.delete(eval_row) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py deleted file mode 100644 index f476c74e..00000000 --- a/app/api/v1/routes/call_imports.py +++ /dev/null @@ -1,4645 +0,0 @@ -"""CSV-driven call import routes. - -Users upload a CSV plus a per-batch column mapping (CSV header -> system -field). The backend persists a CallImport batch + one CallImportRow per -line, then fans the rows out to the Celery ``imports`` queue where each -row is downloaded using the telephony credential pinned on the batch. -Exotel credentialed imports require a ``recording_url`` on every row; -direct-URL imports (no credential) also require a mapped recording URL. -""" - -from __future__ import annotations - -import csv -import io -import json -import re -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta -from typing import Any, Dict, Iterable, List, Optional, Tuple -from uuid import UUID, uuid4 - -from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status -from loguru import logger -from sqlalchemy import desc, func, or_ -from sqlalchemy.orm import Session - -from app.config import settings -from app.core.auth import Principal, get_principal -from app.core.auth.rbac import require_admin -from app.database import get_db -from app.db_sharding.sessions import is_sharding_enabled -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.billing.flexprice_service import record_call_import_batch_created -from app.services.call_imports.audit import ( - actor_emails_for_call_import, - emails_for_user_ids, - stamp_call_import_actor, - user_ids_from_call_imports, -) -from app.services.call_imports.dispatch_diagnostics import ( - build_call_import_dispatch_diagnostics, -) -from app.models.database import ( - CallImport, - CallImportRow, - CallImportSchema, - CallImportSchemaParameter, - CallImportTag, - TelephonyIntegration, -) -from app.models.enums import ( - CallImportParameterType, - CallImportRowStatus, - CallImportStatus, -) -from app.models.schemas import ( - CallImportCancelDiarisationRequest, - CallImportCancelDiarisationResponse, - CallImportDetailResponse, - CallImportDeleteResponse, - CallImportDiarisationPromptDefaultResponse, - CallImportDispatchDiagnosticsResponse, - CallImportInsightsMetric, - CallImportInsightsResponse, - CallImportInsightsRunPoint, - CallImportListResponse, - CallImportMappingUpdate, - CallImportMetricAggregate, - CallImportPreviewResponse, - CallImportPreviewSheet, - CallImportRetryFailedRowsRequest, - CallImportRetryFailedRowsResponse, - CallImportResponse, - CallImportRowIdsResponse, - CallImportRowBulkDelete, - CallImportRowBulkDeleteResponse, - CallImportRowResponse, - CallImportStartRequest, - CallImportTranscribeRequest, - CallImportTranscribeResponse, - CallImportUpdate, - CallImportUploadResponse, - PlatformImportAgentRef, -) - - -router = APIRouter( - prefix="/call-imports", - tags=["Call Imports"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -@dataclass(frozen=True) -class CallImportParseSkip: - """One source row excluded during CSV/Excel parse (identity / recording URL).""" - - source_row: int - reason: str - message: str - - -@dataclass -class CallImportParseResult: - rows: List[Dict[str, Any]] = field(default_factory=list) - skipped: List[CallImportParseSkip] = field(default_factory=list) - - -def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: - """Persistable JSON shape for ``CallImport.source_row_skips``.""" - return [ - { - "source_row": item.source_row, - "reason": item.reason, - "message": item.message, - } - for item in skips - ] - - -def _normalize_dataset(raw: Optional[str]) -> Optional[str]: - """Trim and treat empty strings as 'no dataset' (NULL).""" - if raw is None: - return None - cleaned = raw.strip() - return cleaned or None - - -def _normalize_import_display_name(raw: Optional[str]) -> Optional[str]: - """Trim a user-facing import/batch label; empty clears to NULL.""" - if raw is None: - return None - cleaned = raw.strip() - if not cleaned: - return None - return cleaned[:512] - - -def _imported_agent_partial_id(call_import: CallImport): - from app.services.platform_import_imported_agent import linked_partial_id - - return linked_partial_id(call_import) - - -def _platform_agent_refs(call_import: CallImport) -> list: - """Agent id and name saved with a platform import, for later eval and prompt work.""" - raw = getattr(call_import, "platform_filters", None) or {} - if not isinstance(raw, dict): - return [] - refs = [] - for agent in raw.get("agents") or []: - if not isinstance(agent, dict): - continue - agent_id = str(agent.get("id") or "").strip() - if not agent_id: - continue - medium = agent.get("medium") or raw.get("medium") - refs.append( - { - "id": agent_id, - "name": str(agent.get("name") or agent_id), - "medium": str(medium).strip().lower() if medium else None, - } - ) - if refs: - return refs - medium = raw.get("medium") - for agent_id in raw.get("agent_ids") or []: - text = str(agent_id or "").strip() - if not text: - continue - refs.append( - { - "id": text, - "name": text, - "medium": str(medium).strip().lower() if medium else None, - } - ) - return refs - - -_agent_name_lookup_after: Dict[str, float] = {} - - -def _remember_platform_agent_names(db: Session, call_import: CallImport) -> None: - """Fill in display names for imports that only stored the provider agent id.""" - import time - - refs = _platform_agent_refs(call_import) - if not refs or not call_import.voice_integration_id: - return - if all(ref["name"] and ref["name"] != ref["id"] for ref in refs): - return - lookup_key = str(call_import.id) - now = time.time() - if _agent_name_lookup_after.get(lookup_key, 0) > now: - return - from sqlalchemy.orm.attributes import flag_modified - - from app.core.encryption import decrypt_api_key - from app.models.database import Integration - from app.services.voice_providers import get_voice_provider - - integration = ( - db.query(Integration) - .filter(Integration.id == call_import.voice_integration_id) - .first() - ) - if integration is None or not integration.api_key: - return - try: - from app.api.v1.routes.platform_call_imports import _list_import_agents - - api_key = decrypt_api_key(integration.api_key) - provider_cls = get_voice_provider(integration.platform) - if (integration.platform or "").lower() == "elevenlabs": - client = provider_cls(api_key, base_url=integration.api_base_url) - else: - client = provider_cls(api_key) - listed = _list_import_agents(client, integration.platform) - except Exception: - _agent_name_lookup_after[lookup_key] = time.time() + 300 - return - by_id = {row.id: row for row in listed} - updated = [] - changed = False - for ref in refs: - match = by_id.get(ref["id"]) - name = ref["name"] - medium = ref.get("medium") - if match is not None and match.name and match.name != ref["id"]: - name = match.name - if match is not None and match.medium: - medium = match.medium - if name != ref["name"] or medium != ref.get("medium"): - changed = True - updated.append({"id": ref["id"], "name": name, "medium": medium}) - if not changed: - _agent_name_lookup_after[lookup_key] = time.time() + 300 - return - filters = dict(call_import.platform_filters or {}) - filters["agents"] = updated - call_import.platform_filters = filters - flag_modified(call_import, "platform_filters") - db.commit() - - -def _serialize_call_import( - db: Session, - call_import: CallImport, - *, - user_emails: Optional[Dict[UUID, str]] = None, -) -> CallImportResponse: - """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" - from app.services.call_imports.bulk_ops import ( - _latest_evaluation_progress_for_ui, - rollup_call_import_batch_status, - ) - from app.services.call_imports.progress_counters import ( - clear_import_progress_redis, - read_import_progress, - ) - - redis_completed, redis_failed = read_import_progress(call_import.id) - if ( - redis_completed - or redis_failed - or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) - or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) - ): - rollup_call_import_batch_status(db, call_import) - db.flush() - - clear_import_progress_redis(call_import.id) - db.refresh(call_import) - total = int(call_import.total_rows or 0) - completed = min(int(call_import.completed_rows or 0), total) if total else int( - call_import.completed_rows or 0 - ) - failed = min(int(call_import.failed_rows or 0), total) if total else int( - call_import.failed_rows or 0 - ) - # Shard rows are the live count. The catalog counter lags during download. - source = (getattr(call_import, "source_format", None) or "").lower() - batch_status = ( - call_import.status.value - if isinstance(call_import.status, CallImportStatus) - else str(call_import.status or "") - ) - if source == "platform" and batch_status in {"pending", "processing"}: - from app.services.call_imports.bulk_ops import ( - _aggregate_import_row_status_counts, - ) - - agg_total, agg_completed, agg_failed, _, _ = ( - _aggregate_import_row_status_counts(db, call_import.id) - ) - if agg_total > 0: - total = agg_total - completed = agg_completed - failed = agg_failed - if user_emails is None: - user_emails = emails_for_user_ids( - db, user_ids_from_call_imports([call_import]) - ) - created_email, updated_email = actor_emails_for_call_import( - call_import, user_emails - ) - base = CallImportResponse.model_validate(call_import) - return base.model_copy( - update={ - "total_rows": total, - "completed_rows": completed, - "failed_rows": failed, - "platform_agents": [ - PlatformImportAgentRef.model_validate(ref) - for ref in _platform_agent_refs(call_import) - ], - "imported_agent_partial_id": _imported_agent_partial_id(call_import), - **_latest_evaluation_progress_for_ui(db, call_import.id), - "created_by_email": created_email, - "last_updated_by_email": updated_email, - } - ) - - -def _resolve_tags( - db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] -) -> List[CallImportTag]: - """Look up tag rows by id, scoped to the organization. - - Raises HTTPException(400) if any id is unknown for the org. - """ - if not tag_ids: - return [] - rows = ( - db.query(CallImportTag) - .filter( - CallImportTag.organization_id == organization_id, - CallImportTag.id.in_(tag_ids), - ) - .all() - ) - found_ids = {row.id for row in rows} - missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] - if missing: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown call_import_tag id(s): {missing}", - ) - return rows - - -MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) -MAX_AUDIO_FILES_PER_REQUEST = 100 -MANUAL_AUDIO_BLOB_UPLOAD_WORKERS = 10 - -# File extensions accepted by the upload + preview endpoints. Keep in -# lockstep with the frontend ``accept`` attribute on the file picker. -CSV_EXTENSIONS = (".csv",) -XLSX_EXTENSIONS = (".xlsx", ".xlsm") -ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS - -AUDIO_CONTENT_TYPES = { - "wav": "audio/wav", - "mp3": "audio/mpeg", - "flac": "audio/flac", - "m4a": "audio/mp4", -} - - -def _file_format(filename: Optional[str]) -> Optional[str]: - """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" - if not filename: - return None - name = filename.lower() - if name.endswith(CSV_EXTENSIONS): - return "csv" - if name.endswith(XLSX_EXTENSIONS): - return "xlsx" - return None - - -def _audio_extension(filename: Optional[str]) -> Optional[str]: - """Return the validated lower-case extension for a manual recording.""" - if not filename or "." not in filename: - return None - ext = filename.rsplit(".", 1)[-1].lower().strip() - allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} - return ext if ext in allowed else None - - -def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: - """Prefer the browser-supplied audio content type, with a safe fallback.""" - supplied = (upload_content_type or "").strip() - if supplied and supplied != "application/octet-stream": - return supplied - return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") - - -def _audio_s3_key( - organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str -) -> str: - """Build the canonical S3 key for a manually uploaded recording.""" - from app.services.storage.s3_service import s3_service - - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/{row_id}.{ext}" - ) - - -def _filename_stem(filename: Optional[str]) -> str: - """Extract a cross-platform filename stem from an UploadFile name.""" - raw = (filename or "").strip() - basename = re.split(r"[\\/]", raw)[-1] if raw else "" - if "." in basename: - basename = basename.rsplit(".", 1)[0] - return basename.strip() - - -def _sanitize_conversation_id(raw: str) -> str: - """Turn a filename stem into a stable conversation_id.""" - cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) - cleaned = re.sub(r"_+", "_", cleaned).strip("._-") - return (cleaned or "recording")[:255] - - -def _dedupe_conversation_id( - base: str, counts: Dict[str, int] -) -> str: - """Make conversation ids unique within one manual upload batch.""" - count = counts.get(base, 0) + 1 - counts[base] = count - if count == 1: - return base - suffix = f"-{count}" - return f"{base[: 255 - len(suffix)]}{suffix}" - - -def _conversation_id_base(conversation_id: str) -> str: - match = re.match(r"^(.*)-(\d+)$", (conversation_id or "").strip()) - if match: - return match.group(1) - return (conversation_id or "").strip() - - -def _manual_audio_append_start_state( - db: Session, - call_import: CallImport, -) -> tuple[int, Dict[str, int]]: - """Next row_index and conversation-id suffix counts for audio append.""" - from app.db_sharding.scatter_gather import ( - load_call_import_conversation_ids, - max_call_import_row_index, - ) - - max_index = max_call_import_row_index(db, call_import.id) - start_row_index = max_index + 1 - if start_row_index == 0 and (call_import.total_rows or 0) > 0: - start_row_index = call_import.total_rows - conversation_counts = _seed_conversation_counts( - cid for cid in load_call_import_conversation_ids(db, call_import.id) if cid - ) - return start_row_index, conversation_counts - - -def _seed_conversation_counts(existing_conversation_ids: Iterable[str]) -> Dict[str, int]: - counts: Dict[str, int] = {} - for conversation_id in existing_conversation_ids: - base = _conversation_id_base(conversation_id) - if not base: - continue - counts[base] = counts.get(base, 0) + 1 - return counts - - -def _enforce_audio_upload_file_limit(file_count: int) -> None: - if file_count > MAX_AUDIO_FILES_PER_REQUEST: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"At most {MAX_AUDIO_FILES_PER_REQUEST} audio files may be uploaded " - "per request. Split large batches into multiple requests." - ), - ) - - -async def _prepare_audio_upload_files( - files: List[UploadFile], - conversation_counts: Dict[str, int], -) -> List[Dict[str, Any]]: - if not files: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="At least one audio file is required.", - ) - _enforce_audio_upload_file_limit(len(files)) - - max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 - prepared: List[Dict[str, Any]] = [] - - for idx, upload in enumerate(files): - filename = upload.filename or f"recording-{idx + 1}" - ext = _audio_extension(filename) - if not ext: - allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", - ) - - contents = await upload.read() - if not contents: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Audio file '{filename}' is empty.", - ) - if len(contents) > max_bytes: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=( - f"Audio file '{filename}' exceeds " - f"{settings.MAX_FILE_SIZE_MB} MB." - ), - ) - - base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) - conversation_id = _dedupe_conversation_id( - base_conversation_id, - conversation_counts, - ) - prepared.append( - { - "filename": filename, - "extension": ext, - "content_type": _audio_content_type(ext, upload.content_type), - "contents": contents, - "conversation_id": conversation_id, - } - ) - return prepared - - -def _upload_manual_audio_blobs_parallel( - upload_specs: List[Tuple[str, bytes, str]], -) -> None: - """Upload prepared manual-audio blobs concurrently (key, body, content_type).""" - if not upload_specs: - return - - from app.services.storage.s3_service import s3_service - - def _upload_one(spec: Tuple[str, bytes, str]) -> None: - key, contents, content_type = spec - s3_service.upload_file_by_key( - contents, - key, - content_type=content_type, - ) - - if len(upload_specs) == 1: - _upload_one(upload_specs[0]) - return - - workers = min(MANUAL_AUDIO_BLOB_UPLOAD_WORKERS, len(upload_specs)) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(_upload_one, spec) for spec in upload_specs] - for future in as_completed(futures): - future.result() - - -def _persist_prepared_audio_rows( - db: Session, - *, - call_import: CallImport, - organization_id: UUID, - workspace_id: UUID, - prepared: List[Dict[str, Any]], - start_row_index: int, -) -> List[str]: - uploaded_keys: List[str] = [] - row_mappings: List[Dict[str, Any]] = [] - upload_specs: List[Tuple[str, bytes, str]] = [] - - for offset, item in enumerate(prepared): - row_id = uuid4() - key = _audio_s3_key( - organization_id, - call_import.id, - row_id, - item["extension"], - ) - uploaded_keys.append(key) - upload_specs.append((key, item["contents"], item["content_type"])) - row_mappings.append( - { - "id": row_id, - "call_import_id": call_import.id, - "organization_id": organization_id, - "workspace_id": workspace_id, - "row_index": start_row_index + offset, - "conversation_id": item["conversation_id"], - "recording_url": None, - "transcript": None, - "transcript_source": None, - "raw_columns": {"conversation_id": item["conversation_id"]}, - "status": CallImportRowStatus.COMPLETED, - "recording_s3_key": key, - "recording_content_type": item["content_type"], - "recording_size_bytes": len(item["contents"]), - } - ) - - _upload_manual_audio_blobs_parallel(upload_specs) - - added_size = sum(len(item["contents"]) for item in prepared) - new_total_rows = (call_import.total_rows or 0) + len(prepared) - - if is_sharding_enabled(): - from app.db_sharding.row_ops import ( - bulk_insert_mappings_on_shards, - register_shard_slices, - ) - - bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) - register_shard_slices(db, call_import.id, new_total_rows) - else: - for mapping in row_mappings: - db.add(CallImportRow(**mapping)) - - call_import.total_rows = new_total_rows - call_import.completed_rows = (call_import.completed_rows or 0) + len(prepared) - call_import.source_size_bytes = (call_import.source_size_bytes or 0) + added_size - return uploaded_keys - - -def _normalize_header(name: str) -> str: - return (name or "").strip().lower() - - -def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: - """Map normalized header -> original header for case-insensitive lookup.""" - return {_normalize_header(h): h for h in fieldnames or []} - - -def _resolve_mapped_header( - mapping_value: Optional[str], header_lookup: Dict[str, str] -) -> Optional[str]: - """Translate a user-supplied CSV header into the actual column key. - - The frontend sends headers exactly as they appear in the source file, - but we still normalize on the server so trailing whitespace / casing - doesn't break matching. Returns the canonical fieldname or ``None`` - if not present in the file. - """ - if not mapping_value: - return None - return header_lookup.get(_normalize_header(mapping_value)) - - -def _xlsx_cell_to_str(value: Any) -> str: - """Coerce an openpyxl cell value to the string the rest of the - pipeline expects. - - openpyxl returns native Python types (int, float, datetime, bool, - None). The CSV path always works with strings, so we mirror that: - integers stringify cleanly (no ``.0`` suffix on whole-number floats), - datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. - """ - if value is None: - return "" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, int): - return str(value) - if isinstance(value, float): - if value.is_integer(): - return str(int(value)) - return str(value) - if isinstance(value, datetime): - return value.isoformat() - if isinstance(value, date): - return value.isoformat() - if isinstance(value, time): - return value.isoformat() - if isinstance(value, timedelta): - return str(value) - return str(value) - - -def _parse_recording_date_cell(cell: str) -> date: - """Parse day-first dates with one/two digit day-month parts.""" - match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) - if match: - day, month, year = (int(part) for part in match.groups()) - return date(year, month, day) - - # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO - # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, - # while keeping plain ISO dates rejected for hand-entered text/CSV cells. - if "T" in cell: - return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() - - raise ValueError("expected D/M/YYYY or D-M-YYYY") - - -def _coerce_parameter_value( - raw: str, - param_type: CallImportParameterType, - *, - row_idx: int, - param_name: str, -) -> Any: - """Validate + coerce a single CSV cell against its declared type. - - Returns the typed Python value to surface in ``raw_columns``. Empty - strings are returned as ``None`` regardless of the parameter type so - optional cells stay null end-to-end. Coercion failures raise a - 400 with a row-anchored message. - """ - cell = (raw or "").strip() - if not cell: - return None - - if param_type == CallImportParameterType.CONVERSATION_ID: - return cell - if param_type == CallImportParameterType.RECORDING_URL: - # Recording URLs are exercised by the worker (which downloads - # them); we only do a light "starts with http" check here so a - # paste-error surfaces immediately at upload time. - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid recording URL (must start with http:// or https://)." - ), - ) - return cell - if param_type == CallImportParameterType.RECORDING_DATE: - try: - parsed_date = _parse_recording_date_cell(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid recording date ({cell!r}); expected day-first " - "D/M/YYYY or D-M-YYYY." - ), - ) - return parsed_date.strftime("%d/%m/%Y") - if param_type == CallImportParameterType.TRANSCRIPT: - return cell - if param_type == CallImportParameterType.TEXT: - return cell - if param_type == CallImportParameterType.NUMBER: - try: - value = float(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid number ({cell!r})." - ), - ) - if value.is_integer(): - return int(value) - return value - if param_type == CallImportParameterType.BOOLEAN: - truthy = {"true", "yes", "y", "1", "t"} - falsy = {"false", "no", "n", "0", "f"} - norm = cell.lower() - if norm in truthy: - return True - if norm in falsy: - return False - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid boolean ({cell!r})." - ), - ) - if param_type == CallImportParameterType.DATETIME: - try: - parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid ISO-8601 date/time ({cell!r})." - ), - ) - return parsed.isoformat() - if param_type == CallImportParameterType.URL: - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid URL (must start with http:// or https://)." - ), - ) - return cell - # Unknown types: store as text and let the next migration catch up. - return cell - - -def _recording_url_cell_is_valid_http(raw: str) -> bool: - cell = (raw or "").strip() - if not cell: - return False - lower = cell.lower() - return lower.startswith("http://") or lower.startswith("https://") - - -def _parameter_is_required(param: CallImportSchemaParameter) -> bool: - """Return whether a schema parameter must be mapped on every upload.""" - if param.is_required: - return True - try: - param_type = CallImportParameterType(param.type) - except ValueError: - return False - return param_type == CallImportParameterType.CONVERSATION_ID - - -def _apply_schema_mapping( - fieldnames: List[str], - rows_iter: Iterable[Dict[str, str]], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], - *, - source_label: str = "CSV", - validate_only: bool = False, -) -> CallImportParseResult: - """Schema-driven row projection: parameter -> CSV header -> typed value. - - Validates that every required schema parameter is mapped to a CSV - header that actually exists in the file, and that every CSV header - is either mapped to a parameter or explicitly listed in - ``skipped_columns``. Returns one dict per non-empty data row with: - - * ``conversation_id`` (str, mandatory) - * ``recording_date`` (Optional[str], DD/MM/YYYY date) - * ``recording_url`` (Optional[str]) - * ``transcript`` (Optional[str]) - * ``parameter_values`` (Dict[str, Any]) of typed values keyed by - parameter name (drives ``raw_columns`` so the export can - reproduce the source). - - ``validate_only=True`` runs the header / mapping / skipped-column - checks (every check that doesn't need to read row data) and then - returns an empty list — used by the MAP stage to validate a - mapping payload against the cached sheet snapshot without - re-fetching the source bytes from S3. - """ - header_lookup = _header_lookup(list(fieldnames)) - - # 1. Look up the conversation_id parameter so we can address it - # directly while building each row. - conv_param = next( - (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), - None, - ) - if conv_param is None: - # The schema invariant should have caught this on create/update, - # but a defensive 400 here keeps us safe against hand-rolled - # API callers that bypassed validation. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema is missing the mandatory conversation_id parameter.", - ) - # 2. Resolve every mapped parameter to a canonical fieldname. - # Required parameters MUST resolve; optional ones may resolve to - # None if the user left them blank (no mapping). - canonical_by_param: Dict[str, Optional[str]] = {} - recording_date_param_name: Optional[str] = None - rec_url_param_name: Optional[str] = None - transcript_param_name: Optional[str] = None - for param in parameters: - mapped_header = parameter_mapping.get(param.name) - canonical = ( - _resolve_mapped_header(mapped_header, header_lookup) - if mapped_header - else None - ) - if _parameter_is_required(param) and canonical is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} does not contain the column " - f"'{mapped_header or ''}' mapped to required parameter " - f"'{param.name}'." - ), - ) - canonical_by_param[param.name] = canonical - if param.type == CallImportParameterType.RECORDING_DATE: - recording_date_param_name = param.name - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_param_name = param.name - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_param_name = param.name - - # 3. Every CSV column must either be mapped to a parameter or - # explicitly skipped. Catches "I forgot to skip the email - # column" gracefully instead of dropping data silently. - mapped_canonicals = {c for c in canonical_by_param.values() if c} - skipped_canonicals = { - _resolve_mapped_header(h, header_lookup) - for h in skipped_columns - } - skipped_canonicals.discard(None) - unhandled = [ - h - for h in fieldnames - if h not in mapped_canonicals and h not in skipped_canonicals - ] - if unhandled: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} columns must either be mapped to a schema " - f"parameter or explicitly skipped. Unhandled: {unhandled}." - ), - ) - - conv_canonical = canonical_by_param[conv_param.name] - rec_canonical = ( - canonical_by_param.get(rec_url_param_name) - if rec_url_param_name - else None - ) - recording_date_canonical = ( - canonical_by_param.get(recording_date_param_name) - if recording_date_param_name - else None - ) - transcript_canonical = ( - canonical_by_param.get(transcript_param_name) - if transcript_param_name - else None - ) - - if validate_only: - # MAP-stage validation: every header check above has already - # run; the row loop only matters at IMPORT time. Skip it (and - # the "no data rows" guard at the bottom of the function) so - # the caller gets a clean pass when the mapping is shaped right. - return CallImportParseResult() - - parsed: List[Dict[str, Any]] = [] - skipped: List[CallImportParseSkip] = [] - for idx, row in enumerate(rows_iter): - # Drop fully-blank lines - matches the legacy parser behavior so - # trailing-newline edge cases don't fail an otherwise-good upload. - non_blank = any( - (row.get(c) or "").strip() - for c in mapped_canonicals - if c - ) - if not non_blank: - continue - - source_row = idx + 1 - conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" - if not conv_value: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_conversation_id", - message=( - f"Row {source_row} is missing the '{conv_param.name}' " - "(conversation_id) value." - ), - ) - ) - continue - - if rec_canonical and rec_url_param_name: - rec_param = next( - (p for p in parameters if p.name == rec_url_param_name), - None, - ) - if rec_param is not None: - rec_raw = (row.get(rec_canonical) or "").strip() - if _parameter_is_required(rec_param) and not rec_raw: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{rec_url_param_name}' value." - ), - ) - ) - continue - if rec_raw and not _recording_url_cell_is_valid_http(rec_raw): - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="invalid_recording_url", - message=( - f"Row {source_row}: value for " - f"'{rec_url_param_name}' is not a valid recording " - "URL (must start with http:// or https://)." - ), - ) - ) - continue - - # Materialize every mapped parameter into the per-row snapshot, - # running per-type coercion so a bad cell aborts the upload - # rather than silently storing garbage. - parameter_values: Dict[str, Any] = {} - row_skipped = False - for param in parameters: - canonical = canonical_by_param[param.name] - if canonical is None: - continue - try: - param_type = CallImportParameterType(param.type) - except ValueError: - param_type = CallImportParameterType.TEXT - raw_cell = row.get(canonical) or "" - if ( - param_type == CallImportParameterType.RECORDING_URL - and (raw_cell or "").strip() - and not _recording_url_cell_is_valid_http(raw_cell) - ): - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="invalid_recording_url", - message=( - f"Row {source_row}: value for '{param.name}' is not a " - "valid recording URL (must start with http:// or https://)." - ), - ) - ) - row_skipped = True - break - coerced = _coerce_parameter_value( - raw_cell, - param_type, - row_idx=idx, - param_name=param.name, - ) - if _parameter_is_required(param) and coerced is None: - if param_type == CallImportParameterType.RECORDING_URL: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - ) - row_skipped = True - break - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - parameter_values[param.name] = coerced - if row_skipped: - continue - - rec_value = ( - (row.get(rec_canonical) or "").strip() if rec_canonical else "" - ) - transcript_value = ( - (row.get(transcript_canonical) or "").strip() - if transcript_canonical - else "" - ) - recording_date_value = ( - parameter_values.get(recording_date_param_name) - if recording_date_param_name - else None - ) - - parsed.append( - { - "conversation_id": conv_value, - "recording_date": recording_date_value, - "recording_url": rec_value or None, - "transcript": transcript_value or None, - "parameter_values": parameter_values, - } - ) - - if not parsed and not skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - return CallImportParseResult(rows=parsed, skipped=skipped) - - -def _raise_if_no_importable_rows( - result: CallImportParseResult, *, source_label: str = "CSV" -) -> None: - """Sync upload / API callers fail fast when every data row was skipped.""" - if result.rows: - return - if result.skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"No importable rows. {len(result.skipped)} row(s) skipped due " - "to missing or invalid conversation ID or recording URL." - ), - ) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - -def _parse_csv( - file_bytes: bytes, - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a CSV file using the resolved schema parameters.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - - return _apply_schema_mapping( - list(reader.fieldnames), - reader, - parameters, - parameter_mapping, - skipped_columns, - source_label="CSV", - ) - - -def _open_xlsx_workbook(file_bytes: bytes): - """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). - - Imports openpyxl lazily so the module loads even in environments that - haven't installed the optional dep yet (e.g. lightweight tooling - images). Surfaces a clean 400 if openpyxl is missing or the file is - not a valid Office Open XML workbook. - """ - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded Excel file is empty.", - ) - try: - from openpyxl import load_workbook # type: ignore - from openpyxl.utils.exceptions import InvalidFileException # type: ignore - except ImportError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=( - "Excel uploads require the 'openpyxl' package which is " - "not installed in this environment." - ), - ) from exc - - try: - return load_workbook( - io.BytesIO(file_bytes), - read_only=True, - data_only=True, - ) - except InvalidFileException as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"File is not a valid .xlsx workbook: {exc}", - ) from exc - except Exception as exc: # zipfile.BadZipFile etc. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Could not open Excel workbook: {exc}", - ) from exc - - -def _xlsx_sheet_headers_and_rows( - worksheet, -) -> Tuple[List[str], List[Dict[str, str]]]: - """Read row 1 as headers and the rest as dicts of stringified cells. - - Empty trailing header cells are dropped. Duplicate headers preserve - the first occurrence (matches ``csv.DictReader`` behavior, which - silently drops duplicates). - """ - iterator = worksheet.iter_rows(values_only=True) - try: - header_row = next(iterator) - except StopIteration: - return [], [] - - headers: List[str] = [] - seen: set[str] = set() - for cell in header_row: - name = _xlsx_cell_to_str(cell).strip() - if not name: - # Stop at the first blank header — treats trailing empty - # columns as not part of the table (matches typical Excel - # workbook conventions). - break - norm = name.lower() - if norm in seen: - continue - seen.add(norm) - headers.append(name) - - rows: List[Dict[str, str]] = [] - for row in iterator: - if row is None: - continue - # Pad / truncate to the header length so dict construction is - # stable even when a row has fewer / extra cells than the header. - cells = list(row[: len(headers)]) - if len(cells) < len(headers): - cells.extend([None] * (len(headers) - len(cells))) - if not any(_xlsx_cell_to_str(c).strip() for c in cells): - # Skip fully-blank rows (openpyxl read_only routinely yields - # trailing empties when the worksheet's used range exceeds - # the actual data). - continue - rows.append( - { - header: _xlsx_cell_to_str(value) - for header, value in zip(headers, cells) - } - ) - - return headers, rows - - -def _parse_xlsx( - file_bytes: bytes, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a single worksheet from an xlsx/xlsm workbook. - - ``sheet_name`` must match one of the workbook's sheets (case - insensitive whitespace-trimmed match). Returns the same shape as - :func:`_parse_csv` so the upload handler can persist either format - through the same code path. - """ - if not sheet_name or not sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - workbook = _open_xlsx_workbook(file_bytes) - try: - sheet_names = list(workbook.sheetnames) - target_norm = sheet_name.strip().lower() - match = next( - (s for s in sheet_names if s.strip().lower() == target_norm), - None, - ) - if match is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{sheet_name}' not found in workbook. " - f"Available sheets: {sheet_names}" - ), - ) - worksheet = workbook[match] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - finally: - workbook.close() - - if not headers: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Sheet '{sheet_name}' is missing a header row.", - ) - - return _apply_schema_mapping( - headers, - rows, - parameters, - parameter_mapping, - skipped_columns, - source_label=f"Sheet '{sheet_name}'", - ) - - -def _csv_preview_sheets( - file_bytes: bytes, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Build the synthetic single-sheet preview entry for a CSV upload.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - headers = list(reader.fieldnames) - row_count = 0 - for row in reader: - # Match the parse-time skip: ignore fully blank rows so the - # count the user sees lines up with what /upload will ingest. - if any((v or "").strip() for v in row.values()): - row_count += 1 - - sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" - return [ - CallImportPreviewSheet( - name=sheet_label, - headers=headers, - row_count=row_count, - ) - ] - - -def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: - """List every worksheet in the workbook with its headers and row count.""" - workbook = _open_xlsx_workbook(file_bytes) - sheets: List[CallImportPreviewSheet] = [] - try: - for name in workbook.sheetnames: - worksheet = workbook[name] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - sheets.append( - CallImportPreviewSheet( - name=name, - headers=headers, - row_count=len(rows), - ) - ) - finally: - workbook.close() - return sheets - - -def _parse_json_form_field(name: str, raw: Optional[str], default): - """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" - if raw is None or raw == "": - return default - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{name} must be valid JSON: {exc}", - ) - - -# --------------------------------------------------------------------------- -# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the -# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the -# back-compat path operate on the exact same validation + persistence code. -# --------------------------------------------------------------------------- - - -def _source_content_type(fmt: str) -> str: - """Return the canonical ``Content-Type`` for a parsed file format.""" - if fmt == "csv": - return "text/csv" - if fmt == "xlsx": - return ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - return "application/octet-stream" - - -def _source_s3_key( - organization_id: UUID, call_import_id: UUID, fmt: str -) -> str: - """Build the canonical S3 key for an upload's source file. - - Mirrors the per-row recording key convention used by - ``process_call_import_row`` so a single prefix sweep on delete still - cleans up both the source artefact and every fetched recording. - """ - from app.services.storage.s3_service import s3_service - - ext = "xlsx" if fmt == "xlsx" else "csv" - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/source.{ext}" - ) - - -def _build_available_sheets( - file_bytes: bytes, fmt: str, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" - if fmt == "csv": - return _csv_preview_sheets(file_bytes, filename) - return _xlsx_preview_sheets(file_bytes) - - -def _resolve_schema( - db: Session, - organization_id: UUID, - workspace_id: UUID, - schema_id: UUID, -) -> CallImportSchema: - """Fetch + validate a schema row in the active workspace. - - Eager-loads ``parameters`` so callers can iterate without re-querying. - """ - from sqlalchemy.orm import selectinload as _selectinload - - schema = ( - db.query(CallImportSchema) - .options(_selectinload(CallImportSchema.parameters)) - .filter( - CallImportSchema.id == schema_id, - CallImportSchema.organization_id == organization_id, - CallImportSchema.workspace_id == workspace_id, - ) - .first() - ) - if not schema: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Call import schema not found in the active workspace.", - ) - if not list(schema.parameters): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema has no parameters defined.", - ) - return schema - - -def _validate_direct_url_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure direct-URL import has a mapped recording_url when required.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - return - if not _parameter_is_required(rec_url_param): - return - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _validate_credentialed_import_ready( - provider: str, - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure credentialed import has a mapped recording_url column.""" - provider_key = (provider or "").lower() - provider_label = provider_key.capitalize() - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{provider_label} import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{provider_label} import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _validate_exotel_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure Exotel credentialed import has a mapped recording_url column.""" - _validate_credentialed_import_ready("exotel", parameters, parameter_mapping) - - -def _is_manual_audio_call_import(call_import: CallImport) -> bool: - """True for batches created via manual audio upload (recordings already in S3).""" - return (call_import.source_format or "").lower() == "audio" - - -def _is_platform_call_import(call_import: CallImport) -> bool: - return (call_import.source_format or "").lower() == "platform" - - -def platform_import_hydration_complete(call_import: CallImport) -> bool: - if not _is_platform_call_import(call_import): - return True - discovery = (call_import.discovery_status or "").lower() - if discovery not in {"completed", "capped", "failed", "partial"}: - return False - from sqlalchemy.orm import object_session - - from app.services.call_imports.bulk_ops import _aggregate_import_row_status_counts - - db = object_session(call_import) - if db is None: - return False - _total, _completed, _failed, pending, processing = _aggregate_import_row_status_counts( - db, call_import.id - ) - return pending == 0 and processing == 0 - - -def _validate_diarised_eval_recording_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure diarised evaluation can fetch recordings from the mapped batch.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Diarize then evaluate requires a schema parameter of type " - "'recording_url'. Add one to the schema and map it to a " - "source column." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Diarize then evaluate requires the " - f"'{rec_url_param.name}' recording URL parameter to be " - "mapped to a source column." - ), - ) - - -def _resolve_telephony_integration( - db: Session, - organization_id: UUID, - telephony_integration_id: UUID, - provider: str, -) -> TelephonyIntegration: - """Fetch + validate a telephony credential against the requested provider.""" - integration = ( - db.query(TelephonyIntegration) - .filter( - TelephonyIntegration.id == telephony_integration_id, - TelephonyIntegration.organization_id == organization_id, - ) - .first() - ) - if not integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Telephony credential not found for this organization.", - ) - if (integration.provider or "").lower() != provider.lower(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Selected credential is for provider '{integration.provider}', " - f"but request specified '{provider}'." - ), - ) - if not integration.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected telephony credential is inactive.", - ) - return integration - - -_STUCK_PENDING_ERROR_MARKERS = ( - "rejected credentials", - "telephony credentials rejected", - "auth failed", - "recording fetch failed after", -) - - -def _is_stuck_pending_import_row(row: CallImportRow) -> bool: - """True when a pending row exhausted retries or hit credential errors.""" - from app.models.enums import CallImportRowStatus - - if row.status != CallImportRowStatus.PENDING: - return False - if (row.attempts or 0) > 0: - return True - message = (row.error_message or "").lower() - return any(marker in message for marker in _STUCK_PENDING_ERROR_MARKERS) - - -def _validate_telephony_credentials_live( - db: Session, - organization_id: UUID, - integration: TelephonyIntegration, -) -> None: - """Verify telephony credentials against the provider API before import work.""" - from app.services.telephony.telephony_service import telephony_service - - provider_label = (integration.provider or "telephony").title() - try: - client = telephony_service.get_provider_client( - organization_id, - db, - provider=integration.provider, - credential_id=integration.id, - ) - client.test_connection() - except HTTPException: - raise - except Exception as exc: - detail = str(exc).strip() or "Unknown error" - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{provider_label} credentials could not be verified: {detail}. " - "Update credentials in Integrations and retry." - ), - ) from exc - - -def _clean_parameter_mapping( - mapping_payload: Any, - parameters: List[CallImportSchemaParameter], - schema_name: str, -) -> Dict[str, str]: - """Trim values and drop empties; reject unknown parameter names. - - Accepts an already-decoded value (dict-shaped) so the same helper - works for the JSON-form upload path and the JSON-body PATCH path. - """ - if not isinstance(mapping_payload, dict) or not all( - isinstance(k, str) and (v is None or isinstance(v, str)) - for k, v in mapping_payload.items() - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "parameter_mapping must be an object of " - "{parameter_name: csv_header}." - ), - ) - - valid_param_names = {p.name for p in parameters} - cleaned: Dict[str, str] = {} - for raw_name, raw_header in mapping_payload.items(): - if raw_name not in valid_param_names: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"parameter_mapping references unknown parameter " - f"'{raw_name}' on schema '{schema_name}'." - ), - ) - header = (raw_header or "").strip() - if header: - cleaned[raw_name] = header - return cleaned - - -def _clean_skipped_columns(skipped_payload: Any) -> List[str]: - """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" - if not isinstance(skipped_payload, list) or not all( - isinstance(item, str) for item in skipped_payload - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="skipped_columns must be a list of header strings.", - ) - cleaned: List[str] = [] - seen: set[str] = set() - for item in skipped_payload: - norm = _normalize_header(item) - if not norm or norm in seen: - continue - seen.add(norm) - cleaned.append(item) - return cleaned - - -def _parse_source_file( - file_bytes: bytes, - fmt: str, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - cleaned_mapping: Dict[str, str], - cleaned_skipped: List[str], -) -> CallImportParseResult: - """Run the format-appropriate parser against a buffer of file bytes.""" - if fmt == "csv": - return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) - return _parse_xlsx( - file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped - ) - - -def _materialize_rows( - db: Session, - call_import: CallImport, - parsed_rows: List[Dict[str, Any]], - organization_id: UUID, -) -> List[CallImportRow]: - """Insert one ``CallImportRow`` per parsed row, returning the new models.""" - row_models: List[CallImportRow] = [] - for idx, row in enumerate(parsed_rows): - # Stamp ``transcript_source='csv'`` when the upload actually - # provided a transcript so the UI badge ("From CSV") works from - # day one. Blank cells stay NULL so the row reads as "no - # production transcript yet". - csv_transcript = row["transcript"] - row_model = CallImportRow( - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - row_index=idx, - conversation_id=row["conversation_id"], - recording_date=( - _parse_recording_date_cell(row["recording_date"]) - if row.get("recording_date") - else None - ), - recording_url=row["recording_url"], - transcript=csv_transcript, - transcript_source=( - "csv" if csv_transcript and csv_transcript.strip() else None - ), - raw_columns=row["parameter_values"] or None, - status=CallImportRowStatus.PENDING, - ) - db.add(row_model) - row_models.append(row_model) - return row_models - - -def _enqueue_row_tasks( - db: Session, - call_import: CallImport, - row_models: List[CallImportRow], -) -> None: - """Schedule fair round-robin dispatch for pending import rows.""" - del db, call_import, row_models - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - schedule_fair_import_dispatch(max_workspace_turns=999) - - -def _ensure_blob_storage_enabled() -> None: - """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - err = ( - s3_service.get_status_message() - or "Cloud blob storage is not enabled or not configured" - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=( - "Call uploads require cloud blob storage so the file can be " - f"persisted between stages: {err}" - ), - ) - - -def _validate_sheet_choice( - fmt: str, - sheet_name: Optional[str], - available_sheets: Optional[List[Dict[str, Any]]], -) -> Optional[str]: - """Normalize / validate ``sheet_name`` against the persisted snapshot. - - Returns the canonical sheet name (matching the workbook's casing) - so downstream parsing addresses the right worksheet. - """ - if fmt == "csv": - if sheet_name and sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - return None - - cleaned = (sheet_name or "").strip() or None - if cleaned is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when the source is an Excel workbook.", - ) - - if not available_sheets: - # Nothing to validate against (e.g. legacy batch without snapshot); - # let downstream parsing error out instead of silently importing. - return cleaned - - target = cleaned.strip().lower() - for entry in available_sheets: - name = entry.get("name") if isinstance(entry, dict) else None - if isinstance(name, str) and name.strip().lower() == target: - return name - sheet_names = [ - entry.get("name") for entry in available_sheets if isinstance(entry, dict) - ] - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{cleaned}' not found in the staged file. " - f"Available sheets: {sheet_names}" - ), - ) - - -def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: - """Shape a CallImport's tag relationship for the upload response.""" - return [ - { - "id": tag.id, - "name": tag.name, - "color": tag.color, - "created_at": tag.created_at, - "updated_at": tag.updated_at, - } - for tag in (tags or []) - ] - - -@router.post( - "/preview", - response_model=CallImportPreviewResponse, - operation_id="previewCallImportFile", -) -async def preview_call_import_file( - file: UploadFile = File(...), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportPreviewResponse: - """Inspect an uploaded CSV / Excel file and return its sheets + headers. - - Drives the column-mapping UI without forcing the frontend to parse - CSV / xlsx itself — keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. CSVs return a single - synthetic sheet named after the filename; Excel workbooks return one - entry per worksheet (in workbook order). - """ - del api_key, organization_id, workspace_id, db # auth only - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - if fmt == "csv": - sheets = _csv_preview_sheets(file_bytes, file.filename) - else: - sheets = _xlsx_preview_sheets(file_bytes) - - return CallImportPreviewResponse(format=fmt, sheets=sheets) - - -@router.post( - "", - response_model=CallImportResponse, - status_code=status.HTTP_201_CREATED, - operation_id="createCallImport", -) -async def create_call_import( - file: UploadFile = File( - ..., - description="CSV / Excel file to stage. Persisted to S3 between stages.", - ), - dataset: str = Form( - ..., - description=( - "Required free-text dataset label. Collected up-front so the " - "batch is filterable from the moment it lands." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - schema_id: Optional[UUID] = Form( - None, - description=( - "Optional schema pre-pick. The user can still change it during " - "the MAP stage; provided here only so the detail page can pre-" - "select the schema dropdown." - ), - ), - content_modality: Optional[str] = Form( - None, - description="Optional 'chat' for transcript-only post-prod chat imports; omit for voice/call imports.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """UPLOAD stage of the staged call-import flow. - - Persists the source file to S3 and creates a ``CallImport`` row with - ``status='uploaded'``. No mapping, no provider, no rows yet — the - user moves through MAP and IMPORT as separate idempotent steps. - - Dataset is collected here (rather than at IMPORT) so the batch is - filterable from the moment it appears in the list view. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - # Parse-now so we (a) reject garbage uploads up-front instead of - # later in the MAP step, and (b) capture the sheets snapshot the - # MAP UI needs without having to re-fetch the file from S3. - sheets = _build_available_sheets(file_bytes, fmt, file.filename) - - # Optional schema pre-pick: validated only if supplied (the user is - # allowed to set it for the first time during MAP). - if schema_id is not None: - _resolve_schema(db, organization_id, workspace_id, schema_id) - - normalized_modality: Optional[str] = None - if content_modality is not None and content_modality.strip(): - mod = content_modality.strip().lower() - if mod not in ("chat", "voice"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="content_modality must be 'chat' or 'voice' when provided.", - ) - normalized_modality = mod - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - _ensure_blob_storage_enabled() - - # Pre-generate the id so we can compute a deterministic S3 key - # before the row is persisted, keeping ``source_s3_key`` consistent - # with the prefix sweep used at delete-time. - import uuid as _uuid - - call_import_id = _uuid.uuid4() - s3_key = _source_s3_key(organization_id, call_import_id, fmt) - content_type = _source_content_type(fmt) - - from app.services.storage.s3_service import s3_service, StorageError - - try: - s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) - except StorageError as exc: - logger.exception( - "Failed to upload source file to S3 for new call import {}", - call_import_id, - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Failed to persist upload to S3: {exc}", - ) - - call_import = CallImport( - id=call_import_id, - organization_id=organization_id, - workspace_id=workspace_id, - # Provider + credential aren't known until the IMPORT stage; leave - # them NULL so the staged-vs-legacy distinction is visible at a - # glance from the DB. - provider=None, - telephony_integration_id=None, - original_filename=file.filename, - sheet_name=None, - dataset=normalized_dataset, - schema_id=schema_id, - parameter_mapping={}, - skipped_columns=[], - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - source_s3_key=s3_key, - source_format=fmt, - source_size_bytes=len(file_bytes), - source_content_type=content_type, - available_sheets=[sheet.model_dump() for sheet in sheets], - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.UPLOADED, - content_modality=normalized_modality, - ) - if tag_rows: - call_import.tags = tag_rows - - stamp_call_import_actor(call_import, principal, creating=True) - db.add(call_import) - try: - db.commit() - except Exception: - db.rollback() - # Best-effort cleanup of the uploaded S3 object so a failed - # commit doesn't leak storage. - try: - s3_service.delete_file_by_key(s3_key) - except Exception as cleanup_exc: # noqa: BLE001 - logger.warning( - "Failed to clean up orphaned S3 object {} after DB rollback: {}", - s3_key, - cleanup_exc, - ) - raise - - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.patch( - "/{call_import_id}/mapping", - response_model=CallImportResponse, - operation_id="updateCallImportMapping", -) -async def update_call_import_mapping( - call_import_id: UUID, - payload: CallImportMappingUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """MAP stage of the staged call-import flow. - - Validates ``parameter_mapping`` + ``skipped_columns`` against the - sheet headers captured at UPLOAD time and persists them on the - batch. Idempotent: callers may submit this multiple times while - the batch is in ``uploaded`` or ``mapped`` state. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot edit mapping on a batch in status " - f"'{call_import.status.value}'. Mapping can only be edited " - "before the IMPORT stage." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch was not uploaded through the staged flow and " - "cannot have its mapping edited." - ), - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, payload.schema_id - ) - parameters = list(schema.parameters) - - canonical_sheet = _validate_sheet_choice( - call_import.source_format, - payload.sheet_name, - call_import.available_sheets, - ) - - # Pull the headers for the selected sheet straight out of the - # snapshot so we don't have to re-download the file from S3 just to - # validate the mapping. - headers: List[str] = [] - if call_import.available_sheets: - if canonical_sheet is None: - # CSV: single synthetic sheet. - entry = call_import.available_sheets[0] - headers = list(entry.get("headers") or []) - else: - for entry in call_import.available_sheets: - if not isinstance(entry, dict): - continue - name = entry.get("name") - if isinstance(name, str) and name == canonical_sheet: - headers = list(entry.get("headers") or []) - break - - cleaned_mapping = _clean_parameter_mapping( - payload.parameter_mapping, parameters, schema.name - ) - cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) - - # Run the same per-column validation as the parse path so the user - # gets an immediate 400 if a required parameter is left unmapped or - # a header is neither mapped nor skipped — without needing to read - # the file. ``validate_only`` skips the row loop (and the empty-rows - # guard) since the row data lives in S3, not in this request. - if headers: - _apply_schema_mapping( - headers, - iter(()), - parameters, - cleaned_mapping, - cleaned_skipped, - source_label=( - f"Sheet '{canonical_sheet}'" - if canonical_sheet is not None - else "CSV" - ), - validate_only=True, - ) - - call_import.schema_id = schema.id - call_import.parameter_mapping = dict(cleaned_mapping) - call_import.skipped_columns = list(cleaned_skipped) - call_import.sheet_name = canonical_sheet - call_import.status = CallImportStatus.MAPPED - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.post( - "/{call_import_id}/import", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="startCallImport", -) -async def start_call_import( - call_import_id: UUID, - payload: CallImportStartRequest, - background_tasks: BackgroundTasks, - legacy: bool = Query( - False, - description=( - "Deprecated escape hatch for import-only processing. " - "New batches should use Run Evaluation instead." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Deprecated IMPORT stage — use Run Evaluation for new batches. - - Recording fetch is part of the unified evaluation pipeline. This - endpoint remains available only with ``?legacy=true`` for backward - compatibility. - """ - del api_key, background_tasks - - if not legacy: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Standalone import is deprecated. Use Run Evaluation — " - "recording fetch is part of the evaluation pipeline. " - "Append ?legacy=true to use the import-only path." - ), - ) - - from sqlalchemy.orm import selectinload as _selectinload - - call_import = ( - db.query(CallImport) - .options(_selectinload(CallImport.tags)) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status != CallImportStatus.MAPPED: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot start import for a batch in status " - f"'{call_import.status.value}'. Map the columns first." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file and cannot be imported " - "through the staged flow." - ), - ) - - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot start import without a mapped schema.", - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - _validate_telephony_credentials_live(db, organization_id, integration) - if (integration.provider or "").lower() in {"exotel", "plivo"}: - _validate_credentialed_import_ready( - integration.provider, - parameters, - dict(call_import.parameter_mapping or {}), - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_rows_task, - ) - - materialize_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - str(workspace_id), - schedule_import_dispatch=True, - ) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=0, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - "Import accepted. Rows are being materialized in the background; " - "recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="uploadCallImportCsv", - deprecated=True, -) -async def upload_call_import_csv( - background_tasks: BackgroundTasks, - file: UploadFile = File(...), - provider: Optional[str] = Form( - None, - description=( - "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " - "selected telephony_integration_id's provider. Omit together " - "with telephony_integration_id for direct-URL import." - ), - ), - telephony_integration_id: Optional[UUID] = Form( - None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "provider for direct-URL import." - ), - ), - schema_id: UUID = Form( - ..., - description=( - "Reusable Input Parameter schema this upload is mapped against. " - "Must belong to the active workspace." - ), - ), - parameter_mapping: str = Form( - ..., - description=( - "JSON-encoded ``{schema_parameter_name: source_header}`` map " - "covering every required schema parameter. Optional parameters " - "may be omitted or set to an empty string." - ), - ), - skipped_columns: Optional[str] = Form( - None, - description=( - "JSON-encoded list of source header strings the uploader has " - "explicitly skipped. Every header in the file must either be " - "mapped or appear here; otherwise the upload is rejected so a " - "forgotten column never silently drops." - ), - ), - dataset: Optional[str] = Form( - None, - description=( - "Optional free-text dataset label for high-level segregation. " - "Empty strings are stored as NULL." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - sheet_name: Optional[str] = Form( - None, - description=( - "Worksheet to import when the file is an Excel workbook " - "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " - "uploads (rejected with 400 if non-empty so typos surface " - "instead of silently importing the wrong source)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Legacy one-shot upload kept for backward compatibility. - - DEPRECATED: prefer the staged flow - (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so - each step is idempotent and resumable. This endpoint runs all three - stages inline in a single transaction so existing scripts / - integrations keep working unchanged. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - sheet_name_clean = (sheet_name or "").strip() or None - if fmt == "csv" and sheet_name_clean is not None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - if fmt == "xlsx" and sheet_name_clean is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - schema = _resolve_schema(db, organization_id, workspace_id, schema_id) - parameters = list(schema.parameters) - - mapping_payload = _parse_json_form_field( - "parameter_mapping", parameter_mapping, {} - ) - cleaned_mapping = _clean_parameter_mapping( - mapping_payload, parameters, schema.name - ) - - skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) - cleaned_skipped = _clean_skipped_columns(skipped_payload) - - has_provider = bool((provider or "").strip()) - has_integration = telephony_integration_id is not None - if has_provider != has_integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ), - ) - - if telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, organization_id, telephony_integration_id, provider or "" - ) - if (integration.provider or "").lower() in {"exotel", "plivo"}: - _validate_credentialed_import_ready( - integration.provider, parameters, cleaned_mapping - ) - else: - _validate_direct_url_import_ready(parameters, cleaned_mapping) - integration = None - - parsed_rows = _parse_source_file( - file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped - ) - _raise_if_no_importable_rows(parsed_rows, source_label=fmt) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=integration.provider if integration is not None else None, - telephony_integration_id=integration.id if integration is not None else None, - original_filename=file.filename, - sheet_name=sheet_name_clean, - dataset=_normalize_dataset(dataset), - schema_id=schema.id, - parameter_mapping=dict(cleaned_mapping), - skipped_columns=list(cleaned_skipped), - # Legacy columns are left empty on new uploads; the detail page - # falls back to ``parameter_mapping`` when ``schema_id`` is set. - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - total_rows=len(parsed_rows.rows), - completed_rows=0, - failed_rows=0, - status=CallImportStatus.PENDING, - source_row_skips=parse_skips_to_json(parsed_rows.skipped), - ) - if tag_rows: - call_import.tags = tag_rows - stamp_call_import_actor(call_import, principal, creating=True) - db.add(call_import) - db.flush() # populate call_import.id - if integration is None: - # The model's historical Python default is "exotel"; direct-URL - # imports intentionally have no telephony provider. - call_import.provider = None - - row_models = _materialize_rows( - db, call_import, parsed_rows.rows, organization_id - ) - - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="csv", - provider=call_import.provider, - ) - - _enqueue_row_tasks(db, call_import, row_models) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Accepted {call_import.total_rows} rows for import. " - "Recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/audio-upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_201_CREATED, - operation_id="uploadCallImportAudio", -) -async def upload_call_import_audio( - background_tasks: BackgroundTasks, - files: List[UploadFile] = File( - ..., - description="One or more manual call recording audio files.", - ), - dataset: str = Form( - ..., - description="Required free-text dataset label for the manual upload batch.", - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - batch_name: Optional[str] = Form( - None, - description=( - "Optional display name for this manual upload batch. " - "When omitted, a generic label is used for multi-file uploads." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Persist manually uploaded recordings as completed CallImport rows. - - The rows skip the provider-download worker entirely because the audio - bytes are already in hand. From this point onward they behave exactly - like completed CSV-import rows: playback reads ``recording_s3_key`` and - the existing diarisation/evaluation endpoints can operate on them. - """ - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - - _ensure_blob_storage_enabled() - tag_rows = _resolve_tags(db, organization_id, tag_ids) - prepared = await _prepare_audio_upload_files(files, conversation_counts={}) - - normalized_batch_name = _normalize_import_display_name(batch_name) - if normalized_batch_name: - original_filename = normalized_batch_name - elif len(prepared) == 1: - original_filename = prepared[0]["filename"] - else: - original_filename = "Manual recordings" - uploaded_keys: List[str] = [] - - from app.services.storage.s3_service import s3_service - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=None, - telephony_integration_id=None, - original_filename=original_filename, - source_format="audio", - source_size_bytes=0, - source_content_type="audio/*", - dataset=normalized_dataset, - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.COMPLETED, - ) - if tag_rows: - call_import.tags = tag_rows - - stamp_call_import_actor(call_import, principal, creating=True) - try: - db.add(call_import) - db.flush() - # The model's historical Python default is "exotel"; manual uploads - # intentionally have no telephony provider. - call_import.provider = None - - uploaded_keys = _persist_prepared_audio_rows( - db, - call_import=call_import, - organization_id=organization_id, - workspace_id=workspace_id, - prepared=prepared, - start_row_index=0, - ) - db.commit() - if is_sharding_enabled(): - from app.db_sharding.registry import publish_slice_registry_change - - publish_slice_registry_change(call_import.id) - except Exception as exc: - db.rollback() - if uploaded_keys and s3_service.is_enabled(): - try: - s3_service.delete_keys(uploaded_keys) - except Exception: - logger.exception( - "Failed to clean up manual audio upload keys after error" - ) - logger.exception("Failed to persist manual call recording upload") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to upload manual recordings: {exc}", - ) from exc - - db.refresh(call_import) - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="audio", - provider=None, - ) - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Uploaded {call_import.total_rows} manual recording" - f"{'' if call_import.total_rows == 1 else 's'}." - ), - ) - - -@router.post( - "/{call_import_id}/audio-append", - response_model=CallImportUploadResponse, - status_code=status.HTTP_200_OK, - operation_id="appendCallImportAudio", -) -async def append_call_import_audio( - call_import_id: UUID, - files: List[UploadFile] = File( - ..., - description="Additional manual call recording audio files for an existing batch.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Append manually uploaded recordings to an existing audio batch.""" - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - if (call_import.source_format or "").lower() != "audio": - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Audio append is only supported for manual audio upload batches.", - ) - if call_import.status != CallImportStatus.COMPLETED: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot append audio while batch is in status " - f"'{call_import.status.value}'." - ), - ) - - _ensure_blob_storage_enabled() - - start_row_index, conversation_counts = _manual_audio_append_start_state( - db, - call_import, - ) - - prepared = await _prepare_audio_upload_files(files, conversation_counts) - uploaded_keys: List[str] = [] - - from app.services.storage.s3_service import s3_service - - try: - uploaded_keys = _persist_prepared_audio_rows( - db, - call_import=call_import, - organization_id=organization_id, - workspace_id=workspace_id, - prepared=prepared, - start_row_index=start_row_index, - ) - db.commit() - if is_sharding_enabled(): - from app.db_sharding.registry import publish_slice_registry_change - - publish_slice_registry_change(call_import.id) - except Exception as exc: - db.rollback() - if uploaded_keys and s3_service.is_enabled(): - try: - s3_service.delete_keys(uploaded_keys) - except Exception: - logger.exception( - "Failed to clean up manual audio append keys after error" - ) - logger.exception("Failed to append manual call recording upload") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to append manual recordings: {exc}", - ) from exc - - db.refresh(call_import) - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Uploaded {len(prepared)} additional manual recording" - f"{'' if len(prepared) == 1 else 's'} " - f"({call_import.total_rows} total)." - ), - ) - - -@router.get( - "", - response_model=CallImportListResponse, - operation_id="listCallImports", -) -async def list_call_imports( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - status_filter: Optional[CallImportStatus] = Query(None, alias="status"), - dataset: Optional[str] = Query( - None, - description=( - "Filter by exact dataset string (case-insensitive). Pass the " - "literal value '__none__' to filter to imports with no dataset." - ), - ), - tag_id: Optional[List[UUID]] = Query( - None, - description="Filter to imports tagged with ALL of the given tag ids.", - ), - source_format: Optional[str] = Query( - None, - description=( - "Filter by source format. Use 'audio' for manual recordings or " - "'__non_audio__' for CSV/Excel/legacy imports." - ), - ), - content_modality: Optional[str] = Query( - None, - description="Filter by content_modality (e.g. 'chat'). Use '__voice__' for non-chat batches.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportListResponse: - """List call-import batches for the active workspace, newest first. - - Scoped to (organization_id, workspace_id) so users only see imports - for the workspace they're currently in. Supports a high-level - ``dataset`` filter (powers the segregation dropdown at the top of - the imports page) plus an AND-style multi-tag filter via repeated - ``tag_id`` parameters. - """ - - query = ( - db.query(CallImport) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - ) - if status_filter is not None: - query = query.filter(CallImport.status == status_filter) - - source_filter = (source_format or "").strip().lower() - if source_filter == "__non_audio__": - query = query.filter( - or_( - CallImport.source_format.is_(None), - CallImport.source_format.notin_(["audio", "platform"]), - ) - ) - elif source_filter: - query = query.filter(func.lower(CallImport.source_format) == source_filter) - - modality_filter = (content_modality or "").strip().lower() - if modality_filter == "chat": - query = query.filter(func.lower(CallImport.content_modality) == "chat") - elif modality_filter in ("__voice__", "voice"): - query = query.filter( - or_(CallImport.content_modality.is_(None), func.lower(CallImport.content_modality) != "chat") - ) - - if dataset is not None: - if dataset == "__none__": - query = query.filter(CallImport.dataset.is_(None)) - elif dataset.strip(): - query = query.filter( - func.lower(CallImport.dataset) == dataset.strip().lower() - ) - - if tag_id: - from app.models.database import CallImportTagAssignment - - for single_tag_id in tag_id: - sub = ( - db.query(CallImportTagAssignment.call_import_id) - .filter(CallImportTagAssignment.tag_id == single_tag_id) - .subquery() - ) - query = query.filter(CallImport.id.in_(sub)) - - total = query.count() - items = ( - query.order_by(desc(CallImport.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - - email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) - return CallImportListResponse( - items=[ - _serialize_call_import(db, item, user_emails=email_map) - for item in items - ], - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/dispatch-diagnostics", - response_model=CallImportDispatchDiagnosticsResponse, - operation_id="getCallImportDispatchDiagnostics", - dependencies=[Depends(require_admin)], -) -async def get_call_import_dispatch_diagnostics( - workspace_id: Optional[UUID] = Query( - None, - description=( - "Optional workspace filter. When omitted, returns every workspace " - "in the organization with active eval dispatch state." - ), - ), - include_idle_workspaces: bool = Query( - False, - description=( - "When true, include org workspaces with zero pending rows and " - "zero in-flight slots." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDispatchDiagnosticsResponse: - """Live eval slot usage and fair-dispatch state for operators. - - Org admins use this to diagnose cross-workspace starvation (e.g. one - workspace's 10k run blocking another's pending eval rows) by inspecting - Redis in-flight counters, pending dispatch rows, and scheduler cursors. - """ - del api_key - payload = build_call_import_dispatch_diagnostics( - db, - organization_id, - workspace_id=workspace_id, - include_idle_workspaces=include_idle_workspaces, - ) - return CallImportDispatchDiagnosticsResponse.model_validate(payload) - - -@router.get( - "/datasets", - response_model=List[str], - operation_id="listCallImportDatasets", -) -async def list_call_import_datasets( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> List[str]: - """Return the distinct, non-null dataset labels in use for the active - workspace. - - Scoped per-workspace so each workspace's Dataset dropdown only shows - its own segregation labels. - """ - rows = ( - db.query(CallImport.dataset) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImport.dataset.isnot(None), - CallImport.dataset != "", - ) - .distinct() - .order_by(CallImport.dataset.asc()) - .all() - ) - return [row[0] for row in rows if row[0]] - - -@router.get( - "/diarisation-prompt-default", - response_model=CallImportDiarisationPromptDefaultResponse, - operation_id="getCallImportDiarisationPromptDefault", -) -async def get_call_import_diarisation_prompt_default( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), -) -> CallImportDiarisationPromptDefaultResponse: - """Return the canonical LLM diariser prompt. - - The Transcribe / Run Evaluation modals call this on open so they - can pre-fill the prompt textarea. Returning the constant from the - backend (rather than hard-coding it in the frontend) keeps the - fallback used by the worker and the placeholder shown in the UI - in lock-step — operators always see the *actual* default they'd - get if they leave the field blank. - - Registered before ``GET /{call_import_id}`` so the static path is - not mistaken for a UUID import id (which would 422). - """ - del api_key, organization_id - from app.workers.tasks.helpers.llm_diarisation import ( - DEFAULT_DIARIZATION_PROMPT, - ) - - return CallImportDiarisationPromptDefaultResponse( - prompt=DEFAULT_DIARIZATION_PROMPT - ) - - -@router.patch( - "/{call_import_id}", - response_model=CallImportResponse, - operation_id="updateCallImport", -) -async def update_call_import( - call_import_id: UUID, - payload: CallImportUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """Edit dataset / tag assignments (and schema, pre-import) on a batch. - - ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag - assignments. Fields omitted from the body are left untouched. - - ``schema_id`` is only honoured while the batch is in - ``uploaded`` / ``mapped`` state — once rows have been materialised - the schema is locked. Changing the schema resets any persisted - mapping (the user must re-MAP) and rewinds status to ``uploaded``. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - body = payload.model_dump(exclude_unset=True) - if "original_filename" in body: - call_import.original_filename = _normalize_import_display_name( - body["original_filename"] - ) - - if "dataset" in body: - call_import.dataset = _normalize_dataset(body["dataset"]) - - if "tag_ids" in body: - tag_ids = body["tag_ids"] or [] - call_import.tags = _resolve_tags(db, organization_id, tag_ids) - - if "schema_id" in body and body["schema_id"] is not None: - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot reassign schema on a batch in status " - f"'{call_import.status.value}'." - ), - ) - new_schema = _resolve_schema( - db, organization_id, workspace_id, body["schema_id"] - ) - if call_import.schema_id != new_schema.id: - # Switching schemas invalidates the persisted mapping — - # parameter names won't line up with the new schema, so - # reset to UPLOADED and force a fresh MAP. - call_import.schema_id = new_schema.id - call_import.parameter_mapping = {} - call_import.skipped_columns = [] - call_import.sheet_name = None - call_import.status = CallImportStatus.UPLOADED - - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.get( - "/{call_import_id}", - response_model=CallImportDetailResponse, - operation_id="getCallImportDetail", -) -async def get_call_import_detail( - call_import_id: UUID, - row_limit: int = Query(500, ge=0, le=5000), - row_offset: int = Query(0, ge=0), - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. When set, ``filtered_total_rows`` in " - "the response reflects the post-filter row count so the UI " - "can paginate against the filtered slice." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts one of ``pending``, ``running``, ``completed``, " - "``failed``. When set, ``filtered_total_rows`` reflects the " - "post-filter row count (combined with the ``q`` filter when " - "both are supplied) so the UI can paginate against the same " - "slice it's displaying." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDetailResponse: - """Fetch a single import batch with a slice of its rows. - - ``row_limit=0`` is intentionally allowed so callers that only need the - batch metadata (e.g. the evaluation-detail page rendering the parent's - column mapping) can skip the rows payload entirely. - """ - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status == CallImportStatus.PROCESSING: - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - prior_status = call_import.status - rollup_call_import_batch_status(db, call_import) - if call_import.status != prior_status: - db.commit() - db.refresh(call_import) - - if ( - (call_import.source_format or "") == "platform" - and _imported_agent_partial_id(call_import) is None - ): - from app.services.platform_import_imported_agent import ( - ensure_imported_agent_for_platform_import, - ) - - try: - if ensure_imported_agent_for_platform_import(db, call_import, None): - db.commit() - db.refresh(call_import) - except Exception: - db.rollback() - - search_term = (q or "").strip() - diarised_status_filter = (diarised_status or "").strip() or None - filtered_total_rows: Optional[int] = None - has_row_filters = bool(search_term or diarised_status_filter) - - if has_row_filters: - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import count_call_import_rows_filtered - - filtered_total_rows = count_call_import_rows_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - filtered_total_rows = rows_query.count() - - if row_limit == 0: - rows: List[CallImportRow] = [] - elif is_sharding_enabled(): - from app.db_sharding.scatter_gather import ( - fetch_call_import_rows_filtered_page, - fetch_call_import_rows_page, - ) - - if has_row_filters: - rows = fetch_call_import_rows_filtered_page( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - offset=row_offset, - limit=row_limit, - ) - else: - rows = fetch_call_import_rows_page( - db, - call_import.id, - offset=row_offset, - limit=row_limit, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - from app.db_sharding.scatter_gather import defer_call_import_row_text - - rows = ( - rows_query.options(*defer_call_import_row_text()) - .order_by(CallImportRow.row_index) - .offset(row_offset) - .limit(row_limit) - .all() - ) - - # Batch-wide diarisation status aggregate. One ``GROUP BY`` query - # across the whole batch — much cheaper than paging through every - # row to recount on the client and lets the UI render a - # transcribe/diarise progress bar without a separate roundtrip. - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts - - diarised_status_counts = aggregate_diarised_transcript_counts( - db, call_import.id - ) - else: - diarised_status_counts: Dict[str, int] = {} - for status_value, count in ( - db.query(CallImportRow.diarised_transcript_status, func.count()) - .filter(CallImportRow.call_import_id == call_import.id) - .group_by(CallImportRow.diarised_transcript_status) - .all() - ): - if isinstance(status_value, str): - diarised_status_counts[status_value] = int(count or 0) - - if (call_import.source_format or "").lower() == "platform": - _remember_platform_agent_names(db, call_import) - detail = CallImportDetailResponse.model_validate( - _serialize_call_import(db, call_import).model_dump() - ) - detail.rows = [_call_import_row_list_response(r) for r in rows] - detail.filtered_total_rows = filtered_total_rows - detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) - detail.diarised_running_rows = diarised_status_counts.get("running", 0) - detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) - detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) - if (call_import.source_format or "").lower() == "platform": - from app.services.call_imports.platform_import_progress import ( - build_platform_import_progress, - ) - - detail.platform_progress = build_platform_import_progress(db, call_import) - return detail - - -@router.get( - "/{call_import_id}/row-ids", - response_model=CallImportRowIdsResponse, - operation_id="listCallImportRowIds", -) -async def list_call_import_row_ids( - call_import_id: UUID, - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. Same semantics as the detail endpoint." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowIdsResponse: - """Return every matching ``CallImportRow.id`` for cross-page bulk select. - - Lightweight companion to ``GET /{call_import_id}`` — the detail - endpoint caps ``row_limit`` at 5000 and ships the entire row body - on each page, so harvesting ids that way is wasteful when the - user just wants to bulk-delete or bulk-transcribe everything that - matches the current filters. This endpoint applies the same ``q`` - and ``diarised_status`` filters and returns only the ids. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - search_term = (q or "").strip() - status_filter = (diarised_status or "").strip() or None - - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered - - ids = list_call_import_row_ids_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=status_filter, - ) - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - rows_query = db.query(CallImportRow.id).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == status_filter - ) - - ids = [ - row_id - for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() - ] - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - -def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: - """Best-effort revoke of in-flight Celery tasks for the given rows. - - Failures are logged and swallowed — Celery's control plane is async and - best-effort by design, and we always do an idempotent S3 cleanup - afterwards so a missed revoke can't leak storage. - """ - task_ids = [ - r.celery_task_id - for r in rows - if r.celery_task_id - and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) - ] - if not task_ids: - return - - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_ids, terminate=False) - logger.info("Revoked {} pending call-import tasks", len(task_ids)) - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to revoke pending call-import tasks: {}", exc) - - -def _delete_s3_objects( - organization_id: UUID, - call_import_id: UUID, - rows: List[CallImportRow], -) -> tuple[int, int]: - """Delete every recording associated with ``rows`` plus a prefix sweep. - - The prefix sweep also cleans up the staged source file written at - UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the - per-row recording keys and the source artefact share the same - organization-scoped prefix, so a single sweep covers them all. - - Returns ``(deleted_count, error_count)``. Never raises — callers proceed - with the DB delete regardless; orphans, if any, can be cleaned up by - re-running the same delete (it's idempotent). - """ - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - return 0, 0 - - keys = [r.recording_s3_key for r in rows if r.recording_s3_key] - deleted = 0 - errors = 0 - - if keys: - try: - d, errs = s3_service.delete_keys(keys) - deleted += d - errors += len(errs) - if errs: - logger.warning( - "S3 bulk-delete reported {} errors for call_import {}", - len(errs), - call_import_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc - ) - errors += len(keys) - - # Belt-and-braces sweep: catch anything that landed under the import's - # prefix but never made it into a row's recording_s3_key (narrow - # window where the S3 upload succeeded but the DB commit didn't). - sweep_prefix = ( - f"{s3_service.prefix}organizations/{organization_id}/" - f"call_imports/{call_import_id}/" - ) - try: - d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) - deleted += d - errors += len(errs) - except Exception as exc: # noqa: BLE001 - logger.exception( - "S3 prefix sweep failed for {}: {}", sweep_prefix, exc - ) - - return deleted, errors - - -@router.delete( - "/{call_import_id}", - response_model=CallImportDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="deleteCallImport", -) -async def delete_call_import( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportDeleteResponse: - """Delete a call-import batch asynchronously. - - Flips the batch to ``deleting`` and enqueues background teardown so - large imports (thousands of rows + S3 objects) do not block the API. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - return CallImportDeleteResponse( - id=call_import_id, - status="completed", - ) - - if call_import.status == CallImportStatus.DELETING: - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - call_import.status = CallImportStatus.DELETING - call_import.error_message = None - stamp_call_import_actor(call_import, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import delete_call_import_task - - delete_call_import_task.delay( - str(call_import_id), - str(organization_id), - ) - - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - -def _call_import_row_list_response(row: CallImportRow) -> CallImportRowResponse: - """List payload. Transcript text is loaded by the single-row GET.""" - return CallImportRowResponse( - id=row.id, - row_index=row.row_index, - conversation_id=row.conversation_id, - recording_url=row.recording_url, - recording_date=row.recording_date, - transcript=None, - transcript_source=row.transcript_source, - transcript_provider=row.transcript_provider, - transcript_model=row.transcript_model, - transcript_status=row.transcript_status, - transcript_error=row.transcript_error, - transcribed_at=row.transcribed_at, - diarised_transcript=None, - diarised_transcript_provider=row.diarised_transcript_provider, - diarised_transcript_model=row.diarised_transcript_model, - diarised_transcript_status=row.diarised_transcript_status, - diarised_transcript_error=row.diarised_transcript_error, - diarised_at=row.diarised_at, - diarised_llm_provider=row.diarised_llm_provider, - diarised_llm_model=row.diarised_llm_model, - diarised_prompt=None, - diarised_segments=None, - diarised_speaker_swap=bool(row.diarised_speaker_swap), - status=row.status, - recording_s3_key=row.recording_s3_key, - recording_content_type=row.recording_content_type, - recording_size_bytes=row.recording_size_bytes, - error_message=row.error_message, - attempts=row.attempts, - raw_columns=None, - created_at=row.created_at, - updated_at=row.updated_at, - ) - - -def _locate_call_import_row_or_404( - catalog_db: Session, - *, - call_import_id: UUID, - row_id: UUID, - organization_id: UUID, -) -> Tuple[Session, CallImportRow, Optional[Session]]: - """Find a call import row on the correct DB session for mutation. - - When sharding is enabled rows live on shard databases; ``get_db`` only - opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` - where ``extra_catalog_to_close`` is the catalog session opened by - :func:`locate_call_import_row` (distinct from the route's catalog - session) and must be closed via :func:`close_row_sessions`. - """ - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - except LookupError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) from None - if ( - row.call_import_id != call_import_id - or row.organization_id != organization_id - ): - close_row_sessions( - row_db, - located_catalog if located_catalog is not row_db else None, - ) - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - extra_catalog = located_catalog if located_catalog is not row_db else None - return row_db, row, extra_catalog - - -@router.get( - "/{call_import_id}/rows/{row_id}", - response_model=CallImportRowResponse, - operation_id="getCallImportRow", -) -async def get_call_import_row( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """One import row, including transcript text omitted from the list.""" - del api_key - from app.db_sharding.sessions import is_sharding_enabled - - if not is_sharding_enabled(): - row = ( - db.query(CallImportRow) - .filter( - CallImportRow.id == row_id, - CallImportRow.call_import_id == call_import_id, - CallImportRow.organization_id == organization_id, - ) - .first() - ) - if row is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - return CallImportRowResponse.model_validate(row) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -@router.delete( - "/{call_import_id}/rows/{row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportRow", -) -async def delete_call_import_row( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - """Delete a single CallImportRow and its S3 recording. - - The parent ``CallImport`` is left in place. After deletion we recompute - its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` - so the UI's progress bar stays consistent with reality. - """ - from app.services.storage.s3_service import s3_service - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import.id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _revoke_pending_tasks([row]) - - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) - - row_db.delete(row) - row_db.commit() - - _recompute_call_import_counters(db, call_import) - stamp_call_import_actor(call_import, principal) - db.commit() - finally: - close_row_sessions(row_db, extra_catalog) - - logger.info( - "Deleted call_import_row {} (call_import={}, org={})", - row_id, - call_import.id, - organization_id, - ) - - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -def _recompute_call_import_counters( - db: Session, call_import: CallImport -) -> None: - """Resync ``total/completed/failed_rows`` + status on the parent batch. - - Called after row-level mutations (single delete, bulk delete) so the - UI's progress bar stays consistent with the actual row set. The - rules mirror :func:`delete_call_import_row` so behavior doesn't - diverge between the per-row and bulk paths. - """ - - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "/{call_import_id}/retry-failed", - response_model=CallImportRetryFailedRowsResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryFailedCallImportRows", -) -async def retry_failed_call_import_rows( - call_import_id: UUID, - payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRetryFailedRowsResponse: - """Re-enqueue every failed import row in this batch. - - Useful when transient provider issues are resolved and the operator wants - a one-click "try failed downloads again" pass without re-uploading the CSV. - - Pass ``provider`` + ``telephony_integration_id`` (or both omitted for - direct-URL retry) to change how recordings are fetched on this pass. - When the body is omitted entirely, the batch keeps its existing pinned - credentials. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if payload is not None: - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - _validate_telephony_credentials_live(db, organization_id, integration) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - if (integration.provider or "").lower() in {"exotel", "plivo"}: - schema = _resolve_schema( - db, - organization_id, - call_import.workspace_id, - call_import.schema_id, - ) - _validate_credentialed_import_ready( - integration.provider, - list(schema.parameters), - dict(call_import.parameter_mapping or {}), - ) - else: - call_import.provider = None - call_import.telephony_integration_id = None - stamp_call_import_actor(call_import, principal) - db.commit() - - from app.services.call_imports.bulk_ops import ( - mark_import_rows_enqueue_failed, - requeue_failed_import_rows, - ) - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - reset_ids = requeue_failed_import_rows(db, call_import.id) - if not reset_ids: - return CallImportRetryFailedRowsResponse( - requeued=0, - enqueue_failed=0, - skipped=0, - ) - - _recompute_call_import_counters(db, call_import) - if payload is None: - stamp_call_import_actor(call_import, principal) - db.commit() - - try: - schedule_fair_import_dispatch(max_workspace_turns=999) - requeued = len(reset_ids) - enqueue_failed = 0 - skipped = 0 - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to schedule fair import dispatch for import {}", - call_import.id, - ) - requeued = 0 - enqueue_failed, skipped = mark_import_rows_enqueue_failed( - db, - call_import.id, - reset_ids, - f"Failed to enqueue retry: {exc}", - ) - _recompute_call_import_counters(db, call_import) - db.commit() - - return CallImportRetryFailedRowsResponse( - requeued=requeued, - enqueue_failed=enqueue_failed, - skipped=skipped, - ) - - -@router.post( - "/{call_import_id}/rows/bulk-delete", - response_model=CallImportRowBulkDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="bulkDeleteCallImportRows", -) -async def bulk_delete_call_import_rows( - call_import_id: UUID, - payload: CallImportRowBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowBulkDeleteResponse: - """Delete multiple ``CallImportRow`` rows in one request. - - Unknown / cross-tenant row ids are silently skipped — the response - reports how many actually went away so a UI that holds onto stale - ids (e.g. after another tab already deleted a row) doesn't 404 - the entire bulk action. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if not payload.row_ids: - return CallImportRowBulkDeleteResponse(deleted=0, status="completed") - - from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task - - row_id_strs = [str(rid) for rid in payload.row_ids] - - stamp_call_import_actor(call_import, principal) - db.commit() - - bulk_delete_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - row_id_strs, - ) - - return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") - - -# --------------------------------------------------------------------------- -# Diarization / transcription endpoints -# --------------------------------------------------------------------------- - - -def _select_rows_for_transcription( - db: Session, - call_import: CallImport, - payload: CallImportTranscribeRequest, - requested_row_ids: Optional[List[UUID]] = None, -) -> tuple[List[CallImportRow], Dict[str, int]]: - """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" - from app.services.call_imports.bulk_ops import select_rows_for_transcription - - try: - return select_rows_for_transcription( - db, call_import, payload, requested_row_ids=requested_row_ids - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - -@router.post( - "/{call_import_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImport", -) -async def transcribe_call_import( - call_import_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Fan out diarization tasks for many rows in a single call. - - Returns a summary with how many rows were queued and how many were - skipped (broken down by reason) so the UI can show a meaningful - toast even when nothing actually got enqueued (e.g. "All 12 rows - already have transcripts"). - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task - - stamp_call_import_actor(call_import, principal) - db.commit() - - bulk_diarize_call_import_task.delay( - str(call_import_id), - str(organization_id), - payload.model_dump(mode="json"), - [str(rid) for rid in payload.row_ids] if payload.row_ids else None, - ) - - return CallImportTranscribeResponse( - queued=0, - skipped_rows=0, - skipped_reason_counts={}, - accepted=True, - ) - - -@router.post( - "/{call_import_id}/rows/{row_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImportRow", -) -async def transcribe_call_import_row( - call_import_id: UUID, - row_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Diarize / transcribe a single row. - - Thin wrapper over the batch endpoint that hard-codes a single - ``row_ids`` filter. Skip counts still surface so the UI can render - "Skipped — transcript present" diagnostics consistently. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.services.call_imports.bulk_ops import execute_bulk_diarization - - try: - result = execute_bulk_diarization( - db, - call_import, - payload, - requested_row_ids=[row_id], - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - stamp_call_import_actor(call_import, principal) - db.commit() - - return CallImportTranscribeResponse( - queued=result.queued, - skipped_rows=result.skipped_rows, - skipped_reason_counts=result.skipped_reason_counts, - ) - - -# --------------------------------------------------------------------------- -# Cancel-in-flight diarisation -# --------------------------------------------------------------------------- -# -# Long-running multimodal LLM diarisation calls (especially LLM-only mode on -# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an -# upstream provider stalls. Without an abort affordance the operator's only -# recourse is to wait for Celery's ``time_limit`` to fire — which can be -# several minutes — or to manually mutate the DB. These helpers + the two -# endpoints below give the UI a first-class "Stop diarisation" button. -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` -# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the -# default but we spell it out so the intent is obvious to reviewers. - -# Sentinel error message stamped on cancelled rows. Read by the transcribe -# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) -# to detect a row that was cancelled mid-flight and AVOID overwriting it -# with whatever partial result the worker had managed to compute before the -# SIGTERM landed. -CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" - - -def _cancellable_diarisation_states() -> Tuple[str, ...]: - """States that a diarisation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_diarisation_task(row: CallImportRow) -> None: - """Best-effort revoke of a single row's diarisation Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`CANCELLED_BY_USER_ERROR`). - """ - task_id = (row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked diarisation task {} for call-import row {}", - task_id, - row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke diarisation task {} for row {}: {}", - task_id, - row.id, - exc, - ) - - -def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: - """Cancel diarisation on every cancellable row in ``rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_diarisation_states() - cancelled = 0 - skipped = 0 - for row in rows: - if (row.diarised_transcript_status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = CANCELLED_BY_USER_ERROR - _revoke_diarisation_task(row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -@router.post( - "/{call_import_id}/rows/{row_id}/cancel-diarisation", - response_model=CallImportRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportRowDiarisation", -) -async def cancel_call_import_row_diarisation( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Abort an in-flight (or queued) diarisation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed`` / ``idle``) returns the row unchanged with a 200, so - the UI can fire this from a "Stop" button without having to - pre-check the state. - - Race notes: - - * The row's ``diarised_transcript_status`` is flipped to ``failed`` - with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, - so the polling UI sees the cancel immediately. - * If the worker happens to finish between our DB flip and the - SIGTERM landing, its finaliser will detect the cancelled - sentinel on the row and skip its own status / score writes - (see :mod:`app.workers.tasks.transcribe_call_import_row`). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _apply_diarisation_cancel([row]) - row_db.commit() - stamp_call_import_actor(call_import, principal) - db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -@router.post( - "/{call_import_id}/cancel-diarisation", - response_model=CallImportCancelDiarisationResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportDiarisation", -) -async def cancel_call_import_diarisation( - call_import_id: UUID, - payload: Optional[CallImportCancelDiarisationRequest] = None, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportCancelDiarisationResponse: - """Abort in-flight diarisation for many rows in a single call. - - Default body (no ``row_ids``) cancels every row in this import - whose ``diarised_transcript_status`` is ``pending`` or - ``running`` — the "stop everything" button. Pass ``row_ids`` to - scope the cancel to the rows the operator has selected. - - Returns ``(cancelled, skipped)`` so the UI can render a tight - toast ("Cancelled 3 rows · 1 skipped (already completed)"). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - base_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import_id - ) - - requested_ids = ( - payload.row_ids if payload and payload.row_ids is not None else None - ) - if requested_ids is not None: - if not requested_ids: - # Empty list is "no rows requested" — treat as a no-op - # 200 rather than 400 so the UI can pass through an empty - # selection without a special-case. - return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) - rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() - found_ids = {r.id for r in rows} - # Treat requested-but-not-found ids as ``skipped`` so the UI's - # numbers reconcile (a stale selection that includes deleted - # rows shouldn't 404 the whole call). - missing = [rid for rid in requested_ids if rid not in found_ids] - skipped_missing = len(missing) - else: - # Implicit "cancel every cancellable row in this import" path. - rows = base_query.filter( - CallImportRow.diarised_transcript_status.in_( - list(_cancellable_diarisation_states()) - ) - ).all() - skipped_missing = 0 - - cancelled, skipped = _apply_diarisation_cancel(rows) - stamp_call_import_actor(call_import, principal) - db.commit() - return CallImportCancelDiarisationResponse( - cancelled=cancelled, - skipped=skipped + skipped_missing, - ) - - -def _render_diarised_segments_text( - segments: Optional[List[Dict[str, Any]]], - *, - swap: bool = False, -) -> str: - """Render ``CallImportRow.diarised_segments`` as ``: `` lines. - - Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so - the route doesn't need to import a Celery task module just to - rebuild the rendered transcript). Only ``agent`` and ``user`` are - swapped — multi-party calls keep their ``speaker_N`` labels through - a swap so we don't silently collapse a third speaker into the user - side. - """ - if not segments: - return "" - out: List[str] = [] - for turn in segments: - if not isinstance(turn, dict): - continue - speaker = (turn.get("speaker") or "").strip() - text = (turn.get("text") or "").strip() - if not speaker or not text: - continue - if swap: - if speaker == "agent": - speaker = "user" - elif speaker == "user": - speaker = "agent" - out.append(f"{speaker}: {text}") - return "\n".join(out) - - -@router.post( - "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", - response_model=CallImportRowResponse, - operation_id="toggleCallImportRowSpeakerSwap", -) -async def toggle_call_import_row_speaker_swap( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Flip the user <-> agent mapping on a diarised row. - - The worker's "first speaker is the agent" heuristic is right most of - the time but does fail on inbound recordings where the customer - greets first, on recordings where the agent stays silent for the - intro, etc. Rather than rerun the (expensive) STT + pyannote - pipeline for those cases, we let reviewers flip the mapping in - place: the structured ``diarised_segments`` are the source of truth - and we re-render the plain-text ``diarised_transcript`` from them - with the swap applied. The next CSV export will then show the - corrected labels. - - Returns the updated row so the frontend can refresh without an - extra round-trip. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - segments = ( - row.diarised_segments if isinstance(row.diarised_segments, list) else None - ) - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), - ) - - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None - ) - row_db.commit() - stamp_call_import_actor(call_import, principal) - db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -# --------------------------------------------------------------------------- -# Cross-run insights for the import detail page -# --------------------------------------------------------------------------- - - -@router.get( - "/{call_import_id}/insights", - response_model=CallImportInsightsResponse, - operation_id="getCallImportInsights", -) -async def get_call_import_insights( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportInsightsResponse: - """Aggregate signals across every evaluation run on this import. - - Powers the Insights tab on the call-import detail page: returns - per-metric "latest run" summaries plus a trend series of mean values - across runs so the UI can render a small line chart per metric. Also - bundles transcript coverage stats since those are the cheapest - pre-eval health-check (e.g. "30 of 50 rows still missing - transcripts"). - """ - - del api_key - - from app.models.database import CallImportEvaluation, Metric - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.scatter_gather import ( - aggregate_transcript_coverage, - load_evaluation_score_rows, - ) - - ( - total_rows, - rows_with_transcript, - rows_without_transcript, - source_counts, - ) = aggregate_transcript_coverage(db, call_import_id) - - evaluations = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(CallImportEvaluation.created_at.asc()) - .all() - ) - - # Defer heavy lifting to the aggregation helper so this endpoint and - # the per-run aggregate endpoint share the exact same metric - # bucketing math (no chance of "trend" disagreeing with "latest" on - # the same data set). - from app.api.v1.routes.call_import_evaluations import ( - _compute_metric_aggregates, - ) - - metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} - metric_meta: Dict[str, Metric] = {} - metric_latest: Dict[str, CallImportMetricAggregate] = {} - - for evaluation in evaluations: - eval_rows = load_evaluation_score_rows(db, evaluation.id) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - for agg in aggregates: - if agg.metric_id not in metric_meta: - # ``agg.metric_id`` is normally a UUID string, but the - # aggregator also emits ids that surface in row scores - # without a matching ``Metric`` row (e.g. a metric the - # user deleted mid-run, or LLM-discovered slugs). Those - # are not valid UUIDs, so coerce defensively and skip - # the metric registry lookup when the cast fails — the - # ``meta is None`` branch below already handles the - # display via the values stored on ``agg`` itself. - try: - metric_uuid = UUID(agg.metric_id) - except (ValueError, AttributeError, TypeError): - metric_uuid = None - if metric_uuid is not None: - metric_obj = ( - db.query(Metric) - .filter( - Metric.id == metric_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if metric_obj is not None: - metric_meta[agg.metric_id] = metric_obj - history = metric_history.setdefault(agg.metric_id, []) - history.append( - CallImportInsightsRunPoint( - evaluation_id=evaluation.id, - name=evaluation.name, - created_at=evaluation.created_at, - mean=agg.mean, - completed_rows=agg.count, - ) - ) - metric_latest[agg.metric_id] = agg - - metrics_payload: List[CallImportInsightsMetric] = [] - for metric_id, latest in metric_latest.items(): - meta = metric_meta.get(metric_id) - metrics_payload.append( - CallImportInsightsMetric( - metric_id=metric_id, - metric_name=(meta.name if meta else latest.metric_name), - metric_type=(meta.metric_type if meta else latest.metric_type), - latest=latest, - trend=metric_history.get(metric_id, []), - ) - ) - - return CallImportInsightsResponse( - call_import_id=call_import_id, - total_rows=total_rows, - rows_with_transcript=rows_with_transcript, - rows_without_transcript=rows_without_transcript, - transcript_source_counts=source_counts, - evaluation_count=len(evaluations), - metrics=metrics_payload, - ) - - -from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=CALLS_VIEW, - manage_capability=CALLS_IMPORT, - delete_capability=CALLS_DELETE, -) diff --git a/app/api/v1/routes/call_imports/__init__.py b/app/api/v1/routes/call_imports/__init__.py new file mode 100644 index 00000000..2ae10180 --- /dev/null +++ b/app/api/v1/routes/call_imports/__init__.py @@ -0,0 +1,126 @@ +"""CSV-driven call import routes. + +Users upload a CSV plus a per-batch column mapping (CSV header -> system +field). The backend persists a CallImport batch + one CallImportRow per +line, then fans the rows out to the Celery ``imports`` queue where each +row is downloaded using the telephony credential pinned on the batch. +Exotel credentialed imports require a ``recording_url`` on every row; +direct-URL imports (no credential) also require a mapped recording URL. +""" + +from __future__ import annotations + +from app.services.call_imports import diarisation_route_helpers as diarisation_helpers +from app.services.call_imports import manual_audio_route_helpers as manual_audio_helpers +from app.services.call_imports import present as call_import_present +from app.services.call_imports import source_parse as call_import_source_parse + +from app.api.v1.routes.call_imports._router import router + +# Re-exports for tests and bulk_ops (import path stability) +CallImportParseSkip = call_import_source_parse.CallImportParseSkip +CallImportParseResult = call_import_source_parse.CallImportParseResult +parse_skips_to_json = call_import_source_parse.parse_skips_to_json +MAX_UPLOAD_BYTES = call_import_source_parse.MAX_UPLOAD_BYTES +CSV_EXTENSIONS = call_import_source_parse.CSV_EXTENSIONS +XLSX_EXTENSIONS = call_import_source_parse.XLSX_EXTENSIONS +ALLOWED_EXTENSIONS = call_import_source_parse.ALLOWED_EXTENSIONS +_file_format = call_import_source_parse._file_format +_normalize_header = call_import_source_parse._normalize_header +_parse_csv = call_import_source_parse._parse_csv +_parse_xlsx = call_import_source_parse._parse_xlsx +_parse_recording_date_cell = call_import_source_parse._parse_recording_date_cell +_parse_source_file = call_import_source_parse._parse_source_file +_apply_schema_mapping = call_import_source_parse._apply_schema_mapping +_clean_parameter_mapping = call_import_source_parse._clean_parameter_mapping +_clean_skipped_columns = call_import_source_parse._clean_skipped_columns +_resolve_schema = call_import_source_parse._resolve_schema +_validate_sheet_choice = call_import_source_parse._validate_sheet_choice +_source_s3_key = call_import_source_parse._source_s3_key +_source_content_type = call_import_source_parse._source_content_type +_build_available_sheets = call_import_source_parse._build_available_sheets +_csv_preview_sheets = call_import_source_parse._csv_preview_sheets +_xlsx_preview_sheets = call_import_source_parse._xlsx_preview_sheets +_insert_staged_call_import_record = call_import_source_parse._insert_staged_call_import_record +_parse_json_form_field = call_import_source_parse._parse_json_form_field +_raise_if_no_importable_rows = call_import_source_parse._raise_if_no_importable_rows +_parameter_is_required = call_import_source_parse._parameter_is_required + +_normalize_dataset = call_import_present._normalize_dataset +_normalize_import_display_name = call_import_present._normalize_import_display_name +_imported_agent_partial_id = call_import_present._imported_agent_partial_id +_platform_agent_refs = call_import_present._platform_agent_refs +_remember_platform_agent_names = call_import_present._remember_platform_agent_names +_serialize_call_import = call_import_present._serialize_call_import +_resolve_tags = call_import_present._resolve_tags +_tag_response_payload = call_import_present._tag_response_payload +_call_import_row_list_response = call_import_present._call_import_row_list_response + +MAX_AUDIO_FILES_PER_REQUEST = manual_audio_helpers.MAX_AUDIO_FILES_PER_REQUEST +_enforce_audio_upload_file_limit = manual_audio_helpers._enforce_audio_upload_file_limit +_audio_extension = manual_audio_helpers._audio_extension +_audio_content_type = manual_audio_helpers._audio_content_type +_audio_s3_key = manual_audio_helpers._audio_s3_key +_manual_audio_append_start_state = manual_audio_helpers._manual_audio_append_start_state +_prepare_audio_upload_files = manual_audio_helpers._prepare_audio_upload_files +_prepare_audio_presign_items = manual_audio_helpers._prepare_audio_presign_items +_upload_manual_audio_blobs_parallel = manual_audio_helpers._upload_manual_audio_blobs_parallel +_persist_prepared_audio_rows = manual_audio_helpers._persist_prepared_audio_rows + +CANCELLED_BY_USER_ERROR = diarisation_helpers.CANCELLED_BY_USER_ERROR +_cancellable_diarisation_states = diarisation_helpers._cancellable_diarisation_states +_revoke_diarisation_task = diarisation_helpers._revoke_diarisation_task +_apply_diarisation_cancel = diarisation_helpers._apply_diarisation_cancel +_render_diarised_segments_text = diarisation_helpers._render_diarised_segments_text + +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) + +from app.api.v1.routes.call_imports import ( # noqa: F401 + audio_routes, + core_routes, + diarisation_routes, +) + +from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=CALLS_VIEW, + manage_capability=CALLS_IMPORT, + delete_capability=CALLS_DELETE, +) + +from app.api.v1.routes.call_imports.core_routes import ( # noqa: E402 + _STUCK_PENDING_ERROR_MARKERS, + _delete_s3_objects, + _ensure_blob_storage_enabled, + _enqueue_row_tasks, + _is_manual_audio_call_import, + _is_platform_call_import, + _is_stuck_pending_import_row, + _locate_call_import_row_or_404, + _materialize_rows, + _normalize_content_modality, + _recompute_call_import_counters, + _resolve_telephony_integration, + _revoke_pending_tasks, + _validate_credentialed_import_ready, + _validate_diarised_eval_recording_ready, + _validate_direct_url_import_ready, + _validate_exotel_import_ready, + _validate_telephony_credentials_live, + get_call_import_diarisation_prompt_default, + platform_import_hydration_complete, +) + +from app.api.v1.routes.call_imports.diarisation_routes import ( # noqa: E402 + _select_rows_for_transcription, + cancel_call_import_diarisation, + cancel_call_import_row_diarisation, + toggle_call_import_row_speaker_swap, + transcribe_call_import, + transcribe_call_import_row, +) diff --git a/app/api/v1/routes/call_imports/_router.py b/app/api/v1/routes/call_imports/_router.py new file mode 100644 index 00000000..abdb9ad0 --- /dev/null +++ b/app/api/v1/routes/call_imports/_router.py @@ -0,0 +1,11 @@ +"""Shared APIRouter for call import endpoints.""" + +from fastapi import APIRouter, Depends + +from app.dependencies import require_enterprise_feature + +router = APIRouter( + prefix="/call-imports", + tags=["Call Imports"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) diff --git a/app/api/v1/routes/call_imports/audio_routes.py b/app/api/v1/routes/call_imports/audio_routes.py new file mode 100644 index 00000000..92c63e46 --- /dev/null +++ b/app/api/v1/routes/call_imports/audio_routes.py @@ -0,0 +1,640 @@ +"""CSV-driven call import routes (package).""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.services.call_imports import diarisation_route_helpers as diarisation_helpers +from app.services.call_imports import manual_audio_route_helpers as manual_audio_helpers +from app.services.call_imports import present as call_import_present +from app.services.call_imports import source_parse as call_import_source_parse +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportManualAudioCompleteRequest, + CallImportManualAudioCompleteResponse, + CallImportManualAudioPresignRequest, + CallImportManualAudioPresignResponse, + CallImportManualAudioPresignUploadItem, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportSourceStageCompleteRequest, + CallImportSourceStagePresignRequest, + CallImportSourceStagePresignResponse, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, + PlatformImportAgentRef, +) + +from app.api.v1.routes.call_imports._router import router + +from app.api.v1.routes.call_imports.core_routes import ( + _audio_s3_key, + _ensure_blob_storage_enabled, + _enforce_audio_upload_file_limit, + _manual_audio_append_start_state, + _normalize_dataset, + _normalize_import_display_name, + _persist_prepared_audio_rows, + _prepare_audio_presign_items, + _prepare_audio_upload_files, + _resolve_tags, + _tag_response_payload, +) + +@router.post( + "/audio-upload/presign", + response_model=CallImportManualAudioPresignResponse, + status_code=status.HTTP_201_CREATED, + operation_id="presignCallImportAudioUpload", +) +async def presign_call_import_audio_upload( + payload: CallImportManualAudioPresignRequest, + background_tasks: BackgroundTasks, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportManualAudioPresignResponse: + del api_key + _ensure_blob_storage_enabled() + _enforce_audio_upload_file_limit(len(payload.files)) + + normalized_dataset = _normalize_dataset(payload.dataset or "") + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + tag_rows = _resolve_tags(db, organization_id, payload.tag_ids) + prepared = _prepare_audio_presign_items(payload.files, conversation_counts={}) + + normalized_batch_name = _normalize_import_display_name(payload.batch_name) + if normalized_batch_name: + original_filename = normalized_batch_name + elif len(prepared) == 1: + original_filename = prepared[0]["filename"] + else: + original_filename = "Manual recordings" + + from app.services.call_imports import manual_audio_presign as presign_mod + from app.services.storage.blob_storage_service import blob_storage_service + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=original_filename, + source_format="audio", + source_size_bytes=0, + source_content_type="audio/*", + dataset=normalized_dataset, + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + if tag_rows: + call_import.tags = tag_rows + stamp_call_import_actor(call_import, principal, creating=True) + + try: + db.add(call_import) + db.flush() + call_import.provider = None + uploads_raw, _keys = presign_mod.build_manual_audio_presign_uploads( + organization_id=organization_id, + call_import_id=call_import.id, + prepared=prepared, + audio_s3_key=_audio_s3_key, + generate_presigned_put_url_by_key=blob_storage_service.generate_presigned_put_url_by_key, + ) + presign_mod.persist_manual_audio_row_mappings( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + uploads=uploads_raw, + start_row_index=0, + ) + db.commit() + if is_sharding_enabled(): + from app.db_sharding.registry import publish_slice_registry_change + + publish_slice_registry_change(call_import.id) + except HTTPException: + db.rollback() + raise + except Exception as exc: + db.rollback() + logger.exception("Failed to presign manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to prepare manual recording upload: {exc}", + ) from exc + + db.refresh(call_import) + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="audio", + provider=None, + ) + uploads = [ + CallImportManualAudioPresignUploadItem( + row_id=item["row_id"], + recording_s3_key=item["recording_s3_key"], + conversation_id=item["conversation_id"], + upload_url=item["upload_url"], + content_type=item["content_type"], + filename=item["filename"], + ) + for item in uploads_raw + ] + return CallImportManualAudioPresignResponse( + call_import_id=call_import.id, + uploads=uploads, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Presigned {len(uploads)} recording upload" + f"{'' if len(uploads) == 1 else 's'}." + ), + ) + + +@router.post( + "/{call_import_id}/audio-append/presign", + response_model=CallImportManualAudioPresignResponse, + status_code=status.HTTP_200_OK, + operation_id="presignAppendCallImportAudio", +) +async def presign_append_call_import_audio( + call_import_id: UUID, + payload: CallImportManualAudioPresignRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportManualAudioPresignResponse: + del api_key + _ensure_blob_storage_enabled() + _enforce_audio_upload_file_limit(len(payload.files)) + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + if (call_import.source_format or "").lower() != "audio": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Audio append is only supported for manual audio upload batches.", + ) + if call_import.status != CallImportStatus.COMPLETED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot append audio while batch is in status " + f"'{call_import.status.value}'." + ), + ) + + start_row_index, conversation_counts = _manual_audio_append_start_state( + db, call_import + ) + prepared = _prepare_audio_presign_items(payload.files, conversation_counts) + + from app.services.call_imports import manual_audio_presign as presign_mod + from app.services.storage.blob_storage_service import blob_storage_service + + try: + uploads_raw, _keys = presign_mod.build_manual_audio_presign_uploads( + organization_id=organization_id, + call_import_id=call_import.id, + prepared=prepared, + audio_s3_key=_audio_s3_key, + generate_presigned_put_url_by_key=blob_storage_service.generate_presigned_put_url_by_key, + ) + presign_mod.persist_manual_audio_row_mappings( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + uploads=uploads_raw, + start_row_index=start_row_index, + ) + db.commit() + if is_sharding_enabled(): + from app.db_sharding.registry import publish_slice_registry_change + + publish_slice_registry_change(call_import.id) + except HTTPException: + db.rollback() + raise + except Exception as exc: + db.rollback() + logger.exception("Failed to presign manual audio append") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to prepare audio append: {exc}", + ) from exc + + db.refresh(call_import) + uploads = [ + CallImportManualAudioPresignUploadItem( + row_id=item["row_id"], + recording_s3_key=item["recording_s3_key"], + conversation_id=item["conversation_id"], + upload_url=item["upload_url"], + content_type=item["content_type"], + filename=item["filename"], + ) + for item in uploads_raw + ] + return CallImportManualAudioPresignResponse( + call_import_id=call_import.id, + uploads=uploads, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Presigned {len(uploads)} recording upload" + f"{'' if len(uploads) == 1 else 's'} for append." + ), + ) + + +@router.post( + "/{call_import_id}/manual-audio/complete", + response_model=CallImportManualAudioCompleteResponse, + status_code=status.HTTP_200_OK, + operation_id="completeCallImportManualAudioUpload", +) +async def complete_call_import_manual_audio_upload( + call_import_id: UUID, + payload: CallImportManualAudioCompleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportManualAudioCompleteResponse: + del api_key + _ensure_blob_storage_enabled() + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + + from app.services.call_imports import manual_audio_presign as presign_mod + from app.services.storage.blob_storage_service import blob_storage_service + + verified = presign_mod.verify_manual_audio_uploads( + db, + call_import_id=call_import.id, + organization_id=organization_id, + row_ids=payload.row_ids, + head_object_by_key=blob_storage_service.head_object_by_key, + ) + db.commit() + db.refresh(call_import) + return CallImportManualAudioCompleteResponse( + verified=verified, + call_import_id=call_import.id, + total_rows=int(call_import.total_rows or 0), + ) + + +@router.post( + "/audio-upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_201_CREATED, + operation_id="uploadCallImportAudio", +) +async def upload_call_import_audio( + background_tasks: BackgroundTasks, + files: List[UploadFile] = File( + ..., + description="One or more manual call recording audio files.", + ), + dataset: str = Form( + ..., + description="Required free-text dataset label for the manual upload batch.", + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + batch_name: Optional[str] = Form( + None, + description=( + "Optional display name for this manual upload batch. " + "When omitted, a generic label is used for multi-file uploads." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Persist manually uploaded recordings as completed CallImport rows. + + The rows skip the provider-download worker entirely because the audio + bytes are already in hand. From this point onward they behave exactly + like completed CSV-import rows: playback reads ``recording_s3_key`` and + the existing diarisation/evaluation endpoints can operate on them. + """ + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + _ensure_blob_storage_enabled() + tag_rows = _resolve_tags(db, organization_id, tag_ids) + prepared = await _prepare_audio_upload_files(files, conversation_counts={}) + + normalized_batch_name = _normalize_import_display_name(batch_name) + if normalized_batch_name: + original_filename = normalized_batch_name + elif len(prepared) == 1: + original_filename = prepared[0]["filename"] + else: + original_filename = "Manual recordings" + uploaded_keys: List[str] = [] + + from app.services.storage.blob_storage_service import blob_storage_service + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=original_filename, + source_format="audio", + source_size_bytes=0, + source_content_type="audio/*", + dataset=normalized_dataset, + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + try: + db.add(call_import) + db.flush() + # The model's historical Python default is "exotel"; manual uploads + # intentionally have no telephony provider. + call_import.provider = None + + uploaded_keys = _persist_prepared_audio_rows( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + prepared=prepared, + start_row_index=0, + ) + db.commit() + if is_sharding_enabled(): + from app.db_sharding.registry import publish_slice_registry_change + + publish_slice_registry_change(call_import.id) + except Exception as exc: + db.rollback() + if uploaded_keys and blob_storage_service.is_enabled(): + try: + blob_storage_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio upload keys after error" + ) + logger.exception("Failed to persist manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to upload manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="audio", + provider=None, + ) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {call_import.total_rows} manual recording" + f"{'' if call_import.total_rows == 1 else 's'}." + ), + ) + + +@router.post( + "/{call_import_id}/audio-append", + response_model=CallImportUploadResponse, + status_code=status.HTTP_200_OK, + operation_id="appendCallImportAudio", +) +async def append_call_import_audio( + call_import_id: UUID, + files: List[UploadFile] = File( + ..., + description="Additional manual call recording audio files for an existing batch.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Append manually uploaded recordings to an existing audio batch.""" + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + if (call_import.source_format or "").lower() != "audio": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Audio append is only supported for manual audio upload batches.", + ) + if call_import.status != CallImportStatus.COMPLETED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot append audio while batch is in status " + f"'{call_import.status.value}'." + ), + ) + + _ensure_blob_storage_enabled() + + start_row_index, conversation_counts = _manual_audio_append_start_state( + db, + call_import, + ) + + prepared = await _prepare_audio_upload_files(files, conversation_counts) + uploaded_keys: List[str] = [] + + from app.services.storage.blob_storage_service import blob_storage_service + + try: + uploaded_keys = _persist_prepared_audio_rows( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + prepared=prepared, + start_row_index=start_row_index, + ) + db.commit() + if is_sharding_enabled(): + from app.db_sharding.registry import publish_slice_registry_change + + publish_slice_registry_change(call_import.id) + except Exception as exc: + db.rollback() + if uploaded_keys and blob_storage_service.is_enabled(): + try: + blob_storage_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio append keys after error" + ) + logger.exception("Failed to append manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to append manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {len(prepared)} additional manual recording" + f"{'' if len(prepared) == 1 else 's'} " + f"({call_import.total_rows} total)." + ), + ) + diff --git a/app/api/v1/routes/call_imports/core_routes.py b/app/api/v1/routes/call_imports/core_routes.py new file mode 100644 index 00000000..9ec1837f --- /dev/null +++ b/app/api/v1/routes/call_imports/core_routes.py @@ -0,0 +1,1187 @@ +"""CSV-driven call import routes (package).""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports import import_queries as call_import_queries +from app.services.call_imports.import_queries import ( + locate_call_import_row_or_404 as _locate_call_import_row_or_404, +) +from app.services.call_imports.import_row_mutations import ( + bulk_delete_call_import_rows as _bulk_delete_call_import_rows, + delete_call_import as _delete_call_import_service, + delete_call_import_row as _delete_call_import_row_service, + delete_s3_objects as _delete_s3_objects, + recompute_call_import_counters as _recompute_call_import_counters, + retry_failed_call_import_rows as _retry_failed_call_import_rows, + revoke_pending_tasks as _revoke_pending_tasks, + start_legacy_call_import as _start_legacy_call_import, +) +from app.services.call_imports import diarisation_route_helpers as diarisation_helpers +from app.services.call_imports import manual_audio_route_helpers as manual_audio_helpers +from app.services.call_imports import present as call_import_present +from app.services.call_imports import source_parse as call_import_source_parse +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportManualAudioCompleteRequest, + CallImportManualAudioCompleteResponse, + CallImportManualAudioPresignRequest, + CallImportManualAudioPresignResponse, + CallImportManualAudioPresignUploadItem, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportSourceStageCompleteRequest, + CallImportSourceStagePresignRequest, + CallImportSourceStagePresignResponse, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, + PlatformImportAgentRef, +) + +from app.api.v1.routes.call_imports._router import router + +CallImportParseSkip = call_import_source_parse.CallImportParseSkip +CallImportParseResult = call_import_source_parse.CallImportParseResult +parse_skips_to_json = call_import_source_parse.parse_skips_to_json +MAX_UPLOAD_BYTES = call_import_source_parse.MAX_UPLOAD_BYTES +CSV_EXTENSIONS = call_import_source_parse.CSV_EXTENSIONS +XLSX_EXTENSIONS = call_import_source_parse.XLSX_EXTENSIONS +ALLOWED_EXTENSIONS = call_import_source_parse.ALLOWED_EXTENSIONS +_file_format = call_import_source_parse._file_format +_normalize_header = call_import_source_parse._normalize_header +_parse_csv = call_import_source_parse._parse_csv +_parse_xlsx = call_import_source_parse._parse_xlsx +_parse_recording_date_cell = call_import_source_parse._parse_recording_date_cell +_parse_source_file = call_import_source_parse._parse_source_file +_apply_schema_mapping = call_import_source_parse._apply_schema_mapping +_clean_parameter_mapping = call_import_source_parse._clean_parameter_mapping +_clean_skipped_columns = call_import_source_parse._clean_skipped_columns +_resolve_schema = call_import_source_parse._resolve_schema +_validate_sheet_choice = call_import_source_parse._validate_sheet_choice +_source_s3_key = call_import_source_parse._source_s3_key +_source_content_type = call_import_source_parse._source_content_type +_build_available_sheets = call_import_source_parse._build_available_sheets +_csv_preview_sheets = call_import_source_parse._csv_preview_sheets +_xlsx_preview_sheets = call_import_source_parse._xlsx_preview_sheets +_insert_staged_call_import_record = call_import_source_parse._insert_staged_call_import_record +_parse_json_form_field = call_import_source_parse._parse_json_form_field +_raise_if_no_importable_rows = call_import_source_parse._raise_if_no_importable_rows +_parameter_is_required = call_import_source_parse._parameter_is_required + +_normalize_dataset = call_import_present._normalize_dataset +_normalize_import_display_name = call_import_present._normalize_import_display_name +_imported_agent_partial_id = call_import_present._imported_agent_partial_id +_platform_agent_refs = call_import_present._platform_agent_refs +_remember_platform_agent_names = call_import_present._remember_platform_agent_names +_serialize_call_import = call_import_present._serialize_call_import +_resolve_tags = call_import_present._resolve_tags +_tag_response_payload = call_import_present._tag_response_payload +_call_import_row_list_response = call_import_present._call_import_row_list_response + +MAX_AUDIO_FILES_PER_REQUEST = manual_audio_helpers.MAX_AUDIO_FILES_PER_REQUEST +_enforce_audio_upload_file_limit = manual_audio_helpers._enforce_audio_upload_file_limit +_audio_extension = manual_audio_helpers._audio_extension +_audio_content_type = manual_audio_helpers._audio_content_type +_audio_s3_key = manual_audio_helpers._audio_s3_key +_manual_audio_append_start_state = manual_audio_helpers._manual_audio_append_start_state +_prepare_audio_upload_files = manual_audio_helpers._prepare_audio_upload_files +_prepare_audio_presign_items = manual_audio_helpers._prepare_audio_presign_items +_upload_manual_audio_blobs_parallel = manual_audio_helpers._upload_manual_audio_blobs_parallel +_persist_prepared_audio_rows = manual_audio_helpers._persist_prepared_audio_rows + +CANCELLED_BY_USER_ERROR = diarisation_helpers.CANCELLED_BY_USER_ERROR +_cancellable_diarisation_states = diarisation_helpers._cancellable_diarisation_states +_revoke_diarisation_task = diarisation_helpers._revoke_diarisation_task +_apply_diarisation_cancel = diarisation_helpers._apply_diarisation_cancel +_render_diarised_segments_text = diarisation_helpers._render_diarised_segments_text + +from app.services.call_imports.import_batch_helpers import ( + STUCK_PENDING_ERROR_MARKERS as _STUCK_PENDING_ERROR_MARKERS, + ensure_blob_storage_enabled as _ensure_blob_storage_enabled, + enqueue_row_tasks as _enqueue_row_tasks, + is_manual_audio_call_import as _is_manual_audio_call_import, + is_platform_call_import as _is_platform_call_import, + is_stuck_pending_import_row as _is_stuck_pending_import_row, + materialize_rows as _materialize_rows, + normalize_content_modality as _normalize_content_modality, + platform_import_hydration_complete, + resolve_telephony_integration as _resolve_telephony_integration, + validate_credentialed_import_ready as _validate_credentialed_import_ready, + validate_diarised_eval_recording_ready as _validate_diarised_eval_recording_ready, + validate_direct_url_import_ready as _validate_direct_url_import_ready, + validate_exotel_import_ready as _validate_exotel_import_ready, + validate_telephony_credentials_live as _validate_telephony_credentials_live, +) + + +@router.post( + "/preview", + response_model=CallImportPreviewResponse, + operation_id="previewCallImportFile", +) +async def preview_call_import_file( + file: UploadFile = File(...), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportPreviewResponse: + """Inspect an uploaded CSV / Excel file and return its sheets + headers. + + Drives the column-mapping UI without forcing the frontend to parse + CSV / xlsx itself — keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. CSVs return a single + synthetic sheet named after the filename; Excel workbooks return one + entry per worksheet (in workbook order). + """ + del api_key, organization_id, workspace_id, db # auth only + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + if fmt == "csv": + sheets = _csv_preview_sheets(file_bytes, file.filename) + else: + sheets = _xlsx_preview_sheets(file_bytes) + + return CallImportPreviewResponse(format=fmt, sheets=sheets) + + +@router.post( + "/stage/presign", + response_model=CallImportSourceStagePresignResponse, + status_code=status.HTTP_201_CREATED, + operation_id="presignCallImportSourceStage", +) +async def presign_call_import_source_stage( + payload: CallImportSourceStagePresignRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportSourceStagePresignResponse: + del api_key, db + _ensure_blob_storage_enabled() + + from app.services.call_imports import staged_source_presign as stage_mod + from app.services.storage.blob_storage_service import blob_storage_service + + fmt = stage_mod.validate_staged_source_file( + payload.filename, + payload.size_bytes, + file_format=_file_format, + max_upload_bytes=MAX_UPLOAD_BYTES, + ) + normalized_dataset = _normalize_dataset(payload.dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + call_import_id = uuid4() + content_type = _source_content_type(fmt) + s3_key, upload_url = stage_mod.build_staged_source_presign( + organization_id=organization_id, + call_import_id=call_import_id, + fmt=fmt, + content_type=content_type, + source_s3_key=_source_s3_key, + generate_presigned_put_url_by_key=blob_storage_service.generate_presigned_put_url_by_key, + ) + return CallImportSourceStagePresignResponse( + call_import_id=call_import_id, + source_s3_key=s3_key, + source_format=fmt, + upload_url=upload_url, + content_type=content_type, + ) + + +@router.post( + "/stage/complete", + response_model=CallImportResponse, + status_code=status.HTTP_201_CREATED, + operation_id="completeCallImportSourceStage", +) +async def complete_call_import_source_stage( + payload: CallImportSourceStageCompleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + del api_key + _ensure_blob_storage_enabled() + + from app.services.call_imports import staged_source_presign as stage_mod + from app.services.storage.blob_storage_service import blob_storage_service + + fmt = stage_mod.validate_staged_source_file( + payload.filename, + payload.size_bytes, + file_format=_file_format, + max_upload_bytes=MAX_UPLOAD_BYTES, + ) + normalized_dataset = _normalize_dataset(payload.dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + normalized_modality = _normalize_content_modality(payload.content_modality) + if payload.schema_id is not None: + _resolve_schema(db, organization_id, workspace_id, payload.schema_id) + tag_rows = _resolve_tags(db, organization_id, payload.tag_ids) + + s3_key = _source_s3_key(organization_id, payload.call_import_id, fmt) + content_type = _source_content_type(fmt) + + try: + file_bytes = stage_mod.load_staged_source_bytes( + source_s3_key=s3_key, + expected_size=payload.size_bytes, + head_object_by_key=blob_storage_service.head_object_by_key, + download_file_by_key=blob_storage_service.download_file_by_key, + ) + except HTTPException: + raise + except Exception as exc: + logger.exception( + "Failed to load staged source from S3 for call import {}", + payload.call_import_id, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Failed to read staged upload from storage: {exc}", + ) from exc + + existing = ( + db.query(CallImport) + .filter( + CallImport.id == payload.call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="This staged import id is already registered.", + ) + + try: + call_import = _insert_staged_call_import_record( + db, + call_import_id=payload.call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + principal=principal, + filename=payload.filename, + fmt=fmt, + s3_key=s3_key, + content_type=content_type, + file_bytes=file_bytes, + normalized_dataset=normalized_dataset, + schema_id=payload.schema_id, + normalized_modality=normalized_modality, + tag_rows=tag_rows, + ) + except Exception: + db.rollback() + raise + + return _serialize_call_import(db, call_import) + + +@router.post( + "", + response_model=CallImportResponse, + status_code=status.HTTP_201_CREATED, + operation_id="createCallImport", + deprecated=True, +) +async def create_call_import( + file: UploadFile = File( + ..., + description="CSV / Excel file to stage. Persisted to S3 between stages.", + ), + dataset: str = Form( + ..., + description=( + "Required free-text dataset label. Collected up-front so the " + "batch is filterable from the moment it lands." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + schema_id: Optional[UUID] = Form( + None, + description=( + "Optional schema pre-pick. The user can still change it during " + "the MAP stage; provided here only so the detail page can pre-" + "select the schema dropdown." + ), + ), + content_modality: Optional[str] = Form( + None, + description="Optional 'chat' for transcript-only post-prod chat imports; omit for voice/call imports.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """UPLOAD stage (legacy multipart). + + Prefer ``POST /stage/presign`` then browser PUT then ``POST /stage/complete`` + so the API never buffers the source file. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + if schema_id is not None: + _resolve_schema(db, organization_id, workspace_id, schema_id) + + normalized_modality = _normalize_content_modality(content_modality) + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + _ensure_blob_storage_enabled() + + call_import_id = uuid4() + s3_key = _source_s3_key(organization_id, call_import_id, fmt) + content_type = _source_content_type(fmt) + + from app.services.storage.s3_service import s3_service, StorageError + + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + logger.exception( + "Failed to upload source file to S3 for new call import {}", + call_import_id, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Failed to persist upload to S3: {exc}", + ) + + try: + call_import = _insert_staged_call_import_record( + db, + call_import_id=call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + principal=principal, + filename=file.filename or "upload.csv", + fmt=fmt, + s3_key=s3_key, + content_type=content_type, + file_bytes=file_bytes, + normalized_dataset=normalized_dataset, + schema_id=schema_id, + normalized_modality=normalized_modality, + tag_rows=tag_rows, + ) + except Exception: + db.rollback() + try: + s3_service.delete_file_by_key(s3_key) + except Exception as cleanup_exc: # noqa: BLE001 + logger.warning( + "Failed to clean up orphaned S3 object {} after DB rollback: {}", + s3_key, + cleanup_exc, + ) + raise + + return _serialize_call_import(db, call_import) + + +@router.patch( + "/{call_import_id}/mapping", + response_model=CallImportResponse, + operation_id="updateCallImportMapping", +) +async def update_call_import_mapping( + call_import_id: UUID, + payload: CallImportMappingUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """MAP stage of the staged call-import flow. + + Validates ``parameter_mapping`` + ``skipped_columns`` against the + sheet headers captured at UPLOAD time and persists them on the + batch. Idempotent: callers may submit this multiple times while + the batch is in ``uploaded`` or ``mapped`` state. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot edit mapping on a batch in status " + f"'{call_import.status.value}'. Mapping can only be edited " + "before the IMPORT stage." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch was not uploaded through the staged flow and " + "cannot have its mapping edited." + ), + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, payload.schema_id + ) + parameters = list(schema.parameters) + + canonical_sheet = _validate_sheet_choice( + call_import.source_format, + payload.sheet_name, + call_import.available_sheets, + ) + + # Pull the headers for the selected sheet straight out of the + # snapshot so we don't have to re-download the file from S3 just to + # validate the mapping. + headers: List[str] = [] + if call_import.available_sheets: + if canonical_sheet is None: + # CSV: single synthetic sheet. + entry = call_import.available_sheets[0] + headers = list(entry.get("headers") or []) + else: + for entry in call_import.available_sheets: + if not isinstance(entry, dict): + continue + name = entry.get("name") + if isinstance(name, str) and name == canonical_sheet: + headers = list(entry.get("headers") or []) + break + + cleaned_mapping = _clean_parameter_mapping( + payload.parameter_mapping, parameters, schema.name + ) + cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) + + # Run the same per-column validation as the parse path so the user + # gets an immediate 400 if a required parameter is left unmapped or + # a header is neither mapped nor skipped — without needing to read + # the file. ``validate_only`` skips the row loop (and the empty-rows + # guard) since the row data lives in S3, not in this request. + if headers: + _apply_schema_mapping( + headers, + iter(()), + parameters, + cleaned_mapping, + cleaned_skipped, + source_label=( + f"Sheet '{canonical_sheet}'" + if canonical_sheet is not None + else "CSV" + ), + validate_only=True, + ) + + call_import.schema_id = schema.id + call_import.parameter_mapping = dict(cleaned_mapping) + call_import.skipped_columns = list(cleaned_skipped) + call_import.sheet_name = canonical_sheet + call_import.status = CallImportStatus.MAPPED + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.post( + "/{call_import_id}/import", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="startCallImport", +) +async def start_call_import( + call_import_id: UUID, + payload: CallImportStartRequest, + background_tasks: BackgroundTasks, + legacy: bool = Query( + False, + description=( + "Deprecated escape hatch for import-only processing. " + "New batches should use Run Evaluation instead." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Deprecated IMPORT stage — use Run Evaluation for new batches. + + Recording fetch is part of the unified evaluation pipeline. This + endpoint remains available only with ``?legacy=true`` for backward + compatibility. + """ + del api_key, background_tasks + + if not legacy: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Standalone import is deprecated. Use Run Evaluation — " + "recording fetch is part of the evaluation pipeline. " + "Append ?legacy=true to use the import-only path." + ), + ) + + return _start_legacy_call_import( + db, + call_import_id, + organization_id, + workspace_id, + principal, + payload, + ) + + +@router.post( + "/upload", + status_code=status.HTTP_410_GONE, + operation_id="uploadCallImportCsv", + deprecated=True, +) +async def upload_call_import_csv_removed() -> None: + """Removed: use staged presign then MAP/IMPORT.""" + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail=( + "POST /call-imports/upload was removed. Stage the CSV with " + "POST /call-imports/stage/presign and POST /call-imports/stage/complete, " + "then PATCH /{id}/mapping and POST /{id}/import?legacy=true." + ), + ) + +@router.get( + "", + response_model=CallImportListResponse, + operation_id="listCallImports", +) +async def list_call_imports( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + status_filter: Optional[CallImportStatus] = Query(None, alias="status"), + dataset: Optional[str] = Query( + None, + description=( + "Filter by exact dataset string (case-insensitive). Pass the " + "literal value '__none__' to filter to imports with no dataset." + ), + ), + tag_id: Optional[List[UUID]] = Query( + None, + description="Filter to imports tagged with ALL of the given tag ids.", + ), + source_format: Optional[str] = Query( + None, + description=( + "Filter by source format. Use 'audio' for manual recordings or " + "'__non_audio__' for CSV/Excel/legacy imports." + ), + ), + content_modality: Optional[str] = Query( + None, + description="Filter by content_modality (e.g. 'chat'). Use '__voice__' for non-chat batches.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportListResponse: + """List call-import batches for the active workspace, newest first. + + Scoped to (organization_id, workspace_id) so users only see imports + for the workspace they're currently in. Supports a high-level + ``dataset`` filter (powers the segregation dropdown at the top of + the imports page) plus an AND-style multi-tag filter via repeated + ``tag_id`` parameters. + """ + del api_key + return call_import_queries.list_call_imports_page( + db, + organization_id, + workspace_id, + page=page, + page_size=page_size, + status_filter=status_filter, + dataset=dataset, + tag_id=tag_id, + source_format=source_format, + content_modality=content_modality, + ) + + +@router.get( + "/dispatch-diagnostics", + response_model=CallImportDispatchDiagnosticsResponse, + operation_id="getCallImportDispatchDiagnostics", + dependencies=[Depends(require_admin)], +) +async def get_call_import_dispatch_diagnostics( + workspace_id: Optional[UUID] = Query( + None, + description=( + "Optional workspace filter. When omitted, returns every workspace " + "in the organization with active eval dispatch state." + ), + ), + include_idle_workspaces: bool = Query( + False, + description=( + "When true, include org workspaces with zero pending rows and " + "zero in-flight slots." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDispatchDiagnosticsResponse: + """Live eval slot usage and fair-dispatch state for operators. + + Org admins use this to diagnose cross-workspace starvation (e.g. one + workspace's 10k run blocking another's pending eval rows) by inspecting + Redis in-flight counters, pending dispatch rows, and scheduler cursors. + """ + del api_key + payload = call_import_queries.get_call_import_dispatch_diagnostics_payload( + db, + organization_id, + workspace_id=workspace_id, + include_idle_workspaces=include_idle_workspaces, + ) + return CallImportDispatchDiagnosticsResponse.model_validate(payload) + + +@router.get( + "/datasets", + response_model=List[str], + operation_id="listCallImportDatasets", +) +async def list_call_import_datasets( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> List[str]: + """Return the distinct, non-null dataset labels in use for the active + workspace. + + Scoped per-workspace so each workspace's Dataset dropdown only shows + its own segregation labels. + """ + del api_key + return call_import_queries.list_call_import_dataset_labels( + db, organization_id, workspace_id + ) + + +@router.get( + "/diarisation-prompt-default", + response_model=CallImportDiarisationPromptDefaultResponse, + operation_id="getCallImportDiarisationPromptDefault", +) +async def get_call_import_diarisation_prompt_default( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), +) -> CallImportDiarisationPromptDefaultResponse: + """Return the canonical LLM diariser prompt. + + The Transcribe / Run Evaluation modals call this on open so they + can pre-fill the prompt textarea. Returning the constant from the + backend (rather than hard-coding it in the frontend) keeps the + fallback used by the worker and the placeholder shown in the UI + in lock-step — operators always see the *actual* default they'd + get if they leave the field blank. + + Registered before ``GET /{call_import_id}`` so the static path is + not mistaken for a UUID import id (which would 422). + """ + del api_key, organization_id + from app.workers.tasks.helpers.llm_diarisation import ( + DEFAULT_DIARIZATION_PROMPT, + ) + + return CallImportDiarisationPromptDefaultResponse( + prompt=DEFAULT_DIARIZATION_PROMPT + ) + + +@router.patch( + "/{call_import_id}", + response_model=CallImportResponse, + operation_id="updateCallImport", +) +async def update_call_import( + call_import_id: UUID, + payload: CallImportUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """Edit dataset / tag assignments (and schema, pre-import) on a batch. + + ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag + assignments. Fields omitted from the body are left untouched. + + ``schema_id`` is only honoured while the batch is in + ``uploaded`` / ``mapped`` state — once rows have been materialised + the schema is locked. Changing the schema resets any persisted + mapping (the user must re-MAP) and rewinds status to ``uploaded``. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + body = payload.model_dump(exclude_unset=True) + if "original_filename" in body: + call_import.original_filename = _normalize_import_display_name( + body["original_filename"] + ) + + if "dataset" in body: + call_import.dataset = _normalize_dataset(body["dataset"]) + + if "tag_ids" in body: + tag_ids = body["tag_ids"] or [] + call_import.tags = _resolve_tags(db, organization_id, tag_ids) + + if "schema_id" in body and body["schema_id"] is not None: + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot reassign schema on a batch in status " + f"'{call_import.status.value}'." + ), + ) + new_schema = _resolve_schema( + db, organization_id, workspace_id, body["schema_id"] + ) + if call_import.schema_id != new_schema.id: + # Switching schemas invalidates the persisted mapping — + # parameter names won't line up with the new schema, so + # reset to UPLOADED and force a fresh MAP. + call_import.schema_id = new_schema.id + call_import.parameter_mapping = {} + call_import.skipped_columns = [] + call_import.sheet_name = None + call_import.status = CallImportStatus.UPLOADED + + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.get( + "/{call_import_id}", + response_model=CallImportDetailResponse, + operation_id="getCallImportDetail", +) +async def get_call_import_detail( + call_import_id: UUID, + row_limit: int = Query(500, ge=0, le=5000), + row_offset: int = Query(0, ge=0), + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. When set, ``filtered_total_rows`` in " + "the response reflects the post-filter row count so the UI " + "can paginate against the filtered slice." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts one of ``pending``, ``running``, ``completed``, " + "``failed``. When set, ``filtered_total_rows`` reflects the " + "post-filter row count (combined with the ``q`` filter when " + "both are supplied) so the UI can paginate against the same " + "slice it's displaying." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDetailResponse: + """Fetch a single import batch with a slice of its rows. + + ``row_limit=0`` is intentionally allowed so callers that only need the + batch metadata (e.g. the evaluation-detail page rendering the parent's + column mapping) can skip the rows payload entirely. + """ + del api_key + return call_import_queries.get_call_import_detail( + db, + call_import_id, + organization_id, + row_limit=row_limit, + row_offset=row_offset, + q=q, + diarised_status=diarised_status, + ) + + +@router.get( + "/{call_import_id}/row-ids", + response_model=CallImportRowIdsResponse, + operation_id="listCallImportRowIds", +) +async def list_call_import_row_ids( + call_import_id: UUID, + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. Same semantics as the detail endpoint." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportRowIdsResponse: + """Return every matching ``CallImportRow.id`` for cross-page bulk select. + + Lightweight companion to ``GET /{call_import_id}`` — the detail + endpoint caps ``row_limit`` at 5000 and ships the entire row body + on each page, so harvesting ids that way is wasteful when the + user just wants to bulk-delete or bulk-transcribe everything that + matches the current filters. This endpoint applies the same ``q`` + and ``diarised_status`` filters and returns only the ids. + """ + del api_key + return call_import_queries.list_call_import_row_ids( + db, + call_import_id, + organization_id, + q=q, + diarised_status=diarised_status, + ) + +@router.delete( + "/{call_import_id}", + response_model=CallImportDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="deleteCallImport", +) +async def delete_call_import( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportDeleteResponse: + """Delete a call-import batch asynchronously. + + Flips the batch to ``deleting`` and enqueues background teardown so + large imports (thousands of rows + S3 objects) do not block the API. + """ + del api_key + return _delete_call_import_service( + db, call_import_id, organization_id, principal + ) + + +@router.get( + "/{call_import_id}/rows/{row_id}", + response_model=CallImportRowResponse, + operation_id="getCallImportRow", +) +async def get_call_import_row( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """One import row, including transcript text omitted from the list.""" + del api_key + return call_import_queries.get_call_import_row( + db, + call_import_id, + row_id, + organization_id, + ) + + +@router.delete( + "/{call_import_id}/rows/{row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportRow", +) +async def delete_call_import_row( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single CallImportRow and its S3 recording. + + The parent ``CallImport`` is left in place. After deletion we recompute + its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` + so the UI's progress bar stays consistent with reality. + """ + del api_key + _delete_call_import_row_service( + db, call_import_id, row_id, organization_id, principal + ) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post( + "/{call_import_id}/retry-failed", + response_model=CallImportRetryFailedRowsResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryFailedCallImportRows", +) +async def retry_failed_call_import_rows( + call_import_id: UUID, + payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRetryFailedRowsResponse: + """Re-enqueue every failed import row in this batch. + + Useful when transient provider issues are resolved and the operator wants + a one-click "try failed downloads again" pass without re-uploading the CSV. + + Pass ``provider`` + ``telephony_integration_id`` (or both omitted for + direct-URL retry) to change how recordings are fetched on this pass. + When the body is omitted entirely, the batch keeps its existing pinned + credentials. + """ + del api_key + return _retry_failed_call_import_rows( + db, call_import_id, organization_id, principal, payload + ) + + +@router.post( + "/{call_import_id}/rows/bulk-delete", + response_model=CallImportRowBulkDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="bulkDeleteCallImportRows", +) +async def bulk_delete_call_import_rows( + call_import_id: UUID, + payload: CallImportRowBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowBulkDeleteResponse: + """Delete multiple ``CallImportRow`` rows in one request. + + Unknown / cross-tenant row ids are silently skipped — the response + reports how many actually went away so a UI that holds onto stale + ids (e.g. after another tab already deleted a row) doesn't 404 + the entire bulk action. + """ + del api_key + return _bulk_delete_call_import_rows( + db, call_import_id, organization_id, principal, payload + ) + + +@router.get( + "/{call_import_id}/insights", + response_model=CallImportInsightsResponse, + operation_id="getCallImportInsights", +) +async def get_call_import_insights( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportInsightsResponse: + """Aggregate signals across every evaluation run on this import. + + Powers the Insights tab on the call-import detail page: returns + per-metric "latest run" summaries plus a trend series of mean values + across runs so the UI can render a small line chart per metric. Also + bundles transcript coverage stats since those are the cheapest + pre-eval health-check (e.g. "30 of 50 rows still missing + transcripts"). + """ + + del api_key + return call_import_queries.build_call_import_insights( + db, call_import_id, organization_id + ) + diff --git a/app/api/v1/routes/call_imports/diarisation_routes.py b/app/api/v1/routes/call_imports/diarisation_routes.py new file mode 100644 index 00000000..cd1343cb --- /dev/null +++ b/app/api/v1/routes/call_imports/diarisation_routes.py @@ -0,0 +1,508 @@ +"""CSV-driven call import routes (package).""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.services.call_imports import diarisation_route_helpers as diarisation_helpers +from app.services.call_imports import manual_audio_route_helpers as manual_audio_helpers +from app.services.call_imports import present as call_import_present +from app.services.call_imports import source_parse as call_import_source_parse +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportManualAudioCompleteRequest, + CallImportManualAudioCompleteResponse, + CallImportManualAudioPresignRequest, + CallImportManualAudioPresignResponse, + CallImportManualAudioPresignUploadItem, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportSourceStageCompleteRequest, + CallImportSourceStagePresignRequest, + CallImportSourceStagePresignResponse, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, + PlatformImportAgentRef, +) + +from app.api.v1.routes.call_imports._router import router + +from app.api.v1.routes.call_imports.core_routes import ( + _apply_diarisation_cancel, + _cancellable_diarisation_states, + _locate_call_import_row_or_404, + _render_diarised_segments_text, +) + +def _select_rows_for_transcription( + db: Session, + call_import: CallImport, + payload: CallImportTranscribeRequest, + requested_row_ids: Optional[List[UUID]] = None, +) -> tuple[List[CallImportRow], Dict[str, int]]: + """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" + from app.services.call_imports.bulk_ops import select_rows_for_transcription + + try: + return select_rows_for_transcription( + db, call_import, payload, requested_row_ids=requested_row_ids + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + +@router.post( + "/{call_import_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImport", +) +async def transcribe_call_import( + call_import_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Fan out diarization tasks for many rows in a single call. + + Returns a summary with how many rows were queued and how many were + skipped (broken down by reason) so the UI can show a meaningful + toast even when nothing actually got enqueued (e.g. "All 12 rows + already have transcripts"). + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_diarize_call_import_task.delay( + str(call_import_id), + str(organization_id), + payload.model_dump(mode="json"), + [str(rid) for rid in payload.row_ids] if payload.row_ids else None, + ) + + return CallImportTranscribeResponse( + queued=0, + skipped_rows=0, + skipped_reason_counts={}, + accepted=True, + ) + + +@router.post( + "/{call_import_id}/rows/{row_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImportRow", +) +async def transcribe_call_import_row( + call_import_id: UUID, + row_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Diarize / transcribe a single row. + + Thin wrapper over the batch endpoint that hard-codes a single + ``row_ids`` filter. Skip counts still surface so the UI can render + "Skipped — transcript present" diagnostics consistently. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.services.call_imports.bulk_ops import execute_bulk_diarization + + try: + result = execute_bulk_diarization( + db, + call_import, + payload, + requested_row_ids=[row_id], + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.workers.concurrency.fair_diarization_dispatch import ( + drain_after_diarization_enqueue, + ) + + drain_after_diarization_enqueue( + result.queued, + log_context=( + f"transcribe_call_import_row import={call_import_id} row={row_id}" + ), + ) + + return CallImportTranscribeResponse( + queued=result.queued, + skipped_rows=result.skipped_rows, + skipped_reason_counts=result.skipped_reason_counts, + ) + + +# --------------------------------------------------------------------------- +# Cancel-in-flight diarisation +# --------------------------------------------------------------------------- +# +# Long-running multimodal LLM diarisation calls (especially LLM-only mode on +# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an +# upstream provider stalls. Without an abort affordance the operator's only +# recourse is to wait for Celery's ``time_limit`` to fire — which can be +# several minutes — or to manually mutate the DB. These helpers + the two +# endpoints below give the UI a first-class "Stop diarisation" button. +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` +# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the +# default but we spell it out so the intent is obvious to reviewers. + +# Cancel helpers live in ``app.services.call_imports.diarisation_route_helpers`` +# (re-exported at module top for ``bulk_ops`` / evaluation routes). + +@router.post( + "/{call_import_id}/rows/{row_id}/cancel-diarisation", + response_model=CallImportRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportRowDiarisation", +) +async def cancel_call_import_row_diarisation( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Abort an in-flight (or queued) diarisation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed`` / ``idle``) returns the row unchanged with a 200, so + the UI can fire this from a "Stop" button without having to + pre-check the state. + + Race notes: + + * The row's ``diarised_transcript_status`` is flipped to ``failed`` + with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, + so the polling UI sees the cancel immediately. + * If the worker happens to finish between our DB flip and the + SIGTERM landing, its finaliser will detect the cancelled + sentinel on the row and skip its own status / score writes + (see :mod:`app.workers.tasks.transcribe_call_import_row`). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _apply_diarisation_cancel([row]) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +@router.post( + "/{call_import_id}/cancel-diarisation", + response_model=CallImportCancelDiarisationResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportDiarisation", +) +async def cancel_call_import_diarisation( + call_import_id: UUID, + payload: Optional[CallImportCancelDiarisationRequest] = None, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportCancelDiarisationResponse: + """Abort in-flight diarisation for many rows in a single call. + + Default body (no ``row_ids``) cancels every row in this import + whose ``diarised_transcript_status`` is ``pending`` or + ``running`` — the "stop everything" button. Pass ``row_ids`` to + scope the cancel to the rows the operator has selected. + + Returns ``(cancelled, skipped)`` so the UI can render a tight + toast ("Cancelled 3 rows · 1 skipped (already completed)"). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + requested_ids = ( + payload.row_ids if payload and payload.row_ids is not None else None + ) + row_filter = payload.row_filter if payload else None + if row_filter is not None and requested_ids is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Pass either row_ids or row_filter, not both.", + ) + + from app.services.call_imports.bulk_ops import execute_bulk_diarisation_cancel + + if row_filter is not None: + from app.services.call_imports.row_selection import normalize_row_selection_filter + + search_term, diarised_status = normalize_row_selection_filter(row_filter) + cancelled, skipped = execute_bulk_diarisation_cancel( + db, + call_import_id, + search_term=search_term, + diarised_status_filter=diarised_status, + ) + elif requested_ids is not None: + cancelled, skipped = execute_bulk_diarisation_cancel( + db, + call_import_id, + row_ids=list(requested_ids), + ) + else: + cancelled, skipped = execute_bulk_diarisation_cancel( + db, + call_import_id, + cancel_all_inflight=True, + ) + + stamp_call_import_actor(call_import, principal) + db.commit() + return CallImportCancelDiarisationResponse( + cancelled=cancelled, + skipped=skipped, + ) + + +@router.post( + "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", + response_model=CallImportRowResponse, + operation_id="toggleCallImportRowSpeakerSwap", +) +async def toggle_call_import_row_speaker_swap( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Flip the user <-> agent mapping on a diarised row. + + The worker's "first speaker is the agent" heuristic is right most of + the time but does fail on inbound recordings where the customer + greets first, on recordings where the agent stays silent for the + intro, etc. Rather than rerun the (expensive) STT + pyannote + pipeline for those cases, we let reviewers flip the mapping in + place: the structured ``diarised_segments`` are the source of truth + and we re-render the plain-text ``diarised_transcript`` from them + with the swap applied. The next CSV export will then show the + corrected labels. + + Returns the updated row so the frontend can refresh without an + extra round-trip. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + segments = ( + row.diarised_segments if isinstance(row.diarised_segments, list) else None + ) + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), + ) + + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None + ) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + diff --git a/app/api/v1/routes/evaluator_result_metric_clusters.py b/app/api/v1/routes/evaluator_result_metric_clusters.py index b9cb46fb..25847696 100644 --- a/app/api/v1/routes/evaluator_result_metric_clusters.py +++ b/app/api/v1/routes/evaluator_result_metric_clusters.py @@ -189,9 +189,9 @@ def _revoke_cluster_task(job) -> None: if not task_id: return try: - from app.workers.celery_app import celery_app + from app.workers.celery_revoke import revoke_celery_tasks - celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") + revoke_celery_tasks([task_id]) logger.info("Revoked evaluator metric-clusters task {} for job {}", task_id, job.id) except Exception as exc: # noqa: BLE001 logger.warning( diff --git a/app/api/v1/routes/knowledge_bases.py b/app/api/v1/routes/knowledge_bases.py new file mode 100644 index 00000000..616bb596 --- /dev/null +++ b/app/api/v1/routes/knowledge_bases.py @@ -0,0 +1,653 @@ +"""Knowledge base CRUD and document upload.""" + +from __future__ import annotations + +from typing import List, Optional +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status +from fastapi.responses import Response +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.core.auth import Principal, get_principal +from app.dependencies import get_db, get_organization_id, get_workspace_id +from app.models.database import AgentKnowledgeBase, KnowledgeBase, KnowledgeDocument +from app.models.schemas import ( + EmbeddingConfigHelpResponse, + EmbeddingModelOption, + EmbeddingProviderOption, + KnowledgeBaseCreate, + KnowledgeBaseDetailResponse, + KnowledgeBaseSummaryResponse, + KnowledgeBaseUpdate, + KnowledgeDocumentPagePreview, + KnowledgeDocumentResponse, + KnowledgeDocumentViewResponse, +) +from app.services.knowledge_bases.document_view import build_document_view_payload +from app.services.knowledge_bases.embedding_config import ( + embedding_profile_changed, + parse_embedding_fields, + profile_from_kb, +) +from app.services.knowledge_bases.embeddings_catalog import ( + DEFAULT_EMBEDDING_MODEL, + DEFAULT_EMBEDDING_PROVIDER, + EMBEDDING_CAPABLE_PROVIDERS, + EMBEDDING_DIMENSIONS, + EMBEDDING_MODEL_CHOICES, + PROVIDER_LABELS, + SUGGESTED_EMBEDDING_MODELS, +) +from app.services.knowledge_bases.paths import ( + MAX_KB_UPLOAD_BYTES, + knowledge_document_blob_key, + validate_kb_upload_filename, +) +from app.services.storage.blob_storage_service import blob_storage_service +from app.utils.upload_limits import UploadTooLargeError, read_upload_with_limit + +router = APIRouter(prefix="/knowledge-bases", tags=["knowledge-bases"]) + + +def _kb_detail_fields(row: KnowledgeBase) -> dict: + profile = profile_from_kb(row) + return { + "embedding_provider": profile.provider, + "embedding_model": profile.model, + "embedding_credential_id": profile.credential_id, + } + + +def _apply_embedding_patch( + db: Session, + *, + row: KnowledgeBase, + organization_id: UUID, + data: dict, +) -> bool: + """Apply embedding fields from PATCH payload; return True if re-index needed.""" + embedding_keys = { + "embedding_provider", + "embedding_model", + "embedding_credential_id", + } + if not embedding_keys.intersection(data.keys()): + return False + current = profile_from_kb(row) + provider = ( + data["embedding_provider"] + if "embedding_provider" in data and data["embedding_provider"] is not None + else current.provider + ) + model = ( + data["embedding_model"] + if "embedding_model" in data and data["embedding_model"] is not None + else current.model + ) + if "embedding_credential_id" in data: + credential_id = data["embedding_credential_id"] + else: + credential_id = current.credential_id + try: + profile = parse_embedding_fields( + db, + organization_id=organization_id, + embedding_provider=provider, + embedding_model=model, + embedding_credential_id=credential_id, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + changed = embedding_profile_changed(row, profile) + row.embedding_provider = profile.provider + row.embedding_model = profile.model + row.embedding_credential_id = profile.credential_id + return changed + + +def _queue_reindex_documents(db: Session, kb_id: UUID) -> int: + # Skip documents already ingesting: ingest_knowledge_document re-queues itself if + # the KB embedding profile changes mid-run; overlapping tasks caused duplicate chunks + # before per-document advisory locks (see ingest.py). + docs = ( + db.query(KnowledgeDocument) + .filter( + KnowledgeDocument.knowledge_base_id == kb_id, + KnowledgeDocument.status.in_(("ready", "failed")), + ) + .all() + ) + if not docs: + return 0 + from app.workers.tasks.ingest_knowledge_document import ingest_knowledge_document_task + + for doc in docs: + doc.status = "pending" + doc.error_text = None + db.commit() + for doc in docs: + ingest_knowledge_document_task.delay(str(doc.id)) + return len(docs) + + +def _get_document_or_404( + db: Session, + *, + knowledge_base_id: UUID, + document_id: UUID, + organization_id: UUID, + workspace_id: UUID, +) -> KnowledgeDocument: + doc = ( + db.query(KnowledgeDocument) + .filter( + KnowledgeDocument.id == document_id, + KnowledgeDocument.knowledge_base_id == knowledge_base_id, + KnowledgeDocument.organization_id == organization_id, + KnowledgeDocument.workspace_id == workspace_id, + ) + .first() + ) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + return doc + + +def _document_counts(db: Session, kb_id: UUID) -> tuple[int, int, int]: + total = ( + db.query(func.count(KnowledgeDocument.id)) + .filter(KnowledgeDocument.knowledge_base_id == kb_id) + .scalar() + or 0 + ) + ready = ( + db.query(func.count(KnowledgeDocument.id)) + .filter( + KnowledgeDocument.knowledge_base_id == kb_id, + KnowledgeDocument.status == "ready", + ) + .scalar() + or 0 + ) + failed = ( + db.query(func.count(KnowledgeDocument.id)) + .filter( + KnowledgeDocument.knowledge_base_id == kb_id, + KnowledgeDocument.status == "failed", + ) + .scalar() + or 0 + ) + return int(total), int(ready), int(failed) + + +def _linked_agent_count(db: Session, kb_id: UUID) -> int: + return ( + db.query(func.count(func.distinct(AgentKnowledgeBase.agent_id))) + .filter(AgentKnowledgeBase.knowledge_base_id == kb_id) + .scalar() + or 0 + ) + + +def _get_kb_or_404( + db: Session, + *, + knowledge_base_id: UUID, + organization_id: UUID, + workspace_id: UUID, +) -> KnowledgeBase: + row = ( + db.query(KnowledgeBase) + .filter( + KnowledgeBase.id == knowledge_base_id, + KnowledgeBase.organization_id == organization_id, + KnowledgeBase.workspace_id == workspace_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Knowledge base not found") + return row + + +@router.post("", response_model=KnowledgeBaseDetailResponse, status_code=status.HTTP_201_CREATED) +async def create_knowledge_base( + payload: KnowledgeBaseCreate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + _principal: Principal = Depends(get_principal), +): + try: + profile = parse_embedding_fields( + db, + organization_id=organization_id, + embedding_provider=payload.embedding_provider, + embedding_model=payload.embedding_model, + embedding_credential_id=payload.embedding_credential_id, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + row = KnowledgeBase( + organization_id=organization_id, + workspace_id=workspace_id, + name=payload.name.strip(), + description=(payload.description or "").strip() or None, + embedding_provider=profile.provider, + embedding_model=profile.model, + embedding_credential_id=profile.credential_id, + status="active", + ) + db.add(row) + db.commit() + db.refresh(row) + return KnowledgeBaseDetailResponse( + id=row.id, + name=row.name, + description=row.description, + status=row.status, + documents=[], + linked_agent_count=0, + created_at=row.created_at, + updated_at=row.updated_at, + **_kb_detail_fields(row), + ) + + +@router.get("/embedding-models", response_model=List[EmbeddingModelOption]) +async def list_embedding_models(): + return [EmbeddingModelOption(id=model_id, label=label) for model_id, label in EMBEDDING_MODEL_CHOICES] + + +@router.get("/embedding-config", response_model=EmbeddingConfigHelpResponse) +async def get_embedding_config(): + providers: List[EmbeddingProviderOption] = [] + for provider in EMBEDDING_CAPABLE_PROVIDERS: + pid = provider.value + suggested = SUGGESTED_EMBEDDING_MODELS.get(pid, []) + if not suggested: + continue + providers.append( + EmbeddingProviderOption( + id=pid, + label=PROVIDER_LABELS.get(pid, pid), + suggested_models=[ + EmbeddingModelOption(id=model_id, label=label) + for model_id, label in suggested + ], + ) + ) + return EmbeddingConfigHelpResponse( + default_provider=DEFAULT_EMBEDDING_PROVIDER, + default_model=DEFAULT_EMBEDDING_MODEL, + required_dimensions=EMBEDDING_DIMENSIONS, + providers=providers, + ) + + +@router.get("", response_model=List[KnowledgeBaseSummaryResponse]) +async def list_knowledge_bases( + skip: int = 0, + limit: int = 100, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + rows = ( + db.query(KnowledgeBase) + .filter( + KnowledgeBase.organization_id == organization_id, + KnowledgeBase.workspace_id == workspace_id, + ) + .order_by(KnowledgeBase.updated_at.desc()) + .offset(skip) + .limit(limit) + .all() + ) + summaries: List[KnowledgeBaseSummaryResponse] = [] + for row in rows: + total, ready, failed = _document_counts(db, row.id) + summaries.append( + KnowledgeBaseSummaryResponse( + id=row.id, + name=row.name, + description=row.description, + status=row.status, + **_kb_detail_fields(row), + document_count=total, + ready_document_count=ready, + failed_document_count=failed, + linked_agent_count=_linked_agent_count(db, row.id), + created_at=row.created_at, + updated_at=row.updated_at, + ) + ) + return summaries + + +@router.get("/{knowledge_base_id}", response_model=KnowledgeBaseDetailResponse) +async def get_knowledge_base( + knowledge_base_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + row = _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + documents = ( + db.query(KnowledgeDocument) + .filter(KnowledgeDocument.knowledge_base_id == row.id) + .order_by(KnowledgeDocument.created_at.desc()) + .all() + ) + return KnowledgeBaseDetailResponse( + id=row.id, + name=row.name, + description=row.description, + status=row.status, + documents=[KnowledgeDocumentResponse.model_validate(doc) for doc in documents], + linked_agent_count=_linked_agent_count(db, row.id), + created_at=row.created_at, + updated_at=row.updated_at, + **_kb_detail_fields(row), + ) + + +@router.patch("/{knowledge_base_id}", response_model=KnowledgeBaseDetailResponse) +async def update_knowledge_base( + knowledge_base_id: UUID, + payload: KnowledgeBaseUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + row = _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + data = payload.model_dump(exclude_unset=True) + if "name" in data and data["name"] is not None: + row.name = data["name"].strip() + if "description" in data: + row.description = (data["description"] or "").strip() or None + if "status" in data and data["status"] is not None: + row.status = data["status"] + if _apply_embedding_patch( + db, row=row, organization_id=organization_id, data=data + ): + _queue_reindex_documents(db, row.id) + db.commit() + db.refresh(row) + return await get_knowledge_base(knowledge_base_id, organization_id, workspace_id, db) + + +@router.delete("/{knowledge_base_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_knowledge_base( + knowledge_base_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + row = _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + docs = ( + db.query(KnowledgeDocument) + .filter(KnowledgeDocument.knowledge_base_id == row.id) + .all() + ) + for doc in docs: + try: + blob_storage_service.delete_file_by_key(doc.blob_key) + except Exception: + pass + db.delete(row) + db.commit() + return None + + +@router.post( + "/{knowledge_base_id}/documents", + response_model=KnowledgeDocumentResponse, + status_code=status.HTTP_201_CREATED, +) +async def upload_knowledge_document( + knowledge_base_id: UUID, + file: UploadFile = File(...), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + _principal: Principal = Depends(get_principal), +): + kb = _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + filename = file.filename or "document.txt" + try: + extension, content_type = validate_kb_upload_filename(filename) + file_bytes = await read_upload_with_limit(file, MAX_KB_UPLOAD_BYTES) + except UploadTooLargeError as exc: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds maximum size of {exc.max_bytes} bytes", + ) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + document_id = uuid4() + blob_key = knowledge_document_blob_key( + organization_id, + workspace_id, + kb.id, + document_id, + extension, + ) + blob_storage_service.upload_file_by_key(file_bytes, blob_key, content_type=content_type) + + doc = KnowledgeDocument( + id=document_id, + knowledge_base_id=kb.id, + organization_id=organization_id, + workspace_id=workspace_id, + source_type="file", + original_filename=filename, + blob_key=blob_key, + content_type=content_type, + byte_size=len(file_bytes), + status="pending", + ) + db.add(doc) + db.commit() + db.refresh(doc) + + from app.workers.tasks.ingest_knowledge_document import ingest_knowledge_document_task + + ingest_knowledge_document_task.delay(str(doc.id)) + return KnowledgeDocumentResponse.model_validate(doc) + + +@router.post( + "/{knowledge_base_id}/documents/{document_id}/retry", + response_model=KnowledgeDocumentResponse, +) +async def retry_knowledge_document( + knowledge_base_id: UUID, + document_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + doc = ( + db.query(KnowledgeDocument) + .filter( + KnowledgeDocument.id == document_id, + KnowledgeDocument.knowledge_base_id == knowledge_base_id, + KnowledgeDocument.organization_id == organization_id, + KnowledgeDocument.workspace_id == workspace_id, + ) + .first() + ) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + if doc.status not in ("failed", "ready"): + raise HTTPException(status_code=400, detail="Document is not eligible for retry") + doc.status = "pending" + doc.error_text = None + db.commit() + db.refresh(doc) + from app.workers.tasks.ingest_knowledge_document import ingest_knowledge_document_task + + ingest_knowledge_document_task.delay(str(doc.id)) + return KnowledgeDocumentResponse.model_validate(doc) + + +@router.get( + "/{knowledge_base_id}/documents/{document_id}/view", + response_model=KnowledgeDocumentViewResponse, +) +async def view_knowledge_document( + knowledge_base_id: UUID, + document_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + kb = _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + doc = _get_document_or_404( + db, + knowledge_base_id=knowledge_base_id, + document_id=document_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + try: + payload = build_document_view_payload( + original_filename=doc.original_filename, + content_type=doc.content_type, + blob_key=doc.blob_key, + ) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Could not load document: {exc}") from exc + pages = payload.get("pages") + page_models = None + if pages: + page_models = [ + KnowledgeDocumentPagePreview(page=p.get("page"), text=p.get("text") or "") + for p in pages + ] + return KnowledgeDocumentViewResponse( + knowledge_base_id=kb.id, + knowledge_base_name=kb.name, + document_id=doc.id, + view_mode=str(payload.get("view_mode") or "binary"), + original_filename=payload.get("original_filename"), + content_type=payload.get("content_type"), + text_content=payload.get("text_content"), + pages=page_models, + page_count=payload.get("page_count"), + ) + + +@router.get("/{knowledge_base_id}/documents/{document_id}/download") +async def download_knowledge_document( + knowledge_base_id: UUID, + document_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + doc = _get_document_or_404( + db, + knowledge_base_id=knowledge_base_id, + document_id=document_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + try: + file_bytes = blob_storage_service.download_file_by_key(doc.blob_key) + except Exception as exc: + raise HTTPException(status_code=404, detail="File not found in storage") from exc + filename = doc.original_filename or f"document-{doc.id}" + media_type = doc.content_type or "application/octet-stream" + headers = {"Content-Disposition": f'inline; filename="{filename}"'} + return Response(content=file_bytes, media_type=media_type, headers=headers) + + +@router.delete( + "/{knowledge_base_id}/documents/{document_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_knowledge_document( + knowledge_base_id: UUID, + document_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + _get_kb_or_404( + db, + knowledge_base_id=knowledge_base_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + doc = ( + db.query(KnowledgeDocument) + .filter( + KnowledgeDocument.id == document_id, + KnowledgeDocument.knowledge_base_id == knowledge_base_id, + KnowledgeDocument.organization_id == organization_id, + KnowledgeDocument.workspace_id == workspace_id, + ) + .first() + ) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + try: + blob_storage_service.delete_file_by_key(doc.blob_key) + except Exception: + pass + db.delete(doc) + db.commit() + return None + + +from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=SIM_VIEW, + manage_capability=SIM_MANAGE, +) diff --git a/app/api/v1/routes/metrics.py b/app/api/v1/routes/metrics.py index a2ea93aa..ecb311b8 100644 --- a/app/api/v1/routes/metrics.py +++ b/app/api/v1/routes/metrics.py @@ -16,6 +16,17 @@ from app.core.oss_quotas import enforce_oss_quota from app.services.billing.flexprice_service import record_metrics_llm_assist from app.models.database import Metric, MetricCategory, MetricType, MetricTrigger, ModelProvider +from app.models.enums import MetricScoringMethod +from app.services.metrics.evaluation_scope import ( + agent_ids_for_response, + replace_metric_agents, + replace_metric_scenarios, + scenario_ids_for_response, +) +from app.services.metrics.option_labels import ( + coerce_custom_config_for_storage, + custom_config_for_api, +) from app.models.schemas import ( MetricCreate, MetricCreateWithChildren, @@ -196,7 +207,9 @@ def _serialize_metric_tree(metric: Metric) -> Dict[str, Any]: "supported_surfaces": metric.supported_surfaces or [], "enabled_surfaces": metric.enabled_surfaces or [], "custom_data_type": metric.custom_data_type, - "custom_config": metric.custom_config, + "custom_config": custom_config_for_api( + metric.custom_data_type, metric.custom_config + ), "tags": metric.tags, "capture_rationale": bool(metric.capture_rationale), "parent_metric_id": metric.parent_metric_id, @@ -205,10 +218,15 @@ def _serialize_metric_tree(metric: Metric) -> Dict[str, Any]: "compare_transcripts": bool( getattr(metric, "compare_transcripts", False) ), + "scoring_method": ( + getattr(metric, "scoring_method", None) or "llm" + ), "lifecycle": getattr(metric, "lifecycle", None) or "active", "promoted_from_draft_at": getattr(metric, "promoted_from_draft_at", None), "studio_notes": getattr(metric, "studio_notes", None), "children": children_payload, + "scenario_ids": [str(sid) for sid in scenario_ids_for_response(metric)], + "agent_ids": [str(aid) for aid in agent_ids_for_response(metric)], "created_at": metric.created_at, "updated_at": metric.updated_at, "created_by": metric.created_by, @@ -317,6 +335,16 @@ def create_metric( effective_metric_type = metric_data.metric_type if metric_data.parent_metric_id is not None: effective_metric_type = MetricType.BOOLEAN + if metric_data.parent_metric_id is not None and metric_data.scenario_ids: + raise HTTPException( + status_code=400, + detail="Child metrics cannot have scenario links; set links on the parent.", + ) + if metric_data.parent_metric_id is not None and metric_data.agent_ids: + raise HTTPException( + status_code=400, + detail="Child metrics cannot have agent links; set links on the parent.", + ) metric = Metric( organization_id=organization_id, @@ -333,7 +361,9 @@ def create_metric( supported_surfaces=metric_data.supported_surfaces or ["agent"], enabled_surfaces=enabled_surfaces, custom_data_type=metric_data.custom_data_type, - custom_config=metric_data.custom_config, + custom_config=coerce_custom_config_for_storage( + metric_data.custom_data_type, metric_data.custom_config + ), tags=metric_data.tags, capture_rationale=bool(metric_data.capture_rationale), parent_metric_id=metric_data.parent_metric_id, @@ -342,6 +372,28 @@ def create_metric( compare_transcripts=bool(metric_data.compare_transcripts), ) db.add(metric) + db.flush() + if metric.parent_metric_id is None: + try: + replace_metric_scenarios( + db, + organization_id=organization_id, + metric=metric, + scenario_ids=( + metric_data.scenario_ids if metric_data.scenario_ids is not None else [] + ), + ) + replace_metric_agents( + db, + organization_id=organization_id, + metric=metric, + agent_ids=( + metric_data.agent_ids if metric_data.agent_ids is not None else [] + ), + ) + except ValueError as exc: + db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc db.commit() db.refresh(metric) @@ -417,6 +469,16 @@ def create_metric_draft( effective_metric_type = metric_data.metric_type if metric_data.parent_metric_id is not None: effective_metric_type = MetricType.BOOLEAN + if metric_data.parent_metric_id is not None and metric_data.scenario_ids: + raise HTTPException( + status_code=400, + detail="Child metrics cannot have scenario links; set links on the parent.", + ) + if metric_data.parent_metric_id is not None and metric_data.agent_ids: + raise HTTPException( + status_code=400, + detail="Child metrics cannot have agent links; set links on the parent.", + ) metric = Metric( organization_id=organization_id, @@ -433,7 +495,9 @@ def create_metric_draft( supported_surfaces=metric_data.supported_surfaces or ["agent"], enabled_surfaces=[], custom_data_type=metric_data.custom_data_type, - custom_config=metric_data.custom_config, + custom_config=coerce_custom_config_for_storage( + metric_data.custom_data_type, metric_data.custom_config + ), tags=metric_data.tags, capture_rationale=bool(metric_data.capture_rationale), parent_metric_id=metric_data.parent_metric_id, @@ -444,6 +508,28 @@ def create_metric_draft( studio_notes=metric_data.studio_notes, ) db.add(metric) + db.flush() + if metric.parent_metric_id is None: + try: + replace_metric_scenarios( + db, + organization_id=organization_id, + metric=metric, + scenario_ids=( + metric_data.scenario_ids if metric_data.scenario_ids is not None else [] + ), + ) + replace_metric_agents( + db, + organization_id=organization_id, + metric=metric, + agent_ids=( + metric_data.agent_ids if metric_data.agent_ids is not None else [] + ), + ) + except ValueError as exc: + db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc db.commit() db.refresh(metric) return _serialize_metric_tree(metric) @@ -553,6 +639,22 @@ def _create_metric_with_children( ) db.add(parent) db.flush() + try: + replace_metric_scenarios( + db, + organization_id=organization_id, + metric=parent, + scenario_ids=payload.scenario_ids if payload.scenario_ids is not None else [], + ) + replace_metric_agents( + db, + organization_id=organization_id, + metric=parent, + agent_ids=payload.agent_ids if payload.agent_ids is not None else [], + ) + except ValueError as exc: + db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc for child_draft in payload.children: child_enabled = bool(child_draft.enabled) and len(parent_enabled_surfaces) > 0 @@ -1025,7 +1127,9 @@ def promote_discovered_metric( supported_surfaces=supported_surfaces, enabled_surfaces=enabled_surfaces, custom_data_type=custom_data_type, - custom_config=body.custom_config or None, + custom_config=coerce_custom_config_for_storage( + custom_data_type, body.custom_config or None + ), tags=None, # Standalone metric — rationale is per-metric (default true so # the LLM keeps producing rationales for promoted candidates, @@ -1273,7 +1377,14 @@ def update_metric( metric.custom_data_type = metric_data.custom_data_type if metric_data.custom_config is not None: - metric.custom_config = metric_data.custom_config + effective_dtype = ( + metric_data.custom_data_type + if metric_data.custom_data_type is not None + else metric.custom_data_type + ) + metric.custom_config = coerce_custom_config_for_storage( + effective_dtype, metric_data.custom_config + ) if metric_data.tags is not None: metric.tags = metric_data.tags @@ -1402,6 +1513,42 @@ def update_metric( db.rollback() raise + if metric_data.scenario_ids is not None: + if metric.parent_metric_id is not None: + db.rollback() + raise HTTPException( + status_code=400, + detail="Child metrics cannot have scenario links; update the parent.", + ) + try: + replace_metric_scenarios( + db, + organization_id=organization_id, + metric=metric, + scenario_ids=metric_data.scenario_ids, + ) + except ValueError as exc: + db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if metric_data.agent_ids is not None: + if metric.parent_metric_id is not None: + db.rollback() + raise HTTPException( + status_code=400, + detail="Child metrics cannot have agent links; update the parent.", + ) + try: + replace_metric_agents( + db, + organization_id=organization_id, + metric=metric, + agent_ids=metric_data.agent_ids, + ) + except ValueError as exc: + db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + db.commit() db.refresh(metric) @@ -1606,6 +1753,41 @@ def seed_default_metrics( "supported_surfaces": ["voice_playground"], "enabled_surfaces": ["voice_playground"], }, + # ========================================================================= + # Transcript agreement (deterministic — no LLM cost) + # ========================================================================= + { + "name": "User WER", + "description": ( + "Word error rate of diarised user turns (hypothesis) against " + "production user turns (reference). Agent turns are ignored. " + "Secondary to User CER. Lower is better." + ), + "metric_type": MetricType.NUMBER, + "trigger": MetricTrigger.ALWAYS, + "enabled": True, + "metric_origin": "default", + "supported_surfaces": ["agent"], + "enabled_surfaces": ["agent"], + "tags": ["transcript_accuracy"], + "scoring_method": MetricScoringMethod.USER_WER.value, + }, + { + "name": "User CER", + "description": ( + "Character error rate of diarised user turns (hypothesis) " + "against production user turns (reference). Agent turns are " + "ignored. Headline transcript-agreement metric. Lower is better." + ), + "metric_type": MetricType.NUMBER, + "trigger": MetricTrigger.ALWAYS, + "enabled": True, + "metric_origin": "default", + "supported_surfaces": ["agent"], + "enabled_surfaces": ["agent"], + "tags": ["transcript_accuracy"], + "scoring_method": MetricScoringMethod.USER_CER.value, + }, ] # Names of default voice metrics that must always be enabled on the @@ -1648,6 +1830,10 @@ def seed_default_metrics( metric_origin=metric_data.get("metric_origin", "default"), supported_surfaces=metric_data.get("supported_surfaces", ["agent"]), enabled_surfaces=metric_data.get("enabled_surfaces", ["agent"]), + tags=metric_data.get("tags"), + scoring_method=metric_data.get( + "scoring_method", MetricScoringMethod.LLM.value + ), ) db.add(metric) created_metrics.append(metric) @@ -1655,6 +1841,14 @@ def seed_default_metrics( # Keep default acoustic metric toggles aligned with product defaults. if existing.enabled != metric_data["enabled"]: existing.enabled = metric_data["enabled"] + if metric_data.get("scoring_method"): + existing.scoring_method = metric_data["scoring_method"] + if metric_data["name"] in {"User WER", "User CER"} and metric_data.get( + "description" + ): + existing.description = metric_data["description"] + if metric_data.get("tags") and not existing.tags: + existing.tags = metric_data["tags"] # Re-assert voice_playground surface enrollment for the four required # voice metrics so existing orgs pick up the new default behavior. if metric_data["name"] in voice_playground_required_defaults: @@ -1920,6 +2114,8 @@ def generate_metric( if custom_data_type == "boolean": custom_config = {} + custom_config = coerce_custom_config_for_storage(custom_data_type, custom_config) + name = (parsed.get("name") or "Custom Metric").strip()[:60] existing = db.query(Metric).filter( and_(Metric.name == name, Metric.organization_id == organization_id) diff --git a/app/api/v1/routes/org_usage.py b/app/api/v1/routes/org_usage.py index 53dbbb27..82f4ca8b 100644 --- a/app/api/v1/routes/org_usage.py +++ b/app/api/v1/routes/org_usage.py @@ -5,7 +5,7 @@ from datetime import date, datetime, timezone from app.services.usage.access import UsageAccessPolicy from app.services.usage.dates import usage_date_filter_bounds, usage_local_today -from typing import List, Literal, Optional +from typing import Any, List, Literal, Optional from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query @@ -19,8 +19,6 @@ from app.models.database import ( CallImport, CallImportEvaluation, - CallImportEvaluationRow, - CallImportRow, CallImportTag, CallImportTagAssignment, Agent, @@ -28,9 +26,15 @@ Workspace, ) from app.services.usage.read_cache import ( + acquire_read_lock, cache_key_for, + cache_ttl_seconds, get_cached_response, + get_stale_response, + read_cache_version, + release_read_lock, set_cached_response, + wait_for_cached_response, ) from app.services.usage.fx_rates import get_usd_inr_rate from app.services.usage.usage_costs import costs_from_micro @@ -72,7 +76,11 @@ "other": "Other", } -_LABEL_ROW_LIMIT = 5000 +_FILTER_OPTION_LIMIT = 500 +_UUID_RE = ( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + "[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) def _usage_row_weight(): @@ -87,13 +95,6 @@ def _usage_row_weight(): ) -def _label_row_order(): - return ( - _usage_row_weight().desc(), - LLMUsageDaily.call_count.desc(), - ) - - class UsageCosts(BaseModel): input_cost_usd: float = 0 output_cost_usd: float = 0 @@ -220,6 +221,43 @@ def _parse_usage_range( return display_start, display_end, filter_start, filter_end +def _usage_range_is_historical(access) -> bool: + """Closed in UTC. A timezone behind UTC can still be 'today' locally.""" + return access.filter_end < datetime.now(timezone.utc).date() + + +def _prepare_usage_read(organization_id, endpoint, cache_key, model): + """Return a cached payload, or (None, leader, version) when this request should query. + + ``version`` is read before the database work so a flush during the query + cannot store the old totals under the new cache epoch. + """ + version = read_cache_version(organization_id) + cached = get_cached_response(organization_id, endpoint, cache_key, version) + if cached is not None: + try: + return model.model_validate(cached), False, version + except Exception: + pass + if cache_ttl_seconds() <= 0: + return None, False, version + if acquire_read_lock(organization_id, endpoint, cache_key): + return None, True, version + stale = get_stale_response(organization_id, endpoint, cache_key) + if stale is not None: + try: + return model.model_validate(stale), False, version + except Exception: + pass + waited = wait_for_cached_response(organization_id, endpoint, cache_key) + if waited is not None: + try: + return model.model_validate(waited), False, version + except Exception: + pass + return None, False, version + + def _usage_policy_meta(access) -> UsagePolicyMeta: return UsagePolicyMeta( extended_history=access.policy.extended_history, @@ -268,39 +306,8 @@ def _resource_type_expr(): ) -def _call_import_group_expr(): - """Resolve call import id from context, resource row, evaluation, or import row.""" - eval_call_import = ( - select(CallImportEvaluation.call_import_id) - .where( - CallImportEvaluation.id == cast(_evaluation_id_expr(), PG_UUID) - ) - .correlate(LLMUsageDaily) - .scalar_subquery() - ) - row_call_import = ( - select(CallImportRow.call_import_id) - .where( - CallImportRow.id - == cast(LLMUsageDaily.context["call_import_row_id"].astext, PG_UUID) - ) - .correlate(LLMUsageDaily) - .scalar_subquery() - ) - eval_row_call_import = ( - select(CallImportEvaluation.call_import_id) - .select_from(CallImportEvaluationRow) - .join( - CallImportEvaluation, - CallImportEvaluation.id == CallImportEvaluationRow.evaluation_id, - ) - .where( - CallImportEvaluationRow.id - == cast(LLMUsageDaily.context["evaluation_row_id"].astext, PG_UUID) - ) - .correlate(LLMUsageDaily) - .scalar_subquery() - ) +def _direct_call_import_id_expr(): + """Call import id already stored on the rollup. No per-row joins.""" return cast( func.coalesce( LLMUsageDaily.context["call_import_id"].astext, @@ -311,9 +318,24 @@ def _call_import_group_expr(): ), else_=None, ), - cast(eval_call_import, String), - cast(row_call_import, String), - cast(eval_row_call_import, String), + ), + String, + ) + + +def _evaluation_id_for_import_expr(): + """Evaluation id used to resolve a call import in one catalog lookup.""" + return cast( + func.coalesce( + _evaluation_id_expr(), + case( + ( + LLMUsageDaily.context["resource_type"].astext + == "call_import_evaluation", + LLMUsageDaily.context["resource_id"].astext, + ), + else_=None, + ), ), String, ) @@ -333,25 +355,20 @@ def _evaluation_scope_filter( organization_id: UUID, db: Session, ): + """Match an evaluation from indexed context keys. + + Legacy ``evaluation_row_id`` values are copied onto ``evaluation_id`` by + migration 119, so this does not scan call-import row tables. + """ + del organization_id, db eid = str(evaluation_id) - row_ids = [ - str(row[0]) - for row in db.query(CallImportEvaluationRow.id) - .filter( - CallImportEvaluationRow.evaluation_id == evaluation_id, - ) - .all() - ] - clauses = [ + return or_( LLMUsageDaily.context["evaluation_id"].astext == eid, and_( LLMUsageDaily.context["resource_id"].astext == eid, LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", ), - ] - if row_ids: - clauses.append(LLMUsageDaily.context["evaluation_row_id"].astext.in_(row_ids)) - return or_(*clauses) + ) def _call_import_scope_filter( @@ -359,68 +376,60 @@ def _call_import_scope_filter( organization_id: UUID, db: Session, ): - """Match usage tied to a call import (direct context, resource row, or eval runs).""" + """Match usage tied to a call import or one of its evaluations.""" + del db cid = str(call_import_id) - eval_ids = [ - str(row[0]) - for row in db.query(CallImportEvaluation.id) - .filter( - CallImportEvaluation.organization_id == organization_id, - CallImportEvaluation.call_import_id == call_import_id, - ) - .all() - ] - clauses = [ + eval_ids = select(cast(CallImportEvaluation.id, String)).where( + CallImportEvaluation.organization_id == organization_id, + CallImportEvaluation.call_import_id == call_import_id, + ) + return or_( LLMUsageDaily.context["call_import_id"].astext == cid, and_( LLMUsageDaily.context["resource_id"].astext == cid, LLMUsageDaily.context["resource_type"].astext == "call_import", ), - ] - if eval_ids: - clauses.append(LLMUsageDaily.context["evaluation_id"].astext.in_(eval_ids)) - clauses.append( - and_( - LLMUsageDaily.context["resource_id"].astext.in_(eval_ids), - LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", - ) - ) - return or_(*clauses) + LLMUsageDaily.context["evaluation_id"].astext.in_(eval_ids), + and_( + LLMUsageDaily.context["resource_id"].astext.in_(eval_ids), + LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", + ), + ) -def _call_import_ids_for_filters( - db: Session, +def _scoped_call_import_id_select( *, organization_id: UUID, workspace_id: Optional[UUID] = None, dataset: Optional[str] = None, tag_id: Optional[UUID] = None, -) -> List[UUID]: - query = db.query(CallImport.id).filter( - CallImport.organization_id == organization_id, - ) +): + """Call-import ids for dataset/tag scope. Stays a SQL subquery — never a Python list.""" + stmt = select(CallImport.id).where(CallImport.organization_id == organization_id) if workspace_id is not None: - query = query.filter(CallImport.workspace_id == workspace_id) + stmt = stmt.where(CallImport.workspace_id == workspace_id) if dataset: - query = query.filter(CallImport.dataset == dataset) + stmt = stmt.where(CallImport.dataset == dataset) if tag_id is not None: - query = query.filter( + stmt = stmt.where( CallImport.id.in_( - db.query(CallImportTagAssignment.call_import_id).filter( - CallImportTagAssignment.tag_id == tag_id, + select(CallImportTagAssignment.call_import_id).where( + CallImportTagAssignment.tag_id == tag_id ) ) ) - return [row[0] for row in query.all()] + return stmt -def _call_import_ids_scope_filter( - allowed_import_ids: List[UUID], -): - if not allowed_import_ids: - return LLMUsageDaily.id.is_(None) - allowed = [str(uid) for uid in allowed_import_ids] - return cast(_call_import_group_expr(), String).in_(allowed) +def _call_import_ids_scope_filter(import_id_select): + allowed = select(cast(CallImport.id, String)).where(CallImport.id.in_(import_id_select)) + eval_ids = select(cast(CallImportEvaluation.id, String)).where( + CallImportEvaluation.call_import_id.in_(import_id_select) + ) + return or_( + _direct_call_import_id_expr().in_(allowed), + _evaluation_id_for_import_expr().in_(eval_ids), + ) def _workspace_scope_filter( @@ -435,9 +444,18 @@ def _workspace_scope_filter( CallImport.workspace_id == workspace_id, ) ) + ws_eval_ids = ( + select(cast(CallImportEvaluation.id, String)).where( + CallImportEvaluation.organization_id == organization_id, + CallImportEvaluation.workspace_id == workspace_id, + ) + ) legacy_call_import = and_( LLMUsageDaily.workspace_id.is_(None), - _call_import_group_expr().in_(ws_call_import_ids), + or_( + _direct_call_import_id_expr().in_(ws_call_import_ids), + _evaluation_id_for_import_expr().in_(ws_eval_ids), + ), ) return or_(LLMUsageDaily.workspace_id == workspace_id, legacy_call_import) @@ -523,14 +541,16 @@ def _apply_filters( if dataset or tag_id is not None: if db is None: raise ValueError("db required for dataset/tag filters") - allowed = _call_import_ids_for_filters( - db, - organization_id=organization_id, - workspace_id=workspace_id, - dataset=dataset, - tag_id=tag_id, + query = query.filter( + _call_import_ids_scope_filter( + _scoped_call_import_id_select( + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + ) ) - query = query.filter(_call_import_ids_scope_filter(allowed)) return query @@ -602,47 +622,36 @@ def _infer_agent_types_for_label_buckets( grouped[rid] = ("agent", contexts) -def _resource_label_map( +def _resource_labels_for_ids( db: Session, organization_id: UUID, query, + resource_ids: List[str], ) -> dict[str, str]: - """Build resource_id -> hierarchical label from usage rows in query.""" + """Label only the resource ids already returned by the breakdown page.""" + if not resource_ids: + return {} + rid = cast(_resource_id_expr(), String) rows = ( - query.with_entities( - _resource_id_expr(), - _resource_type_expr(), - LLMUsageDaily.context, - ) - .filter( - or_( - LLMUsageDaily.context["resource_id"].astext.isnot(None), - LLMUsageDaily.context["agent_id"].astext.isnot(None), - ) - ) - .order_by(*_label_row_order()) - .limit(_LABEL_ROW_LIMIT) + query.with_entities(rid, _resource_type_expr(), LLMUsageDaily.context) + .filter(rid.in_(resource_ids)) + .distinct(rid) + .order_by(rid, _usage_row_weight().desc()) .all() ) grouped: dict[str, tuple[Optional[str], list]] = {} for raw_id, rtype, ctx in rows: if not raw_id: continue - key = str(raw_id) - if key not in grouped: - grouped[key] = (rtype, []) - grouped[key][1].append(ctx) - + grouped[str(raw_id)] = (rtype, [dict(ctx or {})]) _infer_agent_types_for_label_buckets(db, organization_id, grouped) - - buckets = [(rid, rtype, contexts) for rid, (rtype, contexts) in grouped.items()] + buckets = [(key, rtype, contexts) for key, (rtype, contexts) in grouped.items()] resolver = UsageNameResolver(db, organization_id) contexts_for_preload = [] - for rid, rtype, contexts in buckets: + for key, rtype, contexts in buckets: for ctx in contexts: merged = dict(ctx or {}) - if rid: - merged.setdefault("resource_id", rid) + merged.setdefault("resource_id", key) if rtype: merged.setdefault("resource_type", rtype) contexts_for_preload.append(merged) @@ -650,43 +659,6 @@ def _resource_label_map( return labels_for_resource_buckets(buckets, resolver) -def _resource_filter_meta_map( - db: Session, - organization_id: UUID, - query, -) -> dict[str, dict[str, Optional[str]]]: - """Map resource id -> {type, product_section} for filter dropdowns.""" - rows = ( - query.with_entities( - _resource_id_expr(), - _resource_type_expr(), - LLMUsageDaily.product_section, - ) - .filter( - or_( - LLMUsageDaily.context["resource_id"].astext.isnot(None), - LLMUsageDaily.context["agent_id"].astext.isnot(None), - ) - ) - .distinct() - .limit(_LABEL_ROW_LIMIT) - .all() - ) - meta: dict[str, dict[str, Optional[str]]] = {} - for raw_id, rtype, section in rows: - if not raw_id: - continue - key = str(raw_id) - entry = meta.setdefault(key, {"type": None, "product_section": None}) - if rtype: - entry["type"] = str(rtype) - if section: - entry["product_section"] = str(section) - if not entry["type"] and section == "agents": - entry["type"] = "agent" - return meta - - def _breakdown_resource_label( raw_res_id: Optional[str], res_type: Optional[str], @@ -709,117 +681,44 @@ def _breakdown_resource_label( return label or "Unscoped" -def _collect_call_import_ids_from_usage_rows( +def _call_import_ids_for_usage_query( db: Session, organization_id: UUID, - rows: list, + query, ) -> set[UUID]: - """Extract call import ids referenced in usage row context tuples.""" + """Resolve call import ids from rollup context, then one catalog evaluation lookup.""" + direct = _direct_call_import_id_expr() + evaluation = _evaluation_id_for_import_expr() + rows = ( + query.with_entities(direct, evaluation) + .group_by(direct, evaluation) + .order_by(func.sum(_usage_row_weight()).desc()) + .limit(_FILTER_OPTION_LIMIT) + .all() + ) import_ids: set[UUID] = set() eval_ids: set[UUID] = set() - row_ids: set[UUID] = set() - eval_row_ids: set[UUID] = set() - - for row in rows: - cid_raw = row[0] if len(row) > 0 else None - res_raw = row[1] if len(row) > 1 else None - rtype = row[2] if len(row) > 2 else None - eval_raw = row[3] if len(row) > 3 else None - row_id_raw = row[4] if len(row) > 4 else None - eval_row_raw = row[5] if len(row) > 5 else None - - if cid_raw: - uid = parse_uuid(cid_raw) - if uid: - import_ids.add(uid) - if res_raw and rtype == "call_import": - uid = parse_uuid(res_raw) - if uid: - import_ids.add(uid) - if eval_raw: - uid = parse_uuid(eval_raw) - if uid: - eval_ids.add(uid) - if res_raw and rtype == "call_import_evaluation": - uid = parse_uuid(res_raw) - if uid: - eval_ids.add(uid) - if row_id_raw: - uid = parse_uuid(row_id_raw) - if uid: - row_ids.add(uid) - if eval_row_raw: - uid = parse_uuid(eval_row_raw) - if uid: - eval_row_ids.add(uid) - + for direct_raw, eval_raw in rows: + direct_id = parse_uuid(direct_raw) + if direct_id: + import_ids.add(direct_id) + continue + eval_id = parse_uuid(eval_raw) + if eval_id: + eval_ids.add(eval_id) if eval_ids: - for ev in ( - db.query(CallImportEvaluation) + for row in ( + db.query(CallImportEvaluation.call_import_id) .filter( CallImportEvaluation.organization_id == organization_id, CallImportEvaluation.id.in_(eval_ids), ) .all() ): - import_ids.add(ev.call_import_id) - - if row_ids: - for cid in ( - db.query(CallImportRow.call_import_id) - .filter(CallImportRow.id.in_(row_ids)) - .distinct() - .all() - ): - import_ids.add(cid[0]) - - if eval_row_ids: - for cid in ( - db.query(CallImportEvaluation.call_import_id) - .join( - CallImportEvaluationRow, - CallImportEvaluationRow.evaluation_id == CallImportEvaluation.id, - ) - .filter(CallImportEvaluationRow.id.in_(eval_row_ids)) - .distinct() - .all() - ): - import_ids.add(cid[0]) - + import_ids.add(row[0]) return import_ids -def _call_import_label_map( - db: Session, - organization_id: UUID, - query, -) -> dict[str, str]: - rows = ( - query.with_entities( - LLMUsageDaily.context["call_import_id"].astext, - LLMUsageDaily.context["resource_id"].astext, - LLMUsageDaily.context["resource_type"].astext, - LLMUsageDaily.context["evaluation_id"].astext, - LLMUsageDaily.context["call_import_row_id"].astext, - LLMUsageDaily.context["evaluation_row_id"].astext, - ) - .order_by(*_label_row_order()) - .limit(_LABEL_ROW_LIMIT) - .all() - ) - import_ids = _collect_call_import_ids_from_usage_rows( - db, organization_id, rows - ) - - if not import_ids: - return {} - resolver = UsageNameResolver(db, organization_id) - resolver.preload( - [{"call_import_id": str(uid)} for uid in import_ids] - ) - return labels_for_call_import_ids(list(import_ids), resolver) - - def _call_import_filter_labels( db: Session, organization_id: UUID, @@ -827,30 +726,31 @@ def _call_import_filter_labels( workspace_id: Optional[UUID] = None, dataset: Optional[str] = None, tag_id: Optional[UUID] = None, + call_import_id: Optional[UUID] = None, ) -> dict[str, str]: - """Call imports for filters: usage in range plus workspace imports when scoped.""" - import_ids = _collect_call_import_ids_from_usage_rows( - db, - organization_id, - query.with_entities( - LLMUsageDaily.context["call_import_id"].astext, - LLMUsageDaily.context["resource_id"].astext, - LLMUsageDaily.context["resource_type"].astext, - LLMUsageDaily.context["evaluation_id"].astext, - LLMUsageDaily.context["call_import_row_id"].astext, - LLMUsageDaily.context["evaluation_row_id"].astext, - ).order_by(*_label_row_order()).limit(_LABEL_ROW_LIMIT).all(), - ) + """Call imports for filters: heaviest usage in range, plus recent workspace imports.""" + import_ids = _call_import_ids_for_usage_query(db, organization_id, query) if workspace_id is not None: - scoped_ids = _call_import_ids_for_filters( - db, - organization_id=organization_id, - workspace_id=workspace_id, - dataset=dataset, - tag_id=tag_id, + recent = ( + db.query(CallImport.id) + .filter( + CallImport.id.in_( + _scoped_call_import_id_select( + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + ) + ) + .order_by(CallImport.created_at.desc()) + .limit(_FILTER_OPTION_LIMIT) + .all() ) - import_ids.update(scoped_ids) + import_ids.update(row[0] for row in recent) + if call_import_id is not None: + import_ids.add(call_import_id) if not import_ids: return {} @@ -859,62 +759,6 @@ def _call_import_filter_labels( return labels_for_call_import_ids(list(import_ids), resolver) -def _evaluation_label_map( - db: Session, - organization_id: UUID, - query, -) -> dict[str, str]: - """Evaluation id -> short label (name + id suffix) for filter dropdowns.""" - rows = ( - query.with_entities( - LLMUsageDaily.context["evaluation_id"].astext, - LLMUsageDaily.context["resource_id"].astext, - LLMUsageDaily.context["resource_type"].astext, - LLMUsageDaily.context, - ) - .filter( - or_( - LLMUsageDaily.context["evaluation_id"].astext.isnot(None), - LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", - ) - ) - .order_by(*_label_row_order()) - .limit(_LABEL_ROW_LIMIT) - .all() - ) - grouped: dict[str, tuple[Optional[str], list]] = {} - for eval_raw, res_raw, rtype, ctx in rows: - key_raw = eval_raw - if not key_raw and rtype == "call_import_evaluation" and res_raw: - key_raw = res_raw - if not key_raw: - continue - key = str(key_raw) - if key not in grouped: - grouped[key] = (rtype, []) - grouped[key][1].append(ctx) - - buckets = [(rid, rtype, contexts) for rid, (rtype, contexts) in grouped.items()] - resolver = UsageNameResolver(db, organization_id) - contexts_for_preload = [] - for _, rtype, contexts in buckets: - for ctx in contexts: - merged = dict(ctx or {}) - if rtype and "resource_type" not in merged: - merged["resource_type"] = rtype - contexts_for_preload.append(merged) - resolver.preload(contexts_for_preload) - - labels: dict[str, str] = {} - for raw_id, rtype, contexts in buckets: - ctx = max([dict(c or {}) for c in contexts], key=len, default={}) - if rtype and "resource_type" not in ctx: - ctx["resource_type"] = rtype - eval_key = ctx.get("evaluation_id") or raw_id - labels[str(raw_id)] = resolver.evaluation_name(str(eval_key)) - return labels - - def _unpriced_usage_condition(): return and_( LLMUsageDaily.pricing_rate_id.is_(None), @@ -1006,47 +850,50 @@ def _usage_totals_from_row(row) -> dict: ) -def _breakdown_metrics_from_tuple(metrics: tuple) -> dict: - prompt = int(metrics[0]) - completion = int(metrics[1]) - base = { - "prompt_tokens": prompt, - "completion_tokens": completion, - "total_tokens": prompt + completion, - "cache_read_tokens": int(metrics[2]), - "cache_creation_tokens": int(metrics[3]), - "reasoning_tokens": int(metrics[4]), - "audio_seconds": int(metrics[5]), - "tts_characters": int(metrics[6]), - "call_count": int(metrics[7]), - "input_cost_micro_usd": int(metrics[8]), - "output_cost_micro_usd": int(metrics[9]), - "cache_read_cost_micro_usd": int(metrics[10]), - "cache_creation_cost_micro_usd": int(metrics[11]), - "reasoning_cost_micro_usd": int(metrics[12]), - "audio_cost_micro_usd": int(metrics[13]), - "tts_cost_micro_usd": int(metrics[14]), - "total_cost_micro_usd": int(metrics[15]), - } - has_unpriced = bool(metrics[16]) if len(metrics) > 16 else False - return _attach_costs(base, has_unpriced_usage=has_unpriced) +_CALL_IMPORT_PRODUCT_SECTIONS = ("call_imports", "call_import_evaluations") +_CALL_IMPORT_RESOURCE_TYPES = ("call_import", "call_import_evaluation") -def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: +def _breakdown_sort_exprs(*group_cols): + """Highest token volume first, then calls, then cost. + + Group columns are the final tie-break so OFFSET pages stay stable when + several groups share the same totals. + """ + tokens = func.sum(LLMUsageDaily.prompt_tokens + LLMUsageDaily.completion_tokens) return ( - db.query(func.max(LLMUsageDaily.updated_at)) - .filter(LLMUsageDaily.organization_id == organization_id) - .scalar() + tokens.desc().nulls_last(), + func.sum(LLMUsageDaily.call_count).desc().nulls_last(), + func.sum(LLMUsageDaily.total_cost_micro_usd).desc().nulls_last(), + *(col.asc().nulls_last() for col in group_cols), ) -def _summary_aggregate_query( - db: Session, +def _workspace_other_source_filters(): + """Rows the workspace table shows besides call-import batches.""" + resource_type = cast(_resource_type_expr(), String) + return ( + LLMUsageDaily.product_section.isnot(None), + LLMUsageDaily.product_section.notin_(_CALL_IMPORT_PRODUCT_SECTIONS), + or_( + resource_type.is_(None), + resource_type.notin_(_CALL_IMPORT_RESOURCE_TYPES), + ), + ) + + +def _displayable_usage_having(): + """Match the composite table, which hides groups with no visible usage.""" + return or_( + func.sum(LLMUsageDaily.prompt_tokens + LLMUsageDaily.completion_tokens) > 0, + func.sum(LLMUsageDaily.call_count) > 0, + func.sum(LLMUsageDaily.audio_seconds) > 0, + func.sum(LLMUsageDaily.tts_characters) > 0, + ) + + +def _filters_share_facet_scan( *, - organization_id: UUID, - start: date, - end: date, - enforced_floor: Optional[date] = None, workspace_id: Optional[UUID], product_section: Optional[str], model: Optional[str], @@ -1054,13 +901,228 @@ def _summary_aggregate_query( usage_kind: Optional[str], call_import_id: Optional[UUID], evaluation_id: Optional[UUID], - evaluation_row_id: Optional[UUID], - dataset: Optional[str] = None, - tag_id: Optional[UUID] = None, -): - return _apply_filters( - db.query( - func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), +) -> bool: + """Workspace, section, kind, and model lists scan the same rows when unscoped.""" + return not any( + ( + workspace_id, + product_section, + model, + resource_id, + usage_kind, + call_import_id, + evaluation_id, + ) + ) + + +def _facet_sets(query) -> tuple[set, set, set, set]: + rows = ( + query.with_entities( + LLMUsageDaily.workspace_id, + LLMUsageDaily.product_section, + LLMUsageDaily.usage_kind, + LLMUsageDaily.model, + ) + .distinct() + .all() + ) + workspace_ids: set = set() + sections: set = set() + kinds: set = set() + models: set = set() + for workspace_id, section, kind, model_name in rows: + if workspace_id is not None: + workspace_ids.add(workspace_id) + if section: + sections.add(section) + if kind: + kinds.add(kind) + if model_name: + models.add(model_name) + return workspace_ids, sections, kinds, models + + +def _grouped_resource_filter_rows( + query, + extra_resource_ids: Optional[List[str]] = None, +): + """Top resources by usage, then one context JSON each.""" + resource_id = cast(_resource_id_expr(), String) + ranked = ( + query.with_entities( + resource_id, + func.max(cast(_resource_type_expr(), String)), + func.max(LLMUsageDaily.product_section), + func.max(LLMUsageDaily.context["evaluation_id"].astext), + ) + .filter(resource_id.isnot(None)) + .group_by(resource_id) + .order_by(func.sum(_usage_row_weight()).desc()) + .limit(_FILTER_OPTION_LIMIT) + .all() + ) + by_id: dict[str, tuple] = {} + for row in ranked: + if row[0]: + by_id[str(row[0])] = row + for extra in extra_resource_ids or []: + if not extra or extra in by_id: + continue + extra_row = ( + query.with_entities( + resource_id, + func.max(cast(_resource_type_expr(), String)), + func.max(LLMUsageDaily.product_section), + func.max(LLMUsageDaily.context["evaluation_id"].astext), + ) + .filter(resource_id == extra) + .group_by(resource_id) + .first() + ) + if extra_row and extra_row[0]: + by_id[str(extra_row[0])] = extra_row + if not by_id: + return [] + contexts: dict[str, Any] = {} + for raw_id, ctx in ( + query.with_entities(resource_id, LLMUsageDaily.context) + .filter(resource_id.in_(list(by_id))) + .distinct(resource_id) + .order_by(resource_id, _usage_row_weight().desc()) + .all() + ): + if raw_id: + contexts[str(raw_id)] = ctx + return [ + (key, row[1], row[2], row[3], contexts.get(key)) + for key, row in by_id.items() + ] + + +def _resource_filter_bundle( + db: Session, + organization_id: UUID, + query, + extra_resource_ids: Optional[List[str]] = None, +) -> tuple[dict[str, str], dict[str, dict[str, Optional[str]]], dict[str, str]]: + """Labels, resource meta, and evaluation labels from one grouped scan.""" + grouped: dict[str, tuple[Optional[str], list]] = {} + meta: dict[str, dict[str, Optional[str]]] = {} + eval_grouped: dict[str, tuple[Optional[str], list]] = {} + for raw_id, rtype, section, eval_id, context in _grouped_resource_filter_rows( + query, extra_resource_ids=extra_resource_ids + ): + if not raw_id: + continue + key = str(raw_id) + ctx = dict(context or {}) + if key not in grouped: + grouped[key] = (rtype, []) + grouped[key][1].append(ctx) + entry = meta.setdefault(key, {"type": None, "product_section": None}) + if rtype: + entry["type"] = str(rtype) + if section: + entry["product_section"] = str(section) + if not entry["type"] and section == "agents": + entry["type"] = "agent" + eval_key = eval_id or ctx.get("evaluation_id") + if not eval_key and rtype == "call_import_evaluation": + eval_key = key + if eval_key: + eval_bucket = eval_grouped.setdefault(str(eval_key), (rtype, [])) + eval_bucket[1].append(ctx) + + _infer_agent_types_for_label_buckets(db, organization_id, grouped) + buckets = [(rid, rtype, contexts) for rid, (rtype, contexts) in grouped.items()] + resolver = UsageNameResolver(db, organization_id) + contexts_for_preload = [] + for rid, rtype, contexts in buckets: + for ctx in contexts: + merged = dict(ctx or {}) + if rid: + merged.setdefault("resource_id", rid) + if rtype: + merged.setdefault("resource_type", rtype) + contexts_for_preload.append(merged) + for rtype, contexts in eval_grouped.values(): + for ctx in contexts: + merged = dict(ctx or {}) + if rtype and "resource_type" not in merged: + merged["resource_type"] = rtype + contexts_for_preload.append(merged) + resolver.preload(contexts_for_preload) + resource_labels = labels_for_resource_buckets(buckets, resolver) + evaluation_labels = { + raw_id: resolver.evaluation_name( + str((contexts[0] or {}).get("evaluation_id") or raw_id) + ) + for raw_id, (_rtype, contexts) in eval_grouped.items() + } + return resource_labels, meta, evaluation_labels + + +def _breakdown_metrics_from_tuple(metrics: tuple) -> dict: + prompt = int(metrics[0]) + completion = int(metrics[1]) + base = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "cache_read_tokens": int(metrics[2]), + "cache_creation_tokens": int(metrics[3]), + "reasoning_tokens": int(metrics[4]), + "audio_seconds": int(metrics[5]), + "tts_characters": int(metrics[6]), + "call_count": int(metrics[7]), + "input_cost_micro_usd": int(metrics[8]), + "output_cost_micro_usd": int(metrics[9]), + "cache_read_cost_micro_usd": int(metrics[10]), + "cache_creation_cost_micro_usd": int(metrics[11]), + "reasoning_cost_micro_usd": int(metrics[12]), + "audio_cost_micro_usd": int(metrics[13]), + "tts_cost_micro_usd": int(metrics[14]), + "total_cost_micro_usd": int(metrics[15]), + } + has_unpriced = bool(metrics[16]) if len(metrics) > 16 else False + return _attach_costs(base, has_unpriced_usage=has_unpriced) + + +def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: + return ( + db.query(LLMUsageDaily.updated_at) + .filter( + LLMUsageDaily.organization_id == organization_id, + LLMUsageDaily.updated_at.isnot(None), + ) + .order_by(LLMUsageDaily.updated_at.desc()) + .limit(1) + .scalar() + ) + + +def _summary_aggregate_query( + db: Session, + *, + organization_id: UUID, + start: date, + end: date, + enforced_floor: Optional[date] = None, + workspace_id: Optional[UUID], + product_section: Optional[str], + model: Optional[str], + resource_id: Optional[UUID], + usage_kind: Optional[str], + call_import_id: Optional[UUID], + evaluation_id: Optional[UUID], + evaluation_row_id: Optional[UUID], + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, +): + return _apply_filters( + db.query( + func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), func.coalesce(func.sum(LLMUsageDaily.completion_tokens), 0).label( "completion_tokens" ), @@ -1149,45 +1211,50 @@ def get_usage_summary( dataset=dataset, tag_id=tag_id, ) - cached = get_cached_response(organization_id, "summary", cache_key) - if cached is not None: - try: - return UsageSummaryResponse.model_validate(cached) - except Exception: - pass - - row = _summary_aggregate_query( - db, - organization_id=organization_id, - start=access.filter_start, - end=access.filter_end, - enforced_floor=access.enforced_filter_floor, - workspace_id=workspace_id, - product_section=product_section, - model=model, - resource_id=resource_id, - usage_kind=usage_kind, - call_import_id=call_import_id, - evaluation_id=evaluation_id, - evaluation_row_id=evaluation_row_id, - dataset=dataset, - tag_id=tag_id, - ).one() - totals = _usage_totals_from_row(row) - response = UsageSummaryResponse( - start=display_start, - end=display_end, - totals=UsageTotals(**totals), - usage_policy=_usage_policy_meta(access), - last_updated_at=_last_updated(db, organization_id), - ) - set_cached_response( - organization_id, - "summary", - cache_key, - response.model_dump(mode="json"), + cached_response, cache_leader, cache_version = _prepare_usage_read( + organization_id, "summary", cache_key, UsageSummaryResponse ) - return response + if cached_response is not None: + return cached_response + + try: + row = _summary_aggregate_query( + db, + organization_id=organization_id, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + ).one() + totals = _usage_totals_from_row(row) + response = UsageSummaryResponse( + start=display_start, + end=display_end, + totals=UsageTotals(**totals), + usage_policy=_usage_policy_meta(access), + last_updated_at=_last_updated(db, organization_id), + ) + set_cached_response( + organization_id, + "summary", + cache_key, + response.model_dump(mode="json"), + historical=_usage_range_is_historical(access), + version=cache_version, + ) + return response + finally: + if cache_leader: + release_read_lock(organization_id, "summary", cache_key) @router.get("/breakdown", response_model=UsageBreakdownResponse) @@ -1209,6 +1276,13 @@ def get_usage_breakdown( evaluation_row_id: Optional[UUID] = Query(None), dataset: Optional[str] = Query(None), tag_id: Optional[UUID] = Query(None), + omit_call_import_sources: bool = Query( + False, + description=( + "When grouping by resource, drop call-import batches and their " + "evaluation rows so offsets match the workspace source table." + ), + ), limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0), organization_id: UUID = Depends(get_organization_id), @@ -1233,115 +1307,82 @@ def get_usage_breakdown( evaluation_row_id=evaluation_row_id, dataset=dataset, tag_id=tag_id, + omit_call_import_sources=omit_call_import_sources, limit=limit, offset=offset, ) - cached = get_cached_response(organization_id, "breakdown", cache_key) - if cached is not None: - try: - return UsageBreakdownResponse.model_validate(cached) - except Exception: - pass - - dim = { - "workspace": LLMUsageDaily.workspace_id, - "product_section": LLMUsageDaily.product_section, - "model": LLMUsageDaily.model, - "resource": cast(_resource_id_expr(), String), - "usage_kind": LLMUsageDaily.usage_kind, - "call_import": _call_import_group_expr(), - }[group_by] - - aggregates = [ - func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), - func.coalesce(func.sum(LLMUsageDaily.completion_tokens), 0).label( - "completion_tokens" - ), - func.coalesce(func.sum(LLMUsageDaily.cache_read_tokens), 0).label( - "cache_read_tokens" - ), - func.coalesce(func.sum(LLMUsageDaily.cache_creation_tokens), 0).label( - "cache_creation_tokens" - ), - func.coalesce(func.sum(LLMUsageDaily.reasoning_tokens), 0).label( - "reasoning_tokens" - ), - func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), - func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), - func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), - *_usage_cost_sum_columns(), - _has_unpriced_usage_column(), - ] - - select_cols = [dim] - group_cols = [dim] - if group_by == "resource": - select_cols.append(cast(_resource_type_expr(), String)) - group_cols.append(cast(_resource_type_expr(), String)) - select_cols.append(LLMUsageDaily.product_section) - group_cols.append(LLMUsageDaily.product_section) - - query = _apply_filters( - db.query(*select_cols, *aggregates), - organization_id=organization_id, - start=access.filter_start, - end=access.filter_end, - enforced_floor=access.enforced_filter_floor, - workspace_id=workspace_id, - product_section=product_section, - model=model, - resource_id=resource_id, - usage_kind=usage_kind, - call_import_id=call_import_id, - evaluation_id=evaluation_id, - evaluation_row_id=evaluation_row_id, - dataset=dataset, - tag_id=tag_id, - db=db, - ).group_by(*group_cols) - - results = ( - query.order_by(func.sum(LLMUsageDaily.call_count).desc()) - .offset(offset) - .limit(limit) - .all() + cached_response, cache_leader, cache_version = _prepare_usage_read( + organization_id, "breakdown", cache_key, UsageBreakdownResponse ) + if cached_response is not None: + return cached_response + + try: + dims = { + "workspace": LLMUsageDaily.workspace_id, + "product_section": LLMUsageDaily.product_section, + "model": LLMUsageDaily.model, + "resource": cast(_resource_id_expr(), String), + "usage_kind": LLMUsageDaily.usage_kind, + } - workspace_names: dict = {} - if group_by == "workspace": - ws_ids = {r[0] for r in results if r[0] is not None} - if ws_ids: - workspace_names = { - w.id: w.name - for w in db.query(Workspace) - .filter( - Workspace.organization_id == organization_id, - Workspace.id.in_(ws_ids), + aggregates = [ + func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), + func.coalesce(func.sum(LLMUsageDaily.completion_tokens), 0).label( + "completion_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_read_tokens), 0).label( + "cache_read_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_creation_tokens), 0).label( + "cache_creation_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.reasoning_tokens), 0).label( + "reasoning_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), + func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), + func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), + *_usage_cost_sum_columns(), + _has_unpriced_usage_column(), + ] + + resolved_call_import = None + if group_by == "call_import": + direct_expr = _direct_call_import_id_expr() + eval_expr = _evaluation_id_for_import_expr() + resolved_call_import = func.coalesce( + direct_expr, cast(CallImportEvaluation.call_import_id, String) + ) + base_query = ( + db.query(resolved_call_import, *aggregates) + .select_from(LLMUsageDaily) + .outerjoin( + CallImportEvaluation, + and_( + direct_expr.is_(None), + eval_expr.op("~")(_UUID_RE), + CallImportEvaluation.id == cast(eval_expr, PG_UUID), + CallImportEvaluation.organization_id == organization_id, + ), ) - .all() - } - resource_labels: dict[str, str] = {} - call_import_labels: dict[str, str] = {} - if group_by == "resource": - label_query = _filtered_query( - db, - organization_id=organization_id, - start=access.filter_start, - end=access.filter_end, - enforced_floor=access.enforced_filter_floor, - workspace_id=workspace_id, - product_section=product_section, - model=model, - resource_id=resource_id, - usage_kind=usage_kind, - call_import_id=call_import_id, - evaluation_id=evaluation_id, - evaluation_row_id=evaluation_row_id, - ) - resource_labels = _resource_label_map(db, organization_id, label_query) - elif group_by == "call_import": - label_query = _filtered_query( - db, + ) + group_cols = [resolved_call_import] + else: + dim = dims[group_by] + select_cols = [dim] + group_cols = [dim] + if group_by == "resource": + select_cols.append(cast(_resource_type_expr(), String)) + group_cols.append(cast(_resource_type_expr(), String)) + select_cols.append(LLMUsageDaily.product_section) + group_cols.append(LLMUsageDaily.product_section) + base_query = db.query(*select_cols, *aggregates) + if omit_call_import_sources and group_by == "resource": + base_query = base_query.filter(*_workspace_other_source_filters()) + + query = _apply_filters( + base_query, organization_id=organization_id, start=access.filter_start, end=access.filter_end, @@ -1354,116 +1395,184 @@ def get_usage_breakdown( call_import_id=call_import_id, evaluation_id=evaluation_id, evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + db=db, + ).group_by(*group_cols) + if omit_call_import_sources and group_by == "resource": + query = query.having(_displayable_usage_having()) + + # Read one extra row so a full page of exactly `limit` rows is not + # reported as truncated when nothing follows it. + window = ( + query.order_by(*_breakdown_sort_exprs(*group_cols)) + .offset(offset) + .limit(limit + 1) + .all() ) - call_import_labels = _call_import_label_map(db, organization_id, label_query) + truncated_at_limit = len(window) > limit + results = window[:limit] - rows: List[UsageBreakdownRow] = [] - for result in results: + workspace_names: dict = {} if group_by == "workspace": - ws_id = result[0] - metrics = result[1:] - rows.append( - UsageBreakdownRow( - workspace_id=ws_id, - workspace_name=workspace_names.get(ws_id) if ws_id else "Unknown", - **_breakdown_metrics_from_tuple(metrics), - ) + ws_ids = {r[0] for r in results if r[0] is not None} + if ws_ids: + workspace_names = { + w.id: w.name + for w in db.query(Workspace) + .filter( + Workspace.organization_id == organization_id, + Workspace.id.in_(ws_ids), + ) + .all() + } + resource_labels: dict[str, str] = {} + call_import_labels: dict[str, str] = {} + if group_by == "resource": + resource_ids = [str(row[0]) for row in results if row[0]] + label_query = _filtered_query( + db, + organization_id=organization_id, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, ) - elif group_by == "product_section": - section = result[0] - metrics = result[1:] - rows.append( - UsageBreakdownRow( - product_section=section, - product_section_label=SECTION_LABELS.get(section or "", section), - **_breakdown_metrics_from_tuple(metrics), - ) + resource_labels = _resource_labels_for_ids( + db, organization_id, label_query, resource_ids ) - elif group_by == "model": - model_name = result[0] - metrics = result[1:] - rows.append( - UsageBreakdownRow( - model=model_name, - **_breakdown_metrics_from_tuple(metrics), + elif group_by == "call_import": + import_ids = [] + for row in results: + parsed = parse_uuid(row[0]) + if parsed: + import_ids.append(parsed) + if import_ids: + resolver = UsageNameResolver(db, organization_id) + resolver.preload( + [{"call_import_id": str(uid)} for uid in import_ids] ) - ) - elif group_by == "usage_kind": - kind = result[0] - metrics = result[1:] - rows.append( - UsageBreakdownRow( - usage_kind=kind, - **_breakdown_metrics_from_tuple(metrics), + call_import_labels = labels_for_call_import_ids(import_ids, resolver) + + rows: List[UsageBreakdownRow] = [] + for result in results: + if group_by == "workspace": + ws_id = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + workspace_id=ws_id, + workspace_name=workspace_names.get(ws_id) if ws_id else "Unknown", + **_breakdown_metrics_from_tuple(metrics), + ) ) - ) - elif group_by == "call_import": - raw_cid = result[0] - metrics = result[1:] - cid = None - if raw_cid: - try: - cid = UUID(str(raw_cid)) - except (ValueError, TypeError): - cid = None - label = ( - call_import_labels.get(str(raw_cid), "Unscoped") - if raw_cid - else "Unscoped" - ) - rows.append( - UsageBreakdownRow( - call_import_id=cid, - call_import_label=label, - **_breakdown_metrics_from_tuple(metrics), + elif group_by == "product_section": + section = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + product_section=section, + product_section_label=SECTION_LABELS.get(section or "", section), + **_breakdown_metrics_from_tuple(metrics), + ) ) - ) - else: - raw_res_id, res_type, section = result[0], result[1], result[2] - metrics = result[3:] - res_id = None - if raw_res_id: - try: - res_id = UUID(str(raw_res_id)) - except (ValueError, TypeError): - res_id = None - rows.append( - UsageBreakdownRow( - resource_id=res_id, - resource_type=res_type or ( - "agent" if section == "agents" and raw_res_id else None - ), - resource_label=_breakdown_resource_label( - raw_res_id, - res_type, - section, - resource_labels, - db, - organization_id, - ), - product_section=section, - product_section_label=SECTION_LABELS.get(section or "", section), - **_breakdown_metrics_from_tuple(metrics), + elif group_by == "model": + model_name = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + model=model_name, + **_breakdown_metrics_from_tuple(metrics), + ) + ) + elif group_by == "usage_kind": + kind = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + usage_kind=kind, + **_breakdown_metrics_from_tuple(metrics), + ) + ) + elif group_by == "call_import": + raw_cid = result[0] + metrics = result[1:] + cid = None + if raw_cid: + try: + cid = UUID(str(raw_cid)) + except (ValueError, TypeError): + cid = None + label = ( + call_import_labels.get(str(raw_cid), "Unscoped") + if raw_cid + else "Unscoped" + ) + rows.append( + UsageBreakdownRow( + call_import_id=cid, + call_import_label=label, + **_breakdown_metrics_from_tuple(metrics), + ) + ) + else: + raw_res_id, res_type, section = result[0], result[1], result[2] + metrics = result[3:] + res_id = None + if raw_res_id: + try: + res_id = UUID(str(raw_res_id)) + except (ValueError, TypeError): + res_id = None + rows.append( + UsageBreakdownRow( + resource_id=res_id, + resource_type=res_type or ( + "agent" if section == "agents" and raw_res_id else None + ), + resource_label=_breakdown_resource_label( + raw_res_id, + res_type, + section, + resource_labels, + db, + organization_id, + ), + product_section=section, + product_section_label=SECTION_LABELS.get(section or "", section), + **_breakdown_metrics_from_tuple(metrics), + ) ) - ) - response = UsageBreakdownResponse( - start=display_start, - end=display_end, - group_by=group_by, - rows=rows, - total_count=len(rows), - truncated_at_limit=len(rows) >= limit, - usage_policy=_usage_policy_meta(access), - last_updated_at=_last_updated(db, organization_id), - ) - set_cached_response( - organization_id, - "breakdown", - cache_key, - response.model_dump(mode="json"), - ) - return response + response = UsageBreakdownResponse( + start=display_start, + end=display_end, + group_by=group_by, + rows=rows, + total_count=len(rows), + truncated_at_limit=truncated_at_limit, + usage_policy=_usage_policy_meta(access), + last_updated_at=_last_updated(db, organization_id), + ) + set_cached_response( + organization_id, + "breakdown", + cache_key, + response.model_dump(mode="json"), + historical=_usage_range_is_historical(access), + version=cache_version, + ) + return response + finally: + if cache_leader: + release_read_lock(organization_id, "breakdown", cache_key) @router.get("/filters", response_model=UsageFiltersResponse) @@ -1505,213 +1614,258 @@ def get_usage_filters( tag_id=tag_id, q=q, ) - cached = get_cached_response(organization_id, "filters", cache_key) - if cached is not None: - try: - return UsageFiltersResponse.model_validate(cached) - except Exception: - pass - - scoped_resource_id = resource_id or evaluation_id - - workspace_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - dataset=dataset, - tag_id=tag_id, - ) - section_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - workspace_id=workspace_id, - dataset=dataset, - tag_id=tag_id, - ) - kind_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - workspace_id=workspace_id, - product_section=product_section, - resource_id=scoped_resource_id, - call_import_id=call_import_id, - evaluation_id=evaluation_id, - dataset=dataset, - tag_id=tag_id, - ) - model_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - workspace_id=workspace_id, - product_section=product_section, - resource_id=scoped_resource_id, - usage_kind=usage_kind, - call_import_id=call_import_id, - evaluation_id=evaluation_id, - dataset=dataset, - tag_id=tag_id, - ) - call_import_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - workspace_id=workspace_id, - product_section=product_section, - dataset=dataset, - tag_id=tag_id, - ) - resource_base = _filtered_query( - db, - organization_id=organization_id, - start=filter_start, - end=filter_end, - enforced_floor=enforced_floor, - workspace_id=workspace_id, - product_section=product_section, - model=model, - usage_kind=usage_kind, - call_import_id=call_import_id, - dataset=dataset, - tag_id=tag_id, - ) - - workspace_ids = { - row[0] - for row in workspace_base.with_entities(LLMUsageDaily.workspace_id).distinct().all() - if row[0] is not None - } - workspaces = [ - {"id": str(w.id), "name": w.name} - for w in db.query(Workspace) - .filter(Workspace.id.in_(workspace_ids) if workspace_ids else False) - .order_by(Workspace.name) - .all() - ] - - sections = sorted( - { - row[0] - for row in section_base.with_entities(LLMUsageDaily.product_section).distinct().all() - if row[0] - } - ) - models = sorted( - { - row[0] - for row in model_base.with_entities(LLMUsageDaily.model).distinct().all() - if row[0] - } - ) - kinds = sorted( - { - row[0] - for row in kind_base.with_entities(LLMUsageDaily.usage_kind).distinct().all() - if row[0] - } - ) - - call_import_labels = _call_import_filter_labels( - db, - organization_id, - call_import_base, - workspace_id=workspace_id, - dataset=dataset, - tag_id=tag_id, + cached_response, cache_leader, cache_version = _prepare_usage_read( + organization_id, "filters", cache_key, UsageFiltersResponse ) - call_imports = [ - {"id": cid, "label": label} - for cid, label in sorted(call_import_labels.items(), key=lambda x: x[1].lower()) - ] + if cached_response is not None: + return cached_response - evaluation_labels = _evaluation_label_map(db, organization_id, resource_base) - needle = (q or "").strip().lower() - evaluations = [] - for rid, label in sorted(evaluation_labels.items(), key=lambda x: x[1].lower()): - if needle and needle not in label.lower(): - continue - evaluations.append({"id": rid, "label": label}) + try: + scoped_resource_id = resource_id or evaluation_id - resource_labels = _resource_label_map(db, organization_id, resource_base) - resource_meta = _resource_filter_meta_map(db, organization_id, resource_base) - resources = [] - for rid, label in sorted(resource_labels.items(), key=lambda x: x[1].lower()): - if needle and needle not in label.lower(): - continue - info = resource_meta.get(rid, {}) - rtype = info.get("type") - if rtype == "call_import_evaluation": - continue - resources.append( - { - "id": rid, - "label": label, - "type": rtype, - "product_section": info.get("product_section"), - } + workspace_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, + dataset=dataset, + tag_id=tag_id, ) - - dataset_query = db.query(CallImport.dataset).filter( - CallImport.organization_id == organization_id, - CallImport.dataset.isnot(None), - CallImport.dataset != "", - ) - if workspace_id is not None: - dataset_query = dataset_query.filter(CallImport.workspace_id == workspace_id) - if dataset or tag_id is not None: - scoped_import_ids = _call_import_ids_for_filters( + section_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + kind_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, + workspace_id=workspace_id, + product_section=product_section, + resource_id=scoped_resource_id, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + dataset=dataset, + tag_id=tag_id, + ) + model_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, + workspace_id=workspace_id, + product_section=product_section, + resource_id=scoped_resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + dataset=dataset, + tag_id=tag_id, + ) + call_import_base = _filtered_query( db, organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, + product_section=product_section, dataset=dataset, tag_id=tag_id, ) - if scoped_import_ids: - dataset_query = dataset_query.filter(CallImport.id.in_(scoped_import_ids)) + resource_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + enforced_floor=enforced_floor, + workspace_id=workspace_id, + product_section=product_section, + model=model, + usage_kind=usage_kind, + call_import_id=call_import_id, + dataset=dataset, + tag_id=tag_id, + ) + + if _filters_share_facet_scan( + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=scoped_resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + ): + workspace_ids, section_values, kind_values, model_values = _facet_sets( + workspace_base + ) + sections = sorted(section_values) + models = sorted(model_values) + kinds = sorted(kind_values) else: - dataset_query = dataset_query.filter(CallImport.id.is_(None)) - datasets = sorted({row[0] for row in dataset_query.distinct().all() if row[0]}) - - tags = [ - {"id": str(tag.id), "label": tag.name} - for tag in ( - db.query(CallImportTag) - .filter(CallImportTag.organization_id == organization_id) - .order_by(CallImportTag.name) + workspace_ids = { + row[0] + for row in workspace_base.with_entities(LLMUsageDaily.workspace_id) + .distinct() + .all() + if row[0] is not None + } + sections = sorted( + { + row[0] + for row in section_base.with_entities(LLMUsageDaily.product_section) + .distinct() + .all() + if row[0] + } + ) + models = sorted( + { + row[0] + for row in model_base.with_entities(LLMUsageDaily.model).distinct().all() + if row[0] + } + ) + kinds = sorted( + { + row[0] + for row in kind_base.with_entities(LLMUsageDaily.usage_kind).distinct().all() + if row[0] + } + ) + if ( + workspace_id is not None + and workspace_id not in workspace_ids + ): + selected = ( + db.query(Workspace) + .filter( + Workspace.id == workspace_id, + Workspace.organization_id == organization_id, + ) + .first() + ) + if selected is not None: + workspace_ids.add(selected.id) + workspaces = [ + {"id": str(w.id), "name": w.name} + for w in db.query(Workspace) + .filter(Workspace.id.in_(workspace_ids) if workspace_ids else False) + .order_by(Workspace.name) .all() + ] + + call_import_labels = _call_import_filter_labels( + db, + organization_id, + call_import_base, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + call_import_id=call_import_id, ) - ] + call_imports = [ + {"id": cid, "label": label} + for cid, label in sorted(call_import_labels.items(), key=lambda x: x[1].lower()) + ] + + pinned_resources = [ + str(value) + for value in (resource_id, evaluation_id) + if value is not None + ] + resource_labels, resource_meta, evaluation_labels = _resource_filter_bundle( + db, + organization_id, + resource_base, + extra_resource_ids=pinned_resources, + ) + needle = (q or "").strip().lower() + evaluations = [] + for rid, label in sorted(evaluation_labels.items(), key=lambda x: x[1].lower()): + if needle and needle not in label.lower(): + continue + evaluations.append({"id": rid, "label": label}) + resources = [] + for rid, label in sorted(resource_labels.items(), key=lambda x: x[1].lower()): + if needle and needle not in label.lower(): + continue + info = resource_meta.get(rid, {}) + rtype = info.get("type") + if rtype == "call_import_evaluation": + continue + resources.append( + { + "id": rid, + "label": label, + "type": rtype, + "product_section": info.get("product_section"), + } + ) - response = UsageFiltersResponse( - workspaces=workspaces, - product_sections=[ - {"id": s, "label": SECTION_LABELS.get(s, s)} for s in sections - ], - call_imports=call_imports, - evaluations=evaluations, - models=models, - resources=resources, - usage_kinds=[{"id": k, "label": usage_kind_label(k)} for k in kinds], - datasets=datasets, - tags=tags, - ) - set_cached_response( - organization_id, - "filters", - cache_key, - response.model_dump(mode="json"), - ) - return response + dataset_query = db.query(CallImport.dataset).filter( + CallImport.organization_id == organization_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + if workspace_id is not None: + dataset_query = dataset_query.filter(CallImport.workspace_id == workspace_id) + if dataset or tag_id is not None: + dataset_query = dataset_query.filter( + CallImport.id.in_( + _scoped_call_import_id_select( + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + ) + ) + datasets = sorted({row[0] for row in dataset_query.distinct().all() if row[0]}) + + tags = [ + {"id": str(tag.id), "label": tag.name} + for tag in ( + db.query(CallImportTag) + .filter(CallImportTag.organization_id == organization_id) + .order_by(CallImportTag.name) + .all() + ) + ] + + response = UsageFiltersResponse( + workspaces=workspaces, + product_sections=[ + {"id": s, "label": SECTION_LABELS.get(s, s)} for s in sections + ], + call_imports=call_imports, + evaluations=evaluations, + models=models, + resources=resources, + usage_kinds=[{"id": k, "label": usage_kind_label(k)} for k in kinds], + datasets=datasets, + tags=tags, + ) + set_cached_response( + organization_id, + "filters", + cache_key, + response.model_dump(mode="json"), + historical=_usage_range_is_historical(access), + version=cache_version, + ) + return response + finally: + if cache_leader: + release_read_lock(organization_id, "filters", cache_key) diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 265fd137..d5e642e6 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -1264,6 +1264,7 @@ async def re_evaluate_call_recording( """ import requests as http_requests import uuid as _uuid + from app.services.playground.post_call_processing import merge_playground_call_data from app.services.storage.s3_service import s3_service call_recording = db.query(CallRecording).filter( diff --git a/app/api/v1/routes/prompt_optimization.py b/app/api/v1/routes/prompt_optimization.py index c97bd524..9ad41721 100644 --- a/app/api/v1/routes/prompt_optimization.py +++ b/app/api/v1/routes/prompt_optimization.py @@ -337,7 +337,7 @@ def push_candidate_to_provider( db.commit() return { - "message": f"Prompt pushed to {platform_val} and agent description updated", + "message": f"Prompt pushed to {integration.platform} and agent description updated", "candidate_id": str(candidate.id), "provider_response": result, } diff --git a/app/api/v1/routes/scenario_metrics.py b/app/api/v1/routes/scenario_metrics.py new file mode 100644 index 00000000..21561a44 --- /dev/null +++ b/app/api/v1/routes/scenario_metrics.py @@ -0,0 +1,154 @@ +"""Enterprise scenario-to-metric generation routes.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID, uuid4 + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request +from app.services.billing.flexprice_service import record_metrics_llm_assist +from app.services.testing.scenario_metric_generation import ( + ScenarioMetricInput, + generate_metrics_from_scenarios, +) +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, +) + +router = APIRouter( + prefix="/metrics", + tags=["metrics"], + dependencies=[Depends(require_enterprise_feature("scenario_metrics"))], +) + + +class ScenarioMetricScenarioInput(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + description: str = Field(..., min_length=1) + goal: Optional[str] = None + metric_type: Optional[ + Literal["auto", "boolean", "rating", "number", "text"] + ] = Field( + default="auto", + description="Desired metric type for this scenario; auto lets the LLM choose.", + ) + + +class GenerateMetricsFromScenariosRequest(BaseModel): + agent_name: str = Field(..., min_length=1, max_length=255) + production_prompt: str = Field(..., min_length=1) + call_medium: Optional[str] = None + scenarios: List[ScenarioMetricScenarioInput] = Field(..., min_length=1, max_length=10) + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + + +class GeneratedScenarioMetricDraftResponse(BaseModel): + name: str + description: str + metric_type: Literal["rating", "boolean", "number", "text"] + custom_data_type: Optional[Literal["boolean", "enum", "number_range"]] = None + custom_config: Dict[str, Any] = {} + supported_surfaces: List[str] + enabled_surfaces: List[str] + tags: List[str] = [] + scenario_name: str + + +class GenerateMetricsFromScenariosResponse(BaseModel): + metrics: List[GeneratedScenarioMetricDraftResponse] + provider: str + model: str + + +@router.post("/generate-from-scenarios", response_model=GenerateMetricsFromScenariosResponse) +async def generate_metrics_from_scenarios_route( + data: GenerateMetricsFromScenariosRequest, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Suggest one evaluation metric per scenario; does not persist metrics.""" + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, data.provider, data.model, data.credential_id + ) + + scenario_inputs = [ + ScenarioMetricInput( + name=item.name.strip(), + description=item.description.strip(), + goal=item.goal.strip() if item.goal else None, + metric_type=item.metric_type, + ) + for item in data.scenarios + ] + + try: + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.METRICS, + ) + ): + result = generate_metrics_from_scenarios( + agent_name=data.agent_name.strip(), + production_prompt=data.production_prompt, + call_medium=data.call_medium, + scenarios=scenario_inputs, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + llm_provider=provider_enum, + llm_model=model_str, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=502, detail=f"AI generation failed: {exc}") from exc + + background_tasks.add_task( + record_metrics_llm_assist, + organization_id, + uuid4(), + workspace_id=workspace_id, + mode="scenario_metrics", + ) + + return GenerateMetricsFromScenariosResponse( + metrics=[ + GeneratedScenarioMetricDraftResponse( + name=m.name, + description=m.description, + metric_type=m.metric_type, # type: ignore[arg-type] + custom_data_type=m.custom_data_type, # type: ignore[arg-type] + custom_config=m.custom_config, + supported_surfaces=m.supported_surfaces, + enabled_surfaces=m.enabled_surfaces, + tags=m.tags, + scenario_name=m.scenario_name, + ) + for m in result.metrics + ], + provider=result.provider, + model=result.model, + ) diff --git a/app/api/v1/routes/scenarios.py b/app/api/v1/routes/scenarios.py index f9295b5a..156f2424 100644 --- a/app/api/v1/routes/scenarios.py +++ b/app/api/v1/routes/scenarios.py @@ -44,7 +44,12 @@ async def create_scenario( agent_id=scenario.agent_id, name=scenario.name, description=scenario.description, - required_info=scenario.required_info + required_info=scenario.required_info, + expected_tool_calls=( + [call.model_dump() for call in scenario.expected_tool_calls] + if scenario.expected_tool_calls is not None + else None + ), ) db.add(db_scenario) db.commit() diff --git a/app/api/v1/routes/simulation_runs.py b/app/api/v1/routes/simulation_runs.py new file mode 100644 index 00000000..3f1e8d5c --- /dev/null +++ b/app/api/v1/routes/simulation_runs.py @@ -0,0 +1,729 @@ +"""Harbor simulation runs: launch, monitor and inspect test ↔ production agent runs.""" + +from __future__ import annotations + +import json +import random +from pathlib import Path +from typing import List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi.responses import FileResponse, Response +from sqlalchemy.orm import Session + +from app.core.auth import Principal, get_principal +from app.dependencies import get_db, get_organization_id, get_workspace_id +from app.models.database import ( + Agent, + EvaluatorResult, + Persona, + Scenario, + SimulationRun, + SimulationRunStatus, +) +from app.models.schemas import ( + EvaluationPromptImprovementsState, + PromptImprovementReviewUpdate, + SimulationComposedPromptResponse, + SimulationFailureAnalysisRequest, + SimulationFailureAnalysisResponse, + SimulationRunCreate, + SimulationRunPdfReportRequest, + SimulationRunRename, + SimulationRunResponse, + SimulationTrialDetailResponse, + SimulationTrialResponse, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + metric_clusters_state_from_raw, +) +from app.services.call_import_prompt_improvements import prompt_improvements_state_from_raw +from app.services.harbor.failure_analysis import ( + count_failed_trials, + editable_agent_prompt, + success_rule_labels, +) +from app.services.harbor.result_ingest import ( + LOCAL_ARTIFACT_PREFIX, + attempt_numbers, + compute_summary, + trial_results, +) +from app.services.harbor.success_criteria import ( + criteria_to_storage, + validate_metric_success_criteria, +) +from app.services.harbor.trial_store import llm_override +from app.services.testing.simulation_llms import describe_simulation_llms +from app.services.testing.simulation_run_naming import simulation_run_name_for_rerun + +router =APIRouter(prefix="/simulation-runs", tags=["simulation-runs"]) + + +def _generate_unique_run_id(db: Session) -> str: + for _ in range(100): + candidate = f"{random.randint(100000, 999999)}" + if not db.query(SimulationRun).filter(SimulationRun.run_id == candidate).first(): + return candidate + raise HTTPException(status_code=500, detail="Failed to generate unique run ID") + + +def _get_run(db: Session, run_ref: str, organization_id: UUID, workspace_id: UUID) -> SimulationRun: + query = db.query(SimulationRun).filter( + SimulationRun.organization_id == organization_id, + SimulationRun.workspace_id == workspace_id, + ) + try: + run = query.filter(SimulationRun.id == UUID(run_ref)).first() + except ValueError: + run = query.filter(SimulationRun.run_id == run_ref).first() + if not run: + raise HTTPException(status_code=404, detail="Simulation run not found") + return run + + +def _run_response(db: Session, run: SimulationRun) -> SimulationRunResponse: + response = SimulationRunResponse.model_validate(run) + response.has_artifacts = bool(run.artifacts_s3_key) + if run.status not in SimulationRunStatus.terminal(): + # Live KPIs while trials are still landing; the stored summary is final-only. + response.summary = compute_summary(db, run, trial_results(db, run.id)) + return response + + +def _validate_agent_llms(db: Session, agent: Agent, payload: SimulationRunCreate) -> None: + from app.services.agents.chat_llm_config import resolve_simulation_llm + from app.services.agents.chat_production_leg import uses_live_production_leg + + try: + if not payload.test_llm_model: + resolve_simulation_llm( + db, agent=agent, organization_id=agent.organization_id, leg="test" + ) + if not payload.production_llm_model and not uses_live_production_leg(agent): + resolve_simulation_llm( + db, agent=agent, organization_id=agent.organization_id, leg="main" + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + +def _create_and_queue( + db: Session, + *, + payload: SimulationRunCreate, + organization_id: UUID, + workspace_id: UUID, + created_by: Optional[str], +) -> SimulationRun: + agent = ( + db.query(Agent) + .filter( + Agent.id == payload.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + .first() + ) + if not agent: + raise HTTPException(status_code=404, detail="Agent not found") + _validate_agent_llms(db, agent, payload) + + found_scenarios = ( + db.query(Scenario.id) + .filter( + Scenario.id.in_(payload.scenario_ids), + Scenario.organization_id == organization_id, + Scenario.workspace_id == workspace_id, + ) + .count() + ) + found_personas = ( + db.query(Persona.id) + .filter( + Persona.id.in_(payload.persona_ids), + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ) + .count() + ) + if found_scenarios != len(set(payload.scenario_ids)) or found_personas != len( + set(payload.persona_ids) + ): + raise HTTPException(status_code=404, detail="One or more scenarios or personas not found") + + stored_criteria = None + if payload.metric_success_criteria: + try: + validate_metric_success_criteria( + db, + organization_id=organization_id, + metric_ids=payload.metric_ids or [], + criteria=payload.metric_success_criteria, + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + stored_criteria = criteria_to_storage(payload.metric_success_criteria) + + run = SimulationRun( + run_id=_generate_unique_run_id(db), + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent.id, + name=payload.name, + scenario_ids=[str(s) for s in dict.fromkeys(payload.scenario_ids)], + persona_ids=[str(p) for p in dict.fromkeys(payload.persona_ids)], + metric_ids=[str(m) for m in payload.metric_ids] if payload.metric_ids else None, + metric_success_criteria=stored_criteria, + n_attempts=payload.n_attempts, + n_concurrent=payload.n_concurrent, + pass_threshold=payload.pass_threshold, + production_llm_provider=payload.production_llm_provider, + production_llm_model=payload.production_llm_model, + test_llm_provider=payload.test_llm_provider, + test_llm_model=payload.test_llm_model, + resolved_llms=describe_simulation_llms( + db, + agent=agent, + production_override=llm_override( + payload.production_llm_provider, payload.production_llm_model + ), + test_override=llm_override(payload.test_llm_provider, payload.test_llm_model), + ), + status=SimulationRunStatus.QUEUED.value, + expected_trials=len(set(payload.scenario_ids)) + * len(set(payload.persona_ids)) + * payload.n_attempts, + created_by=created_by, + ) + db.add(run) + db.commit() + db.refresh(run) + + from app.workers.celery_app import run_harbor_simulation_task + + task = run_harbor_simulation_task.delay(str(run.id)) + run.celery_task_id = task.id + db.commit() + return run + + +@router.post("", response_model=SimulationRunResponse, status_code=status.HTTP_201_CREATED) +def create_simulation_run( + payload: SimulationRunCreate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + principal: Principal = Depends(get_principal), +): + run = _create_and_queue( + db, + payload=payload, + organization_id=organization_id, + workspace_id=workspace_id, + created_by=getattr(principal, "user_id", None) and str(principal.user_id), + ) + return _run_response(db, run) + + +@router.get("", response_model=List[SimulationRunResponse]) +def list_simulation_runs( + agent_id: Optional[UUID] = None, + limit: int = Query(50, ge=1, le=200), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + query = db.query(SimulationRun).filter( + SimulationRun.organization_id == organization_id, + SimulationRun.workspace_id == workspace_id, + ) + if agent_id: + query = query.filter(SimulationRun.agent_id == agent_id) + runs = query.order_by(SimulationRun.created_at.desc()).limit(limit).all() + # Listing uses stored counters only; live summaries are computed on the detail view. + responses = [] + for run in runs: + response = SimulationRunResponse.model_validate(run) + response.has_artifacts = bool(run.artifacts_s3_key) + responses.append(response) + return responses + + +@router.get("/{run_ref}", response_model=SimulationRunResponse) +def get_simulation_run( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + return _run_response(db, _get_run(db, run_ref, organization_id, workspace_id)) + + +def _trial_response( + result: EvaluatorResult, + *, + attempt: Optional[int], + scenario_names: dict, + persona_names: dict, + model=SimulationTrialResponse, + **extra, +): + call_data = result.call_data if isinstance(result.call_data, dict) else {} + tool_calls = [c for c in call_data.get("tool_calls") or [] if isinstance(c, dict)] + return model( + id=result.id, + result_id=result.result_id, + harbor_trial_name=result.harbor_trial_name, + scenario_id=result.scenario_id, + scenario_name=scenario_names.get(result.scenario_id), + persona_id=result.persona_id, + persona_name=persona_names.get(result.persona_id), + attempt_index=result.attempt_index or attempt, + status=result.status, + reward=result.reward, + passed=result.passed, + turns=len(result.speaker_segments or []), + metric_scores=result.metric_scores, + error_message=result.error_message, + tool_call_count=len(tool_calls), + tool_error_count=sum(1 for c in tool_calls if c.get("status") == "error"), + tool_expectations_met=(call_data.get("tool_evaluation") or {}).get("all_expectations_met"), + created_at=result.created_at, + updated_at=result.updated_at, + **extra, + ) + + +def _names(db: Session, results: list[EvaluatorResult]) -> tuple[dict, dict]: + scenario_ids = {r.scenario_id for r in results if r.scenario_id} + persona_ids = {r.persona_id for r in results if r.persona_id} + scenarios = ( + db.query(Scenario.id, Scenario.name).filter(Scenario.id.in_(scenario_ids)).all() + if scenario_ids + else [] + ) + personas = ( + db.query(Persona.id, Persona.name).filter(Persona.id.in_(persona_ids)).all() + if persona_ids + else [] + ) + return dict(scenarios), dict(personas) + + +@router.get("/{run_ref}/trials", response_model=List[SimulationTrialResponse]) +def list_simulation_trials( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + run = _get_run(db, run_ref, organization_id, workspace_id) + results = trial_results(db, run.id) + attempts = attempt_numbers(results) + scenario_names, persona_names = _names(db, results) + return [ + _trial_response( + r, + attempt=attempts.get(r.id), + scenario_names=scenario_names, + persona_names=persona_names, + ) + for r in results + ] + + +def _load_trajectory(run: SimulationRun, result: EvaluatorResult) -> Optional[dict]: + from app.services.harbor.job_runner import harbor_job_dir + + if not result.harbor_trial_name: + return None + path = harbor_job_dir(run) / result.harbor_trial_name / "agent" / "trajectory.json" + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return None + + +@router.get("/{run_ref}/trials/{trial_id}", response_model=SimulationTrialDetailResponse) +def get_simulation_trial( + run_ref: str, + trial_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + run = _get_run(db, run_ref, organization_id, workspace_id) + results = trial_results(db, run.id) + result = next((r for r in results if r.id == trial_id), None) + if not result: + raise HTTPException(status_code=404, detail="Trial not found") + scenario_names, persona_names = _names(db, [result]) + call_data = result.call_data if isinstance(result.call_data, dict) else {} + return _trial_response( + result, + attempt=attempt_numbers(results).get(result.id), + scenario_names=scenario_names, + persona_names=persona_names, + model=SimulationTrialDetailResponse, + speaker_segments=result.speaker_segments, + call_analysis=call_data.get("call_analysis"), + trajectory=_load_trajectory(run, result), + tool_calls=call_data.get("tool_calls"), + tool_evaluation=call_data.get("tool_evaluation"), + llms=call_data.get("llms"), + ) + + +@router.patch("/{run_ref}", response_model=SimulationRunResponse) +def rename_simulation_run( + run_ref: str, + body: SimulationRunRename, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Change the display name. Does not affect trials or scores.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + run.name = body.name + db.commit() + db.refresh(run) + return _run_response(db, run) + + +@router.delete("/{run_ref}", status_code=status.HTTP_204_NO_CONTENT) +def delete_simulation_run( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Remove a simulation run and its trials. Revokes an in-flight Celery task when possible.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + task_id = (run.celery_task_id or "").strip() + if task_id: + from app.services.call_imports.bulk_ops import _batch_revoke_celery_task_ids + + _batch_revoke_celery_task_ids([task_id], terminate=True) + db.delete(run) + db.commit() + + +@router.post("/{run_ref}/cancel", response_model=SimulationRunResponse) +def cancel_simulation_run( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + run = _get_run(db, run_ref, organization_id, workspace_id) + if run.status in SimulationRunStatus.terminal(): + raise HTTPException(status_code=409, detail=f"Run is already {run.status}") + run.cancel_requested = True + if run.status == SimulationRunStatus.QUEUED.value: + # Not picked up yet: the task's queued-only guard makes it a no-op. + from datetime import datetime, timezone + + run.status = SimulationRunStatus.CANCELLED.value + run.finished_at = datetime.now(timezone.utc) + db.commit() + return _run_response(db, run) + + +@router.post( + "/{run_ref}/rerun", response_model=SimulationRunResponse, status_code=status.HTTP_201_CREATED +) +def rerun_simulation_run( + run_ref: str, + only_failed: bool = False, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + principal: Principal = Depends(get_principal), +): + """Clone a run's config; ``only_failed`` keeps scenarios/personas with a failing trial.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + scenario_ids, persona_ids = list(run.scenario_ids or []), list(run.persona_ids or []) + if only_failed: + failing = [r for r in trial_results(db, run.id) if not r.passed] + if not failing: + raise HTTPException(status_code=400, detail="No failed trials to re-run") + scenario_ids = list(dict.fromkeys(str(r.scenario_id) for r in failing)) + persona_ids = list(dict.fromkeys(str(r.persona_id) for r in failing)) + rerun_metric_ids = run.metric_ids + rerun_criteria = None + if run.metric_success_criteria: + from app.models.schemas import MetricSuccessCriterion + + rerun_criteria = [ + MetricSuccessCriterion.model_validate(c) for c in run.metric_success_criteria + ] + rerun_metric_ids = [c.metric_id for c in rerun_criteria] + payload = SimulationRunCreate( + agent_id=run.agent_id, + name=simulation_run_name_for_rerun( + run.name, run.run_id, only_failed=only_failed + ), + scenario_ids=scenario_ids, + persona_ids=persona_ids, + metric_ids=[UUID(str(m)) for m in rerun_metric_ids] if rerun_metric_ids else None, + metric_success_criteria=rerun_criteria, + n_attempts=run.n_attempts, + n_concurrent=run.n_concurrent, + pass_threshold=run.pass_threshold, + production_llm_provider=run.production_llm_provider, + production_llm_model=run.production_llm_model, + test_llm_provider=run.test_llm_provider, + test_llm_model=run.test_llm_model, + ) + new_run = _create_and_queue( + db, + payload=payload, + organization_id=organization_id, + workspace_id=workspace_id, + created_by=getattr(principal, "user_id", None) and str(principal.user_id), + ) + return _run_response(db, new_run) + + +@router.get("/{run_ref}/artifacts") +def download_simulation_artifacts( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Harbor job tarball (tasks, trajectories, rewards); open locally with ``harbor view``.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + key = run.artifacts_s3_key + if not key: + raise HTTPException(status_code=404, detail="No artifacts for this run yet") + filename = f"simulation-run-{run.run_id}.tar.gz" + if key.startswith(LOCAL_ARTIFACT_PREFIX): + path = Path(key[len(LOCAL_ARTIFACT_PREFIX) :]) + if not path.exists(): + raise HTTPException(status_code=404, detail="Artifact file no longer exists") + return FileResponse(path, media_type="application/gzip", filename=filename) + + from app.services.storage.blob_storage_service import blob_storage_service + + return Response( + content=blob_storage_service.download_file_by_key(key), + media_type="application/gzip", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _failure_analysis_response(db: Session, run: SimulationRun) -> SimulationFailureAnalysisResponse: + completed_rows = run.completed_trials or 0 + return SimulationFailureAnalysisResponse( + failed_trials=count_failed_trials(db, run), + success_rules=success_rule_labels(db, run), + metric_clusters=metric_clusters_state_from_raw( + run.metric_clusters, completed_rows=completed_rows + ), + prompt_improvements=( + prompt_improvements_state_from_raw( + run.prompt_improvements, completed_rows=completed_rows + ) + if isinstance(run.prompt_improvements, dict) + else None + ), + ) + + +def _analysis_running(run: SimulationRun) -> bool: + clusters = run.metric_clusters if isinstance(run.metric_clusters, dict) else {} + prompts = run.prompt_improvements if isinstance(run.prompt_improvements, dict) else {} + return clusters.get("status") == "running" or prompts.get("status") == "running" + + +@router.get("/{run_ref}/failure-analysis", response_model=SimulationFailureAnalysisResponse) +def get_simulation_failure_analysis( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Failure clusters for trials that missed their success rules, and prompt suggestions.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + return _failure_analysis_response(db, run) + + +@router.post("/{run_ref}/failure-analysis", response_model=SimulationFailureAnalysisResponse) +def generate_simulation_failure_analysis( + run_ref: str, + body: Optional[SimulationFailureAnalysisRequest] = None, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """(Re)generate failure clusters and prompt suggestions in the background.""" + body = body or SimulationFailureAnalysisRequest() + run = _get_run(db, run_ref, organization_id, workspace_id) + if run.status not in SimulationRunStatus.terminal(): + raise HTTPException(status_code=409, detail="Wait for the run to finish first") + if _analysis_running(run): + raise HTTPException(status_code=409, detail="Failure analysis is already running") + if count_failed_trials(db, run) == 0: + raise HTTPException( + status_code=400, detail="No completed trial missed a success rule" + ) + from app.workers.tasks.analyze_simulation_run_failures import ( + ANALYSIS_ID_KEY, + analyze_simulation_run_failures_task, + new_failure_analysis_state, + ) + + # A fresh ownership ID makes any older worker for this run drop its writes. + analysis_state = new_failure_analysis_state() + run.metric_clusters = analysis_state + run.prompt_improvements = None + db.commit() + + analyze_simulation_run_failures_task.delay( + str(run.id), + provider=body.provider, + model=body.model, + credential_id=str(body.credential_id) if body.credential_id else None, + max_llm_calls=body.max_llm_calls, + analysis_id=analysis_state[ANALYSIS_ID_KEY], + ) + db.refresh(run) + return _failure_analysis_response(db, run) + + +@router.post( + "/{run_ref}/failure-analysis/cancel", response_model=SimulationFailureAnalysisResponse +) +def cancel_simulation_failure_analysis( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Stop clustering; the prompt step is a single LLM call and is left to finish.""" + run = _get_run(db, run_ref, organization_id, workspace_id) + clusters = run.metric_clusters if isinstance(run.metric_clusters, dict) else {} + if clusters.get("status") != "running": + raise HTTPException(status_code=409, detail="Failure clustering is not running") + run.metric_clusters = { + **clusters, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + } + db.commit() + return _failure_analysis_response(db, run) + + +@router.patch( + "/{run_ref}/prompt-improvements/review", response_model=EvaluationPromptImprovementsState +) +def review_simulation_prompt_improvement( + run_ref: str, + body: PromptImprovementReviewUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + from app.services.call_import_prompt_push import PromptPushError, set_suggestion_review + + run = _get_run(db, run_ref, organization_id, workspace_id) + try: + set_suggestion_review(run, body.suggestion_id, body.review_status, body.edited_text) + except PromptPushError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + db.commit() + db.refresh(run) + return prompt_improvements_state_from_raw( + run.prompt_improvements, completed_rows=run.completed_trials or 0 + ) + + +@router.get( + "/{run_ref}/prompt-improvements/composed", response_model=SimulationComposedPromptResponse +) +def get_simulation_composed_prompt( + run_ref: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """The production prompt with accepted suggestions applied, ready to copy.""" + from app.services.call_import_prompt_push import ( + PromptPushError, + compose_accepted_prompt, + review_counts, + ) + + run = _get_run(db, run_ref, organization_id, workspace_id) + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + raw = run.prompt_improvements if isinstance(run.prompt_improvements, dict) else {} + base = raw.get("compose_base_prompt") + if not isinstance(base, str): + base = editable_agent_prompt(agent) + suggestions = raw.get("suggestions") if isinstance(raw.get("suggestions"), list) else [] + try: + composed = compose_accepted_prompt(base, suggestions) + except PromptPushError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + return SimulationComposedPromptResponse( + agent_id=agent.id, + agent_name=agent.name, + base_prompt=base, + composed_prompt=composed, + accepted_count=review_counts(suggestions)[0], + ) + + +@router.post("/{run_ref}/pdf-report") +def download_simulation_run_pdf_report( + run_ref: str, + body: Optional[SimulationRunPdfReportRequest] = None, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """PDF with run KPIs, failure diagnostics and prompt recommendations.""" + from app.services.reporting.simulation_run_pdf_report import ( + simulation_run_pdf_report_service, + ) + + body = body or SimulationRunPdfReportRequest() + run = _get_run(db, run_ref, organization_id, workspace_id) + if run.status not in SimulationRunStatus.terminal(): + raise HTTPException(status_code=409, detail="Wait for the run to finish first") + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + summary = run.summary if isinstance(run.summary, dict) else compute_summary( + db, run, trial_results(db, run.id) + ) + pdf_bytes = simulation_run_pdf_report_service.render_pdf( + run=run, + agent=agent, + summary=summary, + success_rules=success_rule_labels(db, run), + metric_clusters=run.metric_clusters if isinstance(run.metric_clusters, dict) else None, + prompt_improvements=( + run.prompt_improvements if isinstance(run.prompt_improvements, dict) else None + ), + sections=body.sections.model_dump(), + platform_base_url=body.platform_base_url, + ) + filename = f"simulation-run-{run.run_id}-report.pdf" + return Response( + content=pdf_bytes, + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=SIM_VIEW, + manage_capability=SIM_MANAGE, +) diff --git a/app/api/v1/routes/tool_connections.py b/app/api/v1/routes/tool_connections.py new file mode 100644 index 00000000..f8554479 --- /dev/null +++ b/app/api/v1/routes/tool_connections.py @@ -0,0 +1,305 @@ +"""Tool connections (MCP servers, HTTP tools, Postgres) and which agents use them.""" + +from __future__ import annotations + +from typing import List +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.core.auth import Principal, get_principal +from app.dependencies import get_db, get_organization_id, get_workspace_id +from app.models.database import Agent, AgentToolConnection, ToolConnection +from app.models.schemas import ( + AgentToolCatalogEntry, + AgentToolConnectionLink, + AgentToolConnectionsUpdate, + ToolConnectionCreate, + ToolConnectionResponse, + ToolConnectionUpdate, +) +from app.services.tools.connections import ( + CONNECTION_TYPE_MCP, + agent_tool_links, + connection_tool_views, + decrypt_secrets, + encrypt_secrets, + merge_secrets, + refresh_connection_tools, + replace_agent_tool_connections, + secret_keys, + unique_slug, + validate_connection, +) + +router = APIRouter(prefix="/tool-connections", tags=["tool-connections"]) + + +def _response(db: Session, connection: ToolConnection) -> ToolConnectionResponse: + agent_count = ( + db.query(func.count(AgentToolConnection.agent_id)) + .filter(AgentToolConnection.tool_connection_id == connection.id) + .scalar() + or 0 + ) + return ToolConnectionResponse( + id=connection.id, + name=connection.name, + slug=connection.slug, + connection_type=connection.connection_type, + preset=connection.preset, + config=connection.config or {}, + simulation_mode=connection.simulation_mode, + secret_keys=secret_keys(connection), + tools=connection_tool_views(connection), + last_tested_at=connection.last_tested_at, + last_error=connection.last_error, + agent_count=int(agent_count), + created_at=connection.created_at, + updated_at=connection.updated_at, + ) + + +def _get_connection( + db: Session, connection_id: UUID, organization_id: UUID, workspace_id: UUID +) -> ToolConnection: + connection = ( + db.query(ToolConnection) + .filter( + ToolConnection.id == connection_id, + ToolConnection.organization_id == organization_id, + ToolConnection.workspace_id == workspace_id, + ) + .first() + ) + if connection is None: + raise HTTPException(status_code=404, detail="Tool connection not found") + return connection + + +def _get_agent(db: Session, agent_id: UUID, organization_id: UUID, workspace_id: UUID) -> Agent: + agent = ( + db.query(Agent) + .filter( + Agent.id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + .first() + ) + if agent is None: + raise HTTPException(status_code=404, detail="Agent not found") + return agent + + +def _check_transport(connection_type: str, config: dict) -> None: + if connection_type != CONNECTION_TYPE_MCP: + return + from app.services.tools.providers.mcp import McpConnectionError, assert_transport_allowed + + try: + assert_transport_allowed(config) + except McpConnectionError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + +@router.get("", response_model=List[ToolConnectionResponse]) +def list_tool_connections( + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + rows = ( + db.query(ToolConnection) + .filter( + ToolConnection.organization_id == organization_id, + ToolConnection.workspace_id == workspace_id, + ) + .order_by(ToolConnection.created_at.desc()) + .all() + ) + return [_response(db, row) for row in rows] + + +@router.post("", response_model=ToolConnectionResponse, status_code=status.HTTP_201_CREATED) +def create_tool_connection( + payload: ToolConnectionCreate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + principal: Principal = Depends(get_principal), +): + """Create the connection and immediately discover its tools (errors land in ``last_error``).""" + try: + validate_connection(payload.connection_type, payload.config, payload.simulation_mode) + secrets = merge_secrets({}, payload.secrets, payload.connection_type) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + _check_transport(payload.connection_type, payload.config) + connection = ToolConnection( + organization_id=organization_id, + workspace_id=workspace_id, + name=payload.name.strip(), + slug=unique_slug(db, workspace_id=workspace_id, name=payload.name), + connection_type=payload.connection_type, + preset=payload.preset, + config=payload.config, + secrets=encrypt_secrets(secrets), + simulation_mode=payload.simulation_mode, + created_by=getattr(principal, "user_id", None) and str(principal.user_id), + ) + db.add(connection) + db.flush() + refresh_connection_tools(db, connection) + db.commit() + db.refresh(connection) + return _response(db, connection) + + +@router.get("/{connection_id}", response_model=ToolConnectionResponse) +def get_tool_connection( + connection_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + return _response(db, _get_connection(db, connection_id, organization_id, workspace_id)) + + +@router.patch("/{connection_id}", response_model=ToolConnectionResponse) +def update_tool_connection( + connection_id: UUID, + payload: ToolConnectionUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + connection = _get_connection(db, connection_id, organization_id, workspace_id) + config = payload.config if payload.config is not None else dict(connection.config or {}) + mode = payload.simulation_mode or connection.simulation_mode + try: + validate_connection(connection.connection_type, config, mode) + secrets = merge_secrets( + decrypt_secrets(connection), payload.secrets, connection.connection_type + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + _check_transport(connection.connection_type, config) + if payload.name is not None: + connection.name = payload.name.strip() + connection.config = config + connection.simulation_mode = mode + connection.secrets = encrypt_secrets(secrets) + # Endpoint, credentials or tool definitions may have changed: re-discover. + if payload.config is not None or payload.secrets: + refresh_connection_tools(db, connection) + db.commit() + db.refresh(connection) + return _response(db, connection) + + +@router.post("/{connection_id}/test", response_model=ToolConnectionResponse) +def test_tool_connection( + connection_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Reconnect, refresh the cached tool list and report any error.""" + connection = _get_connection(db, connection_id, organization_id, workspace_id) + refresh_connection_tools(db, connection) + db.commit() + db.refresh(connection) + return _response(db, connection) + + +@router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_tool_connection( + connection_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + connection = _get_connection(db, connection_id, organization_id, workspace_id) + db.query(AgentToolConnection).filter( + AgentToolConnection.tool_connection_id == connection.id + ).delete() + db.delete(connection) + db.commit() + + +@router.get("/agents/{agent_id}/links", response_model=List[AgentToolConnectionLink]) +def list_agent_tool_connections( + agent_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + agent = _get_agent(db, agent_id, organization_id, workspace_id) + return [ + AgentToolConnectionLink( + tool_connection_id=link.tool_connection_id, enabled_tools=link.enabled_tools + ) + for link in agent_tool_links(db, agent_id=agent.id) + ] + + +@router.put("/agents/{agent_id}/links", response_model=List[AgentToolConnectionLink]) +def update_agent_tool_connections( + agent_id: UUID, + payload: AgentToolConnectionsUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + agent = _get_agent(db, agent_id, organization_id, workspace_id) + try: + replace_agent_tool_connections( + db, agent=agent, links=[link.model_dump() for link in payload.links] + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + db.commit() + return payload.links + + +@router.get("/agents/{agent_id}/catalog", response_model=List[AgentToolCatalogEntry]) +def agent_tool_catalog( + agent_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Every tool the agent's production side can call in a simulation, and whether it is mocked.""" + from app.services.tools.toolset import build_agent_toolset + + agent = _get_agent(db, agent_id, organization_id, workspace_id) + toolset = build_agent_toolset(db, agent=agent, organization_id=organization_id) + try: + entries = [] + for name in toolset.tool_names: + spec = toolset.spec(name) + entries.append( + AgentToolCatalogEntry( + name=name, + description=spec.description, + source=spec.provider, + side_effect=spec.side_effect, + simulation_behavior="mocked" if toolset.should_mock(name) else "live", + ) + ) + return entries + finally: + toolset.close() + + +from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=SIM_VIEW, + manage_capability=SIM_MANAGE, +) diff --git a/app/api/v1/routes/usage_pricing.py b/app/api/v1/routes/usage_pricing.py index dc868b19..d80fb69e 100644 --- a/app/api/v1/routes/usage_pricing.py +++ b/app/api/v1/routes/usage_pricing.py @@ -164,7 +164,7 @@ def list_usage_pricing_overrides( ) -@router.get("/overrides/{model}", response_model=EffectivePricingResponse) +@router.get("/overrides/{model:path}", response_model=EffectivePricingResponse) def get_usage_pricing_override_effective( model: str, usage_kind: str = Query("llm"), @@ -181,7 +181,7 @@ def get_usage_pricing_override_effective( ) -@router.put("/overrides/{model}", response_model=PricingOverrideResponse) +@router.put("/overrides/{model:path}", response_model=PricingOverrideResponse) def upsert_usage_pricing_override( model: str, body: PricingOverrideUpsertRequest, @@ -200,7 +200,7 @@ def upsert_usage_pricing_override( ) -@router.delete("/overrides/{model}", response_model=PricingOverrideDeleteResponse) +@router.delete("/overrides/{model:path}", response_model=PricingOverrideDeleteResponse) def delete_usage_pricing_override( model: str, usage_kind: str = Query("llm"), diff --git a/app/api/v1/routes/vobiz_telephony.py b/app/api/v1/routes/vobiz_telephony.py index ee0c7291..7fa79237 100644 --- a/app/api/v1/routes/vobiz_telephony.py +++ b/app/api/v1/routes/vobiz_telephony.py @@ -772,6 +772,7 @@ async def carrier_media_websocket(websocket: WebSocket): caller_speaks_first=run_params.caller_speaks_first, caller_opening_text=run_params.caller_opening_text, persona_speaks_via_tts=run_params.persona_speaks_via_tts, + enable_knowledge_base_tools=True, ) else: if not context.google_api_key: diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index 95a6b346..586d6a94 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -530,6 +530,7 @@ def resolve_azure_endpoint_for_provider( caller_opening_text=caller_opening_text, persona_speaks_via_tts=persona_speaks_via_tts, tracing_task_kwargs=tracing_task_kwargs, + enable_knowledge_base_tools=True, ) else: call_metadata = await run_bot( @@ -649,7 +650,13 @@ def resolve_azure_endpoint_for_provider( "speaker_segments": speaker_segments, "recording_s3_key": call_metadata.get("s3_key"), "duration_seconds": call_metadata.get("duration"), + "knowledge_base_tools_enabled": call_metadata.get( + "knowledge_base_tools_enabled" + ), } + kb_calls = call_metadata.get("knowledge_base_tool_calls") + if kb_calls: + playground_call_data["knowledge_base_tool_calls"] = kb_calls if ui_surface: playground_call_data["ui_surface"] = ui_surface evaluator_result = EvaluatorResult( diff --git a/app/app_factory.py b/app/app_factory.py index 9ac37a08..4dfefb02 100644 --- a/app/app_factory.py +++ b/app/app_factory.py @@ -160,6 +160,10 @@ def _mount_frontend(app: FastAPI) -> None: static_dir = frontend_dist / "assets" if static_dir.exists(): app.mount("/assets", StaticFiles(directory=str(static_dir)), name="assets") + # The catch-all below only serves top-level files, so public/ subfolders need a mount. + tool_logos_dir = frontend_dist / "tool-logos" + if tool_logos_dir.exists(): + app.mount("/tool-logos", StaticFiles(directory=str(tool_logos_dir)), name="tool-logos") @app.get("/{full_path:path}") async def serve_frontend(full_path: str): diff --git a/app/cli.py b/app/cli.py index 96c0ca30..baff557f 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,1710 +1,1749 @@ -"""CLI for EfficientAI platform.""" - -import click -import yaml -import os -import sys -import subprocess -import threading -import time -from pathlib import Path -from typing import Optional - - -@click.group() -def main(): - """EfficientAI - Voice AI Evaluation Platform CLI.""" - pass - - -class FrontendWatcher: - """Watch frontend files and rebuild on changes.""" - - def __init__(self, frontend_dir: Path): - self.frontend_dir = frontend_dir - self.watching = False - self.thread = None - self.last_build_time = 0 - self.build_lock = threading.Lock() - - def should_rebuild(self) -> bool: - """Check if frontend files have changed.""" - src_dir = self.frontend_dir / "src" - if not src_dir.exists(): - return False - - # Check modification time of source files - max_mtime = 0 - for ext in [".tsx", ".ts", ".css", ".jsx", ".js"]: - for file_path in src_dir.rglob(f"*{ext}"): - if file_path.is_file(): - max_mtime = max(max_mtime, file_path.stat().st_mtime) - - # Also check config files - config_files = [ - self.frontend_dir / "vite.config.ts", - self.frontend_dir / "tailwind.config.js", - self.frontend_dir / "tsconfig.json", - self.frontend_dir / "package.json", - ] - for config_file in config_files: - if config_file.exists(): - max_mtime = max(max_mtime, config_file.stat().st_mtime) - - if max_mtime > self.last_build_time: - self.last_build_time = max_mtime - return True - return False - - def build_frontend(self): - """Rebuild the frontend.""" - with self.build_lock: - try: - click.echo("\n🔄 Frontend files changed, rebuilding...") - result = subprocess.run( - ["npm", "run", "build"], - cwd=self.frontend_dir, - check=False, - capture_output=True, - text=True, - ) - if result.returncode == 0: - click.echo("✅ Frontend rebuilt successfully") - else: - click.echo(f"⚠️ Frontend build had warnings (check logs)", err=True) - if result.stderr: - click.echo(result.stderr[:500], err=True) # Show first 500 chars - except Exception as e: - click.echo(f"❌ Frontend build error: {e}", err=True) - - def watch_loop(self): - """Watch loop that runs in background thread.""" - while self.watching: - try: - if self.should_rebuild(): - self.build_frontend() - time.sleep(1) # Check every second - except Exception as e: - click.echo(f"❌ Watcher error: {e}", err=True) - time.sleep(5) # Wait longer on error - - def start(self): - """Start the watcher in a background thread.""" - if self.watching: - return - self.watching = True - # Set initial build time to avoid rebuilding immediately - self.last_build_time = time.time() - self.thread = threading.Thread(target=self.watch_loop, daemon=True) - self.thread.start() - - def stop(self): - """Stop the watcher.""" - self.watching = False - if self.thread: - self.thread.join(timeout=1) - - -def start_frontend_watcher(frontend_dir: Path) -> FrontendWatcher: - """Start a frontend file watcher.""" - watcher = FrontendWatcher(frontend_dir) - watcher.start() - return watcher - - -def _read_config_media_ws_base_url(config_path: Path) -> Optional[str]: - """Return vobiz.media_ws_base_url from YAML when explicitly configured.""" - try: - with open(config_path, encoding="utf-8") as handle: - data = yaml.safe_load(handle) or {} - value = ((data.get("vobiz") or {}).get("media_ws_base_url") or "").strip() - return value or None - except Exception: - return None - - -@main.command() -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Show detailed migration output", -) -def migrate(verbose: bool): - """Run pending database migrations.""" - import logging - from app.core.migrations import run_migrations, ensure_migrations_directory - - if verbose: - logging.basicConfig(level=logging.INFO) - - click.echo("🔄 Running database migrations...") - ensure_migrations_directory() - - try: - run_migrations() - click.echo("✅ All migrations completed successfully!") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - sys.exit(1) - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--host", - default=None, - help="Host to bind to (overrides config)", -) -@click.option( - "--port", - default=None, - type=int, - help="Port to bind to (overrides config)", -) -@click.option( - "--build-frontend/--no-build-frontend", - default=True, - help="Build frontend before starting (default: True)", -) -@click.option( - "--reload/--no-reload", - default=True, - help="Enable auto-reload for development (default: True)", -) -@click.option( - "--watch-frontend/--no-watch-frontend", - default=False, - help="Watch frontend files and rebuild automatically (default: False)", -) -@click.option( - "--force-rebuild", - is_flag=True, - default=False, - help="Force rebuild of frontend without prompting", -) -@click.option( - "--skip-migrations", - is_flag=True, - default=False, - help="Skip running migrations before starting (not recommended)", -) -def start(config: str, host: Optional[str], port: Optional[int], build_frontend: bool, reload: bool, watch_frontend: bool, force_rebuild: bool, skip_migrations: bool): - """Start the EfficientAI application server.""" - from app.config import apply_service_mode, load_config_from_file, settings - - # Load configuration from YAML file - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - explicit_media_ws = _read_config_media_ws_base_url(config_path) - os.environ["SERVICE_MODE"] = "api" - if not explicit_media_ws: - # Single-process dev: co-locate voice WebSockets on the API port unless - # config.yml sets vobiz.media_ws_base_url (used with eai telephony-worker). - os.environ.pop("MEDIA_WS_BASE_URL", None) - - try: - load_config_from_file(str(config_path)) - apply_service_mode("api") - if not explicit_media_ws: - settings.MEDIA_WS_BASE_URL = "" - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - # Override with CLI options if provided - if host: - settings.HOST = host - if port: - settings.PORT = port - - # Build frontend if requested - # Check if frontend is already built - frontend_dist = Path(__file__).parent.parent / "frontend" / "dist" - if build_frontend and frontend_dist.exists() and any(frontend_dist.iterdir()): - # If watching, we always want to rebuild to catch latest changes at start - if not force_rebuild and not watch_frontend and not click.confirm("Frontend dist directory already exists. Rebuild anyway?"): - build_frontend = False - - if build_frontend: - click.echo("🔨 Building frontend...") - frontend_dir = Path(__file__).parent.parent / "frontend" - if not frontend_dir.exists(): - click.echo(f"❌ Frontend directory not found: {frontend_dir}", err=True) - sys.exit(1) - - try: - # Check if node_modules exists, if not, install dependencies - if not (frontend_dir / "node_modules").exists(): - click.echo("📦 Installing frontend dependencies...") - subprocess.run( - ["npm", "install", "--legacy-peer-deps"], - cwd=frontend_dir, - check=True, - capture_output=True, - ) - - # Build frontend (show output in real-time for better debugging) - click.echo(" Running TypeScript check and Vite build...") - result = subprocess.run( - ["npm", "run", "build"], - cwd=frontend_dir, - check=False, # Don't fail immediately, we'll check return code - capture_output=True, - text=True, - ) - if result.returncode != 0: - click.echo(f"❌ Frontend build failed with return code {result.returncode}", err=True) - if result.stdout: - click.echo(f"\nSTDOUT:\n{result.stdout}", err=True) - if result.stderr: - click.echo(f"\nSTDERR:\n{result.stderr}", err=True) - click.echo("\n💡 Try running 'npm run build' manually in the frontend directory to see full error details.", err=True) - sys.exit(1) - - click.echo("✅ Frontend built successfully") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Error building frontend:", err=True) - if e.stdout: - click.echo(f"STDOUT:\n{e.stdout}", err=True) - if e.stderr: - click.echo(f"STDERR:\n{e.stderr}", err=True) - click.echo(f"\nReturn code: {e.returncode}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ npm not found. Please install Node.js and npm.", err=True) - sys.exit(1) - - # Initialize DB tables and run migrations before starting (unless explicitly skipped) - if not skip_migrations: - click.echo("🔄 Initializing database and running migrations...") - from app.database import init_db - from app.core.migrations import run_migrations, ensure_migrations_directory - try: - init_db() - ensure_migrations_directory() - run_migrations() - click.echo("✅ Database initialized and migrations completed") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) - sys.exit(1) - else: - click.echo("⚠️ Skipping migrations (not recommended - migrations will run on startup)") - - # Start frontend watcher if requested - frontend_watcher = None - if watch_frontend: - click.echo("👀 Starting frontend file watcher...") - frontend_dir = Path(__file__).parent.parent / "frontend" - frontend_watcher = start_frontend_watcher(frontend_dir) - - # Start the server - import uvicorn - - from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs - - click.echo(f"🚀 Starting EfficientAI server...") - click.echo(f" Host: {settings.HOST}") - click.echo(f" Port: {settings.PORT}") - click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") - click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") - click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") - if watch_frontend: - click.echo(f" Frontend watcher: Active (rebuilding on file changes)") - - # Use import string for reload to work properly - try: - uvicorn.run( - "app.main:app", - host=settings.HOST, - port=settings.PORT, - reload=reload, - **uvicorn_run_extra_kwargs(), - ) - finally: - # Clean up watcher on exit - if frontend_watcher: - frontend_watcher.stop() - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery worker", -) -@click.option( - "--queues", - "-Q", - "queues", - default=None, - help=( - "Comma-separated list of Celery queues this worker should consume " - "(forwarded to celery's -Q flag). Defaults to the default queue." - ), -) -@click.option( - "--concurrency", - default=None, - type=int, - help="Number of concurrent worker processes/threads (Celery --concurrency).", -) -@click.option( - "--pool", - "-P", - "pool", - default=None, - type=click.Choice(["prefork", "threads", "solo", "eventlet", "gevent"], case_sensitive=False), - help="Celery worker pool implementation (forwarded to celery's -P flag).", -) -def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optional[int], pool: Optional[str]): - """Start the Celery worker for background task processing.""" - from app.config import load_config_from_file - - # Load configuration from YAML file - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - click.echo(f"🚀 Starting Celery worker...") - click.echo(f" Log level: {loglevel}") - if queues: - click.echo(f" Queues: {queues}") - if concurrency is not None: - click.echo(f" Concurrency: {concurrency}") - if pool: - click.echo(f" Pool: {pool}") - - # Start Celery worker - try: - import subprocess - cmd = ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"] - if queues: - cmd.append(f"--queues={queues}") - if concurrency is not None: - cmd.append(f"--concurrency={concurrency}") - if pool: - cmd.append(f"--pool={pool}") - subprocess.run(cmd, check=True) - except KeyboardInterrupt: - click.echo("\n👋 Celery worker stopped") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Celery worker failed: {e}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - - -@main.command("beat") -@click.option( - "--config", - "-c", - default="config.yml", - help="Path to configuration file", -) -@click.option( - "--loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery beat", -) -@click.option( - "--platform-worker-concurrency", - default=2, - type=int, - help="Concurrency for the co-located platform task worker (default: 2; thread pool).", -) -def beat(config: str, loglevel: str, platform_worker_concurrency: int): - """Start Celery Beat + platform task worker (alerts, FX, prune) — single replica only.""" - from app.config import load_config_from_file - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - from app.workers.config import PLATFORM_WORKER_QUEUE - - click.echo( - "🚀 Starting Celery Beat + platform worker " - f"(queue={PLATFORM_WORKER_QUEUE}, concurrency={platform_worker_concurrency})" - ) - platform_proc = None - try: - import subprocess - - platform_proc = subprocess.Popen( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--queues={PLATFORM_WORKER_QUEUE}", - "--pool=threads", - f"--concurrency={platform_worker_concurrency}", - f"--loglevel={loglevel}", - ], - ) - subprocess.run( - [ - "celery", - "-A", - "app.workers.celery_app", - "beat", - f"--loglevel={loglevel}", - ], - check=True, - ) - except KeyboardInterrupt: - click.echo("\n👋 Celery Beat stopped") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Celery Beat failed: {e}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - finally: - if platform_proc is not None and platform_proc.poll() is None: - platform_proc.terminate() - try: - platform_proc.wait(timeout=5) - except subprocess.TimeoutExpired: - platform_proc.kill() - - -@main.command("telephony-worker") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--host", default=None, help="Host to bind (default from config)") -@click.option("--port", default=None, type=int, help="Media server port (default 8001)") -def telephony_worker(config: str, host: Optional[str], port: Optional[int]): - """Start the media server for live voice WebSocket connections.""" - os.environ["SERVICE_MODE"] = "media" - from app.config import apply_service_mode, load_config_from_file, settings - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - apply_service_mode("media") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - bind_host = host or settings.HOST - bind_port = port or settings.MEDIA_PORT - - from app.app_factory import create_app - - app = create_app() - print( - f"[MEDIA] telephony-worker ready on {bind_host}:{bind_port} " - f"(SERVICE_MODE={settings.SERVICE_MODE}, media_routes mounted)", - flush=True, - ) - - import uvicorn - - from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs - - click.echo(f"🎙️ Starting EfficientAI media server on {bind_host}:{bind_port}") - uvicorn.run( - app, - host=bind_host, - port=bind_port, - reload=False, - **uvicorn_run_extra_kwargs(), - ) - - -@main.command("start-worker-all") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--loglevel", "-l", default="info", help="Celery log level") -@click.option("--media-port", default=None, type=int, help="Media server port (default 8001)") -def start_worker_all(config: str, loglevel: str, media_port: Optional[int]): - """Start Celery worker and media server together. - - Deprecated: prefer separate ``eai telephony-worker`` and ``eai worker`` services - (see docker-compose ``media`` + ``worker``). - """ - import signal - import atexit - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - from app.config import load_config_from_file, settings - - load_config_from_file(str(config_path)) - - media_proc = None - celery_proc = None - - def cleanup(): - nonlocal media_proc, celery_proc - for proc, label in ((media_proc, "media server"), (celery_proc, "Celery worker")): - if proc is None or proc.poll() is not None: - continue - try: - proc.terminate() - proc.wait(timeout=5) - click.echo(f"✅ {label} stopped") - except Exception: - proc.kill() - - atexit.register(cleanup) - - def _handle_signal(sig, frame): - cleanup() - sys.exit(0) - - signal.signal(signal.SIGINT, _handle_signal) - signal.signal(signal.SIGTERM, _handle_signal) - - port = media_port or settings.MEDIA_PORT - telephony_env = os.environ.copy() - telephony_env["SERVICE_MODE"] = "media" - media_proc = subprocess.Popen( - [sys.executable, "-m", "app.cli", "telephony-worker", "--config", str(config_path), "--port", str(port)], - env=telephony_env, - ) - celery_proc = subprocess.Popen( - ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"], - ) - click.echo(f"🚀 Started media server (pid={media_proc.pid}) and Celery worker (pid={celery_proc.pid})") - - try: - while True: - if media_proc.poll() is not None: - click.echo("❌ Media server exited", err=True) - break - if celery_proc.poll() is not None: - click.echo("❌ Celery worker exited", err=True) - break - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - cleanup() - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--host", - default=None, - help="Host to bind to (overrides config)", -) -@click.option( - "--port", - default=None, - type=int, - help="Port to bind to (overrides config)", -) -@click.option( - "--build-frontend/--no-build-frontend", - default=True, - help="Build frontend before starting (default: True)", -) -@click.option( - "--reload/--no-reload", - default=True, - help="Enable auto-reload for development (default: True)", -) -@click.option( - "--watch-frontend/--no-watch-frontend", - default=False, - help="Watch frontend files and rebuild automatically (default: False)", -) -@click.option( - "--force-rebuild", - is_flag=True, - default=False, - help="Force rebuild of frontend without prompting", -) -@click.option( - "--skip-migrations", - is_flag=True, - default=False, - help="Skip running migrations before starting (not recommended)", -) -@click.option( - "--worker-loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery worker", -) -@click.option( - "--imports-worker/--no-imports-worker", - default=True, - help=( - "Also start a dedicated worker for the `imports` queue used by call " - "import CSV processing (default: True). Disable to keep the previous " - "single-worker behavior." - ), -) -@click.option( - "--imports-worker-concurrency", - default=12, - type=int, - help=( - "Concurrency for the imports+diarization+evaluations worker " - "(default: 12; thread pool). Each shard basket follows this number, " - "plus a little spare, unless database.sharding pool_size and " - "max_overflow are both set." - ), -) -@click.option( - "--usage-worker/--no-usage-worker", - default=True, - help=( - "Also start a dedicated worker for the `usage` queue (flush + cost recompute; " - "default: True)." - ), -) -@click.option( - "--usage-worker-concurrency", - default=4, - type=int, - help="Concurrency for the usage worker (default: 4; thread pool).", -) -@click.option( - "--traces-worker-concurrency", - default=8, - type=int, - help="Concurrency for the traces worker (default: 8; thread pool).", -) -@click.option( - "--beat/--no-beat", - default=True, - help=( - "Start Celery Beat for platform periodic tasks (usage flush, alerts, etc.; " - "default: True — single replica only in production)." - ), -) -@click.option( - "--beat-loglevel", - default=None, - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery Beat (defaults to --worker-loglevel).", -) -@click.option( - "--telephony-worker/--no-telephony-worker", - default=True, - help="Spawn telephony media server for live voice WebSockets (default: on).", -) -@click.option( - "--media-port", - default=None, - type=int, - help="Telephony media server port (default: MEDIA_PORT / 8001).", -) -@click.option( - "--clickhouse/--no-clickhouse", - default=True, - help="Start ClickHouse via docker compose when clickhouse.url is configured (default: on).", -) -def start_all( - config: str, - host: Optional[str], - port: Optional[int], - build_frontend: bool, - reload: bool, - watch_frontend: bool, - force_rebuild: bool, - skip_migrations: bool, - worker_loglevel: str, - imports_worker: bool, - imports_worker_concurrency: int, - usage_worker: bool, - usage_worker_concurrency: int, - traces_worker_concurrency: int, - beat: bool, - beat_loglevel: Optional[str], - telephony_worker: bool, - media_port: Optional[int], - clickhouse: bool, -): - """Start the application server and Celery worker(s) together. - - By default this also spawns a telephony media server (``eai telephony-worker``) - and a second Celery worker that consumes the ``imports`` queue (call-import CSV - fan-out). A dedicated `traces` queue worker always runs (async OTLP ingest). - Use --no-telephony-worker or --no-imports-worker to skip those optional services. - """ - import signal - import atexit - - click.echo("🚀 Starting EfficientAI (App + Worker)...") - if telephony_worker: - click.echo(" Telephony media server will run on a separate port (SERVICE_MODE=media).") - if imports_worker: - click.echo( - " This will start the API server, the default Celery worker, " - "and a dedicated worker for the `imports` queue." - ) - else: - click.echo(" This will start both the API server and Celery worker") - click.echo(" Press Ctrl+C to stop all services\n") - - # Load configuration - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - try: - from app.config import load_config_from_file, settings - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - bind_media_port = media_port or settings.MEDIA_PORT - - if clickhouse and getattr(settings, "CLICKHOUSE_URL", None): - compose_file = Path(__file__).resolve().parent.parent / "docker-compose.yml" - if compose_file.exists(): - click.echo("🗄️ Starting ClickHouse (docker compose up -d clickhouse)...") - subprocess.run( - ["docker", "compose", "up", "-d", "clickhouse"], - cwd=str(compose_file.parent), - check=False, - ) - - os.environ["SERVICE_MODE"] = "api" - os.environ["EFFICIENTAI_CONFIG_PATH"] = str(config_path.resolve()) - - # Store worker processes for cleanup. - worker_process = None - worker_imports_process = None - worker_usage_process = None - worker_traces_process = None - beat_process = None - platform_worker_process = None - telephony_process = None - - def _terminate(proc, label: str): - """Best-effort terminate -> wait -> kill for a worker subprocess.""" - if proc is None or proc.poll() is not None: - return - try: - click.echo(f"\n👋 Stopping {label}...") - proc.terminate() - proc.wait(timeout=5) - click.echo(f"✅ {label} stopped") - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - except Exception: - pass - - def cleanup_processes(): - """Clean up spawned processes.""" - nonlocal worker_process, worker_imports_process, worker_usage_process - nonlocal worker_traces_process, beat_process, platform_worker_process, telephony_process - _terminate(telephony_process, "Telephony media server") - _terminate(beat_process, "Celery Beat") - _terminate(platform_worker_process, "Celery platform worker") - _terminate(worker_process, "Celery worker (default)") - _terminate(worker_imports_process, "Celery worker (imports)") - _terminate(worker_usage_process, "Celery worker (usage)") - _terminate(worker_traces_process, "Celery worker (traces)") - - # Register cleanup on exit - atexit.register(cleanup_processes) - - def signal_handler(sig, frame): - """Handle Ctrl+C gracefully.""" - cleanup_processes() - sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # Initialize DB tables and run migrations before starting (unless explicitly skipped) - if not skip_migrations: - click.echo("🔄 Initializing database and running migrations...") - from app.database import init_db - from app.core.migrations import run_migrations, ensure_migrations_directory - try: - init_db() - ensure_migrations_directory() - run_migrations() - click.echo("✅ Database initialized and migrations completed") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) - sys.exit(1) - - # Build frontend if needed - if build_frontend: - frontend_dir = Path(__file__).parent.parent / "frontend" - if frontend_dir.exists(): - click.echo("🔨 Building frontend...") - try: - if not (frontend_dir / "node_modules").exists(): - click.echo(" Installing frontend dependencies...") - subprocess.run(["npm", "install", "--legacy-peer-deps"], cwd=frontend_dir, check=True, capture_output=True) - subprocess.run(["npm", "run", "build"], cwd=frontend_dir, check=True, capture_output=True) - click.echo("✅ Frontend built successfully") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Frontend build failed: {e}", err=True) - sys.exit(1) - - # Start frontend watcher if requested - frontend_watcher = None - if watch_frontend: - click.echo("👀 Starting frontend file watcher...") - frontend_dir = Path(__file__).parent.parent / "frontend" - frontend_watcher = start_frontend_watcher(frontend_dir) - - def _spawn_worker(args: list[str], label: str, prefix: str) -> subprocess.Popen: - """Spawn a Celery worker subprocess and stream its stdout with a prefix.""" - proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - click.echo(f"✅ {label} started") - - def _stream(): - if proc.stdout: - for line in iter(proc.stdout.readline, ""): - if line: - click.echo(f"{prefix} {line.rstrip()}", err=False) - proc.stdout.close() - - threading.Thread(target=_stream, daemon=True).start() - return proc - - if telephony_worker: - try: - telephony_env = os.environ.copy() - telephony_env["SERVICE_MODE"] = "media" - telephony_process = subprocess.Popen( - [ - sys.executable, - "-m", - "app.cli", - "telephony-worker", - "--config", - str(config_path), - "--port", - str(bind_media_port), - ], - env=telephony_env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - click.echo( - f"✅ Telephony media server started on port {bind_media_port} " - f"(pid={telephony_process.pid})" - ) - - def _stream_telephony(): - if telephony_process.stdout: - for line in iter(telephony_process.stdout.readline, ""): - if line: - click.echo(f"[MEDIA] {line.rstrip()}", err=False) - telephony_process.stdout.close() - - threading.Thread(target=_stream_telephony, daemon=True).start() - except FileNotFoundError: - click.echo("❌ Python interpreter not found for telephony-worker subprocess", err=True) - sys.exit(1) - except Exception as e: - click.echo(f"❌ Failed to start telephony media server: {e}", err=True) - sys.exit(1) - - # Start Celery workers as subprocess(es) with output streaming - try: - worker_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - "celery,audio-metrics", - "-c", - "8", - ], - label="Celery worker (celery + audio-metrics queues, concurrency=8)", - prefix="[WORKER]", - ) - - if imports_worker: - from app.workers.config import IMPORTS_WORKER_QUEUES - - try: - import redis as redis_lib - - from app.config import settings as app_settings - - legacy_q = redis_lib.from_url(app_settings.REDIS_URL, decode_responses=True) - backlog = int(legacy_q.llen("platform-imports") or 0) - if backlog > 0: - click.echo( - f"⚠️ {backlog} Celery task(s) still on legacy queue " - f"'platform-imports' (worker consumes this list to drain them).", - err=True, - ) - except Exception: - pass - - worker_imports_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - IMPORTS_WORKER_QUEUES, - "-P", - "threads", - "-c", - str(imports_worker_concurrency), - ], - label=( - f"Celery worker ({IMPORTS_WORKER_QUEUES} queues, " - f"pool=threads, concurrency={imports_worker_concurrency})" - ), - prefix="[WORKER-IMPORTS]", - ) - - if usage_worker: - from app.workers.config import USAGE_WORKER_QUEUE - - worker_usage_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - USAGE_WORKER_QUEUE, - "-P", - "threads", - "-c", - str(usage_worker_concurrency), - ], - label=( - f"Celery worker ({USAGE_WORKER_QUEUE} queue, " - f"pool=threads, concurrency={usage_worker_concurrency})" - ), - prefix="[WORKER-USAGE]", - ) - - from app.workers.config import TRACES_WORKER_QUEUE - - worker_traces_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - TRACES_WORKER_QUEUE, - "-P", - "threads", - "-c", - str(traces_worker_concurrency), - ], - label=( - f"Celery worker ({TRACES_WORKER_QUEUE} queue, " - f"pool=threads, concurrency={traces_worker_concurrency})" - ), - prefix="[WORKER-TRACES]", - ) - - if beat: - from app.workers.config import PLATFORM_WORKER_QUEUE - - beat_level = beat_loglevel or worker_loglevel - platform_worker_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={beat_level}", - "-Q", - PLATFORM_WORKER_QUEUE, - "-P", - "threads", - "-c", - "2", - ], - label=f"Celery platform worker ({PLATFORM_WORKER_QUEUE} queue)", - prefix="[BEAT-WORKER]", - ) - beat_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "beat", - f"--loglevel={beat_level}", - ], - label=f"Celery Beat (scheduler, loglevel={beat_level})", - prefix="[BEAT]", - ) - - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - except Exception as e: - click.echo(f"❌ Failed to start worker: {e}", err=True) - sys.exit(1) - - # Small delay to let worker start - time.sleep(1) - - # Start the application server in the main process - # This allows uvicorn's reload to work properly (it needs to spawn child processes) - try: - from app.config import settings - import uvicorn - - # Override with CLI options if provided - if host: - settings.HOST = host - if port: - settings.PORT = port - - click.echo("✅ Application server starting...") - click.echo(f" Host: {settings.HOST}") - click.echo(f" Port: {settings.PORT}") - click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") - click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") - click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") - if watch_frontend: - click.echo(f" Frontend watcher: Active (rebuilding on file changes)") - if imports_worker: - from app.workers.config import IMPORTS_WORKER_QUEUES - - click.echo( - f" Workers: default queue + {IMPORTS_WORKER_QUEUES} " - f"(concurrency={imports_worker_concurrency}; imports preferred)" - ) - else: - click.echo(" Workers: default queue only (--no-imports-worker)") - if usage_worker: - from app.workers.config import USAGE_WORKER_QUEUE - - click.echo( - f" Usage worker: {USAGE_WORKER_QUEUE} queue " - f"(concurrency={usage_worker_concurrency})" - ) - else: - click.echo(" Usage worker: disabled (--no-usage-worker)") - if beat: - click.echo(" Celery Beat: scheduler + platform worker (alerts, FX, prune); flush on worker-usage") - else: - click.echo(" Celery Beat: disabled (--no-beat)") - if telephony_worker: - telephony_public = (settings.VOBIZ_WEBHOOK_BASE_URL or "").strip() - click.echo(f" Telephony edge: http://localhost:{bind_media_port} (local)") - if telephony_public: - click.echo(f" Vobiz webhook_base_url: {telephony_public}") - else: - click.echo( - " Set vobiz.webhook_base_url to your telephony public URL " - f"(e.g. ngrok http {bind_media_port}) for inbound PSTN" - ) - if (settings.MEDIA_WS_BASE_URL or "").strip(): - click.echo(f" Browser voice-agent WS: {settings.MEDIA_WS_BASE_URL}") - click.echo("\n📝 All services are running. Press Ctrl+C to stop.\n") - - # Run uvicorn in the main process (allows reload to work) - from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs - - uvicorn.run( - "app.main:app", - host=settings.HOST, - port=settings.PORT, - reload=reload, - **uvicorn_run_extra_kwargs(), - ) - except KeyboardInterrupt: - pass - except Exception as e: - click.echo(f"❌ App error: {e}", err=True) - finally: - cleanup_processes() - if frontend_watcher: - frontend_watcher.stop() - - -@main.command() -@click.option( - "--output", - "-o", - type=click.Path(), - default="config.yml", - help="Output file path for example config", -) -def init_config(output: str): - """Generate an example configuration file.""" - example_config = """# EfficientAI Configuration File - -# Application Settings -app: - name: "Voice AI Evaluation Platform" - version: "0.1.0" - debug: true - secret_key: "your-secret-key-here-change-in-production" - -# Server Settings -server: - host: "0.0.0.0" - port: 8000 - -# Database Configuration -database: - url: "postgresql://efficientai:password@localhost:5432/efficientai" - # Alternative: specify individual components - # user: "efficientai" - # password: "password" - # host: "localhost" - # port: 5432 - # db: "efficientai" - -# Redis Configuration -redis: - url: "redis://localhost:6379/0" - # Alternative: specify individual components - # host: "localhost" - # port: 6379 - # db: 0 - -# Celery Configuration -celery: - broker_url: "redis://localhost:6379/0" - result_backend: "redis://localhost:6379/0" - -# File Storage -storage: - upload_dir: "./uploads" - max_file_size_mb: 500 - allowed_audio_formats: - - "wav" - - "mp3" - - "flac" - - "m4a" - -# CORS Settings -cors: - origins: - - "http://localhost:3000" - - "http://localhost:8000" - -# API Settings -api: - prefix: "/api/v1" - key_header: "X-API-Key" - rate_limit_per_minute: 60 -""" - - output_path = Path(output) - if output_path.exists(): - if not click.confirm(f"File {output} already exists. Overwrite?"): - click.echo("Cancelled.") - return - - try: - output_path.write_text(example_config) - click.echo(f"✅ Created example configuration file: {output}") - click.echo(f"💡 Edit {output} with your settings, then run: eai start --config {output}") - except Exception as e: - click.echo(f"❌ Error creating config file: {e}", err=True) - sys.exit(1) - - -def _bootstrap_sharding_config(config_path: str) -> None: - from app.config import load_config_from_file - from app.db_sharding.pool_manager import db_pool_manager - - load_config_from_file(config_path) - db_pool_manager.reset() - - -@click.group() -def sharding(): - """Call-import data-plane sharding admin (rebalance / registry).""" - pass - - -main.add_command(sharding) - - -@sharding.command("list-slices") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--call-import-id", - required=True, - help="Call import UUID whose slice registry to inspect", -) -def sharding_list_slices(config: str, call_import_id: str): - """Show catalog slice registry rows for a call import.""" - from uuid import UUID - - from app.db_sharding.pool_manager import open_catalog_session - from app.db_sharding.rebalance import RebalanceError, list_shard_slices, require_sharding_enabled - - try: - _bootstrap_sharding_config(config) - require_sharding_enabled() - cid = UUID(call_import_id) - except (ValueError, RebalanceError) as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - - catalog = open_catalog_session() - try: - slices = list_shard_slices(catalog, cid) - if not slices: - click.echo(f"No registry slices for call_import {cid}") - return - click.echo(f"call_import_id: {cid}") - click.echo(f"slices: {len(slices)}") - for item in slices: - click.echo( - f" slice {item.slice_id}: shard={item.shard_id} " - f"rows [{item.row_index_min}, {item.row_index_max}] " - f"(count={item.row_count})" - ) - finally: - catalog.close() - - -@sharding.command("rebalance-slices") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--call-import-id", required=True, help="Call import UUID to rebalance") -@click.option("--from-shard", required=True, help="Source shard id (e.g. data-shard-01)") -@click.option("--to-shard", required=True, help="Target shard id (e.g. data-shard-02)") -@click.option( - "--slice-id", - "slice_ids", - multiple=True, - type=int, - help="Move only these slice ids (default: all slices on --from-shard)", -) -@click.option( - "--dry-run", - is_flag=True, - help="Plan only: print row counts without copying or updating registry", -) -@click.option( - "--force", - is_flag=True, - help="Skip quiescence checks (use only after pausing workers)", -) -def sharding_rebalance_slices( - config: str, - call_import_id: str, - from_shard: str, - to_shard: str, - slice_ids: tuple[int, ...], - dry_run: bool, - force: bool, -): - """Copy call-import slice rows (and eval rows) from one shard to another.""" - from uuid import UUID - - from app.db_sharding.pool_manager import open_catalog_session - from app.db_sharding.rebalance import ( - RebalanceError, - require_sharding_enabled, - build_rebalance_plan, - execute_rebalance_slices, - ) - - try: - _bootstrap_sharding_config(config) - require_sharding_enabled() - cid = UUID(call_import_id) - except (ValueError, RebalanceError) as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - - catalog = open_catalog_session() - try: - plan = build_rebalance_plan( - catalog, - cid, - from_shard_id=from_shard, - to_shard_id=to_shard, - slice_ids=slice_ids or None, - ) - click.echo("Rebalance plan:") - click.echo(f" call_import_id: {plan.call_import_id}") - click.echo(f" from_shard: {plan.from_shard_id}") - click.echo(f" to_shard: {plan.to_shard_id}") - click.echo(f" slices: {len(plan.slices)}") - for item in plan.slices: - click.echo( - f" slice {item.slice_id}: rows [{item.row_index_min}, {item.row_index_max}]" - ) - click.echo(f" import_rows: {plan.import_row_count}") - click.echo(f" eval_rows: {plan.eval_row_count}") - - result = execute_rebalance_slices( - catalog, - plan, - dry_run=dry_run, - force=force, - ) - if result.dry_run: - click.echo("✅ Dry run complete (no changes made)") - else: - click.echo( - "✅ Rebalance complete: " - f"slices={result.slices_moved}, " - f"import_rows={result.import_rows_moved}, " - f"eval_rows={result.eval_rows_moved}" - ) - except RebalanceError as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - finally: - catalog.close() - - -@click.group() -def usage(): - """Usage pricing ops (seed rates, diff catalog, recompute costs).""" - pass - - -main.add_command(usage) - - -def _load_cli_config(config: str) -> Path: - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - from app.config import load_config_from_file - - load_config_from_file(str(config_path)) - return config_path - - -@usage.command("seed-rates") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--effective-from", - type=click.DateTime(formats=["%Y-%m-%d"]), - default=None, - help="Effective date for seeded rates (default: 2020-01-01)", -) -def usage_seed_rates(config: str, effective_from): - """Upsert model_pricing_rates from models.json pricing blocks.""" - _load_cli_config(config) - from app.database import SessionLocal - from app.services.usage.pricing_ops import seed_rates_from_models_json - - day = effective_from.date() if effective_from else None - db = SessionLocal() - try: - count = seed_rates_from_models_json(db, effective_from=day) - db.commit() - click.echo(f"✅ Seeded/updated {count} pricing rate row(s)") - finally: - db.close() - - -@usage.command("diff-rates") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--effective-from", - type=click.DateTime(formats=["%Y-%m-%d"]), - default=None, - help="Compare rates at this effective_from date (default: 2020-01-01)", -) -@click.option("--json", "as_json", is_flag=True, help="Print machine-readable JSON") -def usage_diff_rates(config: str, effective_from, as_json: bool): - """Diff models.json pricing blocks vs model_pricing_rates in Postgres.""" - import json as json_module - - _load_cli_config(config) - from app.database import SessionLocal - from app.services.usage.pricing_ops import diff_models_json_vs_db - - day = effective_from.date() if effective_from else None - db = SessionLocal() - try: - report = diff_models_json_vs_db(db, effective_from=day) - finally: - db.close() - - if as_json: - click.echo(json_module.dumps(report, indent=2, default=str)) - return - - click.echo(f"effective_from: {report['effective_from']}") - click.echo( - f"models.json priced: {report['models_json_count']} | " - f"database rows: {report['database_count']} | " - f"in_sync: {report['in_sync']}" - ) - if report["only_in_models_json"]: - click.echo(f"\nOnly in models.json ({len(report['only_in_models_json'])}):") - for item in report["only_in_models_json"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - if report["only_in_database"]: - click.echo(f"\nOnly in database ({len(report['only_in_database'])}):") - for item in report["only_in_database"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - if report["mismatches"]: - click.echo(f"\nMismatched rates ({len(report['mismatches'])}):") - for item in report["mismatches"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - for field, values in item["fields"].items(): - click.echo( - f" {field}: json={values['models_json']} db={values['database']}" - ) - missing = report["missing_pricing_blocks"] - if missing: - click.echo(f"\nmodels.json entries missing pricing blocks ({len(missing)}):") - for model in missing[:20]: - click.echo(f" - {model}") - unresolved = report["litellm_unresolved"] - if unresolved: - click.echo(f"\nLiteLLM unresolved ({len(unresolved)}):") - for item in unresolved[:20]: - click.echo(f" - {item.get('model')} ({item.get('reason', 'unresolved')})") - - -@usage.command("recompute") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--organization-id", default=None, help="Scope recompute to one org UUID") -@click.option("--model", default=None, help="Scope recompute to one model") -@click.option("--usage-kind", default=None, help="Scope recompute to llm/stt/tts") -@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) -@click.option("--end-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) -@click.option( - "--async/--sync", - "run_async", - default=True, - help="Enqueue Celery task (default) or run synchronously in this process", -) -def usage_recompute( - config: str, - organization_id: Optional[str], - model: Optional[str], - usage_kind: Optional[str], - start_date, - end_date, - run_async: bool, -): - """Backfill or recompute stored usage costs on llm_usage_daily rollups.""" - from uuid import UUID - - _load_cli_config(config) - start = start_date.date() if start_date else None - end = end_date.date() if end_date else None - org_uuid = UUID(organization_id) if organization_id else None - - if run_async: - if org_uuid is None: - click.echo( - "❌ --organization-id is required for async recompute (creates a tracked job).", - err=True, - ) - click.echo( - "💡 Use --sync to recompute in this process without an org scope, or pass --organization-id.", - err=True, - ) - sys.exit(1) - - from app.database import SessionLocal - from app.services.usage.pricing_jobs import ( - create_recompute_job, - enqueue_recompute_job, - job_to_dict, - ) - - db = SessionLocal() - try: - job = create_recompute_job( - db, - organization_id=org_uuid, - model=model, - usage_kind=usage_kind, - start_date=start, - end_date=end, - ) - enqueue_recompute_job(db, job) - db.refresh(job) - payload = job_to_dict(job) - click.echo(f"✅ Enqueued recompute job {payload['id']} (status={payload['status']})") - if payload.get("celery_task_id"): - click.echo(f" Celery task: {payload['celery_task_id']}") - except Exception as exc: - click.echo(f"❌ Failed to enqueue recompute job: {exc}", err=True) - sys.exit(1) - finally: - db.close() - return - - from app.database import SessionLocal - from app.services.usage.pricing import recompute_usage_costs - - db = SessionLocal() - try: - updated = recompute_usage_costs( - db, - organization_id=org_uuid, - model=model, - usage_kind=usage_kind, - start_date=start, - end_date=end, - ) - click.echo(f"✅ Recomputed costs for {updated} rollup row(s)") - finally: - db.close() - - -@usage.command("sync-litellm") -@click.option("--local", is_flag=True, help="Use bundled LiteLLM model_cost JSON") -@click.option( - "--write-models", - is_flag=True, - help="Merge generated pricing into app/config/models.json", -) -@click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") -@click.option( - "--import-missing-fireworks", - is_flag=True, - help="Add current Fireworks serverless chat models missing from models.json", -) -def usage_sync_litellm( - local: bool, - write_models: bool, - stdout: bool, - import_missing_fireworks: bool, -): - """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" - import subprocess - import sys as sys_module - - script = Path(__file__).resolve().parent.parent / "scripts" / "sync_pricing_catalog_from_litellm.py" - cmd = [sys_module.executable, str(script)] - if local: - cmd.append("--local") - if write_models: - cmd.append("--write-models") - if stdout: - cmd.append("--stdout") - if import_missing_fireworks: - cmd.append("--import-missing-fireworks") - subprocess.run(cmd, check=True) - - -if __name__ == "__main__": - main() - +"""CLI for EfficientAI platform.""" + +import click +import yaml +import os +import sys +import subprocess +import threading +import time +from pathlib import Path +from typing import Optional + + +@click.group() +def main(): + """EfficientAI - Voice AI Evaluation Platform CLI.""" + pass + + +class FrontendWatcher: + """Watch frontend files and rebuild on changes.""" + + def __init__(self, frontend_dir: Path): + self.frontend_dir = frontend_dir + self.watching = False + self.thread = None + self.last_build_time = 0 + self.build_lock = threading.Lock() + + def should_rebuild(self) -> bool: + """Check if frontend files have changed.""" + src_dir = self.frontend_dir / "src" + if not src_dir.exists(): + return False + + # Check modification time of source files + max_mtime = 0 + for ext in [".tsx", ".ts", ".css", ".jsx", ".js"]: + for file_path in src_dir.rglob(f"*{ext}"): + if file_path.is_file(): + max_mtime = max(max_mtime, file_path.stat().st_mtime) + + # Also check config files + config_files = [ + self.frontend_dir / "vite.config.ts", + self.frontend_dir / "tailwind.config.js", + self.frontend_dir / "tsconfig.json", + self.frontend_dir / "package.json", + ] + for config_file in config_files: + if config_file.exists(): + max_mtime = max(max_mtime, config_file.stat().st_mtime) + + if max_mtime > self.last_build_time: + self.last_build_time = max_mtime + return True + return False + + def build_frontend(self): + """Rebuild the frontend.""" + with self.build_lock: + try: + click.echo("\n🔄 Frontend files changed, rebuilding...") + result = subprocess.run( + ["npm", "run", "build"], + cwd=self.frontend_dir, + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + click.echo("✅ Frontend rebuilt successfully") + else: + click.echo(f"⚠️ Frontend build had warnings (check logs)", err=True) + if result.stderr: + click.echo(result.stderr[:500], err=True) # Show first 500 chars + except Exception as e: + click.echo(f"❌ Frontend build error: {e}", err=True) + + def watch_loop(self): + """Watch loop that runs in background thread.""" + while self.watching: + try: + if self.should_rebuild(): + self.build_frontend() + time.sleep(1) # Check every second + except Exception as e: + click.echo(f"❌ Watcher error: {e}", err=True) + time.sleep(5) # Wait longer on error + + def start(self): + """Start the watcher in a background thread.""" + if self.watching: + return + self.watching = True + # Set initial build time to avoid rebuilding immediately + self.last_build_time = time.time() + self.thread = threading.Thread(target=self.watch_loop, daemon=True) + self.thread.start() + + def stop(self): + """Stop the watcher.""" + self.watching = False + if self.thread: + self.thread.join(timeout=1) + + +def start_frontend_watcher(frontend_dir: Path) -> FrontendWatcher: + """Start a frontend file watcher.""" + watcher = FrontendWatcher(frontend_dir) + watcher.start() + return watcher + + +def _read_config_media_ws_base_url(config_path: Path) -> Optional[str]: + """Return vobiz.media_ws_base_url from YAML when explicitly configured.""" + try: + with open(config_path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + value = ((data.get("vobiz") or {}).get("media_ws_base_url") or "").strip() + return value or None + except Exception: + return None + + +@main.command() +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed migration output", +) +def migrate(verbose: bool): + """Run pending database migrations.""" + import logging + from app.core.migrations import run_migrations, ensure_migrations_directory + + if verbose: + logging.basicConfig(level=logging.INFO) + + click.echo("🔄 Running database migrations...") + ensure_migrations_directory() + + try: + run_migrations() + click.echo("✅ All migrations completed successfully!") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + sys.exit(1) + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--host", + default=None, + help="Host to bind to (overrides config)", +) +@click.option( + "--port", + default=None, + type=int, + help="Port to bind to (overrides config)", +) +@click.option( + "--build-frontend/--no-build-frontend", + default=True, + help="Build frontend before starting (default: True)", +) +@click.option( + "--reload/--no-reload", + default=True, + help="Enable auto-reload for development (default: True)", +) +@click.option( + "--watch-frontend/--no-watch-frontend", + default=False, + help="Watch frontend files and rebuild automatically (default: False)", +) +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Force rebuild of frontend without prompting", +) +@click.option( + "--skip-migrations", + is_flag=True, + default=False, + help="Skip running migrations before starting (not recommended)", +) +def start(config: str, host: Optional[str], port: Optional[int], build_frontend: bool, reload: bool, watch_frontend: bool, force_rebuild: bool, skip_migrations: bool): + """Start the EfficientAI application server.""" + from app.config import apply_service_mode, load_config_from_file, settings + + # Load configuration from YAML file + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + explicit_media_ws = _read_config_media_ws_base_url(config_path) + os.environ["SERVICE_MODE"] = "api" + if not explicit_media_ws: + # Single-process dev: co-locate voice WebSockets on the API port unless + # config.yml sets vobiz.media_ws_base_url (used with eai telephony-worker). + os.environ.pop("MEDIA_WS_BASE_URL", None) + + try: + load_config_from_file(str(config_path)) + apply_service_mode("api") + if not explicit_media_ws: + settings.MEDIA_WS_BASE_URL = "" + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + # Override with CLI options if provided + if host: + settings.HOST = host + if port: + settings.PORT = port + + # Build frontend if requested + # Check if frontend is already built + frontend_dist = Path(__file__).parent.parent / "frontend" / "dist" + if build_frontend and frontend_dist.exists() and any(frontend_dist.iterdir()): + # If watching, we always want to rebuild to catch latest changes at start + if not force_rebuild and not watch_frontend and not click.confirm("Frontend dist directory already exists. Rebuild anyway?"): + build_frontend = False + + if build_frontend: + click.echo("🔨 Building frontend...") + frontend_dir = Path(__file__).parent.parent / "frontend" + if not frontend_dir.exists(): + click.echo(f"❌ Frontend directory not found: {frontend_dir}", err=True) + sys.exit(1) + + try: + # Check if node_modules exists, if not, install dependencies + if not (frontend_dir / "node_modules").exists(): + click.echo("📦 Installing frontend dependencies...") + subprocess.run( + ["npm", "install", "--legacy-peer-deps"], + cwd=frontend_dir, + check=True, + capture_output=True, + ) + + # Build frontend (show output in real-time for better debugging) + click.echo(" Running TypeScript check and Vite build...") + result = subprocess.run( + ["npm", "run", "build"], + cwd=frontend_dir, + check=False, # Don't fail immediately, we'll check return code + capture_output=True, + text=True, + ) + if result.returncode != 0: + click.echo(f"❌ Frontend build failed with return code {result.returncode}", err=True) + if result.stdout: + click.echo(f"\nSTDOUT:\n{result.stdout}", err=True) + if result.stderr: + click.echo(f"\nSTDERR:\n{result.stderr}", err=True) + click.echo("\n💡 Try running 'npm run build' manually in the frontend directory to see full error details.", err=True) + sys.exit(1) + + click.echo("✅ Frontend built successfully") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Error building frontend:", err=True) + if e.stdout: + click.echo(f"STDOUT:\n{e.stdout}", err=True) + if e.stderr: + click.echo(f"STDERR:\n{e.stderr}", err=True) + click.echo(f"\nReturn code: {e.returncode}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ npm not found. Please install Node.js and npm.", err=True) + sys.exit(1) + + # Initialize DB tables and run migrations before starting (unless explicitly skipped) + if not skip_migrations: + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db + from app.core.migrations import run_migrations, ensure_migrations_directory + try: + init_db() + ensure_migrations_directory() + run_migrations() + click.echo("✅ Database initialized and migrations completed") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) + sys.exit(1) + else: + click.echo("⚠️ Skipping migrations (not recommended - migrations will run on startup)") + + # Start frontend watcher if requested + frontend_watcher = None + if watch_frontend: + click.echo("👀 Starting frontend file watcher...") + frontend_dir = Path(__file__).parent.parent / "frontend" + frontend_watcher = start_frontend_watcher(frontend_dir) + + # Start the server + import uvicorn + + from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs + + click.echo(f"🚀 Starting EfficientAI server...") + click.echo(f" Host: {settings.HOST}") + click.echo(f" Port: {settings.PORT}") + click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") + click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") + click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") + if watch_frontend: + click.echo(f" Frontend watcher: Active (rebuilding on file changes)") + + # Use import string for reload to work properly + try: + uvicorn.run( + "app.main:app", + host=settings.HOST, + port=settings.PORT, + reload=reload, + **uvicorn_run_extra_kwargs(), + ) + finally: + # Clean up watcher on exit + if frontend_watcher: + frontend_watcher.stop() + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery worker", +) +@click.option( + "--queues", + "-Q", + "queues", + default=None, + help=( + "Comma-separated list of Celery queues this worker should consume " + "(forwarded to celery's -Q flag). Defaults to the default queue." + ), +) +@click.option( + "--concurrency", + default=None, + type=int, + help="Number of concurrent worker processes/threads (Celery --concurrency).", +) +@click.option( + "--pool", + "-P", + "pool", + default=None, + type=click.Choice(["prefork", "threads", "solo", "eventlet", "gevent"], case_sensitive=False), + help="Celery worker pool implementation (forwarded to celery's -P flag).", +) +def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optional[int], pool: Optional[str]): + """Start the Celery worker for background task processing.""" + from app.config import load_config_from_file + + # Load configuration from YAML file + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + click.echo(f"🚀 Starting Celery worker...") + click.echo(f" Log level: {loglevel}") + if queues: + click.echo(f" Queues: {queues}") + if concurrency is not None: + click.echo(f" Concurrency: {concurrency}") + if pool: + click.echo(f" Pool: {pool}") + + # Start Celery worker + try: + import subprocess + cmd = ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"] + if queues: + cmd.append(f"--queues={queues}") + if concurrency is not None: + cmd.append(f"--concurrency={concurrency}") + if pool: + cmd.append(f"--pool={pool}") + subprocess.run(cmd, check=True) + except KeyboardInterrupt: + click.echo("\n👋 Celery worker stopped") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Celery worker failed: {e}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + + +@main.command("beat") +@click.option( + "--config", + "-c", + default="config.yml", + help="Path to configuration file", +) +@click.option( + "--loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery beat", +) +@click.option( + "--platform-worker-concurrency", + default=2, + type=int, + help="Concurrency for the co-located platform task worker (default: 2; thread pool).", +) +def beat(config: str, loglevel: str, platform_worker_concurrency: int): + """Start Celery Beat + platform task worker (alerts, FX, prune) — single replica only.""" + from app.config import load_config_from_file + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + from app.workers.config import PLATFORM_WORKER_QUEUE + + click.echo( + "🚀 Starting Celery Beat + platform worker " + f"(queue={PLATFORM_WORKER_QUEUE}, concurrency={platform_worker_concurrency})" + ) + platform_proc = None + try: + import subprocess + + platform_proc = subprocess.Popen( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--queues={PLATFORM_WORKER_QUEUE}", + "--pool=threads", + f"--concurrency={platform_worker_concurrency}", + f"--loglevel={loglevel}", + ], + ) + subprocess.run( + [ + "celery", + "-A", + "app.workers.celery_app", + "beat", + f"--loglevel={loglevel}", + ], + check=True, + ) + except KeyboardInterrupt: + click.echo("\n👋 Celery Beat stopped") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Celery Beat failed: {e}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + finally: + if platform_proc is not None and platform_proc.poll() is None: + platform_proc.terminate() + try: + platform_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + platform_proc.kill() + + +@main.command("telephony-worker") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--host", default=None, help="Host to bind (default from config)") +@click.option("--port", default=None, type=int, help="Media server port (default 8001)") +def telephony_worker(config: str, host: Optional[str], port: Optional[int]): + """Start the media server for live voice WebSocket connections.""" + os.environ["SERVICE_MODE"] = "media" + from app.config import apply_service_mode, load_config_from_file, settings + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + apply_service_mode("media") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + bind_host = host or settings.HOST + bind_port = port or settings.MEDIA_PORT + + from app.app_factory import create_app + + app = create_app() + print( + f"[MEDIA] telephony-worker ready on {bind_host}:{bind_port} " + f"(SERVICE_MODE={settings.SERVICE_MODE}, media_routes mounted)", + flush=True, + ) + + import uvicorn + + from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs + + click.echo(f"🎙️ Starting EfficientAI media server on {bind_host}:{bind_port}") + uvicorn.run( + app, + host=bind_host, + port=bind_port, + reload=False, + **uvicorn_run_extra_kwargs(), + ) + + +@main.command("start-worker-all") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--loglevel", "-l", default="info", help="Celery log level") +@click.option("--media-port", default=None, type=int, help="Media server port (default 8001)") +def start_worker_all(config: str, loglevel: str, media_port: Optional[int]): + """Start Celery worker and media server together. + + Deprecated: prefer separate ``eai telephony-worker`` and ``eai worker`` services + (see docker-compose ``media`` + ``worker``). + """ + import signal + import atexit + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + from app.config import load_config_from_file, settings + + load_config_from_file(str(config_path)) + + media_proc = None + celery_proc = None + + def cleanup(): + nonlocal media_proc, celery_proc + for proc, label in ((media_proc, "media server"), (celery_proc, "Celery worker")): + if proc is None or proc.poll() is not None: + continue + try: + proc.terminate() + proc.wait(timeout=5) + click.echo(f"✅ {label} stopped") + except Exception: + proc.kill() + + atexit.register(cleanup) + + def _handle_signal(sig, frame): + cleanup() + sys.exit(0) + + signal.signal(signal.SIGINT, _handle_signal) + signal.signal(signal.SIGTERM, _handle_signal) + + port = media_port or settings.MEDIA_PORT + telephony_env = os.environ.copy() + telephony_env["SERVICE_MODE"] = "media" + media_proc = subprocess.Popen( + [sys.executable, "-m", "app.cli", "telephony-worker", "--config", str(config_path), "--port", str(port)], + env=telephony_env, + ) + celery_proc = subprocess.Popen( + ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"], + ) + click.echo(f"🚀 Started media server (pid={media_proc.pid}) and Celery worker (pid={celery_proc.pid})") + + try: + while True: + if media_proc.poll() is not None: + click.echo("❌ Media server exited", err=True) + break + if celery_proc.poll() is not None: + click.echo("❌ Celery worker exited", err=True) + break + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + cleanup() + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--host", + default=None, + help="Host to bind to (overrides config)", +) +@click.option( + "--port", + default=None, + type=int, + help="Port to bind to (overrides config)", +) +@click.option( + "--build-frontend/--no-build-frontend", + default=True, + help="Build frontend before starting (default: True)", +) +@click.option( + "--reload/--no-reload", + default=True, + help="Enable auto-reload for development (default: True)", +) +@click.option( + "--watch-frontend/--no-watch-frontend", + default=False, + help="Watch frontend files and rebuild automatically (default: False)", +) +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Force rebuild of frontend without prompting", +) +@click.option( + "--skip-migrations", + is_flag=True, + default=False, + help="Skip running migrations before starting (not recommended)", +) +@click.option( + "--worker-loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery worker", +) +@click.option( + "--imports-worker/--no-imports-worker", + default=True, + help=( + "Also start a dedicated worker for the `imports` queue used by call " + "import CSV processing (default: True). Disable to keep the previous " + "single-worker behavior." + ), +) +@click.option( + "--imports-worker-concurrency", + default=12, + type=int, + help=( + "Concurrency for the imports+diarization+evaluations worker " + "(default: 12; thread pool). Each shard basket follows this number, " + "plus a little spare, unless database.sharding pool_size and " + "max_overflow are both set." + ), +) +@click.option( + "--usage-worker/--no-usage-worker", + default=True, + help=( + "Also start a dedicated worker for the `usage` queue (flush + cost recompute; " + "default: True)." + ), +) +@click.option( + "--usage-worker-concurrency", + default=4, + type=int, + help="Concurrency for the usage worker (default: 4; thread pool).", +) +@click.option( + "--traces-worker-concurrency", + default=8, + type=int, + help="Concurrency for the traces worker (default: 8; thread pool).", +) +@click.option( + "--beat/--no-beat", + default=True, + help=( + "Start Celery Beat for platform periodic tasks (usage flush, alerts, etc.; " + "default: True — single replica only in production)." + ), +) +@click.option( + "--beat-loglevel", + default=None, + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery Beat (defaults to --worker-loglevel).", +) +@click.option( + "--telephony-worker/--no-telephony-worker", + default=True, + help="Spawn telephony media server for live voice WebSockets (default: on).", +) +@click.option( + "--media-port", + default=None, + type=int, + help="Telephony media server port (default: MEDIA_PORT / 8001).", +) +@click.option( + "--clickhouse/--no-clickhouse", + default=True, + help="Start ClickHouse via docker compose when clickhouse.url is configured (default: on).", +) +def start_all( + config: str, + host: Optional[str], + port: Optional[int], + build_frontend: bool, + reload: bool, + watch_frontend: bool, + force_rebuild: bool, + skip_migrations: bool, + worker_loglevel: str, + imports_worker: bool, + imports_worker_concurrency: int, + usage_worker: bool, + usage_worker_concurrency: int, + traces_worker_concurrency: int, + beat: bool, + beat_loglevel: Optional[str], + telephony_worker: bool, + media_port: Optional[int], + clickhouse: bool, +): + """Start the application server and Celery worker(s) together. + + By default this also spawns a telephony media server (``eai telephony-worker``) + and a second Celery worker that consumes the ``imports`` queue (call-import CSV + fan-out). A dedicated `traces` queue worker always runs (async OTLP ingest). + Use --no-telephony-worker or --no-imports-worker to skip those optional services. + """ + import signal + import atexit + + click.echo("🚀 Starting EfficientAI (App + Worker)...") + if telephony_worker: + click.echo(" Telephony media server will run on a separate port (SERVICE_MODE=media).") + if imports_worker: + click.echo( + " This will start the API server, the default Celery worker, " + "and a dedicated worker for the `imports` queue." + ) + else: + click.echo(" This will start both the API server and Celery worker") + click.echo(" Press Ctrl+C to stop all services\n") + + # Load configuration + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + try: + from app.config import load_config_from_file, settings + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + bind_media_port = media_port or settings.MEDIA_PORT + + if clickhouse and getattr(settings, "CLICKHOUSE_URL", None): + compose_file = Path(__file__).resolve().parent.parent / "docker-compose.yml" + if compose_file.exists(): + click.echo("🗄️ Starting ClickHouse (docker compose up -d clickhouse)...") + subprocess.run( + ["docker", "compose", "up", "-d", "clickhouse"], + cwd=str(compose_file.parent), + check=False, + ) + + os.environ["SERVICE_MODE"] = "api" + os.environ["EFFICIENTAI_CONFIG_PATH"] = str(config_path.resolve()) + + # Store worker processes for cleanup. + worker_process = None + worker_imports_process = None + worker_usage_process = None + worker_traces_process = None + worker_harbor_process = None + beat_process = None + platform_worker_process = None + telephony_process = None + + def _terminate(proc, label: str): + """Best-effort terminate -> wait -> kill for a worker subprocess.""" + if proc is None or proc.poll() is not None: + return + try: + click.echo(f"\n👋 Stopping {label}...") + proc.terminate() + proc.wait(timeout=5) + click.echo(f"✅ {label} stopped") + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + except Exception: + pass + + def cleanup_processes(): + """Clean up spawned processes.""" + nonlocal worker_process, worker_imports_process, worker_usage_process + nonlocal worker_traces_process, beat_process, platform_worker_process, telephony_process + nonlocal worker_harbor_process + _terminate(telephony_process, "Telephony media server") + _terminate(beat_process, "Celery Beat") + _terminate(platform_worker_process, "Celery platform worker") + _terminate(worker_process, "Celery worker (default)") + _terminate(worker_imports_process, "Celery worker (imports)") + _terminate(worker_usage_process, "Celery worker (usage)") + _terminate(worker_traces_process, "Celery worker (traces)") + _terminate(worker_harbor_process, "Celery worker (harbor)") + + # Register cleanup on exit + atexit.register(cleanup_processes) + + def signal_handler(sig, frame): + """Handle Ctrl+C gracefully.""" + cleanup_processes() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Initialize DB tables and run migrations before starting (unless explicitly skipped) + if not skip_migrations: + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db + from app.core.migrations import run_migrations, ensure_migrations_directory + try: + init_db() + ensure_migrations_directory() + run_migrations() + click.echo("✅ Database initialized and migrations completed") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) + sys.exit(1) + + # Build frontend if needed + if build_frontend: + frontend_dir = Path(__file__).parent.parent / "frontend" + if frontend_dir.exists(): + click.echo("🔨 Building frontend...") + try: + if not (frontend_dir / "node_modules").exists(): + click.echo(" Installing frontend dependencies...") + subprocess.run(["npm", "install", "--legacy-peer-deps"], cwd=frontend_dir, check=True, capture_output=True) + subprocess.run(["npm", "run", "build"], cwd=frontend_dir, check=True, capture_output=True) + click.echo("✅ Frontend built successfully") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Frontend build failed: {e}", err=True) + sys.exit(1) + + # Start frontend watcher if requested + frontend_watcher = None + if watch_frontend: + click.echo("👀 Starting frontend file watcher...") + frontend_dir = Path(__file__).parent.parent / "frontend" + frontend_watcher = start_frontend_watcher(frontend_dir) + + def _spawn_worker(args: list[str], label: str, prefix: str) -> subprocess.Popen: + """Spawn a Celery worker subprocess and stream its stdout with a prefix.""" + proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + click.echo(f"✅ {label} started") + + def _stream(): + if proc.stdout: + for line in iter(proc.stdout.readline, ""): + if line: + click.echo(f"{prefix} {line.rstrip()}", err=False) + proc.stdout.close() + + threading.Thread(target=_stream, daemon=True).start() + return proc + + if telephony_worker: + try: + telephony_env = os.environ.copy() + telephony_env["SERVICE_MODE"] = "media" + telephony_process = subprocess.Popen( + [ + sys.executable, + "-m", + "app.cli", + "telephony-worker", + "--config", + str(config_path), + "--port", + str(bind_media_port), + ], + env=telephony_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + click.echo( + f"✅ Telephony media server started on port {bind_media_port} " + f"(pid={telephony_process.pid})" + ) + + def _stream_telephony(): + if telephony_process.stdout: + for line in iter(telephony_process.stdout.readline, ""): + if line: + click.echo(f"[MEDIA] {line.rstrip()}", err=False) + telephony_process.stdout.close() + + threading.Thread(target=_stream_telephony, daemon=True).start() + except FileNotFoundError: + click.echo("❌ Python interpreter not found for telephony-worker subprocess", err=True) + sys.exit(1) + except Exception as e: + click.echo(f"❌ Failed to start telephony media server: {e}", err=True) + sys.exit(1) + + # Start Celery workers as subprocess(es) with output streaming + try: + worker_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + "celery,audio-metrics", + "-c", + "8", + ], + label="Celery worker (celery + audio-metrics queues, concurrency=8)", + prefix="[WORKER]", + ) + + if imports_worker: + from app.workers.config import IMPORTS_WORKER_QUEUES + + try: + import redis as redis_lib + + from app.config import settings as app_settings + + legacy_q = redis_lib.from_url(app_settings.REDIS_URL, decode_responses=True) + backlog = int(legacy_q.llen("platform-imports") or 0) + if backlog > 0: + click.echo( + f"⚠️ {backlog} Celery task(s) still on legacy queue " + f"'platform-imports' (worker consumes this list to drain them).", + err=True, + ) + except Exception: + pass + + worker_imports_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + IMPORTS_WORKER_QUEUES, + "-P", + "threads", + "-c", + str(imports_worker_concurrency), + ], + label=( + f"Celery worker ({IMPORTS_WORKER_QUEUES} queues, " + f"pool=threads, concurrency={imports_worker_concurrency})" + ), + prefix="[WORKER-IMPORTS]", + ) + + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + worker_usage_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + USAGE_WORKER_QUEUE, + "-P", + "threads", + "-c", + str(usage_worker_concurrency), + ], + label=( + f"Celery worker ({USAGE_WORKER_QUEUE} queue, " + f"pool=threads, concurrency={usage_worker_concurrency})" + ), + prefix="[WORKER-USAGE]", + ) + + from app.workers.config import TRACES_WORKER_QUEUE + + worker_traces_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + TRACES_WORKER_QUEUE, + "-P", + "threads", + "-c", + str(traces_worker_concurrency), + ], + label=( + f"Celery worker ({TRACES_WORKER_QUEUE} queue, " + f"pool=threads, concurrency={traces_worker_concurrency})" + ), + prefix="[WORKER-TRACES]", + ) + + import importlib.util + + from app.workers.config import HARBOR_WORKER_QUEUE + + # Harbor simulation runs need the optional ``harbor`` extra (Python 3.12+). + # Each task runs one Harbor job subprocess that fans out its own trials. + if importlib.util.find_spec("harbor") is not None: + worker_harbor_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + HARBOR_WORKER_QUEUE, + "-P", + "threads", + "-c", + "2", + ], + label=f"Celery Harbor worker ({HARBOR_WORKER_QUEUE} queue, concurrency=2)", + prefix="[WORKER-HARBOR]", + ) + else: + click.echo( + f"⚠️ Harbor not installed: simulation runs stay queued on '{HARBOR_WORKER_QUEUE}'. " + "Install with: pip install -e '.[harbor]'" + ) + + if beat: + from app.workers.config import PLATFORM_WORKER_QUEUE + + beat_level = beat_loglevel or worker_loglevel + platform_worker_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={beat_level}", + "-Q", + PLATFORM_WORKER_QUEUE, + "-P", + "threads", + "-c", + "2", + ], + label=f"Celery platform worker ({PLATFORM_WORKER_QUEUE} queue)", + prefix="[BEAT-WORKER]", + ) + beat_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "beat", + f"--loglevel={beat_level}", + ], + label=f"Celery Beat (scheduler, loglevel={beat_level})", + prefix="[BEAT]", + ) + + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + except Exception as e: + click.echo(f"❌ Failed to start worker: {e}", err=True) + sys.exit(1) + + # Small delay to let worker start + time.sleep(1) + + # Start the application server in the main process + # This allows uvicorn's reload to work properly (it needs to spawn child processes) + try: + from app.config import settings + import uvicorn + + # Override with CLI options if provided + if host: + settings.HOST = host + if port: + settings.PORT = port + + click.echo("✅ Application server starting...") + click.echo(f" Host: {settings.HOST}") + click.echo(f" Port: {settings.PORT}") + click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") + click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") + click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") + if watch_frontend: + click.echo(f" Frontend watcher: Active (rebuilding on file changes)") + if imports_worker: + from app.workers.config import IMPORTS_WORKER_QUEUES + + click.echo( + f" Workers: default queue + {IMPORTS_WORKER_QUEUES} " + f"(concurrency={imports_worker_concurrency}; imports preferred)" + ) + else: + click.echo(" Workers: default queue only (--no-imports-worker)") + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + click.echo( + f" Usage worker: {USAGE_WORKER_QUEUE} queue " + f"(concurrency={usage_worker_concurrency})" + ) + else: + click.echo(" Usage worker: disabled (--no-usage-worker)") + if beat: + click.echo(" Celery Beat: scheduler + platform worker (alerts, FX, prune); flush on worker-usage") + else: + click.echo(" Celery Beat: disabled (--no-beat)") + if telephony_worker: + telephony_public = (settings.VOBIZ_WEBHOOK_BASE_URL or "").strip() + click.echo(f" Telephony edge: http://localhost:{bind_media_port} (local)") + if telephony_public: + click.echo(f" Vobiz webhook_base_url: {telephony_public}") + else: + click.echo( + " Set vobiz.webhook_base_url to your telephony public URL " + f"(e.g. ngrok http {bind_media_port}) for inbound PSTN" + ) + if (settings.MEDIA_WS_BASE_URL or "").strip(): + click.echo(f" Browser voice-agent WS: {settings.MEDIA_WS_BASE_URL}") + if reload: + click.echo( + " Note: API auto-reload does not restart Celery workers; " + "after Python worker changes, stop and re-run start-all " + "(or restart the imports/evaluations worker process)." + ) + click.echo("\n📝 All services are running. Press Ctrl+C to stop.\n") + + # Run uvicorn in the main process (allows reload to work) + from app.core.uvicorn_runtime import uvicorn_run_extra_kwargs + + uvicorn.run( + "app.main:app", + host=settings.HOST, + port=settings.PORT, + reload=reload, + **uvicorn_run_extra_kwargs(), + ) + except KeyboardInterrupt: + pass + except Exception as e: + click.echo(f"❌ App error: {e}", err=True) + finally: + cleanup_processes() + if frontend_watcher: + frontend_watcher.stop() + + +@main.command() +@click.option( + "--output", + "-o", + type=click.Path(), + default="config.yml", + help="Output file path for example config", +) +def init_config(output: str): + """Generate an example configuration file.""" + example_config = """# EfficientAI Configuration File + +# Application Settings +app: + name: "Voice AI Evaluation Platform" + version: "0.1.0" + debug: true + secret_key: "your-secret-key-here-change-in-production" + +# Server Settings +server: + host: "0.0.0.0" + port: 8000 + +# Database Configuration +database: + url: "postgresql://efficientai:password@localhost:5432/efficientai" + # Alternative: specify individual components + # user: "efficientai" + # password: "password" + # host: "localhost" + # port: 5432 + # db: "efficientai" + +# Redis Configuration +redis: + url: "redis://localhost:6379/0" + # Alternative: specify individual components + # host: "localhost" + # port: 6379 + # db: 0 + +# Celery Configuration +celery: + broker_url: "redis://localhost:6379/0" + result_backend: "redis://localhost:6379/0" + +# File Storage +storage: + upload_dir: "./uploads" + max_file_size_mb: 500 + allowed_audio_formats: + - "wav" + - "mp3" + - "flac" + - "m4a" + +# CORS Settings +cors: + origins: + - "http://localhost:8000" + - "https://sandbox.efficientai.cloud" + +# API Settings +api: + prefix: "/api/v1" + key_header: "X-API-Key" + rate_limit_per_minute: 60 +""" + + output_path = Path(output) + if output_path.exists(): + if not click.confirm(f"File {output} already exists. Overwrite?"): + click.echo("Cancelled.") + return + + try: + output_path.write_text(example_config) + click.echo(f"✅ Created example configuration file: {output}") + click.echo(f"💡 Edit {output} with your settings, then run: eai start --config {output}") + except Exception as e: + click.echo(f"❌ Error creating config file: {e}", err=True) + sys.exit(1) + + +def _bootstrap_sharding_config(config_path: str) -> None: + from app.config import load_config_from_file + from app.db_sharding.pool_manager import db_pool_manager + + load_config_from_file(config_path) + db_pool_manager.reset() + + +@click.group() +def sharding(): + """Call-import data-plane sharding admin (rebalance / registry).""" + pass + + +main.add_command(sharding) + + +@sharding.command("list-slices") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--call-import-id", + required=True, + help="Call import UUID whose slice registry to inspect", +) +def sharding_list_slices(config: str, call_import_id: str): + """Show catalog slice registry rows for a call import.""" + from uuid import UUID + + from app.db_sharding.pool_manager import open_catalog_session + from app.db_sharding.rebalance import RebalanceError, list_shard_slices, require_sharding_enabled + + try: + _bootstrap_sharding_config(config) + require_sharding_enabled() + cid = UUID(call_import_id) + except (ValueError, RebalanceError) as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + + catalog = open_catalog_session() + try: + slices = list_shard_slices(catalog, cid) + if not slices: + click.echo(f"No registry slices for call_import {cid}") + return + click.echo(f"call_import_id: {cid}") + click.echo(f"slices: {len(slices)}") + for item in slices: + click.echo( + f" slice {item.slice_id}: shard={item.shard_id} " + f"rows [{item.row_index_min}, {item.row_index_max}] " + f"(count={item.row_count})" + ) + finally: + catalog.close() + + +@sharding.command("rebalance-slices") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--call-import-id", required=True, help="Call import UUID to rebalance") +@click.option("--from-shard", required=True, help="Source shard id (e.g. data-shard-01)") +@click.option("--to-shard", required=True, help="Target shard id (e.g. data-shard-02)") +@click.option( + "--slice-id", + "slice_ids", + multiple=True, + type=int, + help="Move only these slice ids (default: all slices on --from-shard)", +) +@click.option( + "--dry-run", + is_flag=True, + help="Plan only: print row counts without copying or updating registry", +) +@click.option( + "--force", + is_flag=True, + help="Skip quiescence checks (use only after pausing workers)", +) +def sharding_rebalance_slices( + config: str, + call_import_id: str, + from_shard: str, + to_shard: str, + slice_ids: tuple[int, ...], + dry_run: bool, + force: bool, +): + """Copy call-import slice rows (and eval rows) from one shard to another.""" + from uuid import UUID + + from app.db_sharding.pool_manager import open_catalog_session + from app.db_sharding.rebalance import ( + RebalanceError, + require_sharding_enabled, + build_rebalance_plan, + execute_rebalance_slices, + ) + + try: + _bootstrap_sharding_config(config) + require_sharding_enabled() + cid = UUID(call_import_id) + except (ValueError, RebalanceError) as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + + catalog = open_catalog_session() + try: + plan = build_rebalance_plan( + catalog, + cid, + from_shard_id=from_shard, + to_shard_id=to_shard, + slice_ids=slice_ids or None, + ) + click.echo("Rebalance plan:") + click.echo(f" call_import_id: {plan.call_import_id}") + click.echo(f" from_shard: {plan.from_shard_id}") + click.echo(f" to_shard: {plan.to_shard_id}") + click.echo(f" slices: {len(plan.slices)}") + for item in plan.slices: + click.echo( + f" slice {item.slice_id}: rows [{item.row_index_min}, {item.row_index_max}]" + ) + click.echo(f" import_rows: {plan.import_row_count}") + click.echo(f" eval_rows: {plan.eval_row_count}") + + result = execute_rebalance_slices( + catalog, + plan, + dry_run=dry_run, + force=force, + ) + if result.dry_run: + click.echo("✅ Dry run complete (no changes made)") + else: + click.echo( + "✅ Rebalance complete: " + f"slices={result.slices_moved}, " + f"import_rows={result.import_rows_moved}, " + f"eval_rows={result.eval_rows_moved}" + ) + except RebalanceError as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + finally: + catalog.close() + + +@click.group() +def usage(): + """Usage pricing ops (seed rates, diff catalog, recompute costs).""" + pass + + +main.add_command(usage) + + +def _load_cli_config(config: str) -> Path: + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + from app.config import load_config_from_file + + load_config_from_file(str(config_path)) + return config_path + + +@usage.command("seed-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Effective date for seeded rates (default: 2020-01-01)", +) +def usage_seed_rates(config: str, effective_from): + """Upsert model_pricing_rates from models.json pricing blocks.""" + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import seed_rates_from_models_json + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + count = seed_rates_from_models_json(db, effective_from=day) + db.commit() + click.echo(f"✅ Seeded/updated {count} pricing rate row(s)") + finally: + db.close() + + +@usage.command("diff-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Compare rates at this effective_from date (default: 2020-01-01)", +) +@click.option("--json", "as_json", is_flag=True, help="Print machine-readable JSON") +def usage_diff_rates(config: str, effective_from, as_json: bool): + """Diff models.json pricing blocks vs model_pricing_rates in Postgres.""" + import json as json_module + + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import diff_models_json_vs_db + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + report = diff_models_json_vs_db(db, effective_from=day) + finally: + db.close() + + if as_json: + click.echo(json_module.dumps(report, indent=2, default=str)) + return + + click.echo(f"effective_from: {report['effective_from']}") + click.echo( + f"models.json priced: {report['models_json_count']} | " + f"database rows: {report['database_count']} | " + f"in_sync: {report['in_sync']}" + ) + if report["only_in_models_json"]: + click.echo(f"\nOnly in models.json ({len(report['only_in_models_json'])}):") + for item in report["only_in_models_json"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["only_in_database"]: + click.echo(f"\nOnly in database ({len(report['only_in_database'])}):") + for item in report["only_in_database"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["mismatches"]: + click.echo(f"\nMismatched rates ({len(report['mismatches'])}):") + for item in report["mismatches"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + for field, values in item["fields"].items(): + click.echo( + f" {field}: json={values['models_json']} db={values['database']}" + ) + missing = report["missing_pricing_blocks"] + if missing: + click.echo(f"\nmodels.json entries missing pricing blocks ({len(missing)}):") + for model in missing[:20]: + click.echo(f" - {model}") + unresolved = report["litellm_unresolved"] + if unresolved: + click.echo(f"\nLiteLLM unresolved ({len(unresolved)}):") + for item in unresolved[:20]: + click.echo(f" - {item.get('model')} ({item.get('reason', 'unresolved')})") + + +@usage.command("recompute") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--organization-id", default=None, help="Scope recompute to one org UUID") +@click.option("--model", default=None, help="Scope recompute to one model") +@click.option("--usage-kind", default=None, help="Scope recompute to llm/stt/tts") +@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option("--end-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option( + "--async/--sync", + "run_async", + default=True, + help="Enqueue Celery task (default) or run synchronously in this process", +) +def usage_recompute( + config: str, + organization_id: Optional[str], + model: Optional[str], + usage_kind: Optional[str], + start_date, + end_date, + run_async: bool, +): + """Backfill or recompute stored usage costs on llm_usage_daily rollups.""" + from uuid import UUID + + _load_cli_config(config) + start = start_date.date() if start_date else None + end = end_date.date() if end_date else None + org_uuid = UUID(organization_id) if organization_id else None + + if run_async: + if org_uuid is None: + click.echo( + "❌ --organization-id is required for async recompute (creates a tracked job).", + err=True, + ) + click.echo( + "💡 Use --sync to recompute in this process without an org scope, or pass --organization-id.", + err=True, + ) + sys.exit(1) + + from app.database import SessionLocal + from app.services.usage.pricing_jobs import ( + create_recompute_job, + enqueue_recompute_job, + job_to_dict, + ) + + db = SessionLocal() + try: + job = create_recompute_job( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + enqueue_recompute_job(db, job) + db.refresh(job) + payload = job_to_dict(job) + click.echo(f"✅ Enqueued recompute job {payload['id']} (status={payload['status']})") + if payload.get("celery_task_id"): + click.echo(f" Celery task: {payload['celery_task_id']}") + except Exception as exc: + click.echo(f"❌ Failed to enqueue recompute job: {exc}", err=True) + sys.exit(1) + finally: + db.close() + return + + from app.database import SessionLocal + from app.services.usage.pricing import recompute_usage_costs + + db = SessionLocal() + try: + updated = recompute_usage_costs( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + click.echo(f"✅ Recomputed costs for {updated} rollup row(s)") + finally: + db.close() + + +@usage.command("sync-litellm") +@click.option("--local", is_flag=True, help="Use bundled LiteLLM model_cost JSON") +@click.option( + "--write-models", + is_flag=True, + help="Merge generated pricing into app/config/models.json", +) +@click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") +@click.option( + "--import-missing-fireworks", + is_flag=True, + help="Add current Fireworks serverless chat models missing from models.json", +) +def usage_sync_litellm( + local: bool, + write_models: bool, + stdout: bool, + import_missing_fireworks: bool, +): + """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" + import subprocess + import sys as sys_module + + script = Path(__file__).resolve().parent.parent / "scripts" / "sync_pricing_catalog_from_litellm.py" + cmd = [sys_module.executable, str(script)] + if local: + cmd.append("--local") + if write_models: + cmd.append("--write-models") + if stdout: + cmd.append("--stdout") + if import_missing_fireworks: + cmd.append("--import-missing-fireworks") + subprocess.run(cmd, check=True) + + +if __name__ == "__main__": + main() + diff --git a/app/config.py b/app/config.py index 42ed3599..6bd4386e 100644 --- a/app/config.py +++ b/app/config.py @@ -57,6 +57,12 @@ class Settings(BaseSettings): MAX_FILE_SIZE_MB: int = 500 ALLOWED_AUDIO_FORMATS: Annotated[List[str], NoDecode] = ["wav", "mp3", "flac", "m4a"] BLOB_STORAGE_PROVIDER: str = "s3" # "s3", "gcs", or "azure" + # Harbor simulation runs: exported tasks + job dirs (one subdir per run) + HARBOR_WORK_DIR: str = "./.data/harbor" + # Tool connections: stdio MCP servers run a command on the worker, so they are + # off unless the deployment is single-tenant / self-hosted. + TOOL_CONNECTIONS_ALLOW_STDIO: bool = False + TOOL_CALL_TIMEOUT_SECONDS: float = 30.0 # S3 Configuration S3_ENABLED: bool = False @@ -86,6 +92,8 @@ class Settings(BaseSettings): # Celery CELERY_BROKER_URL: Optional[str] = None CELERY_RESULT_BACKEND: Optional[str] = None + # worker-imports uses --pool=threads; terminate=True raises NotImplementedError there. + CELERY_REVOKE_USE_TERMINATE: bool = False # Call-import worker concurrency limits (Redis fair-share for evaluations) EVAL_WORKSPACE_INFLIGHT_LIMIT: int = 100 @@ -93,6 +101,26 @@ class Settings(BaseSettings): EVAL_GLOBAL_INFLIGHT_LIMIT: int = 128 EVAL_JOB_INFLIGHT_LIMIT: int = 75 EVAL_FAIR_DISPATCH_BATCH_SIZE: int = 75 + # Redis-buffered evaluation progress counters. Eval workers add + # completed/failed deltas to Redis instead of updating the catalog parent + # row per scored row; a beat task flushes them every + # EVAL_PROGRESS_FLUSH_SECONDS and fenced shard recounts run at most every + # EVAL_PROGRESS_RECOUNT_SECONDS for running evaluations. + EVAL_PROGRESS_FLUSH_SECONDS: int = 30 + EVAL_PROGRESS_RECOUNT_SECONDS: int = 300 + # LLM call path cache for org gateway settings + resolved credentials + # (keyed by the Redis ``llmcfg:ver:{org}`` version, bumped on any change). + # TTL bounds staleness only if a version bump is lost while Redis is down. + LLM_CONFIG_CACHE_TTL_SECONDS: int = 120 + LLM_CONFIG_VERSION_MEMO_SECONDS: float = 1.0 + # Per-process worker catalog cache (app/workers/catalog_cache.py): expired + # entries are swept on write at most every CATALOG_CACHE_SWEEP_SECONDS, and + # the least recently used entry is evicted past CATALOG_CACHE_MAX_ENTRIES. + CATALOG_CACHE_MAX_ENTRIES: int = 2048 + CATALOG_CACHE_SWEEP_SECONDS: float = 30.0 + # Eval workers' cached parent-evaluation snapshot. Settings changes bump + # ``eval:settings_ver:{id}`` and reload at once; this TTL is a backstop. + EVAL_SNAPSHOT_CACHE_TTL_SECONDS: int = 600 DIARIZATION_FAIR_DISPATCH_BATCH_SIZE: int = 75 IMPORT_FAIR_DISPATCH_BATCH_SIZE: int = 75 IMPORT_WORKSPACE_INFLIGHT_LIMIT: int = 8 @@ -126,7 +154,10 @@ class Settings(BaseSettings): PLATFORM_IMPORT_FEATURE_ENABLED: bool = True # CORS - CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"] + CORS_ORIGINS: List[str] = [ + "http://localhost:8000", + "https://sandbox.efficientai.cloud", + ] # API Settings API_KEY_HEADER: str = "X-API-Key" @@ -420,16 +451,16 @@ def parse_cors_origins(cls, v: Union[str, List[str], None]) -> List[str]: """Parse CORS_ORIGINS from various formats.""" # Handle None or empty values if v is None: - return ["http://localhost:3000", "http://localhost:8000"] + return ["http://localhost:8000", "https://sandbox.efficientai.cloud"] if isinstance(v, list): - return v if v else ["http://localhost:3000", "http://localhost:8000"] + return v if v else ["http://localhost:8000", "https://sandbox.efficientai.cloud"] if isinstance(v, str): # Handle empty or whitespace-only strings v = v.strip() if not v: - return ["http://localhost:3000", "http://localhost:8000"] + return ["http://localhost:8000", "https://sandbox.efficientai.cloud"] # Try JSON first if v.startswith("["): @@ -442,9 +473,9 @@ def parse_cors_origins(cls, v: Union[str, List[str], None]) -> List[str]: # Fall back to comma-separated origins = [origin.strip() for origin in v.split(",") if origin.strip()] - return origins if origins else ["http://localhost:3000", "http://localhost:8000"] + return origins if origins else ["http://localhost:8000", "https://sandbox.efficientai.cloud"] - return ["http://localhost:3000", "http://localhost:8000"] # Default + return ["http://localhost:8000", "https://sandbox.efficientai.cloud"] # Default @field_validator("AUTH_PROVIDERS", mode="before") @classmethod @@ -709,6 +740,10 @@ def load_config_from_file(config_path: str) -> None: settings.CELERY_BROKER_URL = celery_config["broker_url"] if "result_backend" in celery_config: settings.CELERY_RESULT_BACKEND = celery_config["result_backend"] + if "revoke_use_terminate" in celery_config: + settings.CELERY_REVOKE_USE_TERMINATE = bool( + celery_config["revoke_use_terminate"] + ) if "workers" in config_data: workers_config = config_data["workers"] @@ -732,6 +767,14 @@ def load_config_from_file(config_path: str) -> None: settings.EVAL_JOB_INFLIGHT_LIMIT = int( workers_config["eval_job_inflight_limit"] ) + if "eval_progress_flush_seconds" in workers_config: + settings.EVAL_PROGRESS_FLUSH_SECONDS = max( + 5, int(workers_config["eval_progress_flush_seconds"]) + ) + if "eval_progress_recount_seconds" in workers_config: + settings.EVAL_PROGRESS_RECOUNT_SECONDS = max( + 30, int(workers_config["eval_progress_recount_seconds"]) + ) if "diarization_fair_dispatch_batch_size" in workers_config: settings.DIARIZATION_FAIR_DISPATCH_BATCH_SIZE = int( workers_config["diarization_fair_dispatch_batch_size"] @@ -1279,7 +1322,7 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None # Create with explicit defaults - manually construct with default values settings = Settings( ALLOWED_AUDIO_FORMATS=["wav", "mp3", "flac", "m4a"], - CORS_ORIGINS=["http://localhost:3000", "http://localhost:8000"], + CORS_ORIGINS=["http://localhost:8000", "https://sandbox.efficientai.cloud"], _env_file=None, # Don't load .env ) else: diff --git a/app/core/api_rate_limit.py b/app/core/api_rate_limit.py index d01feb3b..2885e524 100644 --- a/app/core/api_rate_limit.py +++ b/app/core/api_rate_limit.py @@ -47,6 +47,9 @@ "evaluator", "conversation_evaluation", "manual_evaluation", + "simulation", + "knowledge_base", + "tool_connection", } ) diff --git a/app/core/license.py b/app/core/license.py index fb6d1c9a..a90d24c7 100644 --- a/app/core/license.py +++ b/app/core/license.py @@ -54,6 +54,11 @@ "description": "Batch ad-hoc metric scoring runs against evaluation results and call imports.", "category": "evaluation", }, + "scenario_metrics": { + "title": "Scenario Metrics", + "description": "Auto-generate evaluation metrics from agent scenarios and push them into Metrics after review.", + "category": "evaluation", + }, "db_sharding": { "title": "Call Import DB Sharding", "description": "Horizontally shard call-import row storage across multiple database nodes.", @@ -119,6 +124,7 @@ DEFAULT_ENTERPRISE_OFFERINGS: List[str] = [ "alerts", "metric_studio", + "scenario_metrics", "db_sharding", "llm_gateway", ] diff --git a/app/core/migration_scope_audit.py b/app/core/migration_scope_audit.py new file mode 100644 index 00000000..6a3f4bf0 --- /dev/null +++ b/app/core/migration_scope_audit.py @@ -0,0 +1,121 @@ +""" +Static checks for migration scope vs touched tables (sharded deployments). + +Run via tests/test_core/test_migration_scope_audit.py on every CI build. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Sequence + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + +# Row / payload tables queried on data shards when DB sharding is enabled. +SHARD_DATA_TABLES = frozenset( + { + "call_import_rows", + "call_import_evaluation_rows", + "evaluator_result_payloads", + "call_recording_payloads", + "platform_import_ledger_rows", + } +) + +SCOPE_RE = re.compile(r'MIGRATION_SCOPE\s*=\s*["\'](\w+)["\']') +TABLE_RE = re.compile( + r"\b(?:ALTER\s+TABLE|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" + r"DROP\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?)\s+([a-z_][a-z0-9_]*)", + re.I, +) + + +@dataclass(frozen=True) +class MigrationScopeIssue: + migration: str + scope: str + shard_tables: tuple[str, ...] + message: str + + +def _migration_scope(source: str) -> str: + match = SCOPE_RE.search(source) + return match.group(1) if match else "all" + + +def _upgrade_section(source: str) -> str: + if "def downgrade" in source: + return source.split("def downgrade", 1)[0] + return source + + +def iter_migration_files() -> Iterable[Path]: + return sorted( + path + for path in MIGRATIONS_DIR.glob("[0-9]*.py") + if not path.stem.startswith("__") + ) + + +def find_catalog_scope_shard_table_mismatches() -> List[MigrationScopeIssue]: + """Catalog-only migrations must not ALTER/CREATE shard row tables.""" + issues: List[MigrationScopeIssue] = [] + for path in iter_migration_files(): + source = path.read_text(encoding="utf-8") + scope = _migration_scope(source) + if scope != "catalog": + continue + tables = set(TABLE_RE.findall(_upgrade_section(source))) + shard_touch = sorted(tables & SHARD_DATA_TABLES) + if not shard_touch: + continue + issues.append( + MigrationScopeIssue( + migration=path.name, + scope=scope, + shard_tables=tuple(shard_touch), + message=( + f"{path.name} is catalog-scoped but touches shard data table(s): " + f"{', '.join(shard_touch)}" + ), + ) + ) + return issues + + +def find_duplicate_migration_stems() -> List[str]: + stems: List[str] = [path.stem for path in iter_migration_files()] + seen: set[str] = set() + dups: List[str] = [] + for stem in stems: + if stem in seen: + dups.append(stem) + seen.add(stem) + return sorted(dups) + + +def required_shard_migrations() -> Sequence[tuple[str, str]]: + """(filename prefix, expected MIGRATION_SCOPE) for high-risk call-import DDL.""" + return ( + ("108_call_import_recording_duration_seconds", "all"), + ("103_platform_import_hardening", "all"), + ("054_call_import_sharding", "all"), + ) + + +def assert_required_scopes() -> List[str]: + errors: List[str] = [] + for prefix, expected in required_shard_migrations(): + matches = [p for p in iter_migration_files() if p.stem == prefix] + if not matches: + errors.append(f"Missing migration {prefix}") + continue + source = matches[0].read_text(encoding="utf-8") + scope = _migration_scope(source) + if scope != expected: + errors.append( + f"{matches[0].name}: expected MIGRATION_SCOPE={expected!r}, got {scope!r}" + ) + return errors diff --git a/app/db_sharding/eval_rows.py b/app/db_sharding/eval_rows.py index 065d9bb4..60e85a0f 100644 --- a/app/db_sharding/eval_rows.py +++ b/app/db_sharding/eval_rows.py @@ -434,6 +434,7 @@ def retry_eval_page_query( exclude_ids: Optional[Sequence[UUID]] = None, include_metric_scores: bool = False, ordered: bool = False, + search_term: str = "", ) -> Query: """One status range on (evaluation_id, status). No transcript columns. @@ -453,6 +454,15 @@ def retry_eval_page_query( CallImportEvaluationRow.status == status, ) ) + from app.services.call_imports.row_text_search import apply_conversation_id_search + + needle = (search_term or "").strip() + if needle: + query = query.join( + CallImportRow, + CallImportEvaluationRow.call_import_row_id == CallImportRow.id, + ) + query = apply_conversation_id_search(query, needle) if after_id is not None: query = query.filter(CallImportEvaluationRow.id > after_id) if row_ids: @@ -484,6 +494,7 @@ def load_retry_target_page( include_metric_scores: bool = False, ordered: bool = False, limit: int = RETRY_ROW_PAGE_SIZE, + search_term: str = "", ) -> List[Tuple[CallImportEvaluationRow, Optional[CallImportRow]]]: eval_rows = ( retry_eval_page_query( @@ -495,6 +506,7 @@ def load_retry_target_page( exclude_ids=exclude_ids, include_metric_scores=include_metric_scores, ordered=ordered, + search_term=search_term, ) .limit(limit) .all() @@ -511,6 +523,52 @@ def load_retry_target_page( return [(row, sources.get(row.call_import_row_id)) for row in eval_rows] +def count_evaluation_retry_targets( + catalog_db: Session, + evaluation_id: UUID, + *, + statuses: Sequence[str], + search_term: str = "", +) -> int: + from app.services.call_imports.row_text_search import apply_conversation_id_search + + needle = (search_term or "").strip() + + def _count(session: Session) -> int: + total = 0 + for status in statuses: + query = session.query(func.count(CallImportEvaluationRow.id)).filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status == status, + ) + if needle: + query = query.join( + CallImportRow, + CallImportEvaluationRow.call_import_row_id == CallImportRow.id, + ) + query = apply_conversation_id_search(query, needle) + total += int(query.scalar() or 0) + return total + + if not is_sharding_enabled(): + return _count(catalog_db) + + router = db_pool_manager.router + if router is None: + return _count(catalog_db) + total = 0 + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + total += _count(shard_db) + finally: + shard_db.close() + if total == 0: + return _count(catalog_db) + return total + + def iter_evaluation_row_sessions(catalog_db: Session, call_import_id: UUID): """Yield (session, shard_id) for row writes. shard_id is None on the catalog.""" if not is_sharding_enabled(): diff --git a/app/db_sharding/import_dispatch.py b/app/db_sharding/import_dispatch.py index a7d25734..de009dae 100644 --- a/app/db_sharding/import_dispatch.py +++ b/app/db_sharding/import_dispatch.py @@ -143,6 +143,7 @@ def _diarization_dispatch_row_options(): return load_only( CallImportRow.id, CallImportRow.call_import_id, + CallImportRow.row_index, CallImportRow.diarised_transcript_status, CallImportRow.celery_task_id, ) diff --git a/app/db_sharding/scatter_gather.py b/app/db_sharding/scatter_gather.py index 262c5bf8..1a4dd551 100644 --- a/app/db_sharding/scatter_gather.py +++ b/app/db_sharding/scatter_gather.py @@ -6,7 +6,7 @@ from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Sequence, Tuple, TypeVar, TYPE_CHECKING from uuid import UUID -from sqlalchemy import Integer, case, cast, func, or_ +from sqlalchemy import Integer, and_, case, cast, func, or_ from sqlalchemy.orm import Session, defer, load_only, with_expression from app.db_sharding.pool_manager import db_pool_manager @@ -97,10 +97,9 @@ def _apply_call_import_row_filters( search_term: str, diarised_status_filter: Optional[str], ): - if search_term: - query = query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) + from app.services.call_imports.row_text_search import apply_conversation_id_search + + query = apply_conversation_id_search(query, search_term) if diarised_status_filter: query = query.filter( CallImportRow.diarised_transcript_status == diarised_status_filter @@ -796,78 +795,80 @@ def pending_eval_row_triples( return collected -def aggregate_evaluation_row_counts( +def _evaluation_row_count_columns(): + from sqlalchemy import case, func + + def _status_sum(*statuses: str): + return func.coalesce( + func.sum( + case( + (CallImportEvaluationRow.status.in_(statuses), 1), + else_=0, + ) + ), + 0, + ) + + return ( + func.count(CallImportEvaluationRow.id), + _status_sum("completed"), + _status_sum("failed"), + _status_sum("pending", "running"), + ) + + +def aggregate_evaluation_row_counts_detailed( catalog_db: Session, evaluation_id: UUID, -) -> tuple[int, int, int]: - """Return (total, completed, failed) across all shards.""" - from sqlalchemy import case, func +) -> tuple[int, int, int, int]: + """Return (total, completed, failed, in_progress) across all shards. - if not is_sharding_enabled(): + ``in_progress`` counts ``pending`` + ``running`` directly instead of + deriving it from the other three, so a row in any other status never + looks in flight. + """ + + def _one(session: Session) -> tuple[int, int, int, int]: row = ( - catalog_db.query( - func.count(CallImportEvaluationRow.id), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "completed", 1), - else_=0, - ) - ), - 0, - ), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "failed", 1), - else_=0, - ) - ), - 0, - ), - ) + session.query(*_evaluation_row_count_columns()) .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) .one() ) - return int(row[0] or 0), int(row[1] or 0), int(row[2] or 0) + return ( + int(row[0] or 0), + int(row[1] or 0), + int(row[2] or 0), + int(row[3] or 0), + ) - total = completed = failed = 0 + if not is_sharding_enabled(): + return _one(catalog_db) + + total = completed = failed = in_progress = 0 router = db_pool_manager.router assert router is not None for shard_id in router.shard_ids: factory = db_pool_manager.shard_session_factory(shard_id) shard_db = factory() try: - row = ( - shard_db.query( - func.count(CallImportEvaluationRow.id), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "completed", 1), - else_=0, - ) - ), - 0, - ), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "failed", 1), - else_=0, - ) - ), - 0, - ), - ) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) - .one() - ) - total += int(row[0] or 0) - completed += int(row[1] or 0) - failed += int(row[2] or 0) + t, c, f, p = _one(shard_db) + total += t + completed += c + failed += f + in_progress += p finally: shard_db.close() + return total, completed, failed, in_progress + + +def aggregate_evaluation_row_counts( + catalog_db: Session, + evaluation_id: UUID, +) -> tuple[int, int, int]: + """Return (total, completed, failed) across all shards.""" + total, completed, failed, _in_progress = aggregate_evaluation_row_counts_detailed( + catalog_db, evaluation_id + ) return total, completed, failed @@ -1015,6 +1016,26 @@ def load_call_import_rows_for_delete( ) +def load_call_import_rows_for_cancel( + catalog_db: Session, + call_import_id: UUID, + row_ids: List[UUID], +) -> List[CallImportRow]: + """Rows targeted for bulk stop-diarisation (shard-aware when enabled).""" + columns = ( + CallImportRow.id, + CallImportRow.row_index, + CallImportRow.diarised_transcript_status, + CallImportRow.celery_task_id, + ) + return _load_call_import_rows_scatter( + catalog_db, + call_import_id, + columns=columns, + requested_row_ids=row_ids, + ) + + def _legacy_catalog_rows( catalog_db: Session, call_import_id: UUID, @@ -1219,6 +1240,44 @@ def load(db: Session) -> List[CallImportEvaluationRow]: return merged +def load_evaluation_score_rows_for_aggregate( + catalog_db: Session, + evaluation_id: UUID, +) -> List[CallImportEvaluationRow]: + """Score rows with a terminal status only (for chart aggregates).""" + + def load(db: Session) -> List[CallImportEvaluationRow]: + rows = ( + db.query(CallImportEvaluationRow) + .options(load_only(*_EVAL_SCORE_COLUMNS)) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status.in_(("completed", "failed")), + ) + .all() + ) + for row in rows: + db.expunge(row) + return rows + + if not is_sharding_enabled(): + return load(catalog_db) + + merged: List[CallImportEvaluationRow] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + merged.extend(load(shard_db)) + finally: + shard_db.close() + if not merged: + return load(catalog_db) + return merged + + def count_call_import_rows( catalog_db: Session, call_import_id: UUID, @@ -1293,6 +1352,45 @@ def max_on_shard(db: Session, _shard_id: str) -> int: return max_index +def load_call_import_row_indices( + catalog_db: Session, + call_import_id: UUID, +) -> set[int]: + """Every saved ``row_index`` for an import, across shards.""" + if not is_sharding_enabled(): + rows = ( + catalog_db.query(CallImportRow.row_index) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + return {int(row[0]) for row in rows if row[0] is not None} + + found: set[int] = set() + + def indices_on_shard(db: Session, _shard_id: str) -> list[int]: + rows = ( + db.query(CallImportRow.row_index) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + return [int(row[0]) for row in rows if row[0] is not None] + + for part in scatter_gather_on_shards( + shard_ids_for_import(catalog_db, call_import_id), + indices_on_shard, + ): + found.update(part) + + if not found: + rows = ( + catalog_db.query(CallImportRow.row_index) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + found = {int(row[0]) for row in rows if row[0] is not None} + return found + + def load_call_import_conversation_ids( catalog_db: Session, call_import_id: UUID, @@ -1742,3 +1840,372 @@ def load_evaluation_row_pairs( .all() ) return _sort_evaluation_row_pairs(pairs) + + +DEFAULT_EVAL_ROW_PAIR_SAMPLE = 1500 +INSIGHTS_TLDR_ROW_SAMPLE = 1000 +USER_INSIGHTS_ROW_INPUT_CAP = 3000 +EVALUATION_ANALYSIS_BATCH_SIZE = 250 + + +class EvaluationRowPageCursor(NamedTuple): + """Keyset cursor for completed evaluation rows in ``row_index`` order.""" + + row_index: int + evaluation_row_id: UUID + + +def count_completed_evaluation_row_pairs( + catalog_db: Session, + evaluation_id: UUID, +) -> int: + def count_session(db: Session) -> int: + return int( + db.query(func.count(CallImportEvaluationRow.id)) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status == "completed", + ) + .scalar() + or 0 + ) + + if not is_sharding_enabled(): + return count_session(catalog_db) + + total = 0 + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + total += count_session(shard_db) + finally: + shard_db.close() + if total == 0: + total = count_session(catalog_db) + return total + + +def query_completed_recording_date_bounds( + catalog_db: Session, + evaluation_id: UUID, +) -> tuple[Optional["date"], Optional["date"]]: + """Min/max ``recording_date`` for completed rows (scatter-gather when sharded).""" + from datetime import date as date_type + + def bounds(db: Session) -> tuple[Optional[date_type], Optional[date_type]]: + row = ( + db.query( + func.min(CallImportRow.recording_date), + func.max(CallImportRow.recording_date), + ) + .select_from(CallImportEvaluationRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status == "completed", + CallImportRow.recording_date.isnot(None), + ) + .one() + ) + return row[0], row[1] + + if not is_sharding_enabled(): + return bounds(catalog_db) + + global_min: Optional[date_type] = None + global_max: Optional[date_type] = None + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + shard_min, shard_max = bounds(shard_db) + if shard_min is not None: + global_min = ( + shard_min + if global_min is None + else min(global_min, shard_min) + ) + if shard_max is not None: + global_max = ( + shard_max + if global_max is None + else max(global_max, shard_max) + ) + finally: + shard_db.close() + if global_min is None and global_max is None: + return bounds(catalog_db) + return global_min, global_max + + +def _completed_evaluation_row_pairs_query(db: Session, evaluation_id: UUID): + return ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status == "completed", + ) + ) + + +def _subsample_sorted_evaluation_row_pairs( + pairs: List[tuple[CallImportEvaluationRow, CallImportRow]], + *, + cap: int, +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + ordered = _sort_evaluation_row_pairs(pairs) + if len(ordered) <= cap: + return ordered + stride = max(1, (len(ordered) + cap - 1) // cap) + sampled = [ordered[i] for i in range(0, len(ordered), stride)][:cap] + if sampled: + return sampled + return ordered[:cap] + + +def _load_completed_evaluation_row_pair_keys( + catalog_db: Session, + evaluation_id: UUID, +) -> List[tuple[UUID, int, UUID]]: + """Sort keys only: (eval_row_id, row_index, call_import_row_id).""" + + def load_session(db: Session) -> List[tuple[UUID, int, UUID]]: + rows = ( + db.query( + CallImportEvaluationRow.id, + CallImportRow.row_index, + CallImportEvaluationRow.call_import_row_id, + ) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportEvaluationRow.status == "completed", + ) + .order_by(CallImportRow.row_index, CallImportEvaluationRow.id) + .all() + ) + return [(r[0], int(r[1] or 0), r[2]) for r in rows] + + if not is_sharding_enabled(): + return load_session(catalog_db) + + merged: List[tuple[UUID, int, UUID]] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + merged.extend(load_session(shard_db)) + finally: + shard_db.close() + if not merged: + merged = load_session(catalog_db) + return sorted(merged, key=lambda k: (k[1], str(k[0]))) + + +def _subsample_sorted_evaluation_row_pair_keys( + keys: List[tuple[UUID, int, UUID]], + *, + cap: int, +) -> List[tuple[UUID, int, UUID]]: + ordered = sorted(keys, key=lambda k: (k[1], str(k[0]))) + if len(ordered) <= cap: + return ordered + stride = max(1, (len(ordered) + cap - 1) // cap) + sampled = [ordered[i] for i in range(0, len(ordered), stride)][:cap] + if sampled: + return sampled + return ordered[:cap] + + +def _load_all_completed_evaluation_row_pairs( + catalog_db: Session, + evaluation_id: UUID, +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + def load_session(db: Session) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + return ( + _completed_evaluation_row_pairs_query(db, evaluation_id) + .order_by(CallImportRow.row_index, CallImportEvaluationRow.id) + .all() + ) + + if not is_sharding_enabled(): + return load_session(catalog_db) + + merged: List[tuple[CallImportEvaluationRow, CallImportRow]] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + merged.extend(load_session(shard_db)) + finally: + shard_db.close() + if not merged: + merged = load_session(catalog_db) + return merged + + +def load_completed_evaluation_row_pairs_for_eval_row_ids( + catalog_db: Session, + evaluation_id: UUID, + eval_row_ids: List[UUID], +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + """Completed pairs for explicit evaluation row ids (shard-aware).""" + unique_ids = list(dict.fromkeys(eval_row_ids)) + if not unique_ids: + return [] + + def load_session(db: Session) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + return ( + _completed_evaluation_row_pairs_query(db, evaluation_id) + .filter(CallImportEvaluationRow.id.in_(unique_ids)) + .all() + ) + + if not is_sharding_enabled(): + return _sort_evaluation_row_pairs(load_session(catalog_db)) + + merged: List[tuple[CallImportEvaluationRow, CallImportRow]] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + merged.extend(load_session(shard_db)) + finally: + shard_db.close() + if not merged: + merged = load_session(catalog_db) + return _sort_evaluation_row_pairs(merged) + + +def load_completed_evaluation_row_pairs_sample( + catalog_db: Session, + evaluation_id: UUID, + *, + max_pairs: int = DEFAULT_EVAL_ROW_PAIR_SAMPLE, + row_stride: int | None = None, +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + """Deterministic subsample of completed rows in ``row_index`` order.""" + cap = max(1, int(max_pairs)) + total = count_completed_evaluation_row_pairs(catalog_db, evaluation_id) + if total <= cap: + if not is_sharding_enabled(): + pairs = _completed_evaluation_row_pairs_query( + catalog_db, evaluation_id + ).all() + return _sort_evaluation_row_pairs(pairs) + + pairs: List[tuple[CallImportEvaluationRow, CallImportRow]] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + pairs.extend( + _completed_evaluation_row_pairs_query( + shard_db, evaluation_id + ).all() + ) + finally: + shard_db.close() + if not pairs: + pairs = _completed_evaluation_row_pairs_query( + catalog_db, evaluation_id + ).all() + return _sort_evaluation_row_pairs(pairs) + + _ = row_stride # legacy callers; sampling uses sorted positions only + keys = _load_completed_evaluation_row_pair_keys(catalog_db, evaluation_id) + sampled_keys = _subsample_sorted_evaluation_row_pair_keys(keys, cap=cap) + eval_row_ids = [k[0] for k in sampled_keys] + return load_completed_evaluation_row_pairs_for_eval_row_ids( + catalog_db, + evaluation_id, + eval_row_ids, + ) + + +def _completed_pairs_after_cursor_filter( + query, + after_cursor: Optional[EvaluationRowPageCursor], +): + if after_cursor is None: + return query + return query.filter( + or_( + CallImportRow.row_index > after_cursor.row_index, + and_( + CallImportRow.row_index == after_cursor.row_index, + CallImportEvaluationRow.id > after_cursor.evaluation_row_id, + ), + ) + ) + + +def load_completed_evaluation_row_pairs_page( + catalog_db: Session, + evaluation_id: UUID, + *, + limit: int = EVALUATION_ANALYSIS_BATCH_SIZE, + after_cursor: Optional[EvaluationRowPageCursor] = None, +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + """Next ``limit`` completed pairs in global ``row_index`` order (shard-aware).""" + page_size = max(1, int(limit)) + per_shard_limit = page_size + 50 + + def load_session(db: Session) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + q = _completed_evaluation_row_pairs_query(db, evaluation_id) + q = _completed_pairs_after_cursor_filter(q, after_cursor) + return ( + q.order_by(CallImportRow.row_index, CallImportEvaluationRow.id) + .limit(per_shard_limit) + .all() + ) + + if not is_sharding_enabled(): + return load_session(catalog_db)[:page_size] + + merged: List[tuple[CallImportEvaluationRow, CallImportRow]] = [] + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + merged.extend(load_session(shard_db)) + finally: + shard_db.close() + if not merged: + merged = load_session(catalog_db) + return _sort_evaluation_row_pairs(merged)[:page_size] + + +def evaluation_row_page_cursor_from_pair( + pair: tuple[CallImportEvaluationRow, CallImportRow], +) -> EvaluationRowPageCursor: + eval_row, source_row = pair + return EvaluationRowPageCursor( + row_index=int(source_row.row_index or 0), + evaluation_row_id=eval_row.id, + ) diff --git a/app/migrations/096_knowledge_bases.py b/app/migrations/096_knowledge_bases.py new file mode 100644 index 00000000..013886a1 --- /dev/null +++ b/app/migrations/096_knowledge_bases.py @@ -0,0 +1,139 @@ +"""Knowledge bases, documents, chunks (pgvector), and agent attachments.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "knowledge bases with pgvector chunks and agent links" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS knowledge_bases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + name VARCHAR(255) NOT NULL, + description TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_by VARCHAR + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_knowledge_bases_org_ws + ON knowledge_bases (organization_id, workspace_id) + """ + ) + ) + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS knowledge_documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + knowledge_base_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + source_type VARCHAR(32) NOT NULL DEFAULT 'file', + original_filename VARCHAR(512), + blob_key VARCHAR(1024) NOT NULL, + content_type VARCHAR(255), + byte_size BIGINT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + error_text TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_knowledge_documents_kb + ON knowledge_documents (knowledge_base_id) + """ + ) + ) + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS knowledge_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + text TEXT NOT NULL, + page_number INTEGER, + embedding vector(1536), + embedding_model VARCHAR(128), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + ) + # init_db() may have created knowledge_chunks from SQLAlchemy models before + # this migration runs (models omit the vector column). Ensure embedding exists. + db.execute( + text( + """ + ALTER TABLE knowledge_chunks + ADD COLUMN IF NOT EXISTS embedding vector(1536) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_knowledge_chunks_document + ON knowledge_chunks (document_id) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_knowledge_chunks_embedding + ON knowledge_chunks USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100) + """ + ) + ) + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS agent_knowledge_bases ( + agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + knowledge_base_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (agent_id, knowledge_base_id) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_agent_knowledge_bases_kb + ON agent_knowledge_bases (knowledge_base_id) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS agent_knowledge_bases")) + db.execute(text("DROP TABLE IF EXISTS knowledge_chunks")) + db.execute(text("DROP TABLE IF EXISTS knowledge_documents")) + db.execute(text("DROP TABLE IF EXISTS knowledge_bases")) + db.commit() diff --git a/app/migrations/097_knowledge_chunks_id_default.py b/app/migrations/097_knowledge_chunks_id_default.py new file mode 100644 index 00000000..90763e8c --- /dev/null +++ b/app/migrations/097_knowledge_chunks_id_default.py @@ -0,0 +1,31 @@ +"""Ensure knowledge_chunks.id has a DB default (init_db tables may omit it).""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "knowledge_chunks id default" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE knowledge_chunks + ALTER COLUMN id SET DEFAULT gen_random_uuid() + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE knowledge_chunks + ALTER COLUMN id DROP DEFAULT + """ + ) + ) + db.commit() diff --git a/app/migrations/098_knowledge_base_embedding_model.py b/app/migrations/098_knowledge_base_embedding_model.py new file mode 100644 index 00000000..a454865e --- /dev/null +++ b/app/migrations/098_knowledge_base_embedding_model.py @@ -0,0 +1,25 @@ +"""Add per–knowledge-base embedding model selection.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "knowledge_bases.embedding_model column" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE knowledge_bases + ADD COLUMN IF NOT EXISTS embedding_model VARCHAR(128) + NOT NULL DEFAULT 'text-embedding-3-small' + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS embedding_model")) + db.commit() diff --git a/app/migrations/099_knowledge_base_embedding_provider.py b/app/migrations/099_knowledge_base_embedding_provider.py new file mode 100644 index 00000000..d07ecfc2 --- /dev/null +++ b/app/migrations/099_knowledge_base_embedding_provider.py @@ -0,0 +1,65 @@ +"""Per–knowledge-base embedding provider and credential (any LiteLLM embedding model).""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "knowledge_bases embedding provider + credential; widen model id columns" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE knowledge_bases + ADD COLUMN IF NOT EXISTS embedding_provider VARCHAR(64) + NOT NULL DEFAULT 'openai' + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE knowledge_bases + ADD COLUMN IF NOT EXISTS embedding_credential_id UUID + REFERENCES aiproviders(id) ON DELETE SET NULL + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE knowledge_bases + ALTER COLUMN embedding_model TYPE VARCHAR(512) + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE knowledge_chunks + ALTER COLUMN embedding_model TYPE VARCHAR(512) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE knowledge_bases + DROP COLUMN IF EXISTS embedding_credential_id + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE knowledge_bases + DROP COLUMN IF EXISTS embedding_provider + """ + ) + ) + db.commit() diff --git a/app/migrations/100_simulation_runs.py b/app/migrations/100_simulation_runs.py new file mode 100644 index 00000000..c9ec4b8e --- /dev/null +++ b/app/migrations/100_simulation_runs.py @@ -0,0 +1,81 @@ +"""Harbor simulation runs (job-level) and trial columns on evaluator_results.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "simulation_runs table; evaluator_results simulation_run_id / trial / reward columns" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text(""" + CREATE TABLE IF NOT EXISTS simulation_runs ( + id UUID PRIMARY KEY, + run_id VARCHAR(6) NOT NULL UNIQUE, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + name VARCHAR, + scenario_ids JSON NOT NULL DEFAULT '[]', + persona_ids JSON NOT NULL DEFAULT '[]', + metric_ids JSON, + n_attempts INTEGER NOT NULL DEFAULT 1, + n_concurrent INTEGER NOT NULL DEFAULT 4, + pass_threshold DOUBLE PRECISION NOT NULL DEFAULT 0.7, + production_llm_provider VARCHAR(64), + production_llm_model VARCHAR(255), + test_llm_provider VARCHAR(64), + test_llm_model VARCHAR(255), + status VARCHAR(20) NOT NULL DEFAULT 'queued', + expected_trials INTEGER NOT NULL DEFAULT 0, + completed_trials INTEGER NOT NULL DEFAULT 0, + failed_trials INTEGER NOT NULL DEFAULT 0, + summary JSON, + error_message TEXT, + harbor_job_name VARCHAR(255), + artifacts_s3_key VARCHAR, + celery_task_id VARCHAR, + cancel_requested BOOLEAN NOT NULL DEFAULT false, + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + created_by VARCHAR + ) + """)) + for column in ("organization_id", "workspace_id", "agent_id", "status"): + db.execute( + text( + f"CREATE INDEX IF NOT EXISTS ix_simulation_runs_{column} " + f"ON simulation_runs ({column})" + ) + ) + + db.execute(text(""" + ALTER TABLE evaluator_results + ADD COLUMN IF NOT EXISTS simulation_run_id UUID + REFERENCES simulation_runs(id) ON DELETE CASCADE, + ADD COLUMN IF NOT EXISTS harbor_trial_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS attempt_index INTEGER, + ADD COLUMN IF NOT EXISTS reward DOUBLE PRECISION, + ADD COLUMN IF NOT EXISTS passed BOOLEAN + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_evaluator_results_simulation_run_id + ON evaluator_results (simulation_run_id) + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP INDEX IF EXISTS ix_evaluator_results_simulation_run_id")) + db.execute(text(""" + ALTER TABLE evaluator_results + DROP COLUMN IF EXISTS passed, + DROP COLUMN IF EXISTS reward, + DROP COLUMN IF EXISTS attempt_index, + DROP COLUMN IF EXISTS harbor_trial_name, + DROP COLUMN IF EXISTS simulation_run_id + """)) + db.execute(text("DROP TABLE IF EXISTS simulation_runs")) + db.commit() diff --git a/app/migrations/101_simulation_run_metric_success_criteria.py b/app/migrations/101_simulation_run_metric_success_criteria.py new file mode 100644 index 00000000..4dcee978 --- /dev/null +++ b/app/migrations/101_simulation_run_metric_success_criteria.py @@ -0,0 +1,31 @@ +"""Per-metric success criteria on simulation runs.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "simulation_runs.metric_success_criteria JSON column" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE simulation_runs + ADD COLUMN IF NOT EXISTS metric_success_criteria JSON + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE simulation_runs + DROP COLUMN IF EXISTS metric_success_criteria + """ + ) + ) + db.commit() diff --git a/app/migrations/102_metric_scenarios.py b/app/migrations/102_metric_scenarios.py new file mode 100644 index 00000000..63d41049 --- /dev/null +++ b/app/migrations/102_metric_scenarios.py @@ -0,0 +1,35 @@ +"""Link metrics to scenarios for scenario-scoped evaluation.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "metric_scenarios join table (scenario-scoped metrics)" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS metric_scenarios ( + metric_id UUID NOT NULL REFERENCES metrics(id) ON DELETE CASCADE, + scenario_id UUID NOT NULL REFERENCES scenarios(id) ON DELETE CASCADE, + PRIMARY KEY (metric_id, scenario_id) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_metric_scenarios_scenario_id + ON metric_scenarios (scenario_id) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS metric_scenarios")) + db.commit() diff --git a/app/migrations/102_user_agent_favorites.py b/app/migrations/102_user_agent_favorites.py new file mode 100644 index 00000000..a0039557 --- /dev/null +++ b/app/migrations/102_user_agent_favorites.py @@ -0,0 +1,38 @@ +"""Per-user stars on agents. Each person keeps their own list.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "user_agent_favorites table" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS user_agent_favorites ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (user_id, agent_id) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_user_agent_favorites_user_workspace + ON user_agent_favorites (user_id, workspace_id) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS user_agent_favorites")) + db.commit() diff --git a/app/migrations/103_metric_agents.py b/app/migrations/103_metric_agents.py new file mode 100644 index 00000000..7c94ed0c --- /dev/null +++ b/app/migrations/103_metric_agents.py @@ -0,0 +1,35 @@ +"""Link metrics to agents for agent-scoped evaluation.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "metric_agents join table (agent-scoped metrics)" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + CREATE TABLE IF NOT EXISTS metric_agents ( + metric_id UUID NOT NULL REFERENCES metrics(id) ON DELETE CASCADE, + agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + PRIMARY KEY (metric_id, agent_id) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_metric_agents_agent_id + ON metric_agents (agent_id) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS metric_agents")) + db.commit() diff --git a/app/migrations/104_scenario_expected_tool_calls.py b/app/migrations/104_scenario_expected_tool_calls.py new file mode 100644 index 00000000..a305b029 --- /dev/null +++ b/app/migrations/104_scenario_expected_tool_calls.py @@ -0,0 +1,23 @@ +"""Expected tool calls on scenarios (tool-use checks in simulation runs).""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "scenarios.expected_tool_calls JSON column" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE scenarios + ADD COLUMN IF NOT EXISTS expected_tool_calls JSON + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE scenarios + DROP COLUMN IF EXISTS expected_tool_calls + """)) + db.commit() diff --git a/app/migrations/105_simulation_run_resolved_llms.py b/app/migrations/105_simulation_run_resolved_llms.py new file mode 100644 index 00000000..eb14f2ec --- /dev/null +++ b/app/migrations/105_simulation_run_resolved_llms.py @@ -0,0 +1,23 @@ +"""Snapshot of the LLMs each side of a simulation run uses.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "simulation_runs.resolved_llms JSON column" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE simulation_runs + ADD COLUMN IF NOT EXISTS resolved_llms JSON + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE simulation_runs + DROP COLUMN IF EXISTS resolved_llms + """)) + db.commit() diff --git a/app/migrations/106_agent_knowledge_base_roles.py b/app/migrations/106_agent_knowledge_base_roles.py new file mode 100644 index 00000000..7931e6c8 --- /dev/null +++ b/app/migrations/106_agent_knowledge_base_roles.py @@ -0,0 +1,54 @@ +"""Separate knowledge bases for the production agent and the test agent. + +Existing links were shared by both sides, so each is kept as a production link and +copied as a test link; users can then detach either side independently. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "agent_knowledge_bases.role (production | test)" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + has_role = db.execute(text(""" + SELECT 1 FROM information_schema.columns + WHERE table_name = 'agent_knowledge_bases' AND column_name = 'role' + """)).first() + if has_role: + return + db.execute(text(""" + ALTER TABLE agent_knowledge_bases + ADD COLUMN role VARCHAR(16) NOT NULL DEFAULT 'production' + """)) + db.execute(text(""" + ALTER TABLE agent_knowledge_bases DROP CONSTRAINT IF EXISTS agent_knowledge_bases_pkey + """)) + db.execute(text(""" + ALTER TABLE agent_knowledge_bases + ADD PRIMARY KEY (agent_id, knowledge_base_id, role) + """)) + db.execute(text(""" + INSERT INTO agent_knowledge_bases + (agent_id, knowledge_base_id, organization_id, workspace_id, created_at, role) + SELECT agent_id, knowledge_base_id, organization_id, workspace_id, created_at, 'test' + FROM agent_knowledge_bases + WHERE role = 'production' + ON CONFLICT DO NOTHING + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DELETE FROM agent_knowledge_bases WHERE role <> 'production'")) + db.execute( + text( + "ALTER TABLE agent_knowledge_bases DROP CONSTRAINT IF EXISTS agent_knowledge_bases_pkey" + ) + ) + db.execute(text("ALTER TABLE agent_knowledge_bases DROP COLUMN IF EXISTS role")) + db.execute( + text("ALTER TABLE agent_knowledge_bases ADD PRIMARY KEY (agent_id, knowledge_base_id)") + ) + db.commit() diff --git a/app/migrations/107_tool_connections.py b/app/migrations/107_tool_connections.py new file mode 100644 index 00000000..bf2982ea --- /dev/null +++ b/app/migrations/107_tool_connections.py @@ -0,0 +1,60 @@ +"""External tool connections (MCP servers, HTTP tools, Postgres) and their agent links.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "tool_connections + agent_tool_connections" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text(""" + CREATE TABLE IF NOT EXISTS tool_connections ( + id UUID PRIMARY KEY, + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + name VARCHAR(255) NOT NULL, + slug VARCHAR(64) NOT NULL, + connection_type VARCHAR(32) NOT NULL, + preset VARCHAR(64), + config JSON NOT NULL DEFAULT '{}', + secrets TEXT, + simulation_mode VARCHAR(16) NOT NULL DEFAULT 'mock_writes', + cached_tools JSON, + last_tested_at TIMESTAMPTZ, + last_error TEXT, + created_by VARCHAR, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """)) + db.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS uq_tool_connections_workspace_slug + ON tool_connections (workspace_id, slug) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_tool_connections_org + ON tool_connections (organization_id) + """)) + db.execute(text(""" + CREATE TABLE IF NOT EXISTS agent_tool_connections ( + agent_id UUID NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + tool_connection_id UUID NOT NULL REFERENCES tool_connections(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + enabled_tools JSON, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (agent_id, tool_connection_id) + ) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_agent_tool_connections_connection + ON agent_tool_connections (tool_connection_id) + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS agent_tool_connections")) + db.execute(text("DROP TABLE IF EXISTS tool_connections")) + db.commit() diff --git a/app/migrations/108_call_import_recording_duration_seconds.py b/app/migrations/108_call_import_recording_duration_seconds.py new file mode 100644 index 00000000..ad6d3e97 --- /dev/null +++ b/app/migrations/108_call_import_recording_duration_seconds.py @@ -0,0 +1,51 @@ +"""Store recording duration on call import rows at ingest.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add recording_duration_seconds to call_import_rows" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE call_import_rows + ADD COLUMN IF NOT EXISTS recording_duration_seconds INTEGER + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE call_import_rows + DROP COLUMN IF EXISTS recording_duration_seconds + """ + ) + ) + db.commit() diff --git a/app/migrations/108_simulation_run_failure_analysis.py b/app/migrations/108_simulation_run_failure_analysis.py new file mode 100644 index 00000000..d8aaf8da --- /dev/null +++ b/app/migrations/108_simulation_run_failure_analysis.py @@ -0,0 +1,25 @@ +"""Failure clusters and prompt suggestions for simulation runs.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "simulation_runs.metric_clusters and prompt_improvements JSON columns" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE simulation_runs + ADD COLUMN IF NOT EXISTS metric_clusters JSON, + ADD COLUMN IF NOT EXISTS prompt_improvements JSON + """)) + db.commit() + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE simulation_runs + DROP COLUMN IF EXISTS metric_clusters, + DROP COLUMN IF EXISTS prompt_improvements + """)) + db.commit() diff --git a/app/migrations/109_call_import_evaluation_aggregate_cache.py b/app/migrations/109_call_import_evaluation_aggregate_cache.py new file mode 100644 index 00000000..4eff1ef4 --- /dev/null +++ b/app/migrations/109_call_import_evaluation_aggregate_cache.py @@ -0,0 +1,33 @@ +"""Cached metric aggregates for evaluation visualizations.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Cache live metric aggregates on call_import_evaluations" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + ADD COLUMN IF NOT EXISTS cached_metric_aggregates JSONB, + ADD COLUMN IF NOT EXISTS cached_metric_aggregates_at_count INTEGER + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + DROP COLUMN IF EXISTS cached_metric_aggregates_at_count, + DROP COLUMN IF EXISTS cached_metric_aggregates + """ + ) + ) + db.commit() diff --git a/app/migrations/110_call_import_recording_duration_repair.py b/app/migrations/110_call_import_recording_duration_repair.py new file mode 100644 index 00000000..17e4d600 --- /dev/null +++ b/app/migrations/110_call_import_recording_duration_repair.py @@ -0,0 +1,47 @@ +""" +Idempotent repair for recording_duration_seconds on shard DBs. + +Migration 108 originally used MIGRATION_SCOPE=catalog, so some shard databases +never received the column while the catalog did. After 108 was corrected to +``all``, fresh shard runs apply 108. This migration is a no-op when the column +already exists and is safe on every engine (catalog + shards). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Ensure recording_duration_seconds exists on call_import_rows (shard repair)" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE call_import_rows + ADD COLUMN IF NOT EXISTS recording_duration_seconds INTEGER + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.commit() diff --git a/app/migrations/111_call_import_row_trgm_search.py b/app/migrations/111_call_import_row_trgm_search.py new file mode 100644 index 00000000..22157564 --- /dev/null +++ b/app/migrations/111_call_import_row_trgm_search.py @@ -0,0 +1,68 @@ +"""GIN trigram indexes for call-import row text search.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "pg_trgm indexes on call_import_rows search columns" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm")) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_rows_conversation_id_trgm + ON call_import_rows + USING gin (conversation_id gin_trgm_ops) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_rows_transcript_trgm + ON call_import_rows + USING gin (transcript gin_trgm_ops) + WHERE transcript IS NOT NULL + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_rows_diarised_transcript_trgm + ON call_import_rows + USING gin (diarised_transcript gin_trgm_ops) + WHERE diarised_transcript IS NOT NULL + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute(text("DROP INDEX IF EXISTS ix_call_import_rows_diarised_transcript_trgm")) + db.execute(text("DROP INDEX IF EXISTS ix_call_import_rows_transcript_trgm")) + db.execute(text("DROP INDEX IF EXISTS ix_call_import_rows_conversation_id_trgm")) + db.commit() diff --git a/app/migrations/112_call_import_evaluation_report_period.py b/app/migrations/112_call_import_evaluation_report_period.py new file mode 100644 index 00000000..6f47485b --- /dev/null +++ b/app/migrations/112_call_import_evaluation_report_period.py @@ -0,0 +1,48 @@ +"""Persist ISO-week report period on evaluations for fast baseline lookup.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Report period columns on call_import_evaluations" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + ADD COLUMN IF NOT EXISTS report_period_start DATE, + ADD COLUMN IF NOT EXISTS report_period_end DATE, + ADD COLUMN IF NOT EXISTS report_period_label VARCHAR(64), + ADD COLUMN IF NOT EXISTS report_period_display VARCHAR(128), + ADD COLUMN IF NOT EXISTS report_period_at_count INTEGER + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_evaluations_report_period_start + ON call_import_evaluations (report_period_start) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP INDEX IF EXISTS ix_call_import_evaluations_report_period_start")) + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + DROP COLUMN IF EXISTS report_period_at_count, + DROP COLUMN IF EXISTS report_period_display, + DROP COLUMN IF EXISTS report_period_label, + DROP COLUMN IF EXISTS report_period_end, + DROP COLUMN IF EXISTS report_period_start + """ + ) + ) + db.commit() diff --git a/app/migrations/113_call_import_evaluation_period_delta_cache.py b/app/migrations/113_call_import_evaluation_period_delta_cache.py new file mode 100644 index 00000000..500c0e79 --- /dev/null +++ b/app/migrations/113_call_import_evaluation_period_delta_cache.py @@ -0,0 +1,31 @@ +"""Cache numeric period-over-period metric deltas on the evaluation row.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Cached period deltas for visualization baseline comparisons" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + ADD COLUMN IF NOT EXISTS cached_period_deltas JSONB + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + DROP COLUMN IF EXISTS cached_period_deltas + """ + ) + ) + db.commit() diff --git a/app/migrations/114_metric_scoring_method.py b/app/migrations/114_metric_scoring_method.py new file mode 100644 index 00000000..454a54d8 --- /dev/null +++ b/app/migrations/114_metric_scoring_method.py @@ -0,0 +1,47 @@ +"""Migration: computed metric scoring_method and call-import speaker aliases.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "metric_scoring_method_and_call_import_speaker_aliases" +MIGRATION_SCOPE = "catalog" + + +def upgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS scoring_method VARCHAR(32) NOT NULL DEFAULT 'llm' + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE call_imports + ADD COLUMN IF NOT EXISTS speaker_aliases JSONB NULL + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + db.execute( + text( + """ + ALTER TABLE call_imports + DROP COLUMN IF EXISTS speaker_aliases + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE metrics + DROP COLUMN IF EXISTS scoring_method + """ + ) + ) + db.commit() diff --git a/app/migrations/115_call_import_row_diarised_native_transcript.py b/app/migrations/115_call_import_row_diarised_native_transcript.py new file mode 100644 index 00000000..0e896348 --- /dev/null +++ b/app/migrations/115_call_import_row_diarised_native_transcript.py @@ -0,0 +1,51 @@ +"""Store the raw STT transcript used as the native-script WER/CER hypothesis.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add diarised_native_transcript to call_import_rows" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE call_import_rows + ADD COLUMN IF NOT EXISTS diarised_native_transcript TEXT NULL + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "call_import_rows"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE call_import_rows + DROP COLUMN IF EXISTS diarised_native_transcript + """ + ) + ) + db.commit() diff --git a/app/migrations/116_evaluation_analysis_batches.py b/app/migrations/116_evaluation_analysis_batches.py new file mode 100644 index 00000000..88a18b01 --- /dev/null +++ b/app/migrations/116_evaluation_analysis_batches.py @@ -0,0 +1,67 @@ +"""Durable batch checkpoints for evaluation analysis jobs (insights, clusters, prompts).""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add evaluation_analysis_batches checkpoint table" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if _table_exists(db, "evaluation_analysis_batches"): + db.commit() + return + db.execute( + text( + """ + CREATE TABLE evaluation_analysis_batches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + evaluation_id UUID NOT NULL + REFERENCES call_import_evaluations(id) ON DELETE CASCADE, + job_kind VARCHAR(32) NOT NULL, + job_run_id UUID NOT NULL, + batch_index INTEGER NOT NULL, + cursor_row_index INTEGER NULL, + cursor_evaluation_row_id UUID NULL, + transcript_source VARCHAR(16) NULL, + status VARCHAR(32) NOT NULL DEFAULT 'completed', + payload JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_evaluation_analysis_batch + UNIQUE (evaluation_id, job_kind, job_run_id, batch_index) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_evaluation_analysis_batches_eval_run + ON evaluation_analysis_batches (evaluation_id, job_kind, job_run_id) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "evaluation_analysis_batches"): + db.commit() + return + db.execute(text("DROP TABLE IF EXISTS evaluation_analysis_batches")) + db.commit() diff --git a/app/migrations/117_evaluation_analysis_batch_status_width.py b/app/migrations/117_evaluation_analysis_batch_status_width.py new file mode 100644 index 00000000..27418bb6 --- /dev/null +++ b/app/migrations/117_evaluation_analysis_batch_status_width.py @@ -0,0 +1,55 @@ +"""Widen evaluation_analysis_batches.status so synthesis_completed fits. + +The original column was VARCHAR(16). The insights synthesis checkpoint writes +status 'synthesis_completed' (19 characters) and Postgres rejected the insert. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Widen evaluation_analysis_batches.status to VARCHAR(32)" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "evaluation_analysis_batches"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE evaluation_analysis_batches + ALTER COLUMN status TYPE VARCHAR(32) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "evaluation_analysis_batches"): + db.commit() + return + db.execute( + text( + """ + ALTER TABLE evaluation_analysis_batches + ALTER COLUMN status TYPE VARCHAR(16) + """ + ) + ) + db.commit() diff --git a/app/migrations/118_eval_progress_counter_fence.py b/app/migrations/118_eval_progress_counter_fence.py new file mode 100644 index 00000000..0d5de943 --- /dev/null +++ b/app/migrations/118_eval_progress_counter_fence.py @@ -0,0 +1,81 @@ +"""Fence columns for Redis-buffered evaluation progress counters. + +* ``call_import_evaluations.progress_counter_gen`` is bumped by every fenced + shard recount. A flush carrying an older gen is ignored. +* ``call_import_evaluations.progress_flush_seq`` is the last applied flush + sequence, so re-sending the same pending batch is a no-op. +* ``call_import_evaluation_rows (evaluation_id, status)`` makes the shard + recounts (total / completed / failed / in-progress) index-only friendly. + +Runs on every engine: the catalog gets the columns, shards get the index. +Each step is skipped when its table is not present on that engine. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Eval progress counter gen/seq columns and (evaluation_id, status) index" +MIGRATION_SCOPE = "all" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if _table_exists(db, "call_import_evaluations"): + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + ADD COLUMN IF NOT EXISTS progress_counter_gen INTEGER NOT NULL DEFAULT 0 + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE call_import_evaluations + ADD COLUMN IF NOT EXISTS progress_flush_seq BIGINT NOT NULL DEFAULT 0 + """ + ) + ) + if _table_exists(db, "call_import_evaluation_rows"): + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_evaluation_rows_evaluation_status + ON call_import_evaluation_rows (evaluation_id, status) + """ + ) + ) + db.commit() + + +def downgrade(db: Session): + if _table_exists(db, "call_import_evaluation_rows"): + db.execute( + text("DROP INDEX IF EXISTS ix_call_import_evaluation_rows_evaluation_status") + ) + if _table_exists(db, "call_import_evaluations"): + db.execute( + text( + "ALTER TABLE call_import_evaluations DROP COLUMN IF EXISTS progress_flush_seq" + ) + ) + db.execute( + text( + "ALTER TABLE call_import_evaluations DROP COLUMN IF EXISTS progress_counter_gen" + ) + ) + db.commit() diff --git a/app/migrations/119_usage_query_scalability.py b/app/migrations/119_usage_query_scalability.py new file mode 100644 index 00000000..1d80cfca --- /dev/null +++ b/app/migrations/119_usage_query_scalability.py @@ -0,0 +1,52 @@ +"""Backfill call-import context keys and add rollup indexes for usage reads.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Backfill llm_usage_daily call_import_id/evaluation_id and add read indexes" +) +MIGRATION_SCOPE = "catalog" + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; skipping 119") + db.commit() + return + from app.services.usage.rollup_maintenance import ( + backfill_usage_context_keys, + ensure_usage_read_indexes, + repair_orphaned_row_context, + ) + + backfill_usage_context_keys(db) + repair_orphaned_row_context(db) + ensure_usage_read_indexes(db) + print("Backfilled usage context keys and ensured usage read indexes") + + +def downgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + db.commit() + return + from app.services.usage.rollup_maintenance import READ_INDEX_NAMES + + for index in READ_INDEX_NAMES: + db.execute(text(f"DROP INDEX IF EXISTS {index}")) + db.commit() diff --git a/app/migrations/120_usage_rollup_hardening.py b/app/migrations/120_usage_rollup_hardening.py new file mode 100644 index 00000000..3adb97f8 --- /dev/null +++ b/app/migrations/120_usage_rollup_hardening.py @@ -0,0 +1,45 @@ +"""Repair usage context after 119 and build read indexes without blocking writes.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Collision-safe usage context repair, shard orphan backfill, and concurrent read indexes" +) +MIGRATION_SCOPE = "catalog" + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; skipping 120") + db.commit() + return + from app.services.usage.rollup_maintenance import ( + backfill_usage_context_keys, + ensure_usage_read_indexes, + repair_orphaned_row_context, + ) + + backfill_usage_context_keys(db) + repair_orphaned_row_context(db) + ensure_usage_read_indexes(db) + print("Repaired usage rollup context and read indexes") + + +def downgrade(db: Session): + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 6b3e162e..669572ea 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -1,3464 +1,3844 @@ -"""SQLAlchemy database models.""" - -from sqlalchemy import ( - BigInteger, - Boolean, - Column, - Date, - DateTime, - DDL, - Enum, - event, - Float, - ForeignKey, - Integer, - JSON, - LargeBinary, - String, - Text, - UniqueConstraint, - select, - text, -) -from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.orm import query_expression, relationship -from sqlalchemy.sql import func -import uuid -import enum -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, -) - -def get_enum_values(enum_class): - """Helper to get values from enum class for SQLAlchemy.""" - return [e.value for e in enum_class] - -from app.database import Base - - -# Enums moved to enums.py - - -class Organization(Base): - """Organization model for multi-tenancy.""" - - __tablename__ = "organizations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - name = Column(String(255), nullable=False) - voice_playground_threshold_overrides = Column(JSON, nullable=True) - # AlignEval-style judge alignment thresholds. - # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} - # Falls back to system defaults (20 / 50) when null. - judge_alignment_settings = Column(JSON, nullable=True) - # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). - llm_gateway_settings = Column(JSON, nullable=True) - # Alerting: sync_notification_lifecycle, pagerduty_inbound_token, signing secret - alerting_settings = Column(JSON, nullable=True) - is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) - disabled_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - api_keys = relationship("APIKey", back_populates="organization") - members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") - invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") - workspaces = relationship( - "Workspace", - back_populates="organization", - cascade="all, delete-orphan", - ) - workspace_roles = relationship( - "WorkspaceRole", - back_populates="organization", - cascade="all, delete-orphan", - ) - - -class Workspace(Base): - """Workspace - in-org isolation boundary for call imports and metrics. - - Every organization has at least one workspace (``is_default = True``, - seeded by migration 033). Users pick an "active workspace" in the UI; - list endpoints filter by it so users only see calls/metrics from the - project they're currently working in. Access is governed by - ``workspace_members`` and org-scoped ``workspace_roles`` (capability - bundles); org admins implicitly access all workspaces. - """ - - __tablename__ = "workspaces" - __table_args__ = ( - UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), - ) - - # ``server_default`` is required so that raw-SQL INSERTs (e.g. the - # per-org Default seed in migration 033) can omit ``id`` and let the - # database fill it in. Without it, ``create_all`` produces a column - # with NOT NULL but no DEFAULT, and the migration crashes with - # ``null value in column "id"``. - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - slug = Column(String(255), nullable=False) - # At most one default per org. Enforced on Postgres by the partial - # unique index attached via the after_create event below; on - # SQLite (test runs) we rely on the route-level _check_slug_unique - # check + the Default-workspace conftest fixture instead, because - # SQLite doesn't support partial indexes the same way. - is_default = Column(Boolean, nullable=False, default=False, server_default="false") - is_active = Column(Boolean, nullable=False, default=True, server_default="true") - # Reusable PDF/report branding metadata scoped to this workspace. Images - # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, - # content_type, filename, size_bytes, updated_at}, ...]}. - report_branding = Column(JSON, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspaces") - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceRole(Base): - """Org-scoped workspace role (system or custom) as a capability bundle.""" - - __tablename__ = "workspace_roles" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - capabilities = Column(JSON, nullable=False, default=list) - is_system = Column(Boolean, nullable=False, default=False, server_default="false") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspace_roles") - members = relationship("WorkspaceMember", back_populates="role") - - -class WorkspaceMember(Base): - """User membership in a workspace with an assigned workspace role.""" - - __tablename__ = "workspace_members" - __table_args__ = ( - UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - role_id = Column( - UUID(as_uuid=True), - ForeignKey("workspace_roles.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - added_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - workspace = relationship("Workspace", back_populates="members") - user = relationship("User", foreign_keys=[user_id]) - role = relationship("WorkspaceRole", back_populates="members") - added_by = relationship("User", foreign_keys=[added_by_user_id]) - - -# Partial unique index: "at most one default workspace per org". This -# is attached as an after_create event (rather than declared in -# ``__table_args__``) because SQLAlchemy's ``Index(..., -# postgresql_where=...)`` silently degrades to a *full* unique index on -# SQLite - which then forbids any second workspace per org and breaks -# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL -# a no-op on SQLite while still emitting it on Postgres (prod, CI). -event.listen( - Workspace.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " - "ON workspaces (organization_id) WHERE is_default" - ).execute_if(dialect="postgresql"), -) - - -class User(Base): - """User model for authentication and profile management.""" - - __tablename__ = "users" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - first_name = Column(String(255), nullable=True) - last_name = Column(String(255), nullable=True) - password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation - external_id = Column(String(255), unique=True, nullable=True, index=True) - auth_provider = Column(String(50), nullable=True) - mfa_enabled = Column(Boolean, default=False, nullable=False) - session_epoch = Column(Integer, default=0, nullable=False, server_default="0") - last_login_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") - api_keys = relationship("APIKey", back_populates="user") - invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") - refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") - - -class PlatformAdmin(Base): - """Platform-level administrator (separate from org-scoped users).""" - - __tablename__ = "platform_admins" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - password_hash = Column(String(255), nullable=False) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_login_at = Column(DateTime(timezone=True), nullable=True) - - signup_reference_codes = relationship( - "SignupReferenceCode", - back_populates="created_by_admin", - foreign_keys="SignupReferenceCode.created_by", - ) - - -class SignupReferenceCode(Base): - """Single- or multi-use reference code required for gated self-service signup.""" - - __tablename__ = "signup_reference_codes" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - code_hash = Column(String(64), unique=True, nullable=False) - label = Column(String(255), nullable=True) - max_uses = Column(Integer, nullable=True) - use_count = Column(Integer, default=0, nullable=False) - expires_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False, index=True) - created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - created_by_admin = relationship( - "PlatformAdmin", - back_populates="signup_reference_codes", - foreign_keys=[created_by], - ) - - -class RefreshToken(Base): - """Opaque refresh token for extending local-password sessions.""" - - __tablename__ = "refresh_tokens" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash = Column(String(64), unique=True, nullable=False, index=True) - expires_at = Column(DateTime(timezone=True), nullable=False) - revoked_at = Column(DateTime(timezone=True), nullable=True) - authenticated_org_ids = Column(JSON, nullable=True) - authenticated_org_epochs = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - user = relationship("User", back_populates="refresh_tokens") - - -class OrganizationMember(Base): - """Organization membership with role.""" - - __tablename__ = "organization_members" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) - role = Column(String, nullable=False, default=RoleEnum.READER.value) - - # User preferences for this organization - default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - - joined_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Unique constraint: one membership per user per organization - __table_args__ = ( - UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), - ) - - # Relationships - organization = relationship("Organization", back_populates="members") - user = relationship("User", back_populates="organization_memberships") - default_agent = relationship("Agent", foreign_keys=[default_agent_id]) - - -class OrganizationMemberCredential(Base): - """Per-organization password and session state for a membership.""" - - __tablename__ = "organization_member_credentials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True - ) - user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True - ) - password_hash = Column(String(255), nullable=True) - auth_provider = Column(String(50), nullable=True) - session_epoch = Column(Integer, default=0, nullable=False, server_default="0") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - __table_args__ = ( - UniqueConstraint("organization_id", "user_id", name="uq_org_member_credentials_org_user"), - ) - - -class OrganizationMemberSessionRevocation(Base): - """Minimum valid session epoch after membership removal for a user/org pair.""" - - __tablename__ = "organization_member_session_revocations" - - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), primary_key=True - ) - user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True - ) - min_session_epoch = Column(Integer, nullable=False, server_default="1") - revoked_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class Invitation(Base): - """Invitation model for inviting users to organizations.""" - - __tablename__ = "invitations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet - invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - email = Column(String(255), nullable=False) # Email of invited user - role = Column(String, nullable=False, default=RoleEnum.READER.value) - status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) - - - - token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token - expires_at = Column(DateTime(timezone=True), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - accepted_at = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="invitations") - invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") - invited_by = relationship("User", foreign_keys=[invited_by_id]) - - -class APIKey(Base): - """API Key model for authentication.""" - - __tablename__ = "api_keys" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - key = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="api_keys") - user = relationship("User", back_populates="api_keys") - - -class AudioFile(Base): - """Audio file model.""" - - __tablename__ = "audio_files" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False) - file_path = Column(String(512), nullable=False) - file_size = Column(Integer, nullable=False) # Size in bytes - duration = Column(Float, nullable=True) # Duration in seconds - sample_rate = Column(Integer, nullable=True) - channels = Column(Integer, nullable=True) - format = Column(String(10), nullable=False) # wav, mp3, flac, etc. - uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluations = relationship("Evaluation", back_populates="audio_file") - - -class Evaluation(Base): - """Evaluation job model.""" - - __tablename__ = "evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every legacy audio evaluation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) - reference_text = Column(String, nullable=True) # For WER calculation - evaluation_type = Column(String, nullable=False) - model_name = Column(String(100), nullable=True) - status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) - - - - metrics_requested = Column(JSON, nullable=True) # List of requested metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - error_message = Column(String, nullable=True) - - # Relationships - audio_file = relationship("AudioFile", back_populates="evaluations") - result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) - - -class EvaluationResult(Base): - """Evaluation result model.""" - - __tablename__ = "evaluation_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) - # Workspace isolation: mirrors the parent Evaluation's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - transcript = Column(String, nullable=True) - metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} - raw_output = Column(JSON, nullable=True) # Full model output - processing_time = Column(Float, nullable=True) # Processing time in seconds - model_used = Column(String(100), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluation = relationship("Evaluation", back_populates="result") - - -# ============================================ -# VAIOPS MODELS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -class Agent(Base): - """Test Agent - The voice AI agent being evaluated""" - __tablename__ = "agents" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every agent belongs to a workspace within its - # org. Stamped from the X-Workspace-Id header (falling back to the - # org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - phone_number = Column(String, nullable=True) # Optional, required only for phone_call - language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) - description = Column(String) - provider_prompt = Column(Text, nullable=True) - provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) - call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) - call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) - telephony_phone_number_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - - - - # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) - ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) - - # Voice AI agent integration (Retell, Vapi, etc.) - voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) - voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) - prompt_variables = Column(JSON, nullable=True) - test_agent_template = Column(JSON, nullable=True) - silence_hangup_secs = Column(Integer, nullable=False, server_default="15") - - chat_connection_type = Column(String(32), nullable=True) - main_llm_provider = Column(String(64), nullable=True) - main_llm_model = Column(String(255), nullable=True) - main_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - main_llm_config = Column(JSON, nullable=True) - test_llm_provider = Column(String(64), nullable=True) - test_llm_model = Column(String(255), nullable=True) - test_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - test_llm_config = Column(JSON, nullable=True) - chat_connection_config = Column(JSON, nullable=True) - chat_eval_mode = Column(String(32), nullable=True) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Persona(Base): - """Persona - TTS provider-tied voice identity for testing""" - __tablename__ = "personas" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every persona belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - simulation_medium = Column(String(16), nullable=False, default="voice") - gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) - tts_provider = Column(String(100), nullable=True) - tts_voice_id = Column(String(255), nullable=True) - tts_voice_name = Column(String(255), nullable=True) - is_custom = Column(Boolean, default=False) - description = Column(Text, nullable=True) - tts_config = Column(JSON, nullable=True) - llm_temperature = Column(Float, nullable=True) - llm_max_tokens = Column(Integer, nullable=True) - response_delay_ms = Column(Integer, nullable=True) - max_turns = Column(Integer, nullable=True) - allow_interruptions = Column(Boolean, nullable=True) - background_noise_source = Column(String(20), nullable=False, default="none") - background_noise_preset = Column(String(50), nullable=True) - background_noise_volume = Column(Float, nullable=True, default=0.22) - background_noise_s3_key = Column(String, nullable=True) - background_noise_asset_id = Column( - UUID(as_uuid=True), - ForeignKey("ambient_noise_assets.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class AmbientNoiseAsset(Base): - """Reusable ambient noise bed uploaded for test-agent personas.""" - __tablename__ = "ambient_noise_assets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - s3_key = Column(String, nullable=False) - original_filename = Column(String(255), nullable=True) - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - - -class Scenario(Base): - """Scenario - The conversation scenario/test case""" - __tablename__ = "scenarios" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every scenario belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - name = Column(String, nullable=False) - description = Column(String) - required_info = Column(JSON) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -# Enums moved to enums.py - - -class Integration(Base): - """Integration model for connecting with external voice AI platforms.""" - __tablename__ = "integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - platform = Column(String, nullable=False) - - - - name = Column(String, nullable=True) # Optional friendly name - api_key = Column(String, nullable=False) # Encrypted Private API key for the platform - public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) - # ElevenLabs data-residency API origin (e.g. https://api.eu.residency.elevenlabs.io) - api_base_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple credentials per (org, platform) are allowed. is_default marks - # the row used when a caller does not explicitly select a credential. - # A partial unique index in migration 028 enforces at most one default - # per (org, platform) at the DB level. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -class ManualTranscription(Base): - """Manual transcription model for storing transcriptions from S3 audio files.""" - - __tablename__ = "manual_transcriptions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String(255), nullable=True) # User-friendly name for the transcription - audio_file_key = Column(String(512), nullable=False) # S3 key or file path - transcript = Column(String, nullable=False) # Full transcript text - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") - stt_provider = Column(String, nullable=True) # Provider used - - - - language = Column(String(10), nullable=True) # Detected or specified language - processing_time = Column(Float, nullable=True) # Processing time in seconds - raw_output = Column(JSON, nullable=True) # Full model output for reference - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class ConversationEvaluation(Base): - """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" - - __tablename__ = "conversation_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - - # Evaluation results - objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? - objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result - additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) - overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) - - # LLM metadata - llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) - - llm_model = Column(String(100), nullable=True) - llm_response = Column(JSON, nullable=True) # Full LLM response for reference - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class AIProvider(Base): - """AI Provider - Stores API keys for different AI platforms.""" - __tablename__ = "aiproviders" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String, nullable=False) - - - - api_key = Column(String, nullable=False) # Encrypted API key - name = Column(String, nullable=True) # Optional friendly name - # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). - # Only used when provider is azure; other providers ignore this column. - endpoint_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple AIProvider rows per (org, provider) are allowed. is_default - # marks the row resolved when no explicit credential id is selected. - # A partial unique index in migration 028 enforces at most one default. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Bifrost custom model ID used when routing via gateway - gateway_model = Column(String(255), nullable=True) - # inherit | bifrost | litellm_proxy — gateway backend override - gateway_type = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # inherit | litellm_shim | native_openai — Bifrost API surface override - gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Optional per-credential Bifrost/gateway base URL override - gateway_base_url = Column(String(512), nullable=True) - # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) - gateway_auth_header = Column(String(64), nullable=True) - # Env var name whose value is sent as the gateway auth secret - gateway_auth_secret_env = Column(String(128), nullable=True) - # Encrypted inline gateway auth secret (alternative to env var) - gateway_auth_secret = Column(String, nullable=True) - # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls - gateway_extra_headers = Column(JSON, nullable=True) - # Non-empty list restricts model pickers; null/empty = all catalog models for provider. - enabled_models = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -# Enums moved to enums.py - - -class VoiceBundle(Base): - """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" - __tablename__ = "voicebundles" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String, nullable=False) - description = Column(String, nullable=True) - - # Bundle type: either STT+LLM+TTS or S2S - # Using String instead of Enum to avoid SQLAlchemy enum conversion issues - # The enum conversion is handled in the Pydantic schemas - bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) - - # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - stt_provider = Column(String, nullable=True) - # Optional explicit credential row (aiproviders.id or integrations.id). - # When NULL the credential resolver picks the default row for the - # provider. No FK is set because the target table varies by provider. - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" - - # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - llm_provider = Column(String, nullable=True) - llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - - llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" - llm_temperature = Column(Float, nullable=True, default=0.7) - llm_max_tokens = Column(Integer, nullable=True) - llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) - - # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - tts_provider = Column(String, nullable=True) - tts_credential_id = Column(UUID(as_uuid=True), nullable=True) - - tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" - tts_voice = Column(String, nullable=True) # Voice selection if applicable - tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) - - # S2S Configuration - required for S2S type, optional for STT_LLM_TTS - s2s_provider = Column(String, nullable=True) - s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) - - - - s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model - s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) - - # Additional configuration for extensibility - extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) - - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class TestAgentConversation(Base): - """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" - __tablename__ = "test_agent_conversations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every playground conversation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Configuration - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - # Conversation data - status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) - - - - live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps - conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio - full_transcript = Column(String, nullable=True) # Full conversation transcript - - # Metadata - started_at = Column(DateTime(timezone=True), server_default=func.now()) - ended_at = Column(DateTime(timezone=True), nullable=True) - duration_seconds = Column(Float, nullable=True) - - # Additional metadata - conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorSuite(Base): - """Evaluator suite — one agent + one or more personas × N scenario combinations.""" - - __tablename__ = "evaluator_suites" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String, nullable=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - metric_ids = Column(JSON, nullable=True) - llm_provider = Column(String, nullable=True) - llm_model = Column(String, nullable=True) - llm_config = Column(JSON, nullable=True) - tags = Column(JSON, nullable=True) - default_runs_per_combination = Column(Integer, nullable=False, default=1) - round_robin_index = Column(Integer, nullable=False, default=0) - is_active = Column(Boolean, nullable=False, default=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class Evaluator(Base): - """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" - __tablename__ = "evaluators" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Display name (required for custom evaluators, optional for standard) - name = Column(String, nullable=True) - - # Parent suite (nullable for legacy/custom evaluators) - suite_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_suites.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - - # Standard evaluator configuration (nullable for custom evaluators) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - - # Custom evaluator prompt (used instead of agent/persona/scenario) - custom_prompt = Column(Text, nullable=True) - - # Custom evaluator metric selection. When set, the worker filters the - # enabled-org metrics down to only these IDs (list of metric UUID strings). - # Standard evaluators leave this NULL and use all enabled agent metrics. - metric_ids = Column(JSON, nullable=True) - - # LLM configuration for evaluation (overrides hardcoded defaults) - llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" - llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" - llm_config = Column(JSON, nullable=True) - - # Tags for categorization - tags = Column(JSON, nullable=True) # Array of tag strings - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class Metric(Base): - """Metric - Configuration for evaluation metrics. - - Supports a 2-level hierarchy via ``parent_metric_id``: a "category" - parent metric (e.g. "Call Outcome") owns N child sub-metric labels - (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set - only on parents and controls how the LLM scores children together - (``single_choice`` = pick exactly one; ``multi_label`` = independent - yes/no with logical consistency). Children are always boolean. - """ - __tablename__ = "metrics" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: two-shape column. - # - # * ``workspace_id = `` — workspace-scoped metric. Only - # visible inside that workspace (the default behavior; existing - # rows all look like this). - # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in - # every workspace's listing under this org so users don't have - # to recreate the same metric per workspace. - # - # Children always inherit their parent's ``workspace_id`` (including - # NULL) so a category metric's whole subtree shares one scope; the - # add-child / promote-discovered endpoints enforce this. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - - # Basic information - name = Column(String, nullable=False) - description = Column(String, nullable=True) - # Free-form illustrative example used to sharpen the LLM judge's - # rubric. Today this is consumed by child sub-labels of a - # categorization parent metric so each label can carry "what does - # this look like in a transcript?" text alongside the rubric in - # ``description``. The column lives on every Metric row for - # forward-compat: a standalone metric could later surface its own - # example without another migration. - example = Column(Text, nullable=True) - - # Configuration - metric_type = Column(String, nullable=False, default=MetricType.RATING.value) - metric_category = Column( - String(30), - nullable=False, - default=MetricCategory.QUALITY.value, - server_default=MetricCategory.QUALITY.value, - ) - trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) - metric_origin = Column(String(30), nullable=False, default="default") - supported_surfaces = Column(JSON, nullable=False, default=list) # agent, chat_agent, voice_playground - enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces - custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" - custom_config = Column(JSON, nullable=True) # enum options / number range config - tags = Column(JSON, nullable=True) # ["tone", "latency", ...] - - # Hierarchy: NULL = standalone or parent. When set, this row is a - # child sub-metric of the referenced parent. ON DELETE CASCADE so - # deleting a category removes its children atomically. - parent_metric_id = Column( - UUID(as_uuid=True), - ForeignKey("metrics.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - # Set only on parent rows (``parent_metric_id IS NULL``). Either - # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical - # metric (no children). - selection_mode = Column(String(20), nullable=True) - - # When true on a parent metric (any selection_mode), the LLM is - # invited during call-import evaluation to emit additional - # candidate sub-labels beyond the user-defined children. The - # candidates surface in a "Discovered labels" panel where the user - # manually promotes them into real child Metric rows. For - # ``single_choice`` parents the discovered entries are - # supplemental — the chosen child is still picked from the - # predefined children so the exactly-one-true invariant holds. - # The validator rejects this flag on standalone / child metrics. - allow_discovery = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - # When True, this metric is a "transcript-compare judge": the - # call-import evaluator feeds BOTH the production transcript - # (``call_import_rows.transcript``, CSV-supplied) and the diarised - # transcript (``call_import_rows.diarised_transcript``, worker- - # produced by the STT/diarisation pipeline) to the LLM as a - # labeled pair instead of feeding one transcript. The parent - # evaluation's ``CallImportEvaluation.transcript_source`` is - # ignored for these metrics — they always read both columns. - # Rows where either transcript is missing are skipped per-metric - # with ``skipped="comparison_missing_transcript"`` so the rest of - # the row's metrics still produce scores. The Pydantic validator - # rejects ``compare_transcripts`` combined with ``parent_metric_id`` - # or ``selection_mode`` (i.e. it can't simultaneously be part of - # a parent/child hierarchy). The call-import worker also - # auto-promotes a metric to comparison mode when its description - # references the production / diarised transcripts in well-known - # phrases (see ``_metric_text_references_production`` in - # ``app.workers.tasks.evaluate_call_import_row``). - compare_transcripts = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - parent = relationship( - "Metric", - remote_side=[id], - backref="children", - ) - - # When true, the LLM-judge is asked to also return a short free-form - # rationale alongside the value (stored under ``metric_scores[id].rationale``). - # Adds a second " - LLM Rationale" column in the call-import CSV export. - capture_rationale = Column(Boolean, nullable=False, default=False) - - enabled = Column(Boolean, nullable=False, default=True) - - # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics - # Studio until promoted to ``active``. - lifecycle = Column( - String(20), - nullable=False, - default="active", - server_default="active", - ) - promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) - studio_notes = Column(Text, nullable=True) - - # Metadata - is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResult(Base): - """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" - __tablename__ = "evaluator_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator result belongs to a workspace - # within its org. Stamped from the active workspace at creation time - # (either the X-Workspace-Id header or the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # References - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls - - # Result data - name = Column(String, nullable=True) # Scenario name or test call name (optional) - timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - duration_seconds = Column(Float, nullable=True) # Call duration - status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) - - # Audio and transcription - audio_s3_key = Column(String, nullable=True) # S3 key for audio file - transcription = Column(String, nullable=True) # Full transcription - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - - # Metric scores - JSON object with metric_id as key and score as value - # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} - metric_scores = Column(JSON, nullable=True) - - # Celery task tracking - celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking - - # Error information - error_message = Column(String, nullable=True) - - # Call event tracking (similar to CallRecording) - call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) - - # Data-plane shard routing (payload rows on shard DBs when sharding enabled) - shard_id = Column(String(64), nullable=True, index=True) - synthetic_call_trace_id = Column( - UUID(as_uuid=True), - ForeignKey("synthetic_call_traces.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResultClusterJob(Base): - """Cached failure-cluster state for a filtered evaluator-results scope.""" - - __tablename__ = "evaluator_result_cluster_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - scope_key = Column(String(512), nullable=False) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - suite_id = Column(UUID(as_uuid=True), ForeignKey("evaluator_suites.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - since = Column(DateTime(timezone=True), nullable=True) - until = Column(DateTime(timezone=True), nullable=True) - scenario_ids = Column(JSON, nullable=True) - metric_clusters = Column(JSON, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -# Enums moved to enums.py - - -class CallRecordingSource(str, enum.Enum): - """Source of the call recording data.""" - - PLAYGROUND = "playground" - WEBHOOK = "webhook" - - -class CallRecording(Base): - """Call Recording model for tracking voice provider calls.""" - __tablename__ = "call_recordings" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every recording belongs to a workspace within - # its org. For playground-origin rows this is stamped from the active - # workspace at creation time; for webhook-origin rows the worker - # looks up the recording's agent and inherits its workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) - call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) - source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) - call_data = Column(JSON, nullable=True) # JSON blob for provider response - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent - - # Link to EvaluatorResult for metric evaluations - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - shard_id = Column(String(64), nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class EvaluatorResultPayload(Base): - """Heavy evaluator result fields stored on data shards when sharding is enabled.""" - - __tablename__ = "evaluator_result_payloads" - - evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - audio_s3_key = Column(String, nullable=True) - transcription = Column(String, nullable=True) - speaker_segments = Column(JSON, nullable=True) - metric_scores = Column(JSON, nullable=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallRecordingPayload(Base): - """Heavy call recording fields stored on data shards when sharding is enabled.""" - - __tablename__ = "call_recording_payloads" - - call_recording_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class SyntheticCallTrace(Base): - """Legacy PG trace header (superseded by ClickHouse call_traces; dropped by migration 086).""" - - __tablename__ = "synthetic_call_traces" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id", ondelete="SET NULL"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id", ondelete="SET NULL"), nullable=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True) - call_recording_id = Column( - UUID(as_uuid=True), - ForeignKey("call_recordings.id", ondelete="SET NULL"), - nullable=True, - ) - call_short_id = Column(String(6), nullable=True, index=True) - environment = Column(String(32), nullable=False, default="pre_prod") - provider_platform = Column(String(64), nullable=True) - transport = Column(String(32), nullable=False, default="phone") - tier = Column(String(32), nullable=False, default="black_box") - status = Column(String(32), nullable=False, default="open") - started_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - ended_at = Column(DateTime(timezone=True), nullable=True) - turn_count = Column(Integer, nullable=False, default=0) - response_latency_p50_ms = Column(Float, nullable=True) - response_latency_p90_ms = Column(Float, nullable=True) - response_latency_p95_ms = Column(Float, nullable=True) - component_aggregates = Column(JSON, nullable=True) - failure_flags = Column(JSON, nullable=True) - trace_version = Column(Integer, nullable=False, default=1) - shard_id = Column(String(64), nullable=True, index=True) - spans_s3_key = Column(String(512), nullable=True) - spans_storage = Column(String(16), nullable=False, default="legacy_jsonb") - span_count = Column(Integer, nullable=False, default=0) - last_span_at = Column(DateTime(timezone=True), nullable=True) - derive_pending = Column(Boolean, nullable=False, default=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class SyntheticTraceSpanBatch(Base): - """Append-only OTLP span batch for Phase 2 async ingest.""" - - __tablename__ = "synthetic_trace_span_batches" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - synthetic_call_trace_id = Column( - UUID(as_uuid=True), - ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - seq = Column(BigInteger, nullable=False) - span_count = Column(Integer, nullable=False, default=0) - spans = Column(JSON, nullable=False, default=list) - received_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - - -class SyntheticTraceIngestStaging(Base): - """Short-lived raw OTLP payload staged before worker parse (Layout 3).""" - - __tablename__ = "synthetic_trace_ingest_staging" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), nullable=False, index=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - content_type = Column(String(128), nullable=False, default="") - body = Column(LargeBinary, nullable=False) - body_bytes = Column(Integer, nullable=False, default=0) - header_evaluator_result_id = Column(String(128), nullable=True) - header_agent_id = Column(String(128), nullable=True) - header_call_short_id = Column(String(32), nullable=True) - status = Column(String(16), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - accepted_spans = Column(Integer, nullable=True) - synthetic_call_trace_id = Column( - UUID(as_uuid=True), - ForeignKey("synthetic_call_traces.id", ondelete="SET NULL"), - nullable=True, - ) - correlated = Column(Boolean, nullable=True) - attempt_count = Column(Integer, nullable=False, default=0) - received_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - processed_at = Column(DateTime(timezone=True), nullable=True) - - -class SyntheticTracePayload(Base): - """Per-turn timing payload for a synthetic call trace.""" - - __tablename__ = "synthetic_trace_payloads" - - synthetic_call_trace_id = Column( - UUID(as_uuid=True), - ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), - primary_key=True, - ) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - turns = Column(JSON, nullable=False, default=list) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class SyntheticTraceOtelPayload(Base): - """Full OTLP span tree for a synthetic call trace.""" - - __tablename__ = "synthetic_trace_otel_payloads" - - synthetic_call_trace_id = Column( - UUID(as_uuid=True), - ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), - primary_key=True, - ) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - spans = Column(JSON, nullable=False, default=list) - trace_ids = Column(JSON, nullable=False, default=list) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class Alert(Base): - """Alert model for configuring monitoring alerts.""" - __tablename__ = "alerts" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - - data_source = Column(String, nullable=False, default="evaluations") - - # Metric condition configuration - metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) - aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) - operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) - threshold_value = Column(Float, nullable=False) - time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation - alert_on_missing_data = Column( - Boolean, nullable=False, default=False, server_default=text("false") - ) - - # Agent selection (JSON array of agent UUIDs, null means all agents) - agent_ids = Column(JSON, nullable=True) - - # Notification configuration - notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) - notify_emails = Column(JSON, nullable=True) # Array of email addresses - notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) - notify_pagerduty_routing_keys = Column(JSON, nullable=True) - - # Status - status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) - suppress_reopen_until_ok = Column( - Boolean, nullable=False, default=False, server_default=text("false") - ) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - # Relationships - alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") - - -class AlertHistory(Base): - """Alert history model for tracking triggered alerts.""" - __tablename__ = "alert_history" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) - - # Trigger information - triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert - threshold_value = Column(Float, nullable=False) # The threshold at time of trigger - - # Status tracking - status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) - - # Notification tracking - notified_at = Column(DateTime(timezone=True), nullable=True) - notification_details = Column(JSON, nullable=True) # Details of sent notifications - - # Resolution - acknowledged_at = Column(DateTime(timezone=True), nullable=True) - acknowledged_by = Column(String, nullable=True) - resolved_at = Column(DateTime(timezone=True), nullable=True) - resolved_by = Column(String, nullable=True) - resolution_notes = Column(String, nullable=True) - - # Additional context - context_data = Column(JSON, nullable=True) # Additional data about the trigger - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - alert = relationship("Alert", back_populates="alert_history") - - -class CronJob(Base): - """Cron job model for scheduling automated evaluator runs.""" - __tablename__ = "cron_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=True, index=True) - - # Basic information - name = Column(String(255), nullable=False) - job_type = Column(String(64), nullable=False, default="evaluator_run") - is_system = Column(Boolean, nullable=False, default=False) - config = Column(JSON, nullable=False, default=dict) - cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" - timezone = Column(String(100), nullable=False, default="UTC") - - # Run configuration - max_runs = Column(Integer, nullable=False, default=10) - current_runs = Column(Integer, nullable=False, default=0) - - # Evaluators to trigger (JSON array of evaluator UUIDs) - evaluator_ids = Column(JSON, nullable=False) - - # Status - status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) - - # Run tracking - next_run_at = Column(DateTime(timezone=True), nullable=True) - last_run_at = Column(DateTime(timezone=True), nullable=True) - last_dispatch_celery_task_id = Column(String(255), nullable=True) - last_dispatch_status = Column(String(32), nullable=True) - last_dispatch_error = Column(String, nullable=True) - last_dispatch_at = Column(DateTime(timezone=True), nullable=True) - last_run_status = Column(String(32), nullable=True) - last_run_error = Column(String, nullable=True) - - @property - def interval_days(self) -> int | None: - config = self.config if isinstance(self.config, dict) else {} - raw = config.get("interval_days") - try: - days = int(raw) - except (TypeError, ValueError): - return None - return days if days > 0 else None - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class TTSComparisonStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSSampleStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSReportJobStatus(str, enum.Enum): - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSComparison(Base): - """TTS Comparison session for A/B testing voice providers.""" - __tablename__ = "tts_comparisons" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every voice playground comparison belongs to - # a workspace within its org. Children (samples, report jobs, blind - # test shares) inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - simulation_id = Column(String(6), unique=True, index=True, nullable=True) - - name = Column(String(255), nullable=True) - status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) - - # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). - # 'blind_test_only' = standalone blind test built from existing recordings - # / uploads / past TTS samples; no TTS generation happens. - mode = Column(String(32), nullable=False, default="benchmark") - - provider_a = Column(String(100), nullable=True) - model_a = Column(String(100), nullable=True) - voices_a = Column(JSON, nullable=True) - - provider_b = Column(String(100), nullable=True) - model_b = Column(String(100), nullable=True) - voices_b = Column(JSON, nullable=True) - - sample_texts = Column(JSON, nullable=False) - num_runs = Column(Integer, nullable=False, default=1) - - blind_test_results = Column(JSON, nullable=True) - evaluation_summary = Column(JSON, nullable=True) - - eval_stt_provider = Column(String(100), nullable=True) - eval_stt_model = Column(String(100), nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") - - -class TTSSample(Base): - """Individual TTS audio sample within a comparison.""" - __tablename__ = "tts_samples" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - provider = Column(String(100), nullable=True) - model = Column(String(100), nullable=True) - voice_id = Column(String(255), nullable=True) - voice_name = Column(String(255), nullable=True) - side = Column(String(1), nullable=True) # "A" or "B" - sample_index = Column(Integer, nullable=False) - run_index = Column(Integer, nullable=False, default=0) - - # 'tts' (default, audio is synthesized by a provider), 'recording' (audio - # is reused from a CallImportRow recording), or 'upload' (audio was - # uploaded by the user). Non-tts samples are marked completed up-front - # by the API and skipped by the generation worker. - source_type = Column(String(32), nullable=False, default="tts") - # When source_type == 'recording', references CallImportRow.id (no FK - # constraint to keep cascading deletes simple if a call import is later - # removed; the audio_s3_key is what's actually used). - source_ref_id = Column(UUID(as_uuid=True), nullable=True) - - text = Column(String, nullable=False) - audio_s3_key = Column(String(512), nullable=True) - duration_seconds = Column(Float, nullable=True) - latency_ms = Column(Float, nullable=True) - ttfb_ms = Column(Float, nullable=True) - - evaluation_metrics = Column(JSON, nullable=True) - status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - comparison = relationship("TTSComparison", back_populates="samples") - - -class TTSReportJob(Base): - """Asynchronous PDF report generation jobs for Voice Playground.""" - __tablename__ = "tts_report_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - - status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) - format = Column(String(20), nullable=False, default="pdf") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - error_message = Column(String, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - - -class TTSBlindTestShareStatus(str, enum.Enum): - OPEN = "open" - CLOSED = "closed" - - -class TTSBlindTestShare(Base): - """A publicly sharable blind test for a TTSComparison. - - The share_token is the capability: anyone holding it can open the public - form and submit a response. Each comparison has at most one share row. - """ - __tablename__ = "tts_blind_test_shares" - __table_args__ = ( - UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_comparisons.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - share_token = Column(String(64), unique=True, nullable=False, index=True) - - title = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # Internal notes visible only to the share creator (e.g. which voice - # corresponds to which side, source notes for standalone blind tests). - # Never exposed via the public blind test payload. - creator_notes = Column(Text, nullable=True) - - # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] - custom_metrics = Column(JSON, nullable=False) - - status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - closed_at = Column(DateTime(timezone=True), nullable=True) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - responses = relationship( - "TTSBlindTestResponse", - back_populates="share", - cascade="all, delete-orphan", - ) - - -class TTSBlindTestResponse(Base): - """A single rater's submission against a TTSBlindTestShare.""" - __tablename__ = "tts_blind_test_responses" - __table_args__ = ( - UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - share_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - rater_name = Column(String(255), nullable=False) - rater_email = Column(String(320), nullable=False, index=True) - - # JSON list keyed by sample_index. Server stores in TRUE A/B orientation - # (already de-flipped from whatever the rater's UI showed): - # [{ - # "sample_index": int, - # "preferred": "A" | "B", - # "ratings_a": { metric_key: number }, - # "ratings_b": { metric_key: number }, - # "comment": str? - # }] - responses = Column(JSON, nullable=False) - - ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - - submitted_at = Column(DateTime(timezone=True), server_default=func.now()) - - share = relationship("TTSBlindTestShare", back_populates="responses") - - -class PromptPartial(Base): - """Prompt Partial - Reusable prompt templates with version history.""" - __tablename__ = "prompt_partials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every prompt partial belongs to a workspace - # within its org. Versions inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - content = Column(Text, nullable=False) - tags = Column(JSON, nullable=True) - current_version = Column(Integer, nullable=False, default=1) - # Cached LLM-generated flowchart for imported production agent prompts. - # Shape: AgentFlowGraph JSON (nodes[], edges[]). - agent_flowchart = Column(JSON, nullable=True) - agent_flowchart_status = Column(String(20), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") - pushes = relationship("PromptPush", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPush.created_at.desc()") - - -class PromptPartialVersion(Base): - """Version history for a prompt partial.""" - __tablename__ = "prompt_partial_versions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptPartial's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - version = Column(Integer, nullable=False) - content = Column(Text, nullable=False) - change_summary = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - created_by = Column(String, nullable=True) - - prompt_partial = relationship("PromptPartial", back_populates="versions") - - __table_args__ = ( - UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), - ) - - -class PromptPush(Base): - """Audit row for a live provider prompt push. The prompt body stays on PromptPartialVersion.""" - - __tablename__ = "prompt_pushes" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column(UUID(as_uuid=True), ForeignKey("workspaces.id", ondelete="RESTRICT"), nullable=False, index=True) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - prompt_version = Column(Integer, nullable=False) - provider = Column(String(50), nullable=False) - external_agent_id = Column(String(255), nullable=False) - external_agent_name = Column(String(255), nullable=True) - integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id", ondelete="SET NULL"), nullable=True) - evaluation_id = Column(UUID(as_uuid=True), nullable=True, index=True) - pushed_by = Column(String, nullable=True) - status = Column(String(20), nullable=False) - error_message = Column(Text, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - prompt_partial = relationship("PromptPartial", back_populates="pushes") - - -class CustomTTSVoice(Base): - """Organization-scoped custom TTS voice metadata.""" - __tablename__ = "custom_tts_voices" - __table_args__ = ( - UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(100), nullable=False, index=True) - voice_id = Column(String(255), nullable=False) - name = Column(String(255), nullable=False) - gender = Column(String(50), nullable=True) - accent = Column(String(100), nullable=True) - description = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - -class PromptOptimizationRun(Base): - """A single GEPA prompt optimization run for an agent.""" - __tablename__ = "prompt_optimization_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every optimization run belongs to a workspace - # within its org. Candidates inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - seed_prompt = Column(Text, nullable=False) - best_prompt = Column(Text, nullable=True) - best_score = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) - config = Column(JSON, nullable=True) - reflection_trace = Column(JSON, nullable=True) - metric_history = Column(JSON, nullable=True) - - num_iterations = Column(Integer, nullable=True) - num_metric_calls = Column(Integer, nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") - - -class PromptOptimizationCandidate(Base): - """A candidate prompt generated during an optimization run.""" - __tablename__ = "prompt_optimization_candidates" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - prompt_text = Column(Text, nullable=False) - score = Column(Float, nullable=True) - metric_breakdown = Column(JSON, nullable=True) - reflection_summary = Column(Text, nullable=True) - - parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) - - is_accepted = Column(Boolean, nullable=False, default=False) - pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") - - -class TelephonyIntegration(Base): - """Per-organization telephony provider credentials and configuration. - - Multiple rows per (organization_id, provider) are allowed so that an - organization can keep several Plivo / Exotel accounts side-by-side. - A partial unique index in migration 028 enforces at most one row with - is_default = TRUE per (org, provider); resolution falls back to that - default row when the caller does not pin a specific credential. - """ - - __tablename__ = "telephony_integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(50), nullable=False, default="plivo") - name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials - - auth_id = Column(String(255), nullable=False) - auth_token = Column(String(512), nullable=False) - - verify_app_uuid = Column(String(255), nullable=True) - voice_app_id = Column(String(255), nullable=True) - sip_domain = Column(String(255), nullable=True) - masking_config = Column(JSON, nullable=True) - - is_active = Column(Boolean, default=True, nullable=False) - is_default = Column(Boolean, default=False, nullable=False) - last_tested_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyPhoneNumber(Base): - """Inventory of telephony phone numbers owned by an organization.""" - - __tablename__ = "telephony_phone_numbers" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True - ) - - phone_number = Column(String(20), nullable=False, index=True) - country_iso2 = Column(String(2), nullable=True) - region = Column(String(100), nullable=True) - number_type = Column(String(20), nullable=True) - capabilities = Column(JSON, nullable=True) - provider_app_id = Column(String(255), nullable=True) - - is_masking_pool = Column(Boolean, default=False, nullable=False) - inbound_enabled = Column(Boolean, default=True, nullable=False) - outbound_enabled = Column(Boolean, default=True, nullable=False) - source = Column(String(20), nullable=False, default="imported") - agent_id = Column( - UUID(as_uuid=True), - ForeignKey( - "agents.id", - ondelete="SET NULL", - use_alter=True, - name="fk_telephony_phone_numbers_agent_id", - ), - nullable=True, - index=True, - ) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyDialTarget(Base): - """Org-scoped saved destination numbers for outbound test calls.""" - - __tablename__ = "telephony_dial_targets" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - phone_number = Column(String(20), nullable=False, index=True) - label = Column(String(255), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyVerifySession(Base): - """Tracks voice OTP verification sessions via telephony provider.""" - - __tablename__ = "telephony_verify_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) - recipient_number = Column(String(20), nullable=False) - channel = Column(String(10), nullable=False, default="voice") - status = Column(String(20), nullable=False, default="pending") - initiated_by = Column(String(255), nullable=True) - verify_app_uuid = Column(String(255), nullable=True) - verified_at = Column(DateTime(timezone=True), nullable=True) - expires_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyMaskedSession(Base): - """Number-masking session between two parties through a middle number.""" - - __tablename__ = "telephony_masked_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) - masked_number_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True - ) - masked_number = Column(String(20), nullable=False) - party_a_number = Column(String(20), nullable=False) - party_b_number = Column(String(20), nullable=False) - status = Column(String(20), nullable=False, default="active") - expires_at = Column(DateTime(timezone=True), nullable=True) - ended_at = Column(DateTime(timezone=True), nullable=True) - session_metadata = Column("metadata", JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallImportSchema(Base): - """Reusable Input Parameter schema for the call-uploads flow. - - A schema is workspace-scoped: users define a named bundle of typed - Input Parameters once (e.g. "Standard Voice QA" with conversation_id + - recording_url + transcript + agent_name) and then map those parameters - to CSV/Excel headers each time they upload a new batch. - - Every schema MUST contain exactly one parameter with - ``type='conversation_id'`` and ``is_required=True`` - that's the - mandatory identity field every imported row needs. A schema may - optionally include at most one ``recording_url`` parameter. The - invariant is enforced in app code on create/update (no DB-level - CHECK because the parent + children are written across two tables in - one transaction). - """ - - __tablename__ = "call_import_schemas" - __table_args__ = ( - # Case-insensitive uniqueness is enforced via the matching partial - # index on ``LOWER(name)`` in the migration; this constraint here - # would be case-sensitive and is intentionally omitted to avoid - # confusing the user. - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - parameters = relationship( - "CallImportSchemaParameter", - back_populates="schema", - cascade="all, delete-orphan", - order_by="CallImportSchemaParameter.ordering", - ) - - -class CallImportSchemaParameter(Base): - """A single typed parameter inside a :class:`CallImportSchema`. - - ``type`` is one of the strings tracked by - :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` - is reserved for the mandatory identity parameter every schema must - contain. - """ - - __tablename__ = "call_import_schema_parameters" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - type = Column(String(32), nullable=False) - description = Column(Text, nullable=True) - is_required = Column(Boolean, nullable=False, default=False) - # Stable ordering so the UI renders parameters in the order the - # schema author defined them (matters when conversation_id is pinned - # first and the user re-orders the rest). - ordering = Column(Integer, nullable=False, default=0) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - schema = relationship("CallImportSchema", back_populates="parameters") - - -class CallImport(Base): - """Batch record for a CSV-driven call import job.""" - - __tablename__ = "call_imports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every imported batch belongs to a workspace - # within its org. The /upload endpoint stamps it from the active - # workspace header (or the org's Default if absent). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the - # legacy one-shot ``POST /upload`` endpoint this is supplied with the - # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value - # isn't known until the IMPORT stage, so the column is nullable for - # ``uploaded`` / ``mapped`` batches. - provider = Column(String(50), nullable=True, default="exotel") - # Pin a specific telephony credential for this batch so the worker - # downloads recordings using *that* row instead of the org default. - # NULL preserves legacy behavior (resolve by provider + default). - telephony_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - original_filename = Column(String(512), nullable=True) - # When the source file was a multi-sheet Excel workbook, this records - # the worksheet the rows came from (one batch per sheet). NULL for CSV - # uploads since CSV has no sheet concept. - sheet_name = Column(String(255), nullable=True) - - # --- Source-file staging (UPLOAD stage) --------------------------- - # The raw CSV / Excel file is stored in S3 between stages so the - # user can come back later to MAP and IMPORT without re-uploading. - # ``source_s3_key`` is NULL on legacy batches that were imported via - # the one-shot endpoint (those batches stay read-only post-import). - source_s3_key = Column(Text, nullable=True) - source_format = Column(String(16), nullable=True) - source_size_bytes = Column(BigInteger, nullable=True) - source_content_type = Column(String(255), nullable=True) - - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI doesn't need to re-fetch the source bytes from S3. - # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. - available_sheets = Column(JSON, nullable=True) - - # User's explicit "drop these columns" decision captured at MAP - # time. Was validation-only and ephemeral in the legacy flow; now - # persisted so the IMPORT stage can re-parse the file with the same - # mapping/skip intent. - skipped_columns = Column(JSON, nullable=False, default=list) - # Rows skipped at parse time (missing/invalid conversation_id or URL). - # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. - source_row_skips = Column(JSON, nullable=False, default=list) - - # Free-text high-level segregation label. Powers the "Dataset" filter - # at the top of the imports page; multiple imports can share a value. - dataset = Column(String(255), nullable=True, index=True) - content_modality = Column(String(16), nullable=True) - - # Reusable Input Parameter schema this batch was uploaded against. - # NULL on legacy batches uploaded before the schema-driven flow - # shipped; those still render via ``column_mapping`` + ``extra_columns`` - # + ``custom_column_mapping`` below. - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. - # Populated for new uploads; empty dict on legacy batches. - parameter_mapping = Column(JSON, nullable=False, default=dict) - - # Legacy free-form mapping (pre-schema-flow). Kept on the model so - # batches that were uploaded before the schema feature shipped still - # render correctly on the detail page; new uploads stop writing here. - # Keys: external_call_id (required), transcript, recording_url. - # (DB column ``external_call_id`` is now ``conversation_id``; this - # JSON key stays as-is for historical batches.) - # Values: original CSV header strings (preserve user casing for export). - column_mapping = Column(JSON, nullable=False, default=dict) - # Ordered list of additional CSV header strings the uploader wants - # preserved verbatim into the evaluation export CSV. - extra_columns = Column(JSON, nullable=False, default=list) - # User-defined ``{custom_field_name: csv_header}`` mappings on top of - # the three system fields above. Cells from the mapped CSV columns are - # preserved per row (keyed by the CSV header in ``raw_columns``) and - # surface in the evaluation export under the uploader-chosen name. - custom_column_mapping = Column(JSON, nullable=False, default=dict) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - - status = Column( - Enum(CallImportStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportStatus.PENDING, - index=True, - ) - error_message = Column(Text, nullable=True) - - voice_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - platform_filters = Column(JSON, nullable=False, default=dict) - window_start = Column(DateTime(timezone=True), nullable=True) - window_end = Column(DateTime(timezone=True), nullable=True) - window_timezone = Column(String(64), nullable=True) - discovery_status = Column(String(32), nullable=True) - max_calls = Column(Integer, nullable=True) - next_row_index = Column(Integer, nullable=False, default=0) - skip_counts = Column(JSON, nullable=False, default=dict) - skip_samples = Column(JSON, nullable=False, default=list) - paused_at = Column(DateTime(timezone=True), nullable=True) - pause_reason = Column(Text, nullable=True) - billed_at = Column(DateTime(timezone=True), nullable=True) - last_progress_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - rows = relationship( - "CallImportRow", - back_populates="call_import", - cascade="all, delete-orphan", - order_by="CallImportRow.row_index", - ) - tags = relationship( - "CallImportTag", - secondary="call_import_tag_assignments", - backref="call_imports", - lazy="selectin", - ) - evaluations = relationship( - "CallImportEvaluation", - back_populates="call_import", - cascade="all, delete-orphan", - ) - - -class CallImportDiscoverySlice(Base): - """Time slice for platform historical import discovery.""" - - __tablename__ = "call_import_discovery_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - slice_start = Column(DateTime(timezone=True), nullable=False) - slice_end = Column(DateTime(timezone=True), nullable=False) - cursor = Column(Text, nullable=True) - agent_id = Column(String(255), nullable=True) - status = Column(String(32), nullable=False, default="pending") - listed = Column(Integer, nullable=False, default=0) - accepted = Column(Integer, nullable=False, default=0) - attempts = Column(Integer, nullable=False, default=0) - error = Column(Text, nullable=True) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class PlatformConversationLedger(Base): - """One ledger row per conversation inside a single import.""" - - __tablename__ = "platform_conversation_ledger" - __table_args__ = ( - UniqueConstraint( - "call_import_id", - "conversation_id", - name="uq_platform_ledger_import_conversation", - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - provider = Column(String(50), nullable=False) - conversation_id = Column(String(255), nullable=False) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - row_id = Column(UUID(as_uuid=True), nullable=False) - row_index = Column(Integer, nullable=False) - state = Column(String(32), nullable=False, default="reserved") - recording_date = Column(Date, nullable=True) - raw_columns = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportShardSlice(Base): - """Registry row: which shard stores a slice of rows for an import.""" - - __tablename__ = "call_import_shard_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - shard_id = Column(String(64), nullable=False, index=True) - row_index_min = Column(Integer, nullable=False) - row_index_max = Column(Integer, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportRow(Base): - """A single row within a CallImport batch (one CSV line / one external call).""" - - __tablename__ = "call_import_rows" - __table_args__ = ( - UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - row_index = Column(Integer, nullable=False) - # Was historically named ``external_call_id``; renamed to - # ``conversation_id`` so the new schema-driven upload flow can refer - # to it by a single canonical name across the schema definition, - # exports, and downstream evaluation tables. - conversation_id = Column(String(255), nullable=False, index=True) - # Supplied via CSV for Exotel credentialed imports (required per row). - # Nullable in the schema for legacy rows imported before recording_url - # was mandatory on every Exotel upload. - recording_url = Column(Text, nullable=True) - # Date-only call recording date supplied by the import schema. Used - # for historical report comparisons without timezone/time ambiguity. - recording_date = Column(Date, nullable=True, index=True) - # The "production" transcript: the value supplied via the CSV - # upload mapping. Never overwritten by the diarisation worker — - # the worker writes its output into ``diarised_transcript`` so - # the user keeps both versions side by side. - transcript = Column(Text, nullable=True) - # Snapshot of the original CSV row keyed by the user's headers so the - # evaluation export can reproduce every column the uploader supplied - # (mapped + extra). NULL on legacy rows imported before this column. - raw_columns = Column(JSON, nullable=True) - - # Where the value in ``transcript`` came from. ``csv`` = supplied via - # the upload mapping, ``edited`` = manually changed in the UI. NULL - # on rows that have never had a production transcript. - # (Worker-produced transcripts now live in ``diarised_transcript`` - # and are tracked via ``diarised_transcript_*`` metadata below.) - transcript_source = Column(String(20), nullable=True) - # Provider/model recorded by the (legacy) post-hoc transcription - # worker. New worker runs leave these NULL and write into the - # ``diarised_transcript_*`` columns instead; kept on the model for - # backwards compatibility with pre-split rows that still carry the - # original transcription metadata here. - transcript_provider = Column(String(50), nullable=True) - transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the legacy transcription workflow itself, - # independent of the row's recording-fetch ``status``. ``idle`` = - # no transcribe task has touched this column. New diarisation runs - # update ``diarised_transcript_status`` instead. - transcript_status = Column( - String(20), - nullable=False, - default="idle", - ) - transcript_error = Column(Text, nullable=True) - transcribed_at = Column(DateTime(timezone=True), nullable=True) - - # The "diarised" transcript: produced by the post-hoc - # transcription/diarisation worker. Stored separately so a manual - # diarisation run never clobbers the production transcript above. - # Evaluations can be configured to score against either column - # (see ``CallImportEvaluation.transcript_source``). - diarised_transcript = Column(Text, nullable=True) - transcript_char_length = query_expression() - diarised_transcript_char_length = query_expression() - # Provider/model the diarisation worker used. Surfaced in the UI - # as "Diarised via deepgram/nova-2" next to the diarised - # transcript section. - diarised_transcript_provider = Column(String(50), nullable=True) - diarised_transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the diarisation workflow. - # ``idle`` = no diarisation task has run; ``pending``/``running`` = - # a Celery task is queued or in flight; ``completed``/``failed`` = - # terminal. Independent of ``transcript_status`` so the two - # transcripts can be in different lifecycle states. - diarised_transcript_status = Column( - String(20), - nullable=False, - default="idle", - server_default="idle", - ) - diarised_transcript_error = Column(Text, nullable=True) - diarised_at = Column(DateTime(timezone=True), nullable=True) - - # Structured speaker turns produced by the diarisation worker — - # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", - # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` - # The plain-text ``diarised_transcript`` above is a rendered view - # of this list (``: `` per line). When the worker - # cannot recover structured turns (no pyannote token / single- - # speaker recording / provider that doesn't surface segments) this - # column stays NULL and the plain-text path is still populated. - diarised_segments = Column(JSON, nullable=True) - # When True the ``agent`` <-> ``user`` mapping inside - # ``diarised_segments`` is inverted at render / export time. The - # worker writes the canonical mapping using the "first speaker is - # the agent" heuristic; reviewers can flip the toggle from the row - # detail panel without re-running diarisation. - diarised_speaker_swap = Column( - Boolean, - nullable=False, - default=False, - server_default="false", - ) - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. The legacy diarisation worker used - # pyannote and left these NULL; the current path always runs an - # LLM with the operator-supplied (or default) ``diarised_prompt`` - # below, and records exactly which model + prompt produced each - # row so reviewers can reproduce a specific run. - diarised_llm_provider = Column(String(50), nullable=True) - diarised_llm_model = Column(String(100), nullable=True) - diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarised_prompt = Column(Text, nullable=True) - # Which diarisation pipeline produced this row's turns. - # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. - # ``diarised_transcript_provider``/``_model`` describe the STT - # side; ``diarised_llm_provider``/``_model`` the LLM side. - # * ``"llm_only"`` — single-stage: audio fed straight to a - # multimodal LLM. ``diarised_transcript_provider`` is stamped - # with the sentinel ``"llm_only"``; the real model is on - # ``diarised_llm_*``. - # Persisting it on the row (not just the run) lets the row detail - # panel render the right "Diarised via …" label even for ad-hoc - # standalone transcribes (no parent evaluation). - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - status = Column( - Enum(CallImportRowStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportRowStatus.PENDING, - index=True, - ) - - recording_s3_key = Column(String(1024), nullable=True) - recording_content_type = Column(String(128), nullable=True) - recording_size_bytes = Column(Integer, nullable=True) - - error_message = Column(Text, nullable=True) - attempts = Column(Integer, nullable=False, default=0) - celery_task_id = Column(String(255), nullable=True) - dispatched_at = Column(DateTime(timezone=True), nullable=True) - download_started_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - call_import = relationship("CallImport", back_populates="rows") - - -@event.listens_for(CallImportRow, "before_insert") -def _call_import_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent import when omitted.""" - if target.workspace_id is not None or target.call_import_id is None: - return - workspace_id = connection.execute( - select(CallImport.workspace_id).where( - CallImport.id == target.call_import_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportTag(Base): - """User-defined tag that can be attached to one or more call imports. - - Tags coexist with the free-text ``CallImport.dataset`` column: dataset - is the primary high-level segregation, tags are an optional secondary - classification (an import can have many tags). - """ - - __tablename__ = "call_import_tags" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - name = Column(String(255), nullable=False) - color = Column(String(32), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportTagAssignment(Base): - """Many-to-many join table between CallImport and CallImportTag.""" - - __tablename__ = "call_import_tag_assignments" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - tag_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_tags.id", ondelete="CASCADE"), - primary_key=True, - index=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluation(Base): - """Parent record for an evaluation run over a CallImport batch. - - A user picks a subset of org ``Metric`` rows and triggers an evaluation; - we fan out one ``CallImportEvaluationRow`` per source row and roll up - counters as workers finish. Status mirrors ``CallImportStatus`` plus a - ``RUNNING`` value so the UI can distinguish "queued" from "in flight". - """ - - __tablename__ = "call_import_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent CallImport's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Optional user-supplied label for this run. Lets the UI surface - # something more meaningful than the UUID prefix (e.g. "March QA pass"). - name = Column(String(255), nullable=True) - - # JSON list of Metric UUID strings selected for this run. Stored as text - # in JSON so we don't have to deal with PG arrays of UUIDs / cascade - # delete policies when metrics are removed; the loader filters for - # still-existing org metrics at run time. - selected_metric_ids = Column(JSON, nullable=False, default=list) - # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. - # Captures which children belong to which parent for THIS run so the UI - # / aggregator can reconstruct the tree even when the user selected - # only a subset of children, or after metrics are deleted / renamed. - # NULL on legacy rows means "no hierarchy" → fall back to flat - # ``selected_metric_ids`` semantics. - selected_metric_groups = Column(JSON, nullable=True) - # User-driven merges of LLM-discovered candidate sub-labels for - # ``allow_discovery`` parents. Shape: - # ``{"": {"": "", ...}}``. - # Populated via ``POST .../discovered-labels/merge``; consulted by - # the discovered-labels aggregator, the flow graph builder, and the - # worker so that rows finishing AFTER a merge cannot reintroduce - # the merged-away slug. Empty dict on fresh rows. - discovered_label_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Per-run opt-in for top-level metric discovery. When True, the LLM - # is asked to propose brand-new top-level metrics (boolean / rating / - # category) observed in the transcripts in addition to scoring the - # ``selected_metric_ids`` for the row. Candidates surface in a - # "Discovered metrics" panel on the evaluation's Flow tab and can - # be promoted into real standalone ``Metric`` rows via - # ``POST /metrics/from-discovered``. Defaults to False so existing - # evaluation creation payloads keep their previous behaviour. - discover_new_metrics = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Flat slug-to-slug redirect map for user merges + tombstones of - # discovered top-level metric candidates. Mirrors - # ``discovered_label_aliases`` but is NOT nested per parent — - # top-level metric discovery is not scoped to any parent. Shape:: - # - # {"": "", ...} - # - # An empty-string value tombstones the slug so workers finishing - # later can't re-introduce it. - discovered_metric_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Run-level LLM config picked from the Run Evaluation modal. NULL on - # legacy rows means "use the historical OpenAI/gpt-4o default" — the - # worker checks for this and falls back accordingly. ``llm_credential_id`` - # pins a specific AIProvider row when the org has multiple credentials - # for the same provider. - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - # Optional per-metric LLM override: - # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. - # Each entry overrides the run-level default for that metric only; - # missing keys = use run-level default. Stored as JSON so the UI can - # round-trip arbitrary {provider, model} pairs without migrations. - metric_llm_overrides = Column(JSON, nullable=True) - - # When ``auto_transcribe`` was set on the create payload, record the - # STT provider/model used so the UI can show "Auto-transcribed via - # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is - # untyped (no FK) because STT keys may live in either ``aiproviders`` - # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the - # transcription service handles the lookup. - stt_provider = Column(String(50), nullable=True) - stt_model = Column(String(100), nullable=True) - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - # Run-level LLM diariser config. Used when the create-run / - # retry-run paths chain a ``transcribe_call_import_row_task`` - # because the row is missing a diarised transcript. Persisted on - # the run so a retry uses the same diariser the original create - # call picked (unless the retry payload explicitly overrides). - diarisation_llm_provider = Column(String(50), nullable=True) - diarisation_llm_model = Column(String(100), nullable=True) - diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarisation_prompt = Column(Text, nullable=True) - # Mode the run was *created* with for its auto-transcribe step. - # Retry chains read this to decide whether to enqueue an STT+LLM - # transcribe or a single-stage multimodal LLM transcribe — without - # it we'd have to infer the mode from "stt_provider is NULL", which - # would silently break legacy rows that simply never configured - # auto-transcribe. See migration 041 for the column DDL. - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - # Which of the two transcripts on each ``CallImportRow`` this run - # scored against. ``'production'`` reads ``CallImportRow.transcript`` - # (the CSV-supplied value); ``'diarised'`` reads - # ``CallImportRow.diarised_transcript`` (the worker output). When - # the user ticks both checkboxes in the Run Evaluation modal we - # create two ``CallImportEvaluation`` rows — one per source — so - # the two scorings can be compared side-by-side. Defaults to - # ``'production'`` so legacy runs (which always read the single - # historical ``transcript`` column) keep their semantics. - transcript_source = Column( - String(20), - nullable=False, - default="production", - server_default="production", - ) - - # Cached LLM-generated TLDR rendered above the Visualizations charts. - # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we - # never auto-burn LLM tokens on page load. Shape:: - # {"narrative": str, "patterns": [str, ...], - # "generated_at": iso8601, "generated_at_completed_rows": int, - # "provider": str, "model": str} - # NULL on rows that have never been summarised. - tldr_summary = Column(JSON, nullable=True) - - # Cached LLM-generated user insights for External Audit PDF section 03. - # Populated by a background Celery job triggered alongside TLDR generation. - # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). - user_insights = Column(JSON, nullable=True) - - # Cached per-metric failure clustering for internal diagnostics PDF/UI. - # Shape: EvaluationMetricClustersState JSON (status, groups[], …). - metric_clusters = Column(JSON, nullable=True) - - # Cached LLM-generated prompt improvement suggestions keyed to an - # imported agent (PromptPartial tagged __imported_agent__). - # Shape: EvaluationPromptImprovementsState JSON. - prompt_improvements = Column(JSON, nullable=True) - - # Cached LLM explanations for week-over-week metric deltas keyed by - # baseline evaluation id + completed row counts. - period_delta_explanations = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - # Flexprice pass-level delta billing watermark: rows already emitted - # on ``call_import.evaluation_completed`` for this evaluation run. - billed_completed_rows = Column( - Integer, nullable=False, default=0, server_default="0" - ) - error_message = Column(Text, nullable=True) - celery_group_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - call_import = relationship("CallImport", back_populates="evaluations") - row_results = relationship( - "CallImportEvaluationRow", - back_populates="evaluation", - cascade="all, delete-orphan", - ) - - -class CallImportEvaluationRow(Base): - """Per-source-row scoring output for a CallImportEvaluation parent.""" - - __tablename__ = "call_import_evaluation_rows" - __table_args__ = ( - UniqueConstraint( - "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_row_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_rows.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - status = Column(String(20), nullable=False, default="pending", index=True) - # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - evaluation = relationship("CallImportEvaluation", back_populates="row_results") - source_row = relationship("CallImportRow") - - -@event.listens_for(CallImportEvaluationRow, "before_insert") -def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent evaluation when omitted.""" - if target.workspace_id is not None or target.evaluation_id is None: - return - workspace_id = connection.execute( - select(CallImportEvaluation.workspace_id).where( - CallImportEvaluation.id == target.evaluation_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class MetricStudioRun(Base): - """Ad-hoc metric experiment run in Metrics Studio.""" - - __tablename__ = "metric_studio_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - - name = Column(String(255), nullable=True) - selected_metric_ids = Column(JSON, nullable=False, default=list) - selected_metric_groups = Column(JSON, nullable=True) - transcript_source = Column( - String(20), - nullable=False, - default="diarised", - server_default="diarised", - ) - - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - metric_llm_overrides = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - total_items = Column(Integer, nullable=False, default=0) - completed_items = Column(Integer, nullable=False, default=0) - failed_items = Column(Integer, nullable=False, default=0) - error_message = Column(Text, nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - results = relationship( - "MetricStudioRunResult", - back_populates="run", - cascade="all, delete-orphan", - ) - - -class MetricStudioRunResult(Base): - """Per-source scoring output for a MetricStudioRun.""" - - __tablename__ = "metric_studio_run_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - run_id = Column( - UUID(as_uuid=True), - ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - source_kind = Column(String(40), nullable=False) - source_ref = Column(String(255), nullable=False) - display_label = Column(String(512), nullable=True) - source_metadata = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - run = relationship("MetricStudioRun", back_populates="results") - - -class CallImportEvaluationReportSnapshot(Base): - """Persisted PDF-report aggregate used for period-over-period deltas.""" - - __tablename__ = "call_import_evaluation_report_snapshots" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - period_label = Column(String(64), nullable=True, index=True) - period_start = Column(Date, nullable=True, index=True) - period_end = Column(Date, nullable=True, index=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") - metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - narrative = Column(JSON, nullable=True) - total_calls = Column(Integer, nullable=False, default=0) - selected_metric_count = Column(Integer, nullable=False, default=0) - total_metric_count = Column(Integer, nullable=False, default=0) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportEvaluationPdfReport(Base): - """Stored PDF artifact for a call import evaluation report generation.""" - - __tablename__ = "call_import_evaluation_pdf_reports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - snapshot_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - vendor_name = Column(String(120), nullable=False) - report_type = Column(String(20), nullable=False, default="external") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - cache_fingerprint = Column(String(64), nullable=True) - created_by = Column(String, nullable=True) - created_by_user_id = Column(UUID(as_uuid=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluationExport(Base): - """Background CSV/XLSX export for one evaluation run.""" - - __tablename__ = "call_import_evaluation_exports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - format = Column(String(8), nullable=False, default="csv") - include_transcripts = Column(Boolean, nullable=False, default=False) - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - parts = Column(JSONB, nullable=False, default=list) - claim_token = Column(String(36), nullable=True) - heartbeat_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - finished_at = Column(DateTime(timezone=True), nullable=True) - - -# One active export per evaluation. A full unique index would block the next -# download after the first job finishes, and SQLite turns postgresql_where -# into that full index, so this DDL is postgres-only. -event.listen( - CallImportEvaluationExport.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_call_import_evaluation_exports_active " - "ON call_import_evaluation_exports (evaluation_id) " - "WHERE status IN ('pending', 'running')" - ).execute_if(dialect="postgresql"), -) - - -# --------------------------------------------------------------------------- -# Judge Alignment (AlignEval-style hybrid integration) -# -# Three tables back the "Judge Alignment" surface: -# - JudgeDataset: a labeled dataset materialised from one of three sources -# (voice transcripts, existing Metric/Evaluator outputs, -# or a generic CSV upload). Holds the dataset's source -# config + which fields play the role of input/output. -# - JudgeSample: one row in a dataset (input/output pair plus an -# optional binary pass/fail human label). -# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over -# a subset of samples, with computed alignment metrics -# (precision/recall/F1/Cohen's kappa) and per-sample -# predictions. Optionally links to a GEPA optimization -# run when the user kicks off prompt tuning from a -# dataset. -# --------------------------------------------------------------------------- - - -class JudgeDataset(Base): - """Container for binary-labeled samples used to calibrate an LLM-judge.""" - - __tablename__ = "judge_datasets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: every judge dataset belongs to a workspace - # within its org. Samples and runs inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # One of: "transcript", "metric_output", "csv" - source_type = Column(String(32), nullable=False, index=True) - # Source-specific config. Examples: - # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} - # metric_output: {"metric_id": "...", "evaluator_id": "..."} - # csv: {"s3_key": "...", "filename": "..."} - source_config = Column(JSON, nullable=False, default=dict) - - # Field roles - which textual content is "input" vs "output" for the judge. - # For voice transcripts both default to the transcript text but can be - # tightened (e.g. agent-only turns vs full conversation). - input_field = Column(String(64), nullable=False, default="input") - output_field = Column(String(64), nullable=False, default="output") - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship( - "JudgeSample", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeSample.created_at", - ) - runs = relationship( - "JudgeRun", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeRun.created_at.desc()", - ) - - -class JudgeSample(Base): - """One labelable input/output pair within a JudgeDataset.""" - - __tablename__ = "judge_samples" - __table_args__ = ( - UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Stable identifier within the source (e.g. transcription UUID, CSV row id). - # Used to dedupe re-imports and link back to the originating record. - external_id = Column(String(128), nullable=True, index=True) - - input_text = Column(Text, nullable=False) - output_text = Column(Text, nullable=False) - - # Binary human label: "pass" | "fail" | null (unlabeled). - # Stored as string (rather than enum) so it stays trivially extendable. - label = Column(String(16), nullable=True, index=True) - labeled_by = Column(String(255), nullable=True) - labeled_at = Column(DateTime(timezone=True), nullable=True) - - # Source-specific context (e.g. agent_id, original metric value, csv row). - extra = Column(JSON, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - dataset = relationship("JudgeDataset", back_populates="samples") - - -class JudgeRun(Base): - """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" - - __tablename__ = "judge_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model - # define the judge under test). Nullable so a run may target an inline prompt - # in the future without inflating the Evaluator table. - evaluator_id = Column( - UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True - ) - - # Which subset was scored: "all" | "dev" | "test" - split = Column(String(16), nullable=False, default="all") - - # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). - llm_provider = Column(String(64), nullable=True) - llm_model = Column(String(128), nullable=True) - - # Computed alignment metrics: - # {"precision": float, "recall": float, "f1": float, "kappa": float, - # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} - metrics = Column(JSON, nullable=True) - - # Per-sample predictions, keyed by sample_id (UUID string): - # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} - predictions = Column(JSON, nullable=True) - - # Run lifecycle. - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - # Optional link to a GEPA optimization run kicked off from this dataset. - gepa_optimization_id = Column( - UUID(as_uuid=True), - ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - dataset = relationship("JudgeDataset", back_populates="runs") - - -class UsageCostRecomputeJob(Base): - """Async job tracking for retroactive usage cost recompute.""" - - __tablename__ = "usage_cost_recompute_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - status = Column(String(32), nullable=False, default="pending", server_default="pending") - model = Column(String(255), nullable=True) - usage_kind = Column(String(16), nullable=True) - start_date = Column(Date, nullable=True) - end_date = Column(Date, nullable=True) - updated_rows = Column(BigInteger, nullable=False, default=0, server_default="0") - error_message = Column(String, nullable=True) - celery_task_id = Column(String(255), nullable=True, index=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - completed_at = Column(DateTime(timezone=True), nullable=True) - - -class LLMUsageDaily(Base): - """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" - - __tablename__ = "llm_usage_daily" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - product_section = Column(String(64), nullable=False, index=True) - model = Column(String(255), nullable=False, index=True) - context = Column(JSONB, nullable=False, server_default="{}", default=dict) - usage_date = Column(Date, nullable=False, index=True) - usage_kind = Column(String(16), nullable=False, default="llm", server_default="llm") - prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_creation_tokens = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") - tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") - call_count = Column(BigInteger, nullable=False, default=0, server_default="0") - input_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - output_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_read_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - cache_creation_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - reasoning_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - audio_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - tts_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - total_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - pricing_rate_source = Column(String(16), nullable=True) - pricing_rate_id = Column(UUID(as_uuid=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) +"""SQLAlchemy database models.""" + +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + DDL, + Enum, + event, + Float, + Index, + ForeignKey, + Integer, + JSON, + LargeBinary, + String, + Text, + UniqueConstraint, + select, + text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import query_expression, relationship +from sqlalchemy.sql import func +import uuid +import enum +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, SimulationRunStatus, +) + +def get_enum_values(enum_class): + """Helper to get values from enum class for SQLAlchemy.""" + return [e.value for e in enum_class] + +from app.database import Base + + +# Enums moved to enums.py + + +class Organization(Base): + """Organization model for multi-tenancy.""" + + __tablename__ = "organizations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) + # AlignEval-style judge alignment thresholds. + # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} + # Falls back to system defaults (20 / 50) when null. + judge_alignment_settings = Column(JSON, nullable=True) + # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). + llm_gateway_settings = Column(JSON, nullable=True) + # Alerting: sync_notification_lifecycle, pagerduty_inbound_token, signing secret + alerting_settings = Column(JSON, nullable=True) + is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) + disabled_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + api_keys = relationship("APIKey", back_populates="organization") + members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") + invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") + workspaces = relationship( + "Workspace", + back_populates="organization", + cascade="all, delete-orphan", + ) + workspace_roles = relationship( + "WorkspaceRole", + back_populates="organization", + cascade="all, delete-orphan", + ) + + +class Workspace(Base): + """Workspace - in-org isolation boundary for call imports and metrics. + + Every organization has at least one workspace (``is_default = True``, + seeded by migration 033). Users pick an "active workspace" in the UI; + list endpoints filter by it so users only see calls/metrics from the + project they're currently working in. Access is governed by + ``workspace_members`` and org-scoped ``workspace_roles`` (capability + bundles); org admins implicitly access all workspaces. + """ + + __tablename__ = "workspaces" + __table_args__ = ( + UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), + ) + + # ``server_default`` is required so that raw-SQL INSERTs (e.g. the + # per-org Default seed in migration 033) can omit ``id`` and let the + # database fill it in. Without it, ``create_all`` produces a column + # with NOT NULL but no DEFAULT, and the migration crashes with + # ``null value in column "id"``. + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + slug = Column(String(255), nullable=False) + # At most one default per org. Enforced on Postgres by the partial + # unique index attached via the after_create event below; on + # SQLite (test runs) we rely on the route-level _check_slug_unique + # check + the Default-workspace conftest fixture instead, because + # SQLite doesn't support partial indexes the same way. + is_default = Column(Boolean, nullable=False, default=False, server_default="false") + is_active = Column(Boolean, nullable=False, default=True, server_default="true") + # Reusable PDF/report branding metadata scoped to this workspace. Images + # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, + # content_type, filename, size_bytes, updated_at}, ...]}. + report_branding = Column(JSON, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspaces") + members = relationship( + "WorkspaceMember", + back_populates="workspace", + cascade="all, delete-orphan", + ) + + +class WorkspaceRole(Base): + """Org-scoped workspace role (system or custom) as a capability bundle.""" + + __tablename__ = "workspace_roles" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + capabilities = Column(JSON, nullable=False, default=list) + is_system = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspace_roles") + members = relationship("WorkspaceMember", back_populates="role") + + +class WorkspaceMember(Base): + """User membership in a workspace with an assigned workspace role.""" + + __tablename__ = "workspace_members" + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + role_id = Column( + UUID(as_uuid=True), + ForeignKey("workspace_roles.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + added_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workspace = relationship("Workspace", back_populates="members") + user = relationship("User", foreign_keys=[user_id]) + role = relationship("WorkspaceRole", back_populates="members") + added_by = relationship("User", foreign_keys=[added_by_user_id]) + + +# Partial unique index: "at most one default workspace per org". This +# is attached as an after_create event (rather than declared in +# ``__table_args__``) because SQLAlchemy's ``Index(..., +# postgresql_where=...)`` silently degrades to a *full* unique index on +# SQLite - which then forbids any second workspace per org and breaks +# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL +# a no-op on SQLite while still emitting it on Postgres (prod, CI). +event.listen( + Workspace.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " + "ON workspaces (organization_id) WHERE is_default" + ).execute_if(dialect="postgresql"), +) + + +class User(Base): + """User model for authentication and profile management.""" + + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + first_name = Column(String(255), nullable=True) + last_name = Column(String(255), nullable=True) + password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation + external_id = Column(String(255), unique=True, nullable=True, index=True) + auth_provider = Column(String(50), nullable=True) + mfa_enabled = Column(Boolean, default=False, nullable=False) + session_epoch = Column(Integer, default=0, nullable=False, server_default="0") + last_login_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") + api_keys = relationship("APIKey", back_populates="user") + invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") + refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") + + +class UserAgentFavorite(Base): + """An agent's star, stored for one user. Other people in the workspace do not see it.""" + + __tablename__ = "user_agent_favorites" + + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="CASCADE"), primary_key=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False) + workspace_id = Column(UUID(as_uuid=True), ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class PlatformAdmin(Base): + """Platform-level administrator (separate from org-scoped users).""" + + __tablename__ = "platform_admins" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + password_hash = Column(String(255), nullable=False) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_login_at = Column(DateTime(timezone=True), nullable=True) + + signup_reference_codes = relationship( + "SignupReferenceCode", + back_populates="created_by_admin", + foreign_keys="SignupReferenceCode.created_by", + ) + + +class SignupReferenceCode(Base): + """Single- or multi-use reference code required for gated self-service signup.""" + + __tablename__ = "signup_reference_codes" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + code_hash = Column(String(64), unique=True, nullable=False) + label = Column(String(255), nullable=True) + max_uses = Column(Integer, nullable=True) + use_count = Column(Integer, default=0, nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False, index=True) + created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + created_by_admin = relationship( + "PlatformAdmin", + back_populates="signup_reference_codes", + foreign_keys=[created_by], + ) + + +class RefreshToken(Base): + """Opaque refresh token for extending local-password sessions.""" + + __tablename__ = "refresh_tokens" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) + token_hash = Column(String(64), unique=True, nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + authenticated_org_ids = Column(JSON, nullable=True) + authenticated_org_epochs = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + user = relationship("User", back_populates="refresh_tokens") + + +class OrganizationMember(Base): + """Organization membership with role.""" + + __tablename__ = "organization_members" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) + role = Column(String, nullable=False, default=RoleEnum.READER.value) + + # User preferences for this organization + default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + + joined_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Unique constraint: one membership per user per organization + __table_args__ = ( + UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), + ) + + # Relationships + organization = relationship("Organization", back_populates="members") + user = relationship("User", back_populates="organization_memberships") + default_agent = relationship("Agent", foreign_keys=[default_agent_id]) + + +class OrganizationMemberCredential(Base): + """Per-organization password and session state for a membership.""" + + __tablename__ = "organization_member_credentials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True + ) + user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + password_hash = Column(String(255), nullable=True) + auth_provider = Column(String(50), nullable=True) + session_epoch = Column(Integer, default=0, nullable=False, server_default="0") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + __table_args__ = ( + UniqueConstraint("organization_id", "user_id", name="uq_org_member_credentials_org_user"), + ) + + +class OrganizationMemberSessionRevocation(Base): + """Minimum valid session epoch after membership removal for a user/org pair.""" + + __tablename__ = "organization_member_session_revocations" + + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), primary_key=True + ) + user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + min_session_epoch = Column(Integer, nullable=False, server_default="1") + revoked_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class Invitation(Base): + """Invitation model for inviting users to organizations.""" + + __tablename__ = "invitations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet + invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + email = Column(String(255), nullable=False) # Email of invited user + role = Column(String, nullable=False, default=RoleEnum.READER.value) + status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) + + + + token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + accepted_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="invitations") + invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") + invited_by = relationship("User", foreign_keys=[invited_by_id]) + + +class APIKey(Base): + """API Key model for authentication.""" + + __tablename__ = "api_keys" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + key = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="api_keys") + user = relationship("User", back_populates="api_keys") + + +class AudioFile(Base): + """Audio file model.""" + + __tablename__ = "audio_files" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + filename = Column(String(255), nullable=False) + file_path = Column(String(512), nullable=False) + file_size = Column(Integer, nullable=False) # Size in bytes + duration = Column(Float, nullable=True) # Duration in seconds + sample_rate = Column(Integer, nullable=True) + channels = Column(Integer, nullable=True) + format = Column(String(10), nullable=False) # wav, mp3, flac, etc. + uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluations = relationship("Evaluation", back_populates="audio_file") + + +class Evaluation(Base): + """Evaluation job model.""" + + __tablename__ = "evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every legacy audio evaluation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) + reference_text = Column(String, nullable=True) # For WER calculation + evaluation_type = Column(String, nullable=False) + model_name = Column(String(100), nullable=True) + status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) + + + + metrics_requested = Column(JSON, nullable=True) # List of requested metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + error_message = Column(String, nullable=True) + + # Relationships + audio_file = relationship("AudioFile", back_populates="evaluations") + result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) + + +class EvaluationResult(Base): + """Evaluation result model.""" + + __tablename__ = "evaluation_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) + # Workspace isolation: mirrors the parent Evaluation's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + transcript = Column(String, nullable=True) + metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} + raw_output = Column(JSON, nullable=True) # Full model output + processing_time = Column(Float, nullable=True) # Processing time in seconds + model_used = Column(String(100), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluation = relationship("Evaluation", back_populates="result") + + +# ============================================ +# VAIOPS MODELS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +class Agent(Base): + """Test Agent - The voice AI agent being evaluated""" + __tablename__ = "agents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every agent belongs to a workspace within its + # org. Stamped from the X-Workspace-Id header (falling back to the + # org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + phone_number = Column(String, nullable=True) # Optional, required only for phone_call + language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) + description = Column(String) + provider_prompt = Column(Text, nullable=True) + provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) + call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) + call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) + telephony_phone_number_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + + + + # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) + ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) + + # Voice AI agent integration (Retell, Vapi, etc.) + voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) + voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + prompt_variables = Column(JSON, nullable=True) + test_agent_template = Column(JSON, nullable=True) + silence_hangup_secs = Column(Integer, nullable=False, server_default="15") + + chat_connection_type = Column(String(32), nullable=True) + main_llm_provider = Column(String(64), nullable=True) + main_llm_model = Column(String(255), nullable=True) + main_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + main_llm_config = Column(JSON, nullable=True) + test_llm_provider = Column(String(64), nullable=True) + test_llm_model = Column(String(255), nullable=True) + test_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + test_llm_config = Column(JSON, nullable=True) + chat_connection_config = Column(JSON, nullable=True) + chat_eval_mode = Column(String(32), nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Persona(Base): + """Persona - TTS provider-tied voice identity for testing""" + __tablename__ = "personas" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every persona belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + simulation_medium = Column(String(16), nullable=False, default="voice") + gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) + tts_provider = Column(String(100), nullable=True) + tts_voice_id = Column(String(255), nullable=True) + tts_voice_name = Column(String(255), nullable=True) + is_custom = Column(Boolean, default=False) + description = Column(Text, nullable=True) + tts_config = Column(JSON, nullable=True) + llm_temperature = Column(Float, nullable=True) + llm_max_tokens = Column(Integer, nullable=True) + response_delay_ms = Column(Integer, nullable=True) + max_turns = Column(Integer, nullable=True) + allow_interruptions = Column(Boolean, nullable=True) + background_noise_source = Column(String(20), nullable=False, default="none") + background_noise_preset = Column(String(50), nullable=True) + background_noise_volume = Column(Float, nullable=True, default=0.22) + background_noise_s3_key = Column(String, nullable=True) + background_noise_asset_id = Column( + UUID(as_uuid=True), + ForeignKey("ambient_noise_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class AmbientNoiseAsset(Base): + """Reusable ambient noise bed uploaded for test-agent personas.""" + __tablename__ = "ambient_noise_assets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + s3_key = Column(String, nullable=False) + original_filename = Column(String(255), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + +class Scenario(Base): + """Scenario - The conversation scenario/test case""" + __tablename__ = "scenarios" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every scenario belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + name = Column(String, nullable=False) + description = Column(String) + required_info = Column(JSON) + # Tool-use expectations checked on simulation trials; see app/services/harbor/tool_checks.py + expected_tool_calls = Column(JSON, nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class MetricScenarioAssignment(Base): + """Many-to-many: metric scored only on linked scenarios (empty = golden / all).""" + + __tablename__ = "metric_scenarios" + + metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + primary_key=True, + ) + scenario_id = Column( + UUID(as_uuid=True), + ForeignKey("scenarios.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + + +class MetricAgentAssignment(Base): + """Many-to-many: metric scored only on linked agents (empty = golden / all).""" + + __tablename__ = "metric_agents" + + metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + primary_key=True, + ) + agent_id = Column( + UUID(as_uuid=True), + ForeignKey("agents.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + + +class KnowledgeBase(Base): + """Workspace document library for agent retrieval in simulation.""" + __tablename__ = "knowledge_bases" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + embedding_provider = Column(String(64), nullable=False, default="openai") + embedding_model = Column(String(512), nullable=False, default="text-embedding-3-small") + embedding_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + status = Column(String(32), nullable=False, default="active") + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class KnowledgeDocument(Base): + __tablename__ = "knowledge_documents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + knowledge_base_id = Column( + UUID(as_uuid=True), + ForeignKey("knowledge_bases.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + source_type = Column(String(32), nullable=False, default="file") + original_filename = Column(String(512), nullable=True) + blob_key = Column(String(1024), nullable=False) + content_type = Column(String(255), nullable=True) + byte_size = Column(BigInteger, nullable=True) + status = Column(String(32), nullable=False, default="pending") + error_text = Column(Text, nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + +class KnowledgeChunk(Base): + __tablename__ = "knowledge_chunks" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + document_id = Column( + UUID(as_uuid=True), + ForeignKey("knowledge_documents.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + chunk_index = Column(Integer, nullable=False) + text = Column(Text, nullable=False) + page_number = Column(Integer, nullable=True) + embedding_model = Column(String(512), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + + +class ToolConnection(Base): + """An external tool source an agent can call: MCP server, HTTP tools or Postgres. + + ``secrets`` is a Fernet-encrypted JSON object (headers, env vars, DSN); it is + never returned by the API. ``cached_tools`` is the tool list from the last + successful test, so building an agent's toolset needs no network round-trip. + """ + + __tablename__ = "tool_connections" + __table_args__ = (UniqueConstraint("workspace_id", "slug", name="uq_tool_connections_workspace_slug"),) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + # Prefix for this connection's tool names (``slack__search_messages``). + slug = Column(String(64), nullable=False) + connection_type = Column(String(32), nullable=False) # mcp | http | postgres + preset = Column(String(64), nullable=True) # slack | notion | gdrive | postgres | ... + config = Column(JSON, nullable=False, default=dict) + secrets = Column(Text, nullable=True) + # live | mock_writes | mock_all — how calls behave inside simulations. + simulation_mode = Column(String(16), nullable=False, default="mock_writes") + cached_tools = Column(JSON, nullable=True) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + last_error = Column(Text, nullable=True) + created_by = Column(String, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AgentToolConnection(Base): + """Tool connections attached to a production agent; ``enabled_tools`` NULL = all.""" + + __tablename__ = "agent_tool_connections" + + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="CASCADE"), primary_key=True) + tool_connection_id = Column( + UUID(as_uuid=True), + ForeignKey("tool_connections.id", ondelete="CASCADE"), + primary_key=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column(UUID(as_uuid=True), ForeignKey("workspaces.id", ondelete="RESTRICT"), nullable=False) + enabled_tools = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class AgentKnowledgeBase(Base): + __tablename__ = "agent_knowledge_bases" + + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="CASCADE"), primary_key=True) + knowledge_base_id = Column( + UUID(as_uuid=True), + ForeignKey("knowledge_bases.id", ondelete="CASCADE"), + primary_key=True, + ) + # "production" = the agent under test; "test" = the simulated customer. + role = Column(String(16), primary_key=True, nullable=False, default="production") + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_at = Column(DateTime, server_default=func.now()) + + +# Enums moved to enums.py + + +class Integration(Base): + """Integration model for connecting with external voice AI platforms.""" + __tablename__ = "integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + platform = Column(String, nullable=False) + + + + name = Column(String, nullable=True) # Optional friendly name + api_key = Column(String, nullable=False) # Encrypted Private API key for the platform + public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) + # ElevenLabs data-residency API origin (e.g. https://api.eu.residency.elevenlabs.io) + api_base_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple credentials per (org, platform) are allowed. is_default marks + # the row used when a caller does not explicitly select a credential. + # A partial unique index in migration 028 enforces at most one default + # per (org, platform) at the DB level. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +class ManualTranscription(Base): + """Manual transcription model for storing transcriptions from S3 audio files.""" + + __tablename__ = "manual_transcriptions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String(255), nullable=True) # User-friendly name for the transcription + audio_file_key = Column(String(512), nullable=False) # S3 key or file path + transcript = Column(String, nullable=False) # Full transcript text + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") + stt_provider = Column(String, nullable=True) # Provider used + + + + language = Column(String(10), nullable=True) # Detected or specified language + processing_time = Column(Float, nullable=True) # Processing time in seconds + raw_output = Column(JSON, nullable=True) # Full model output for reference + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class ConversationEvaluation(Base): + """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" + + __tablename__ = "conversation_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + + # Evaluation results + objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? + objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result + additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) + overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) + + # LLM metadata + llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) + + llm_model = Column(String(100), nullable=True) + llm_response = Column(JSON, nullable=True) # Full LLM response for reference + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AIProvider(Base): + """AI Provider - Stores API keys for different AI platforms.""" + __tablename__ = "aiproviders" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String, nullable=False) + + + + api_key = Column(String, nullable=False) # Encrypted API key + name = Column(String, nullable=True) # Optional friendly name + # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). + # Only used when provider is azure; other providers ignore this column. + endpoint_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple AIProvider rows per (org, provider) are allowed. is_default + # marks the row resolved when no explicit credential id is selected. + # A partial unique index in migration 028 enforces at most one default. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | bifrost | litellm_proxy — gateway backend override + gateway_type = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) + # Non-empty list restricts model pickers; null/empty = all catalog models for provider. + enabled_models = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +# Enums moved to enums.py + + +class VoiceBundle(Base): + """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" + __tablename__ = "voicebundles" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String, nullable=False) + description = Column(String, nullable=True) + + # Bundle type: either STT+LLM+TTS or S2S + # Using String instead of Enum to avoid SQLAlchemy enum conversion issues + # The enum conversion is handled in the Pydantic schemas + bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) + + # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + stt_provider = Column(String, nullable=True) + # Optional explicit credential row (aiproviders.id or integrations.id). + # When NULL the credential resolver picks the default row for the + # provider. No FK is set because the target table varies by provider. + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" + + # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + llm_provider = Column(String, nullable=True) + llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + + llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" + llm_temperature = Column(Float, nullable=True, default=0.7) + llm_max_tokens = Column(Integer, nullable=True) + llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) + + # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + tts_provider = Column(String, nullable=True) + tts_credential_id = Column(UUID(as_uuid=True), nullable=True) + + tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" + tts_voice = Column(String, nullable=True) # Voice selection if applicable + tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) + + # S2S Configuration - required for S2S type, optional for STT_LLM_TTS + s2s_provider = Column(String, nullable=True) + s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) + + + + s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model + s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) + + # Additional configuration for extensibility + extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) + + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class TestAgentConversation(Base): + """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" + __tablename__ = "test_agent_conversations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every playground conversation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Configuration + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + # Conversation data + status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) + + + + live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps + conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio + full_transcript = Column(String, nullable=True) # Full conversation transcript + + # Metadata + started_at = Column(DateTime(timezone=True), server_default=func.now()) + ended_at = Column(DateTime(timezone=True), nullable=True) + duration_seconds = Column(Float, nullable=True) + + # Additional metadata + conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorSuite(Base): + """Evaluator suite — one agent + one or more personas × N scenario combinations.""" + + __tablename__ = "evaluator_suites" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String, nullable=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + metric_ids = Column(JSON, nullable=True) + llm_provider = Column(String, nullable=True) + llm_model = Column(String, nullable=True) + llm_config = Column(JSON, nullable=True) + tags = Column(JSON, nullable=True) + default_runs_per_combination = Column(Integer, nullable=False, default=1) + round_robin_index = Column(Integer, nullable=False, default=0) + is_active = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class Evaluator(Base): + """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" + __tablename__ = "evaluators" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Display name (required for custom evaluators, optional for standard) + name = Column(String, nullable=True) + + # Parent suite (nullable for legacy/custom evaluators) + suite_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_suites.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + # Standard evaluator configuration (nullable for custom evaluators) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + + # Custom evaluator prompt (used instead of agent/persona/scenario) + custom_prompt = Column(Text, nullable=True) + + # Custom evaluator metric selection. When set, the worker filters the + # enabled-org metrics down to only these IDs (list of metric UUID strings). + # Standard evaluators leave this NULL and use all enabled agent metrics. + metric_ids = Column(JSON, nullable=True) + + # LLM configuration for evaluation (overrides hardcoded defaults) + llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" + llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" + llm_config = Column(JSON, nullable=True) + + # Tags for categorization + tags = Column(JSON, nullable=True) # Array of tag strings + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class Metric(Base): + """Metric - Configuration for evaluation metrics. + + Supports a 2-level hierarchy via ``parent_metric_id``: a "category" + parent metric (e.g. "Call Outcome") owns N child sub-metric labels + (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set + only on parents and controls how the LLM scores children together + (``single_choice`` = pick exactly one; ``multi_label`` = independent + yes/no with logical consistency). Children are always boolean. + """ + __tablename__ = "metrics" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: two-shape column. + # + # * ``workspace_id = `` — workspace-scoped metric. Only + # visible inside that workspace (the default behavior; existing + # rows all look like this). + # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in + # every workspace's listing under this org so users don't have + # to recreate the same metric per workspace. + # + # Children always inherit their parent's ``workspace_id`` (including + # NULL) so a category metric's whole subtree shares one scope; the + # add-child / promote-discovered endpoints enforce this. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + + # Basic information + name = Column(String, nullable=False) + description = Column(String, nullable=True) + # Free-form illustrative example used to sharpen the LLM judge's + # rubric. Today this is consumed by child sub-labels of a + # categorization parent metric so each label can carry "what does + # this look like in a transcript?" text alongside the rubric in + # ``description``. The column lives on every Metric row for + # forward-compat: a standalone metric could later surface its own + # example without another migration. + example = Column(Text, nullable=True) + + # Configuration + metric_type = Column(String, nullable=False, default=MetricType.RATING.value) + metric_category = Column( + String(30), + nullable=False, + default=MetricCategory.QUALITY.value, + server_default=MetricCategory.QUALITY.value, + ) + trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) + metric_origin = Column(String(30), nullable=False, default="default") + supported_surfaces = Column(JSON, nullable=False, default=list) # agent, chat_agent, voice_playground + enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces + custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" + custom_config = Column(JSON, nullable=True) # enum options / number range config + tags = Column(JSON, nullable=True) # ["tone", "latency", ...] + + # Hierarchy: NULL = standalone or parent. When set, this row is a + # child sub-metric of the referenced parent. ON DELETE CASCADE so + # deleting a category removes its children atomically. + parent_metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + # Set only on parent rows (``parent_metric_id IS NULL``). Either + # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical + # metric (no children). + selection_mode = Column(String(20), nullable=True) + + # When true on a parent metric (any selection_mode), the LLM is + # invited during call-import evaluation to emit additional + # candidate sub-labels beyond the user-defined children. The + # candidates surface in a "Discovered labels" panel where the user + # manually promotes them into real child Metric rows. For + # ``single_choice`` parents the discovered entries are + # supplemental — the chosen child is still picked from the + # predefined children so the exactly-one-true invariant holds. + # The validator rejects this flag on standalone / child metrics. + allow_discovery = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # When True, this metric is a "transcript-compare judge": the + # call-import evaluator feeds BOTH the production transcript + # (``call_import_rows.transcript``, CSV-supplied) and the diarised + # transcript (``call_import_rows.diarised_transcript``, worker- + # produced by the STT/diarisation pipeline) to the LLM as a + # labeled pair instead of feeding one transcript. The parent + # evaluation's ``CallImportEvaluation.transcript_source`` is + # ignored for these metrics — they always read both columns. + # Rows where either transcript is missing are skipped per-metric + # with ``skipped="comparison_missing_transcript"`` so the rest of + # the row's metrics still produce scores. The Pydantic validator + # rejects ``compare_transcripts`` combined with ``parent_metric_id`` + # or ``selection_mode`` (i.e. it can't simultaneously be part of + # a parent/child hierarchy). The call-import worker also + # auto-promotes a metric to comparison mode when its description + # references the production / diarised transcripts in well-known + # phrases (see ``_metric_text_references_production`` in + # ``app.workers.tasks.evaluate_call_import_row``). + compare_transcripts = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # How call-import evaluation scores this metric. ``llm`` is the default + # LLM judge; ``user_wer`` / ``user_cer`` are deterministic transcript + # agreement metrics (diarized reference vs production hypothesis). + scoring_method = Column( + String(32), + nullable=False, + default="llm", + server_default="llm", + ) + + parent = relationship( + "Metric", + remote_side=[id], + backref="children", + ) + scoped_scenarios = relationship( + "Scenario", + secondary="metric_scenarios", + lazy="selectin", + ) + scoped_agents = relationship( + "Agent", + secondary="metric_agents", + lazy="selectin", + ) + + # When true, the LLM-judge is asked to also return a short free-form + # rationale alongside the value (stored under ``metric_scores[id].rationale``). + # Adds a second " - LLM Rationale" column in the call-import CSV export. + capture_rationale = Column(Boolean, nullable=False, default=False) + + enabled = Column(Boolean, nullable=False, default=True) + + # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics + # Studio until promoted to ``active``. + lifecycle = Column( + String(20), + nullable=False, + default="active", + server_default="active", + ) + promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) + studio_notes = Column(Text, nullable=True) + + # Metadata + is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResult(Base): + """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" + __tablename__ = "evaluator_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator result belongs to a workspace + # within its org. Stamped from the active workspace at creation time + # (either the X-Workspace-Id header or the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # References + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls + + # Result data + name = Column(String, nullable=True) # Scenario name or test call name (optional) + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + duration_seconds = Column(Float, nullable=True) # Call duration + status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) + + # Audio and transcription + audio_s3_key = Column(String, nullable=True) # S3 key for audio file + transcription = Column(String, nullable=True) # Full transcription + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + + # Metric scores - JSON object with metric_id as key and score as value + # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} + metric_scores = Column(JSON, nullable=True) + + # Celery task tracking + celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking + + # Error information + error_message = Column(String, nullable=True) + + # Call event tracking (similar to CallRecording) + call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) + + # Data-plane shard routing (payload rows on shard DBs when sharding enabled) + shard_id = Column(String(64), nullable=True, index=True) + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + # Harbor simulation run trial (NULL for regular evaluator runs). + simulation_run_id = Column( + UUID(as_uuid=True), + ForeignKey("simulation_runs.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + harbor_trial_name = Column(String(255), nullable=True) + attempt_index = Column(Integer, nullable=True) + # Aggregate normalized [0, 1] reward reported to Harbor and pass/fail vs run threshold. + reward = Column(Float, nullable=True) + passed = Column(Boolean, nullable=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class SimulationRun(Base): + """Harbor job: every scenario × persona × attempt simulated against one production agent.""" + + __tablename__ = "simulation_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="CASCADE"), nullable=False, index=True) + name = Column(String, nullable=True) + + scenario_ids = Column(JSON, nullable=False, default=list) + persona_ids = Column(JSON, nullable=False, default=list) + metric_ids = Column(JSON, nullable=True) # NULL = all enabled agent metrics + metric_success_criteria = Column(JSON, nullable=True) # per-metric pass rules for this run + + n_attempts = Column(Integer, nullable=False, default=1) + n_concurrent = Column(Integer, nullable=False, default=4) + pass_threshold = Column(Float, nullable=False, default=0.7) + + # Optional LLM overrides; NULL = agent's main / test LLM. + production_llm_provider = Column(String(64), nullable=True) + production_llm_model = Column(String(255), nullable=True) + test_llm_provider = Column(String(64), nullable=True) + test_llm_model = Column(String(255), nullable=True) + # {"production": {...}, "test": {...}} resolved at creation; see app/services/testing/simulation_llms.py + resolved_llms = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default=SimulationRunStatus.QUEUED.value, index=True) + expected_trials = Column(Integer, nullable=False, default=0) + completed_trials = Column(Integer, nullable=False, default=0) + failed_trials = Column(Integer, nullable=False, default=0) + summary = Column(JSON, nullable=True) + error_message = Column(Text, nullable=True) + # Failure analysis of trials that missed their success rules; see app/services/harbor/failure_analysis.py + metric_clusters = Column(JSON, nullable=True) # EvaluationMetricClustersState + prompt_improvements = Column(JSON, nullable=True) # EvaluationPromptImprovementsState + + harbor_job_name = Column(String(255), nullable=True) + artifacts_s3_key = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True) + cancel_requested = Column(Boolean, nullable=False, default=False, server_default="false") + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResultClusterJob(Base): + """Cached failure-cluster state for a filtered evaluator-results scope.""" + + __tablename__ = "evaluator_result_cluster_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + scope_key = Column(String(512), nullable=False) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + suite_id = Column(UUID(as_uuid=True), ForeignKey("evaluator_suites.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + since = Column(DateTime(timezone=True), nullable=True) + until = Column(DateTime(timezone=True), nullable=True) + scenario_ids = Column(JSON, nullable=True) + metric_clusters = Column(JSON, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +# Enums moved to enums.py + + +class CallRecordingSource(str, enum.Enum): + """Source of the call recording data.""" + + PLAYGROUND = "playground" + WEBHOOK = "webhook" + + +class CallRecording(Base): + """Call Recording model for tracking voice provider calls.""" + __tablename__ = "call_recordings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every recording belongs to a workspace within + # its org. For playground-origin rows this is stamped from the active + # workspace at creation time; for webhook-origin rows the worker + # looks up the recording's agent and inherits its workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) + call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) + source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) + call_data = Column(JSON, nullable=True) # JSON blob for provider response + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent + + # Link to EvaluatorResult for metric evaluations + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + shard_id = Column(String(64), nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class EvaluatorResultPayload(Base): + """Heavy evaluator result fields stored on data shards when sharding is enabled.""" + + __tablename__ = "evaluator_result_payloads" + + evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + audio_s3_key = Column(String, nullable=True) + transcription = Column(String, nullable=True) + speaker_segments = Column(JSON, nullable=True) + metric_scores = Column(JSON, nullable=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallRecordingPayload(Base): + """Heavy call recording fields stored on data shards when sharding is enabled.""" + + __tablename__ = "call_recording_payloads" + + call_recording_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticCallTrace(Base): + """Legacy PG trace header (superseded by ClickHouse call_traces; dropped by migration 086).""" + + __tablename__ = "synthetic_call_traces" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id", ondelete="SET NULL"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id", ondelete="SET NULL"), nullable=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True) + call_recording_id = Column( + UUID(as_uuid=True), + ForeignKey("call_recordings.id", ondelete="SET NULL"), + nullable=True, + ) + call_short_id = Column(String(6), nullable=True, index=True) + environment = Column(String(32), nullable=False, default="pre_prod") + provider_platform = Column(String(64), nullable=True) + transport = Column(String(32), nullable=False, default="phone") + tier = Column(String(32), nullable=False, default="black_box") + status = Column(String(32), nullable=False, default="open") + started_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + ended_at = Column(DateTime(timezone=True), nullable=True) + turn_count = Column(Integer, nullable=False, default=0) + response_latency_p50_ms = Column(Float, nullable=True) + response_latency_p90_ms = Column(Float, nullable=True) + response_latency_p95_ms = Column(Float, nullable=True) + component_aggregates = Column(JSON, nullable=True) + failure_flags = Column(JSON, nullable=True) + trace_version = Column(Integer, nullable=False, default=1) + shard_id = Column(String(64), nullable=True, index=True) + spans_s3_key = Column(String(512), nullable=True) + spans_storage = Column(String(16), nullable=False, default="legacy_jsonb") + span_count = Column(Integer, nullable=False, default=0) + last_span_at = Column(DateTime(timezone=True), nullable=True) + derive_pending = Column(Boolean, nullable=False, default=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticTraceSpanBatch(Base): + """Append-only OTLP span batch for Phase 2 async ingest.""" + + __tablename__ = "synthetic_trace_span_batches" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + seq = Column(BigInteger, nullable=False) + span_count = Column(Integer, nullable=False, default=0) + spans = Column(JSON, nullable=False, default=list) + received_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + +class SyntheticTraceIngestStaging(Base): + """Short-lived raw OTLP payload staged before worker parse (Layout 3).""" + + __tablename__ = "synthetic_trace_ingest_staging" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), nullable=False, index=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + content_type = Column(String(128), nullable=False, default="") + body = Column(LargeBinary, nullable=False) + body_bytes = Column(Integer, nullable=False, default=0) + header_evaluator_result_id = Column(String(128), nullable=True) + header_agent_id = Column(String(128), nullable=True) + header_call_short_id = Column(String(32), nullable=True) + status = Column(String(16), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + accepted_spans = Column(Integer, nullable=True) + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="SET NULL"), + nullable=True, + ) + correlated = Column(Boolean, nullable=True) + attempt_count = Column(Integer, nullable=False, default=0) + received_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + processed_at = Column(DateTime(timezone=True), nullable=True) + + +class SyntheticTracePayload(Base): + """Per-turn timing payload for a synthetic call trace.""" + + __tablename__ = "synthetic_trace_payloads" + + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), + primary_key=True, + ) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + turns = Column(JSON, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticTraceOtelPayload(Base): + """Full OTLP span tree for a synthetic call trace.""" + + __tablename__ = "synthetic_trace_otel_payloads" + + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), + primary_key=True, + ) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + spans = Column(JSON, nullable=False, default=list) + trace_ids = Column(JSON, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class Alert(Base): + """Alert model for configuring monitoring alerts.""" + __tablename__ = "alerts" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + + data_source = Column(String, nullable=False, default="evaluations") + + # Metric condition configuration + metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) + aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) + operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) + threshold_value = Column(Float, nullable=False) + time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation + alert_on_missing_data = Column( + Boolean, nullable=False, default=False, server_default=text("false") + ) + + # Agent selection (JSON array of agent UUIDs, null means all agents) + agent_ids = Column(JSON, nullable=True) + + # Notification configuration + notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) + notify_emails = Column(JSON, nullable=True) # Array of email addresses + notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) + notify_pagerduty_routing_keys = Column(JSON, nullable=True) + + # Status + status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) + suppress_reopen_until_ok = Column( + Boolean, nullable=False, default=False, server_default=text("false") + ) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + # Relationships + alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") + + +class AlertHistory(Base): + """Alert history model for tracking triggered alerts.""" + __tablename__ = "alert_history" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) + + # Trigger information + triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert + threshold_value = Column(Float, nullable=False) # The threshold at time of trigger + + # Status tracking + status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) + + # Notification tracking + notified_at = Column(DateTime(timezone=True), nullable=True) + notification_details = Column(JSON, nullable=True) # Details of sent notifications + + # Resolution + acknowledged_at = Column(DateTime(timezone=True), nullable=True) + acknowledged_by = Column(String, nullable=True) + resolved_at = Column(DateTime(timezone=True), nullable=True) + resolved_by = Column(String, nullable=True) + resolution_notes = Column(String, nullable=True) + + # Additional context + context_data = Column(JSON, nullable=True) # Additional data about the trigger + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + alert = relationship("Alert", back_populates="alert_history") + + +class CronJob(Base): + """Cron job model for scheduling automated evaluator runs.""" + __tablename__ = "cron_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=True, index=True) + + # Basic information + name = Column(String(255), nullable=False) + job_type = Column(String(64), nullable=False, default="evaluator_run") + is_system = Column(Boolean, nullable=False, default=False) + config = Column(JSON, nullable=False, default=dict) + cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" + timezone = Column(String(100), nullable=False, default="UTC") + + # Run configuration + max_runs = Column(Integer, nullable=False, default=10) + current_runs = Column(Integer, nullable=False, default=0) + + # Evaluators to trigger (JSON array of evaluator UUIDs) + evaluator_ids = Column(JSON, nullable=False) + + # Status + status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) + + # Run tracking + next_run_at = Column(DateTime(timezone=True), nullable=True) + last_run_at = Column(DateTime(timezone=True), nullable=True) + last_dispatch_celery_task_id = Column(String(255), nullable=True) + last_dispatch_status = Column(String(32), nullable=True) + last_dispatch_error = Column(String, nullable=True) + last_dispatch_at = Column(DateTime(timezone=True), nullable=True) + last_run_status = Column(String(32), nullable=True) + last_run_error = Column(String, nullable=True) + + @property + def interval_days(self) -> int | None: + config = self.config if isinstance(self.config, dict) else {} + raw = config.get("interval_days") + try: + days = int(raw) + except (TypeError, ValueError): + return None + return days if days > 0 else None + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class TTSComparisonStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + EVALUATING = "evaluating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSSampleStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSReportJobStatus(str, enum.Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSComparison(Base): + """TTS Comparison session for A/B testing voice providers.""" + __tablename__ = "tts_comparisons" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every voice playground comparison belongs to + # a workspace within its org. Children (samples, report jobs, blind + # test shares) inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + simulation_id = Column(String(6), unique=True, index=True, nullable=True) + + name = Column(String(255), nullable=True) + status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) + + # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). + # 'blind_test_only' = standalone blind test built from existing recordings + # / uploads / past TTS samples; no TTS generation happens. + mode = Column(String(32), nullable=False, default="benchmark") + + provider_a = Column(String(100), nullable=True) + model_a = Column(String(100), nullable=True) + voices_a = Column(JSON, nullable=True) + + provider_b = Column(String(100), nullable=True) + model_b = Column(String(100), nullable=True) + voices_b = Column(JSON, nullable=True) + + sample_texts = Column(JSON, nullable=False) + num_runs = Column(Integer, nullable=False, default=1) + + blind_test_results = Column(JSON, nullable=True) + evaluation_summary = Column(JSON, nullable=True) + + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") + + +class TTSSample(Base): + """Individual TTS audio sample within a comparison.""" + __tablename__ = "tts_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + provider = Column(String(100), nullable=True) + model = Column(String(100), nullable=True) + voice_id = Column(String(255), nullable=True) + voice_name = Column(String(255), nullable=True) + side = Column(String(1), nullable=True) # "A" or "B" + sample_index = Column(Integer, nullable=False) + run_index = Column(Integer, nullable=False, default=0) + + # 'tts' (default, audio is synthesized by a provider), 'recording' (audio + # is reused from a CallImportRow recording), or 'upload' (audio was + # uploaded by the user). Non-tts samples are marked completed up-front + # by the API and skipped by the generation worker. + source_type = Column(String(32), nullable=False, default="tts") + # When source_type == 'recording', references CallImportRow.id (no FK + # constraint to keep cascading deletes simple if a call import is later + # removed; the audio_s3_key is what's actually used). + source_ref_id = Column(UUID(as_uuid=True), nullable=True) + + text = Column(String, nullable=False) + audio_s3_key = Column(String(512), nullable=True) + duration_seconds = Column(Float, nullable=True) + latency_ms = Column(Float, nullable=True) + ttfb_ms = Column(Float, nullable=True) + + evaluation_metrics = Column(JSON, nullable=True) + status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + comparison = relationship("TTSComparison", back_populates="samples") + + +class TTSReportJob(Base): + """Asynchronous PDF report generation jobs for Voice Playground.""" + __tablename__ = "tts_report_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + + status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) + format = Column(String(20), nullable=False, default="pdf") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + error_message = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + + +class TTSBlindTestShareStatus(str, enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +class TTSBlindTestShare(Base): + """A publicly sharable blind test for a TTSComparison. + + The share_token is the capability: anyone holding it can open the public + form and submit a response. Each comparison has at most one share row. + """ + __tablename__ = "tts_blind_test_shares" + __table_args__ = ( + UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_comparisons.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + share_token = Column(String(64), unique=True, nullable=False, index=True) + + title = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Internal notes visible only to the share creator (e.g. which voice + # corresponds to which side, source notes for standalone blind tests). + # Never exposed via the public blind test payload. + creator_notes = Column(Text, nullable=True) + + # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] + custom_metrics = Column(JSON, nullable=False) + + status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + closed_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + responses = relationship( + "TTSBlindTestResponse", + back_populates="share", + cascade="all, delete-orphan", + ) + + +class TTSBlindTestResponse(Base): + """A single rater's submission against a TTSBlindTestShare.""" + __tablename__ = "tts_blind_test_responses" + __table_args__ = ( + UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + share_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + rater_name = Column(String(255), nullable=False) + rater_email = Column(String(320), nullable=False, index=True) + + # JSON list keyed by sample_index. Server stores in TRUE A/B orientation + # (already de-flipped from whatever the rater's UI showed): + # [{ + # "sample_index": int, + # "preferred": "A" | "B", + # "ratings_a": { metric_key: number }, + # "ratings_b": { metric_key: number }, + # "comment": str? + # }] + responses = Column(JSON, nullable=False) + + ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + + submitted_at = Column(DateTime(timezone=True), server_default=func.now()) + + share = relationship("TTSBlindTestShare", back_populates="responses") + + +class PromptPartial(Base): + """Prompt Partial - Reusable prompt templates with version history.""" + __tablename__ = "prompt_partials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every prompt partial belongs to a workspace + # within its org. Versions inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + content = Column(Text, nullable=False) + tags = Column(JSON, nullable=True) + current_version = Column(Integer, nullable=False, default=1) + # Cached LLM-generated flowchart for imported production agent prompts. + # Shape: AgentFlowGraph JSON (nodes[], edges[]). + agent_flowchart = Column(JSON, nullable=True) + agent_flowchart_status = Column(String(20), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") + pushes = relationship("PromptPush", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPush.created_at.desc()") + + +class PromptPartialVersion(Base): + """Version history for a prompt partial.""" + __tablename__ = "prompt_partial_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptPartial's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + version = Column(Integer, nullable=False) + content = Column(Text, nullable=False) + change_summary = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + created_by = Column(String, nullable=True) + + prompt_partial = relationship("PromptPartial", back_populates="versions") + + __table_args__ = ( + UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), + ) + + +class PromptPush(Base): + """Audit row for a live provider prompt push. The prompt body stays on PromptPartialVersion.""" + + __tablename__ = "prompt_pushes" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column(UUID(as_uuid=True), ForeignKey("workspaces.id", ondelete="RESTRICT"), nullable=False, index=True) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + prompt_version = Column(Integer, nullable=False) + provider = Column(String(50), nullable=False) + external_agent_id = Column(String(255), nullable=False) + external_agent_name = Column(String(255), nullable=True) + integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id", ondelete="SET NULL"), nullable=True) + evaluation_id = Column(UUID(as_uuid=True), nullable=True, index=True) + pushed_by = Column(String, nullable=True) + status = Column(String(20), nullable=False) + error_message = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + prompt_partial = relationship("PromptPartial", back_populates="pushes") + + +class CustomTTSVoice(Base): + """Organization-scoped custom TTS voice metadata.""" + __tablename__ = "custom_tts_voices" + __table_args__ = ( + UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(100), nullable=False, index=True) + voice_id = Column(String(255), nullable=False) + name = Column(String(255), nullable=False) + gender = Column(String(50), nullable=True) + accent = Column(String(100), nullable=True) + description = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + +class PromptOptimizationRun(Base): + """A single GEPA prompt optimization run for an agent.""" + __tablename__ = "prompt_optimization_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every optimization run belongs to a workspace + # within its org. Candidates inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + seed_prompt = Column(Text, nullable=False) + best_prompt = Column(Text, nullable=True) + best_score = Column(Float, nullable=True) + + status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) + config = Column(JSON, nullable=True) + reflection_trace = Column(JSON, nullable=True) + metric_history = Column(JSON, nullable=True) + + num_iterations = Column(Integer, nullable=True) + num_metric_calls = Column(Integer, nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") + + +class PromptOptimizationCandidate(Base): + """A candidate prompt generated during an optimization run.""" + __tablename__ = "prompt_optimization_candidates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + prompt_text = Column(Text, nullable=False) + score = Column(Float, nullable=True) + metric_breakdown = Column(JSON, nullable=True) + reflection_summary = Column(Text, nullable=True) + + parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) + + is_accepted = Column(Boolean, nullable=False, default=False) + pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") + + +class TelephonyIntegration(Base): + """Per-organization telephony provider credentials and configuration. + + Multiple rows per (organization_id, provider) are allowed so that an + organization can keep several Plivo / Exotel accounts side-by-side. + A partial unique index in migration 028 enforces at most one row with + is_default = TRUE per (org, provider); resolution falls back to that + default row when the caller does not pin a specific credential. + """ + + __tablename__ = "telephony_integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(50), nullable=False, default="plivo") + name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials + + auth_id = Column(String(255), nullable=False) + auth_token = Column(String(512), nullable=False) + + verify_app_uuid = Column(String(255), nullable=True) + voice_app_id = Column(String(255), nullable=True) + sip_domain = Column(String(255), nullable=True) + masking_config = Column(JSON, nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + is_default = Column(Boolean, default=False, nullable=False) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyPhoneNumber(Base): + """Inventory of telephony phone numbers owned by an organization.""" + + __tablename__ = "telephony_phone_numbers" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True + ) + + phone_number = Column(String(20), nullable=False, index=True) + country_iso2 = Column(String(2), nullable=True) + region = Column(String(100), nullable=True) + number_type = Column(String(20), nullable=True) + capabilities = Column(JSON, nullable=True) + provider_app_id = Column(String(255), nullable=True) + + is_masking_pool = Column(Boolean, default=False, nullable=False) + inbound_enabled = Column(Boolean, default=True, nullable=False) + outbound_enabled = Column(Boolean, default=True, nullable=False) + source = Column(String(20), nullable=False, default="imported") + agent_id = Column( + UUID(as_uuid=True), + ForeignKey( + "agents.id", + ondelete="SET NULL", + use_alter=True, + name="fk_telephony_phone_numbers_agent_id", + ), + nullable=True, + index=True, + ) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyDialTarget(Base): + """Org-scoped saved destination numbers for outbound test calls.""" + + __tablename__ = "telephony_dial_targets" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + phone_number = Column(String(20), nullable=False, index=True) + label = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyVerifySession(Base): + """Tracks voice OTP verification sessions via telephony provider.""" + + __tablename__ = "telephony_verify_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) + recipient_number = Column(String(20), nullable=False) + channel = Column(String(10), nullable=False, default="voice") + status = Column(String(20), nullable=False, default="pending") + initiated_by = Column(String(255), nullable=True) + verify_app_uuid = Column(String(255), nullable=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + expires_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyMaskedSession(Base): + """Number-masking session between two parties through a middle number.""" + + __tablename__ = "telephony_masked_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) + masked_number_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True + ) + masked_number = Column(String(20), nullable=False) + party_a_number = Column(String(20), nullable=False) + party_b_number = Column(String(20), nullable=False) + status = Column(String(20), nullable=False, default="active") + expires_at = Column(DateTime(timezone=True), nullable=True) + ended_at = Column(DateTime(timezone=True), nullable=True) + session_metadata = Column("metadata", JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallImportSchema(Base): + """Reusable Input Parameter schema for the call-uploads flow. + + A schema is workspace-scoped: users define a named bundle of typed + Input Parameters once (e.g. "Standard Voice QA" with conversation_id + + recording_url + transcript + agent_name) and then map those parameters + to CSV/Excel headers each time they upload a new batch. + + Every schema MUST contain exactly one parameter with + ``type='conversation_id'`` and ``is_required=True`` - that's the + mandatory identity field every imported row needs. A schema may + optionally include at most one ``recording_url`` parameter. The + invariant is enforced in app code on create/update (no DB-level + CHECK because the parent + children are written across two tables in + one transaction). + """ + + __tablename__ = "call_import_schemas" + __table_args__ = ( + # Case-insensitive uniqueness is enforced via the matching partial + # index on ``LOWER(name)`` in the migration; this constraint here + # would be case-sensitive and is intentionally omitted to avoid + # confusing the user. + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + parameters = relationship( + "CallImportSchemaParameter", + back_populates="schema", + cascade="all, delete-orphan", + order_by="CallImportSchemaParameter.ordering", + ) + + +class CallImportSchemaParameter(Base): + """A single typed parameter inside a :class:`CallImportSchema`. + + ``type`` is one of the strings tracked by + :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` + is reserved for the mandatory identity parameter every schema must + contain. + """ + + __tablename__ = "call_import_schema_parameters" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + type = Column(String(32), nullable=False) + description = Column(Text, nullable=True) + is_required = Column(Boolean, nullable=False, default=False) + # Stable ordering so the UI renders parameters in the order the + # schema author defined them (matters when conversation_id is pinned + # first and the user re-orders the rest). + ordering = Column(Integer, nullable=False, default=0) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + schema = relationship("CallImportSchema", back_populates="parameters") + + +class CallImport(Base): + """Batch record for a CSV-driven call import job.""" + + __tablename__ = "call_imports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every imported batch belongs to a workspace + # within its org. The /upload endpoint stamps it from the active + # workspace header (or the org's Default if absent). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the + # legacy one-shot ``POST /upload`` endpoint this is supplied with the + # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value + # isn't known until the IMPORT stage, so the column is nullable for + # ``uploaded`` / ``mapped`` batches. + provider = Column(String(50), nullable=True, default="exotel") + # Pin a specific telephony credential for this batch so the worker + # downloads recordings using *that* row instead of the org default. + # NULL preserves legacy behavior (resolve by provider + default). + telephony_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + original_filename = Column(String(512), nullable=True) + # When the source file was a multi-sheet Excel workbook, this records + # the worksheet the rows came from (one batch per sheet). NULL for CSV + # uploads since CSV has no sheet concept. + sheet_name = Column(String(255), nullable=True) + + # --- Source-file staging (UPLOAD stage) --------------------------- + # The raw CSV / Excel file is stored in S3 between stages so the + # user can come back later to MAP and IMPORT without re-uploading. + # ``source_s3_key`` is NULL on legacy batches that were imported via + # the one-shot endpoint (those batches stay read-only post-import). + source_s3_key = Column(Text, nullable=True) + source_format = Column(String(16), nullable=True) + source_size_bytes = Column(BigInteger, nullable=True) + source_content_type = Column(String(255), nullable=True) + + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI doesn't need to re-fetch the source bytes from S3. + # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. + available_sheets = Column(JSON, nullable=True) + + # User's explicit "drop these columns" decision captured at MAP + # time. Was validation-only and ephemeral in the legacy flow; now + # persisted so the IMPORT stage can re-parse the file with the same + # mapping/skip intent. + skipped_columns = Column(JSON, nullable=False, default=list) + # Rows skipped at parse time (missing/invalid conversation_id or URL). + # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. + source_row_skips = Column(JSON, nullable=False, default=list) + + # Free-text high-level segregation label. Powers the "Dataset" filter + # at the top of the imports page; multiple imports can share a value. + dataset = Column(String(255), nullable=True, index=True) + content_modality = Column(String(16), nullable=True) + + # Reusable Input Parameter schema this batch was uploaded against. + # NULL on legacy batches uploaded before the schema-driven flow + # shipped; those still render via ``column_mapping`` + ``extra_columns`` + # + ``custom_column_mapping`` below. + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. + # Populated for new uploads; empty dict on legacy batches. + parameter_mapping = Column(JSON, nullable=False, default=dict) + + # Legacy free-form mapping (pre-schema-flow). Kept on the model so + # batches that were uploaded before the schema feature shipped still + # render correctly on the detail page; new uploads stop writing here. + # Keys: external_call_id (required), transcript, recording_url. + # (DB column ``external_call_id`` is now ``conversation_id``; this + # JSON key stays as-is for historical batches.) + # Values: original CSV header strings (preserve user casing for export). + column_mapping = Column(JSON, nullable=False, default=dict) + # Ordered list of additional CSV header strings the uploader wants + # preserved verbatim into the evaluation export CSV. + extra_columns = Column(JSON, nullable=False, default=list) + # User-defined ``{custom_field_name: csv_header}`` mappings on top of + # the three system fields above. Cells from the mapped CSV columns are + # preserved per row (keyed by the CSV header in ``raw_columns``) and + # surface in the evaluation export under the uploader-chosen name. + custom_column_mapping = Column(JSON, nullable=False, default=dict) + # Optional per-import overrides for speaker labels in production + # transcripts (e.g. ``{"user": ["Member"], "agent": ["IVR"]}``). + speaker_aliases = Column(JSON, nullable=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + + status = Column( + Enum(CallImportStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportStatus.PENDING, + index=True, + ) + error_message = Column(Text, nullable=True) + + voice_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + platform_filters = Column(JSON, nullable=False, default=dict) + window_start = Column(DateTime(timezone=True), nullable=True) + window_end = Column(DateTime(timezone=True), nullable=True) + window_timezone = Column(String(64), nullable=True) + discovery_status = Column(String(32), nullable=True) + max_calls = Column(Integer, nullable=True) + next_row_index = Column(Integer, nullable=False, default=0) + skip_counts = Column(JSON, nullable=False, default=dict) + skip_samples = Column(JSON, nullable=False, default=list) + paused_at = Column(DateTime(timezone=True), nullable=True) + pause_reason = Column(Text, nullable=True) + billed_at = Column(DateTime(timezone=True), nullable=True) + last_progress_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + rows = relationship( + "CallImportRow", + back_populates="call_import", + cascade="all, delete-orphan", + order_by="CallImportRow.row_index", + ) + tags = relationship( + "CallImportTag", + secondary="call_import_tag_assignments", + backref="call_imports", + lazy="selectin", + ) + evaluations = relationship( + "CallImportEvaluation", + back_populates="call_import", + cascade="all, delete-orphan", + ) + + +class CallImportDiscoverySlice(Base): + """Time slice for platform historical import discovery.""" + + __tablename__ = "call_import_discovery_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + slice_start = Column(DateTime(timezone=True), nullable=False) + slice_end = Column(DateTime(timezone=True), nullable=False) + cursor = Column(Text, nullable=True) + agent_id = Column(String(255), nullable=True) + status = Column(String(32), nullable=False, default="pending") + listed = Column(Integer, nullable=False, default=0) + accepted = Column(Integer, nullable=False, default=0) + attempts = Column(Integer, nullable=False, default=0) + error = Column(Text, nullable=True) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class PlatformConversationLedger(Base): + """One ledger row per conversation inside a single import.""" + + __tablename__ = "platform_conversation_ledger" + __table_args__ = ( + UniqueConstraint( + "call_import_id", + "conversation_id", + name="uq_platform_ledger_import_conversation", + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + provider = Column(String(50), nullable=False) + conversation_id = Column(String(255), nullable=False) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + row_id = Column(UUID(as_uuid=True), nullable=False) + row_index = Column(Integer, nullable=False) + state = Column(String(32), nullable=False, default="reserved") + recording_date = Column(Date, nullable=True) + raw_columns = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportShardSlice(Base): + """Registry row: which shard stores a slice of rows for an import.""" + + __tablename__ = "call_import_shard_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + shard_id = Column(String(64), nullable=False, index=True) + row_index_min = Column(Integer, nullable=False) + row_index_max = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportRow(Base): + """A single row within a CallImport batch (one CSV line / one external call).""" + + __tablename__ = "call_import_rows" + __table_args__ = ( + UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + row_index = Column(Integer, nullable=False) + # Was historically named ``external_call_id``; renamed to + # ``conversation_id`` so the new schema-driven upload flow can refer + # to it by a single canonical name across the schema definition, + # exports, and downstream evaluation tables. + conversation_id = Column(String(255), nullable=False, index=True) + # Supplied via CSV for Exotel credentialed imports (required per row). + # Nullable in the schema for legacy rows imported before recording_url + # was mandatory on every Exotel upload. + recording_url = Column(Text, nullable=True) + # Date-only call recording date supplied by the import schema. Used + # for historical report comparisons without timezone/time ambiguity. + recording_date = Column(Date, nullable=True, index=True) + # The "production" transcript: the value supplied via the CSV + # upload mapping. Never overwritten by the diarisation worker — + # the worker writes its output into ``diarised_transcript`` so + # the user keeps both versions side by side. + transcript = Column(Text, nullable=True) + # Snapshot of the original CSV row keyed by the user's headers so the + # evaluation export can reproduce every column the uploader supplied + # (mapped + extra). NULL on legacy rows imported before this column. + raw_columns = Column(JSON, nullable=True) + + # Where the value in ``transcript`` came from. ``csv`` = supplied via + # the upload mapping, ``edited`` = manually changed in the UI. NULL + # on rows that have never had a production transcript. + # (Worker-produced transcripts now live in ``diarised_transcript`` + # and are tracked via ``diarised_transcript_*`` metadata below.) + transcript_source = Column(String(20), nullable=True) + # Provider/model recorded by the (legacy) post-hoc transcription + # worker. New worker runs leave these NULL and write into the + # ``diarised_transcript_*`` columns instead; kept on the model for + # backwards compatibility with pre-split rows that still carry the + # original transcription metadata here. + transcript_provider = Column(String(50), nullable=True) + transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the legacy transcription workflow itself, + # independent of the row's recording-fetch ``status``. ``idle`` = + # no transcribe task has touched this column. New diarisation runs + # update ``diarised_transcript_status`` instead. + transcript_status = Column( + String(20), + nullable=False, + default="idle", + ) + transcript_error = Column(Text, nullable=True) + transcribed_at = Column(DateTime(timezone=True), nullable=True) + + # The "diarised" transcript: produced by the post-hoc + # transcription/diarisation worker. Stored separately so a manual + # diarisation run never clobbers the production transcript above. + # Evaluations can be configured to score against either column + # (see ``CallImportEvaluation.transcript_source``). + diarised_transcript = Column(Text, nullable=True) + # Raw STT text in the spoken script. Set only when an evaluation that + # selected User WER / User CER ran ``stt_llm`` diarisation. Standalone + # diarisation leaves this null. + diarised_native_transcript = Column(Text, nullable=True) + transcript_char_length = query_expression() + diarised_transcript_char_length = query_expression() + # Provider/model the diarisation worker used. Surfaced in the UI + # as "Diarised via deepgram/nova-2" next to the diarised + # transcript section. + diarised_transcript_provider = Column(String(50), nullable=True) + diarised_transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the diarisation workflow. + # ``idle`` = no diarisation task has run; ``pending``/``running`` = + # a Celery task is queued or in flight; ``completed``/``failed`` = + # terminal. Independent of ``transcript_status`` so the two + # transcripts can be in different lifecycle states. + diarised_transcript_status = Column( + String(20), + nullable=False, + default="idle", + server_default="idle", + ) + diarised_transcript_error = Column(Text, nullable=True) + diarised_at = Column(DateTime(timezone=True), nullable=True) + + # Structured speaker turns produced by the diarisation worker — + # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", + # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` + # The plain-text ``diarised_transcript`` above is a rendered view + # of this list (``: `` per line). When the worker + # cannot recover structured turns (no pyannote token / single- + # speaker recording / provider that doesn't surface segments) this + # column stays NULL and the plain-text path is still populated. + diarised_segments = Column(JSON, nullable=True) + # When True the ``agent`` <-> ``user`` mapping inside + # ``diarised_segments`` is inverted at render / export time. The + # worker writes the canonical mapping using the "first speaker is + # the agent" heuristic; reviewers can flip the toggle from the row + # detail panel without re-running diarisation. + diarised_speaker_swap = Column( + Boolean, + nullable=False, + default=False, + server_default="false", + ) + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. The legacy diarisation worker used + # pyannote and left these NULL; the current path always runs an + # LLM with the operator-supplied (or default) ``diarised_prompt`` + # below, and records exactly which model + prompt produced each + # row so reviewers can reproduce a specific run. + diarised_llm_provider = Column(String(50), nullable=True) + diarised_llm_model = Column(String(100), nullable=True) + diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarised_prompt = Column(Text, nullable=True) + # Which diarisation pipeline produced this row's turns. + # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. + # ``diarised_transcript_provider``/``_model`` describe the STT + # side; ``diarised_llm_provider``/``_model`` the LLM side. + # * ``"llm_only"`` — single-stage: audio fed straight to a + # multimodal LLM. ``diarised_transcript_provider`` is stamped + # with the sentinel ``"llm_only"``; the real model is on + # ``diarised_llm_*``. + # Persisting it on the row (not just the run) lets the row detail + # panel render the right "Diarised via …" label even for ad-hoc + # standalone transcribes (no parent evaluation). + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + status = Column( + Enum(CallImportRowStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportRowStatus.PENDING, + index=True, + ) + + recording_s3_key = Column(String(1024), nullable=True) + recording_content_type = Column(String(128), nullable=True) + recording_size_bytes = Column(Integer, nullable=True) + recording_duration_seconds = Column(Integer, nullable=True) + + error_message = Column(Text, nullable=True) + attempts = Column(Integer, nullable=False, default=0) + celery_task_id = Column(String(255), nullable=True) + dispatched_at = Column(DateTime(timezone=True), nullable=True) + download_started_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + call_import = relationship("CallImport", back_populates="rows") + + +@event.listens_for(CallImportRow, "before_insert") +def _call_import_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent import when omitted.""" + if target.workspace_id is not None or target.call_import_id is None: + return + workspace_id = connection.execute( + select(CallImport.workspace_id).where( + CallImport.id == target.call_import_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportTag(Base): + """User-defined tag that can be attached to one or more call imports. + + Tags coexist with the free-text ``CallImport.dataset`` column: dataset + is the primary high-level segregation, tags are an optional secondary + classification (an import can have many tags). + """ + + __tablename__ = "call_import_tags" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + name = Column(String(255), nullable=False) + color = Column(String(32), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportTagAssignment(Base): + """Many-to-many join table between CallImport and CallImportTag.""" + + __tablename__ = "call_import_tag_assignments" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_tags.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportEvaluation(Base): + """Parent record for an evaluation run over a CallImport batch. + + A user picks a subset of org ``Metric`` rows and triggers an evaluation; + we fan out one ``CallImportEvaluationRow`` per source row and roll up + counters as workers finish. Status mirrors ``CallImportStatus`` plus a + ``RUNNING`` value so the UI can distinguish "queued" from "in flight". + """ + + __tablename__ = "call_import_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent CallImport's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Optional user-supplied label for this run. Lets the UI surface + # something more meaningful than the UUID prefix (e.g. "March QA pass"). + name = Column(String(255), nullable=True) + + # JSON list of Metric UUID strings selected for this run. Stored as text + # in JSON so we don't have to deal with PG arrays of UUIDs / cascade + # delete policies when metrics are removed; the loader filters for + # still-existing org metrics at run time. + selected_metric_ids = Column(JSON, nullable=False, default=list) + # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. + # Captures which children belong to which parent for THIS run so the UI + # / aggregator can reconstruct the tree even when the user selected + # only a subset of children, or after metrics are deleted / renamed. + # NULL on legacy rows means "no hierarchy" → fall back to flat + # ``selected_metric_ids`` semantics. + selected_metric_groups = Column(JSON, nullable=True) + # User-driven merges of LLM-discovered candidate sub-labels for + # ``allow_discovery`` parents. Shape: + # ``{"": {"": "", ...}}``. + # Populated via ``POST .../discovered-labels/merge``; consulted by + # the discovered-labels aggregator, the flow graph builder, and the + # worker so that rows finishing AFTER a merge cannot reintroduce + # the merged-away slug. Empty dict on fresh rows. + discovered_label_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Per-run opt-in for top-level metric discovery. When True, the LLM + # is asked to propose brand-new top-level metrics (boolean / rating / + # category) observed in the transcripts in addition to scoring the + # ``selected_metric_ids`` for the row. Candidates surface in a + # "Discovered metrics" panel on the evaluation's Flow tab and can + # be promoted into real standalone ``Metric`` rows via + # ``POST /metrics/from-discovered``. Defaults to False so existing + # evaluation creation payloads keep their previous behaviour. + discover_new_metrics = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + # Flat slug-to-slug redirect map for user merges + tombstones of + # discovered top-level metric candidates. Mirrors + # ``discovered_label_aliases`` but is NOT nested per parent — + # top-level metric discovery is not scoped to any parent. Shape:: + # + # {"": "", ...} + # + # An empty-string value tombstones the slug so workers finishing + # later can't re-introduce it. + discovered_metric_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Run-level LLM config picked from the Run Evaluation modal. NULL on + # legacy rows means "use the historical OpenAI/gpt-4o default" — the + # worker checks for this and falls back accordingly. ``llm_credential_id`` + # pins a specific AIProvider row when the org has multiple credentials + # for the same provider. + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + # Optional per-metric LLM override: + # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. + # Each entry overrides the run-level default for that metric only; + # missing keys = use run-level default. Stored as JSON so the UI can + # round-trip arbitrary {provider, model} pairs without migrations. + metric_llm_overrides = Column(JSON, nullable=True) + + # When ``auto_transcribe`` was set on the create payload, record the + # STT provider/model used so the UI can show "Auto-transcribed via + # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is + # untyped (no FK) because STT keys may live in either ``aiproviders`` + # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the + # transcription service handles the lookup. + stt_provider = Column(String(50), nullable=True) + stt_model = Column(String(100), nullable=True) + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + # Run-level LLM diariser config. Used when the create-run / + # retry-run paths chain a ``transcribe_call_import_row_task`` + # because the row is missing a diarised transcript. Persisted on + # the run so a retry uses the same diariser the original create + # call picked (unless the retry payload explicitly overrides). + diarisation_llm_provider = Column(String(50), nullable=True) + diarisation_llm_model = Column(String(100), nullable=True) + diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarisation_prompt = Column(Text, nullable=True) + # Mode the run was *created* with for its auto-transcribe step. + # Retry chains read this to decide whether to enqueue an STT+LLM + # transcribe or a single-stage multimodal LLM transcribe — without + # it we'd have to infer the mode from "stt_provider is NULL", which + # would silently break legacy rows that simply never configured + # auto-transcribe. See migration 041 for the column DDL. + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + # Which of the two transcripts on each ``CallImportRow`` this run + # scored against. ``'production'`` reads ``CallImportRow.transcript`` + # (the CSV-supplied value); ``'diarised'`` reads + # ``CallImportRow.diarised_transcript`` (the worker output). When + # the user ticks both checkboxes in the Run Evaluation modal we + # create two ``CallImportEvaluation`` rows — one per source — so + # the two scorings can be compared side-by-side. Defaults to + # ``'production'`` so legacy runs (which always read the single + # historical ``transcript`` column) keep their semantics. + transcript_source = Column( + String(20), + nullable=False, + default="production", + server_default="production", + ) + + # Cached LLM-generated TLDR rendered above the Visualizations charts. + # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we + # never auto-burn LLM tokens on page load. Shape:: + # {"narrative": str, "patterns": [str, ...], + # "generated_at": iso8601, "generated_at_completed_rows": int, + # "provider": str, "model": str} + # NULL on rows that have never been summarised. + tldr_summary = Column(JSON, nullable=True) + + # Cached LLM-generated user insights for External Audit PDF section 03. + # Populated by a background Celery job triggered alongside TLDR generation. + # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). + user_insights = Column(JSON, nullable=True) + + # Cached per-metric failure clustering for internal diagnostics PDF/UI. + # Shape: EvaluationMetricClustersState JSON (status, groups[], …). + metric_clusters = Column(JSON, nullable=True) + + # Cached LLM-generated prompt improvement suggestions keyed to an + # imported agent (PromptPartial tagged __imported_agent__). + # Shape: EvaluationPromptImprovementsState JSON. + prompt_improvements = Column(JSON, nullable=True) + + # Cached LLM explanations for week-over-week metric deltas keyed by + # baseline evaluation id + completed row counts. + period_delta_explanations = Column(JSON, nullable=True) + + cached_metric_aggregates = Column(JSON, nullable=True) + cached_metric_aggregates_at_count = Column(Integer, nullable=True) + + # ISO-week period derived from completed rows' recording_date on the + # parent import. Refreshed whenever ``report_period_at_count`` != + # ``completed_rows`` (same watermark as aggregate cache). + report_period_start = Column(Date, nullable=True, index=True) + report_period_end = Column(Date, nullable=True) + report_period_label = Column(String(64), nullable=True) + report_period_display = Column(String(128), nullable=True) + report_period_at_count = Column(Integer, nullable=True) + + # Numeric ``period_deltas`` keyed by baseline id, row-count watermark, + # and failure-policy version (same key shape as + # ``period_delta_explanations``). Avoids rescanning rows on every + # Visualizations tab load. + cached_period_deltas = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + # Flexprice pass-level delta billing watermark: rows already emitted + # on ``call_import.evaluation_completed`` for this evaluation run. + billed_completed_rows = Column( + Integer, nullable=False, default=0, server_default="0" + ) + # Redis-buffered progress counters (``eval_progress_buffer``). ``gen`` is + # bumped by every fenced shard recount; flushes carrying an older gen are + # ignored. ``flush_seq`` is the last applied flush, so re-sending the same + # pending batch is a no-op. + progress_counter_gen = Column( + Integer, nullable=False, default=0, server_default="0" + ) + progress_flush_seq = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + error_message = Column(Text, nullable=True) + celery_group_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + call_import = relationship("CallImport", back_populates="evaluations") + row_results = relationship( + "CallImportEvaluationRow", + back_populates="evaluation", + cascade="all, delete-orphan", + ) + + +class CallImportEvaluationRow(Base): + """Per-source-row scoring output for a CallImportEvaluation parent.""" + + __tablename__ = "call_import_evaluation_rows" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" + ), + Index( + "ix_call_import_evaluation_rows_evaluation_status", + "evaluation_id", + "status", + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_row_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_rows.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + status = Column(String(20), nullable=False, default="pending", index=True) + # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + evaluation = relationship("CallImportEvaluation", back_populates="row_results") + source_row = relationship("CallImportRow") + + +@event.listens_for(CallImportEvaluationRow, "before_insert") +def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent evaluation when omitted.""" + if target.workspace_id is not None or target.evaluation_id is None: + return + workspace_id = connection.execute( + select(CallImportEvaluation.workspace_id).where( + CallImportEvaluation.id == target.evaluation_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class MetricStudioRun(Base): + """Ad-hoc metric experiment run in Metrics Studio.""" + + __tablename__ = "metric_studio_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + + name = Column(String(255), nullable=True) + selected_metric_ids = Column(JSON, nullable=False, default=list) + selected_metric_groups = Column(JSON, nullable=True) + transcript_source = Column( + String(20), + nullable=False, + default="diarised", + server_default="diarised", + ) + + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + metric_llm_overrides = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + total_items = Column(Integer, nullable=False, default=0) + completed_items = Column(Integer, nullable=False, default=0) + failed_items = Column(Integer, nullable=False, default=0) + error_message = Column(Text, nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + results = relationship( + "MetricStudioRunResult", + back_populates="run", + cascade="all, delete-orphan", + ) + + +class MetricStudioRunResult(Base): + """Per-source scoring output for a MetricStudioRun.""" + + __tablename__ = "metric_studio_run_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column( + UUID(as_uuid=True), + ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + source_kind = Column(String(40), nullable=False) + source_ref = Column(String(255), nullable=False) + display_label = Column(String(512), nullable=True) + source_metadata = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + run = relationship("MetricStudioRun", back_populates="results") + + +class CallImportEvaluationReportSnapshot(Base): + """Persisted PDF-report aggregate used for period-over-period deltas.""" + + __tablename__ = "call_import_evaluation_report_snapshots" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + period_label = Column(String(64), nullable=True, index=True) + period_start = Column(Date, nullable=True, index=True) + period_end = Column(Date, nullable=True, index=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") + metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + narrative = Column(JSON, nullable=True) + total_calls = Column(Integer, nullable=False, default=0) + selected_metric_count = Column(Integer, nullable=False, default=0) + total_metric_count = Column(Integer, nullable=False, default=0) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportEvaluationPdfReport(Base): + """Stored PDF artifact for a call import evaluation report generation.""" + + __tablename__ = "call_import_evaluation_pdf_reports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + snapshot_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + vendor_name = Column(String(120), nullable=False) + report_type = Column(String(20), nullable=False, default="external") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + cache_fingerprint = Column(String(64), nullable=True) + created_by = Column(String, nullable=True) + created_by_user_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class EvaluationAnalysisBatch(Base): + """Durable checkpoint row for paged evaluation analysis (insights / clusters / prompts).""" + + __tablename__ = "evaluation_analysis_batches" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", + "job_kind", + "job_run_id", + "batch_index", + name="uq_evaluation_analysis_batch", + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + job_kind = Column(String(32), nullable=False) + job_run_id = Column(UUID(as_uuid=True), nullable=False, index=True) + batch_index = Column(Integer, nullable=False) + cursor_row_index = Column(Integer, nullable=True) + cursor_evaluation_row_id = Column(UUID(as_uuid=True), nullable=True) + transcript_source = Column(String(16), nullable=True) + status = Column(String(32), nullable=False, default="completed") + payload = Column(JSONB, nullable=False, default=dict, server_default="{}") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportEvaluationExport(Base): + """Background CSV/XLSX export for one evaluation run.""" + + __tablename__ = "call_import_evaluation_exports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + format = Column(String(8), nullable=False, default="csv") + include_transcripts = Column(Boolean, nullable=False, default=False) + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + parts = Column(JSONB, nullable=False, default=list) + claim_token = Column(String(36), nullable=True) + heartbeat_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + finished_at = Column(DateTime(timezone=True), nullable=True) + + +# One active export per evaluation. A full unique index would block the next +# download after the first job finishes, and SQLite turns postgresql_where +# into that full index, so this DDL is postgres-only. +event.listen( + CallImportEvaluationExport.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_call_import_evaluation_exports_active " + "ON call_import_evaluation_exports (evaluation_id) " + "WHERE status IN ('pending', 'running')" + ).execute_if(dialect="postgresql"), +) + + +# --------------------------------------------------------------------------- +# Judge Alignment (AlignEval-style hybrid integration) +# +# Three tables back the "Judge Alignment" surface: +# - JudgeDataset: a labeled dataset materialised from one of three sources +# (voice transcripts, existing Metric/Evaluator outputs, +# or a generic CSV upload). Holds the dataset's source +# config + which fields play the role of input/output. +# - JudgeSample: one row in a dataset (input/output pair plus an +# optional binary pass/fail human label). +# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over +# a subset of samples, with computed alignment metrics +# (precision/recall/F1/Cohen's kappa) and per-sample +# predictions. Optionally links to a GEPA optimization +# run when the user kicks off prompt tuning from a +# dataset. +# --------------------------------------------------------------------------- + + +class JudgeDataset(Base): + """Container for binary-labeled samples used to calibrate an LLM-judge.""" + + __tablename__ = "judge_datasets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: every judge dataset belongs to a workspace + # within its org. Samples and runs inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # One of: "transcript", "metric_output", "csv" + source_type = Column(String(32), nullable=False, index=True) + # Source-specific config. Examples: + # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} + # metric_output: {"metric_id": "...", "evaluator_id": "..."} + # csv: {"s3_key": "...", "filename": "..."} + source_config = Column(JSON, nullable=False, default=dict) + + # Field roles - which textual content is "input" vs "output" for the judge. + # For voice transcripts both default to the transcript text but can be + # tightened (e.g. agent-only turns vs full conversation). + input_field = Column(String(64), nullable=False, default="input") + output_field = Column(String(64), nullable=False, default="output") + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship( + "JudgeSample", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeSample.created_at", + ) + runs = relationship( + "JudgeRun", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeRun.created_at.desc()", + ) + + +class JudgeSample(Base): + """One labelable input/output pair within a JudgeDataset.""" + + __tablename__ = "judge_samples" + __table_args__ = ( + UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Stable identifier within the source (e.g. transcription UUID, CSV row id). + # Used to dedupe re-imports and link back to the originating record. + external_id = Column(String(128), nullable=True, index=True) + + input_text = Column(Text, nullable=False) + output_text = Column(Text, nullable=False) + + # Binary human label: "pass" | "fail" | null (unlabeled). + # Stored as string (rather than enum) so it stays trivially extendable. + label = Column(String(16), nullable=True, index=True) + labeled_by = Column(String(255), nullable=True) + labeled_at = Column(DateTime(timezone=True), nullable=True) + + # Source-specific context (e.g. agent_id, original metric value, csv row). + extra = Column(JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + dataset = relationship("JudgeDataset", back_populates="samples") + + +class JudgeRun(Base): + """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" + + __tablename__ = "judge_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model + # define the judge under test). Nullable so a run may target an inline prompt + # in the future without inflating the Evaluator table. + evaluator_id = Column( + UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # Which subset was scored: "all" | "dev" | "test" + split = Column(String(16), nullable=False, default="all") + + # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). + llm_provider = Column(String(64), nullable=True) + llm_model = Column(String(128), nullable=True) + + # Computed alignment metrics: + # {"precision": float, "recall": float, "f1": float, "kappa": float, + # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} + metrics = Column(JSON, nullable=True) + + # Per-sample predictions, keyed by sample_id (UUID string): + # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} + predictions = Column(JSON, nullable=True) + + # Run lifecycle. + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + # Optional link to a GEPA optimization run kicked off from this dataset. + gepa_optimization_id = Column( + UUID(as_uuid=True), + ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + dataset = relationship("JudgeDataset", back_populates="runs") + + +class UsageCostRecomputeJob(Base): + """Async job tracking for retroactive usage cost recompute.""" + + __tablename__ = "usage_cost_recompute_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + status = Column(String(32), nullable=False, default="pending", server_default="pending") + model = Column(String(255), nullable=True) + usage_kind = Column(String(16), nullable=True) + start_date = Column(Date, nullable=True) + end_date = Column(Date, nullable=True) + updated_rows = Column(BigInteger, nullable=False, default=0, server_default="0") + error_message = Column(String, nullable=True) + celery_task_id = Column(String(255), nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + completed_at = Column(DateTime(timezone=True), nullable=True) + + +class LLMUsageDaily(Base): + """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" + + __tablename__ = "llm_usage_daily" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + product_section = Column(String(64), nullable=False, index=True) + model = Column(String(255), nullable=False, index=True) + context = Column(JSONB, nullable=False, server_default="{}", default=dict) + usage_date = Column(Date, nullable=False, index=True) + usage_kind = Column(String(16), nullable=False, default="llm", server_default="llm") + prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_creation_tokens = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") + call_count = Column(BigInteger, nullable=False, default=0, server_default="0") + input_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + output_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + cache_creation_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + audio_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + total_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + pricing_rate_source = Column(String(16), nullable=True) + pricing_rate_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +# Bump the Redis settings version when an evaluation's worker-visible settings +# change, so eval workers drop their cached copy (see eval_settings_version). +from app.services.call_imports.eval_settings_version import ( # noqa: E402 + register_settings_version_hooks as _register_eval_settings_hooks, +) + +_register_eval_settings_hooks() + +# Bump the Redis LLM-config version when an org's credentials or gateway +# settings change, so cached call-path resolutions refresh. +from app.services.ai.llm_credential_version import ( # noqa: E402 + register_llm_config_version_hooks as _register_llm_config_hooks, +) + +_register_llm_config_hooks() diff --git a/app/models/enums.py b/app/models/enums.py index df3a63fc..b5708fa2 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -1,376 +1,399 @@ - -import enum - -class EvaluationType(str, enum.Enum): - """Evaluation type enumeration.""" - ASR = "asr" - TTS = "tts" - QUALITY = "quality" - -class EvaluationStatus(str, enum.Enum): - """Evaluation status enumeration.""" - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - -class EvaluatorResultStatus(str, enum.Enum): - """Evaluator result status enumeration.""" - QUEUED = "queued" - CALL_INITIATING = "call_initiating" # Creating the web call to Retell/Vapi - CALL_CONNECTING = "call_connecting" # WebRTC connecting to Voice AI agent - CALL_IN_PROGRESS = "call_in_progress" # Call is active - CALL_ENDED = "call_ended" # Call finished - FETCHING_DETAILS = "fetching_details" # Fetching call details from provider - TRANSCRIBING = "transcribing" # Only used for S3-based transcription (legacy) - EVALUATING = "evaluating" # Running LLM evaluation on transcript - COMPLETED = "completed" - FAILED = "failed" - -class RoleEnum(str, enum.Enum): - """User role enumeration for RBAC.""" - READER = "reader" - WRITER = "writer" - ADMIN = "admin" - -class InvitationStatus(str, enum.Enum): - """Invitation status enumeration.""" - PENDING = "pending" - ACCEPTED = "accepted" - DECLINED = "declined" - EXPIRED = "expired" - -class LanguageEnum(str, enum.Enum): - """Supported languages""" - ENGLISH = "en" - SPANISH = "es" - FRENCH = "fr" - GERMAN = "de" - CHINESE = "zh" - JAPANESE = "ja" - HINDI = "hi" - ARABIC = "ar" - -class CallTypeEnum(str, enum.Enum): - """Call direction""" - INBOUND = "inbound" - OUTBOUND = "outbound" - -class CallMediumEnum(str, enum.Enum): - """Call medium""" - PHONE_CALL = "phone_call" - WEB_CALL = "web_call" - SIP_CALL = "sip_call" - CHAT = "chat" - - -class ChatConnectionTypeEnum(str, enum.Enum): - """How the platform reaches the production chat agent under test.""" - INTERNAL_LLM = "internal_llm" - PROVIDER_CHAT = "provider_chat" - CUSTOMER_API = "customer_api" - CUSTOMER_WEBSOCKET = "customer_websocket" - MESSAGING_CHANNELS = "messaging_channels" - - -class ChatEvalModeEnum(str, enum.Enum): - """How chat agents are exercised in evaluator suites vs production data.""" - PRE_PROD_SIM = "pre_prod_sim" - POST_PROD_LIVE = "post_prod_live" - POST_PROD_IMPORT = "post_prod_import" - - -class SimulationMediumEnum(str, enum.Enum): - """Whether a persona is for voice callers or text chat (simulation_medium on personas).""" - VOICE = "voice" - TEXT = "text" - - -class GenderEnum(str, enum.Enum): - """Gender options for personas""" - MALE = "male" - FEMALE = "female" - NEUTRAL = "neutral" - -class AccentEnum(str, enum.Enum): - """Accent options""" - AMERICAN = "american" - BRITISH = "british" - AUSTRALIAN = "australian" - INDIAN = "indian" - CHINESE = "chinese" - SPANISH = "spanish" - FRENCH = "french" - GERMAN = "german" - NEUTRAL = "neutral" - -class BackgroundNoiseSourceEnum(str, enum.Enum): - """Where persona ambient audio is loaded from.""" - NONE = "none" - PLATFORM = "platform" - CUSTOM = "custom" - - -class BackgroundNoiseEnum(str, enum.Enum): - """Platform ambient preset names.""" - NONE = "none" - OFFICE = "office" - STREET = "street" - TRAFFIC = "traffic" - CAFE = "cafe" - CONCERT = "concert" - HOME = "home" - CALL_CENTER = "call_center" - -class IntegrationPlatform(str, enum.Enum): - """Integration platform enumeration.""" - RETELL = "retell" - VAPI = "vapi" - CARTESIA = "cartesia" - ELEVENLABS = "elevenlabs" - DEEPGRAM = "deepgram" - MURF = "murf" - SARVAM = "sarvam" - VOICEMAKER = "voicemaker" - SMALLEST = "smallest" - - -class TelephonyProvider(str, enum.Enum): - """Telephony provider enumeration - extensible for future providers.""" - PLIVO = "plivo" - EXOTEL = "exotel" - VOBIZ = "vobiz" - TWILIO = "twilio" - TELNYX = "telnyx" - META_WHATSAPP = "meta_whatsapp" - - -class CredentialRoutingMode(str, enum.Enum): - """Per-credential LLM routing preference.""" - INHERIT = "inherit" - GATEWAY = "gateway" - DIRECT = "direct" - - -class GatewayInterfaceMode(str, enum.Enum): - """How Bifrost is reached when gateway routing is active.""" - INHERIT = "inherit" - LITELLM_SHIM = "litellm_shim" - NATIVE_OPENAI = "native_openai" - - -class GatewayTypeMode(str, enum.Enum): - """Per-credential gateway backend override.""" - INHERIT = "inherit" - BIFROST = "bifrost" - LITELLM_PROXY = "litellm_proxy" - - -class ModelProvider(str, enum.Enum): - """Model provider enumeration for extensibility.""" - OPENAI = "openai" - OPENROUTER = "openrouter" - ANTHROPIC = "anthropic" - GOOGLE = "google" - XAI = "xai" - FIREWORKS = "fireworks" - COHERE = "cohere" - MISTRAL = "mistral" - META = "meta" - TOGETHER = "together" - TYPESAFE = "typesafe" - PERPLEXITY = "perplexity" - AZURE = "azure" - AWS = "aws" - DEEPGRAM = "deepgram" - CARTESIA = "cartesia" - ELEVENLABS = "elevenlabs" - MURF = "murf" - CUSTOM = "custom" - SARVAM = "sarvam" - VOICEMAKER = "voicemaker" - SMALLEST = "smallest" - -class VoiceBundleType(str, enum.Enum): - """VoiceBundle type enumeration.""" - STT_LLM_TTS = "stt_llm_tts" - S2S = "s2s" - -class TestAgentConversationStatus(str, enum.Enum): - """Test agent conversation status enumeration.""" - INITIALIZING = "initializing" - ACTIVE = "active" - PAUSED = "paused" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - -class MetricType(str, enum.Enum): - """Metric type enumeration. - - ``TEXT`` is for free-form LLM-generated text (summaries, classifications, - explanations) where the value is a string rather than a number/boolean. - Text metrics are *not* aggregated in numeric dashboards. - """ - NUMBER = "number" - BOOLEAN = "boolean" - RATING = "rating" - TEXT = "text" - - -class MetricCategory(str, enum.Enum): - """High-level grouping for report rendering and call-import insights.""" - - QUALITY = "quality" - USER_INSIGHT = "user_insight" - - -class MetricLifecycle(str, enum.Enum): - """Lifecycle state for metrics — drafts are Studio-only until promoted.""" - - ACTIVE = "active" - DRAFT = "draft" - - -class MetricTrigger(str, enum.Enum): - """Metric trigger enumeration.""" - ALWAYS = "always" - -class CallRecordingStatus(str, enum.Enum): - """Call recording status enumeration.""" - PENDING = "pending" - UPDATED = "updated" - - -class AlertDataSource(str, enum.Enum): - """Where alert metrics are computed from.""" - - EVALUATIONS = "evaluations" - PRODUCTION_CALLS = "production_calls" - PRODUCTION_TRACES = "production_traces" - CRON_JOBS = "cron_jobs" - - -class AlertMetricType(str, enum.Enum): - """Alert metric type enumeration.""" - NUMBER_OF_CALLS = "number_of_calls" - CALL_DURATION = "call_duration" - ERROR_RATE = "error_rate" - SUCCESS_RATE = "success_rate" - LATENCY = "latency" - CUSTOM = "custom" - FAILURE_COUNT = "failure_count" - - -class AlertAggregation(str, enum.Enum): - """Alert aggregation type enumeration.""" - SUM = "sum" - AVG = "avg" - COUNT = "count" - MIN = "min" - MAX = "max" - - -class AlertOperator(str, enum.Enum): - """Alert comparison operator enumeration.""" - GREATER_THAN = ">" - LESS_THAN = "<" - GREATER_THAN_OR_EQUAL = ">=" - LESS_THAN_OR_EQUAL = "<=" - EQUAL = "=" - NOT_EQUAL = "!=" - - -class AlertNotifyFrequency(str, enum.Enum): - """Alert notification frequency enumeration.""" - IMMEDIATE = "immediate" - HOURLY = "hourly" - DAILY = "daily" - WEEKLY = "weekly" - - -class AlertStatus(str, enum.Enum): - """Alert status enumeration.""" - ACTIVE = "active" - PAUSED = "paused" - DISABLED = "disabled" - - -class AlertHistoryStatus(str, enum.Enum): - """Alert history status enumeration.""" - TRIGGERED = "triggered" - NOTIFIED = "notified" - ACKNOWLEDGED = "acknowledged" - RESOLVED = "resolved" - - -class CronJobStatus(str, enum.Enum): - """Cron job status enumeration.""" - ACTIVE = "active" - PAUSED = "paused" - COMPLETED = "completed" - - -class PromptOptimizationStatus(str, enum.Enum): - """Prompt optimization run status.""" - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - - -class CallImportStatus(str, enum.Enum): - """Status of a CSV-driven call import batch. - - The lifecycle is now split into three independent stages so the - upload, mapping, and import are each idempotent on their own: - - ``uploaded`` -> source file landed in S3, no mapping yet. - ``mapped`` -> user picked a schema + sheet + column mapping; - no rows materialised yet, no worker enqueued. - ``processing`` -> rows materialised + workers enqueued (today's - post-upload state). - ``deleting`` -> whole-batch teardown queued in background. - - ``pending`` is kept for backward compatibility with the legacy - one-shot ``POST /upload`` endpoint which still flips through it - momentarily before transitioning to ``processing``. - """ - PENDING = "pending" - UPLOADED = "uploaded" - MAPPED = "mapped" - PROCESSING = "processing" - COMPLETED = "completed" - PARTIAL = "partial" - FAILED = "failed" - DELETING = "deleting" - - -class CallImportRowStatus(str, enum.Enum): - """Status of an individual call-import row.""" - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class CallImportParameterType(str, enum.Enum): - """Allowed types for a :class:`CallImportSchemaParameter`. - - The first four values feed dedicated columns on - :class:`CallImportRow` (``conversation_id``, ``recording_url``, - ``recording_date``, ``transcript``); the rest are generic typed text fields whose values - are preserved per row in ``raw_columns`` and surfaced under the - parameter name in the evaluation export. - """ - - CONVERSATION_ID = "conversation_id" - RECORDING_URL = "recording_url" - RECORDING_DATE = "recording_date" - TRANSCRIPT = "transcript" - TEXT = "text" - NUMBER = "number" - BOOLEAN = "boolean" - DATETIME = "datetime" - URL = "url" + +import enum + +class EvaluationType(str, enum.Enum): + """Evaluation type enumeration.""" + ASR = "asr" + TTS = "tts" + QUALITY = "quality" + +class EvaluationStatus(str, enum.Enum): + """Evaluation status enumeration.""" + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + +class EvaluatorResultStatus(str, enum.Enum): + """Evaluator result status enumeration.""" + QUEUED = "queued" + CALL_INITIATING = "call_initiating" # Creating the web call to Retell/Vapi + CALL_CONNECTING = "call_connecting" # WebRTC connecting to Voice AI agent + CALL_IN_PROGRESS = "call_in_progress" # Call is active + CALL_ENDED = "call_ended" # Call finished + FETCHING_DETAILS = "fetching_details" # Fetching call details from provider + TRANSCRIBING = "transcribing" # Only used for S3-based transcription (legacy) + EVALUATING = "evaluating" # Running LLM evaluation on transcript + COMPLETED = "completed" + FAILED = "failed" + +class SimulationRunStatus(str, enum.Enum): + """Harbor simulation run (job) status.""" + QUEUED = "queued" + PREPARING = "preparing" # Exporting Harbor tasks + RUNNING = "running" # Harbor job executing trials + INGESTING = "ingesting" # Reading job results / uploading artifacts + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + @classmethod + def terminal(cls) -> frozenset[str]: + return frozenset({cls.COMPLETED.value, cls.FAILED.value, cls.CANCELLED.value}) + +class RoleEnum(str, enum.Enum): + """User role enumeration for RBAC.""" + READER = "reader" + WRITER = "writer" + ADMIN = "admin" + +class InvitationStatus(str, enum.Enum): + """Invitation status enumeration.""" + PENDING = "pending" + ACCEPTED = "accepted" + DECLINED = "declined" + EXPIRED = "expired" + +class LanguageEnum(str, enum.Enum): + """Supported languages""" + ENGLISH = "en" + SPANISH = "es" + FRENCH = "fr" + GERMAN = "de" + CHINESE = "zh" + JAPANESE = "ja" + HINDI = "hi" + ARABIC = "ar" + +class CallTypeEnum(str, enum.Enum): + """Call direction""" + INBOUND = "inbound" + OUTBOUND = "outbound" + +class CallMediumEnum(str, enum.Enum): + """Call medium""" + PHONE_CALL = "phone_call" + WEB_CALL = "web_call" + SIP_CALL = "sip_call" + CHAT = "chat" + + +class ChatConnectionTypeEnum(str, enum.Enum): + """How the platform reaches the production chat agent under test.""" + INTERNAL_LLM = "internal_llm" + PROVIDER_CHAT = "provider_chat" + CUSTOMER_API = "customer_api" + CUSTOMER_WEBSOCKET = "customer_websocket" + MESSAGING_CHANNELS = "messaging_channels" + + +class ChatEvalModeEnum(str, enum.Enum): + """How chat agents are exercised in evaluator suites vs production data.""" + PRE_PROD_SIM = "pre_prod_sim" + POST_PROD_LIVE = "post_prod_live" + POST_PROD_IMPORT = "post_prod_import" + + +class SimulationMediumEnum(str, enum.Enum): + """Whether a persona is for voice callers or text chat (simulation_medium on personas).""" + VOICE = "voice" + TEXT = "text" + + +class GenderEnum(str, enum.Enum): + """Gender options for personas""" + MALE = "male" + FEMALE = "female" + NEUTRAL = "neutral" + +class AccentEnum(str, enum.Enum): + """Accent options""" + AMERICAN = "american" + BRITISH = "british" + AUSTRALIAN = "australian" + INDIAN = "indian" + CHINESE = "chinese" + SPANISH = "spanish" + FRENCH = "french" + GERMAN = "german" + NEUTRAL = "neutral" + +class BackgroundNoiseSourceEnum(str, enum.Enum): + """Where persona ambient audio is loaded from.""" + NONE = "none" + PLATFORM = "platform" + CUSTOM = "custom" + + +class BackgroundNoiseEnum(str, enum.Enum): + """Platform ambient preset names.""" + NONE = "none" + OFFICE = "office" + STREET = "street" + TRAFFIC = "traffic" + CAFE = "cafe" + CONCERT = "concert" + HOME = "home" + CALL_CENTER = "call_center" + +class IntegrationPlatform(str, enum.Enum): + """Integration platform enumeration.""" + RETELL = "retell" + VAPI = "vapi" + CARTESIA = "cartesia" + ELEVENLABS = "elevenlabs" + DEEPGRAM = "deepgram" + MURF = "murf" + SARVAM = "sarvam" + VOICEMAKER = "voicemaker" + SMALLEST = "smallest" + + +class TelephonyProvider(str, enum.Enum): + """Telephony provider enumeration - extensible for future providers.""" + PLIVO = "plivo" + EXOTEL = "exotel" + VOBIZ = "vobiz" + TWILIO = "twilio" + TELNYX = "telnyx" + META_WHATSAPP = "meta_whatsapp" + + +class CredentialRoutingMode(str, enum.Enum): + """Per-credential LLM routing preference.""" + INHERIT = "inherit" + GATEWAY = "gateway" + DIRECT = "direct" + + +class GatewayInterfaceMode(str, enum.Enum): + """How Bifrost is reached when gateway routing is active.""" + INHERIT = "inherit" + LITELLM_SHIM = "litellm_shim" + NATIVE_OPENAI = "native_openai" + + +class GatewayTypeMode(str, enum.Enum): + """Per-credential gateway backend override.""" + INHERIT = "inherit" + BIFROST = "bifrost" + LITELLM_PROXY = "litellm_proxy" + + +class ModelProvider(str, enum.Enum): + """Model provider enumeration for extensibility.""" + OPENAI = "openai" + OPENROUTER = "openrouter" + ANTHROPIC = "anthropic" + GOOGLE = "google" + XAI = "xai" + FIREWORKS = "fireworks" + COHERE = "cohere" + MISTRAL = "mistral" + META = "meta" + TOGETHER = "together" + TYPESAFE = "typesafe" + PERPLEXITY = "perplexity" + AZURE = "azure" + AWS = "aws" + DEEPGRAM = "deepgram" + CARTESIA = "cartesia" + ELEVENLABS = "elevenlabs" + MURF = "murf" + CUSTOM = "custom" + SARVAM = "sarvam" + VOICEMAKER = "voicemaker" + SMALLEST = "smallest" + +class VoiceBundleType(str, enum.Enum): + """VoiceBundle type enumeration.""" + STT_LLM_TTS = "stt_llm_tts" + S2S = "s2s" + +class TestAgentConversationStatus(str, enum.Enum): + """Test agent conversation status enumeration.""" + INITIALIZING = "initializing" + ACTIVE = "active" + PAUSED = "paused" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + +class MetricType(str, enum.Enum): + """Metric type enumeration. + + ``TEXT`` is for free-form LLM-generated text (summaries, classifications, + explanations) where the value is a string rather than a number/boolean. + Text metrics are *not* aggregated in numeric dashboards. + """ + NUMBER = "number" + BOOLEAN = "boolean" + RATING = "rating" + TEXT = "text" + + +class MetricCategory(str, enum.Enum): + """High-level grouping for report rendering and call-import insights.""" + + QUALITY = "quality" + USER_INSIGHT = "user_insight" + + +class MetricLifecycle(str, enum.Enum): + """Lifecycle state for metrics — drafts are Studio-only until promoted.""" + + ACTIVE = "active" + DRAFT = "draft" + + +class MetricTrigger(str, enum.Enum): + """Metric trigger enumeration.""" + ALWAYS = "always" + + +class MetricScoringMethod(str, enum.Enum): + """How a metric is scored during call-import evaluation.""" + + LLM = "llm" + AUDIO = "audio" + USER_WER = "user_wer" + USER_CER = "user_cer" + +class CallRecordingStatus(str, enum.Enum): + """Call recording status enumeration.""" + PENDING = "pending" + UPDATED = "updated" + + +class AlertDataSource(str, enum.Enum): + """Where alert metrics are computed from.""" + + EVALUATIONS = "evaluations" + PRODUCTION_CALLS = "production_calls" + PRODUCTION_TRACES = "production_traces" + CRON_JOBS = "cron_jobs" + + +class AlertMetricType(str, enum.Enum): + """Alert metric type enumeration.""" + NUMBER_OF_CALLS = "number_of_calls" + CALL_DURATION = "call_duration" + ERROR_RATE = "error_rate" + SUCCESS_RATE = "success_rate" + LATENCY = "latency" + CUSTOM = "custom" + FAILURE_COUNT = "failure_count" + + +class AlertAggregation(str, enum.Enum): + """Alert aggregation type enumeration.""" + SUM = "sum" + AVG = "avg" + COUNT = "count" + MIN = "min" + MAX = "max" + + +class AlertOperator(str, enum.Enum): + """Alert comparison operator enumeration.""" + GREATER_THAN = ">" + LESS_THAN = "<" + GREATER_THAN_OR_EQUAL = ">=" + LESS_THAN_OR_EQUAL = "<=" + EQUAL = "=" + NOT_EQUAL = "!=" + + +class AlertNotifyFrequency(str, enum.Enum): + """Alert notification frequency enumeration.""" + IMMEDIATE = "immediate" + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + + +class AlertStatus(str, enum.Enum): + """Alert status enumeration.""" + ACTIVE = "active" + PAUSED = "paused" + DISABLED = "disabled" + + +class AlertHistoryStatus(str, enum.Enum): + """Alert history status enumeration.""" + TRIGGERED = "triggered" + NOTIFIED = "notified" + ACKNOWLEDGED = "acknowledged" + RESOLVED = "resolved" + + +class CronJobStatus(str, enum.Enum): + """Cron job status enumeration.""" + ACTIVE = "active" + PAUSED = "paused" + COMPLETED = "completed" + + +class PromptOptimizationStatus(str, enum.Enum): + """Prompt optimization run status.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +class CallImportStatus(str, enum.Enum): + """Status of a CSV-driven call import batch. + + The lifecycle is now split into three independent stages so the + upload, mapping, and import are each idempotent on their own: + + ``uploaded`` -> source file landed in S3, no mapping yet. + ``mapped`` -> user picked a schema + sheet + column mapping; + no rows materialised yet, no worker enqueued. + ``processing`` -> rows materialised + workers enqueued (today's + post-upload state). + ``deleting`` -> whole-batch teardown queued in background. + + ``pending`` is kept for backward compatibility with the legacy + one-shot ``POST /upload`` endpoint which still flips through it + momentarily before transitioning to ``processing``. + """ + PENDING = "pending" + UPLOADED = "uploaded" + MAPPED = "mapped" + PROCESSING = "processing" + COMPLETED = "completed" + PARTIAL = "partial" + FAILED = "failed" + DELETING = "deleting" + + +class CallImportRowStatus(str, enum.Enum): + """Status of an individual call-import row.""" + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class CallImportParameterType(str, enum.Enum): + """Allowed types for a :class:`CallImportSchemaParameter`. + + The first four values feed dedicated columns on + :class:`CallImportRow` (``conversation_id``, ``recording_url``, + ``recording_date``, ``transcript``); the rest are generic typed text fields whose values + are preserved per row in ``raw_columns`` and surfaced under the + parameter name in the evaluation export. + """ + + CONVERSATION_ID = "conversation_id" + RECORDING_URL = "recording_url" + RECORDING_DATE = "recording_date" + TRANSCRIPT = "transcript" + TEXT = "text" + NUMBER = "number" + BOOLEAN = "boolean" + DATETIME = "datetime" + URL = "url" diff --git a/app/models/schemas.py b/app/models/schemas.py index e2ecde18..606a8996 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -241,6 +241,9 @@ class AgentCreate(BaseModel): le=600, description="End live calls after this many seconds of silence (0 disables)", ) + # Production agent's knowledge bases; the test agent has its own list. + knowledge_base_ids: Optional[List[UUID]] = None + test_knowledge_base_ids: Optional[List[UUID]] = None @field_validator('phone_number') @classmethod @@ -365,6 +368,8 @@ class AgentUpdate(BaseModel): test_agent_template: Optional[TestAgentTemplateInput] = None prompt_variables: Optional[Dict[str, str]] = None silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) + knowledge_base_ids: Optional[List[UUID]] = None + test_knowledge_base_ids: Optional[List[UUID]] = None @model_validator(mode='after') def validate_voice_config(self): @@ -431,6 +436,25 @@ class GeneratedScenarioDraftResponse(BaseModel): +class GenerateProductionPromptRequest(BaseModel): + """Generate a production system prompt from a short agent brief.""" + agent_name: str = Field(..., min_length=1, max_length=255) + brief: str = Field(..., min_length=1) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + +class GenerateProductionPromptResponse(BaseModel): + production_prompt: str + provider: str + model: str + + class GenerateTestPromptRequest(BaseModel): """Stage 1: generate foundational test agent prompt from production prompt.""" production_prompt: str = Field(..., min_length=1) @@ -545,6 +569,9 @@ class AgentResponse(BaseModel): description="End live calls after this many seconds of silence (0 disables)", ) provider_prompt_synced_at: Optional[datetime] = None + knowledge_base_ids: List[UUID] = Field(default_factory=list) + test_knowledge_base_ids: List[UUID] = Field(default_factory=list) + is_favorite: bool = False created_at: datetime updated_at: datetime @@ -808,12 +835,25 @@ class GeneratePersonaPromptResponse(BaseModel): # Scenario Schemas +class ExpectedToolCall(BaseModel): + """One tool-use expectation checked on simulation trials.""" + tool: str = Field(..., min_length=1, max_length=255) + args_contains: Dict[str, str] = Field(default_factory=dict) + min_calls: int = Field(1, ge=1, le=50) + max_calls: Optional[int] = Field(None, ge=0, le=50) + must_succeed: bool = False + forbidden: bool = False + # Returned instead of running the tool (simulations only). + mock_response: Optional[Any] = None + + class ScenarioCreate(BaseModel): """Schema for creating a new scenario""" name: str = Field(..., min_length=1, max_length=255) agent_id: Optional[UUID] = None description: Optional[str] = None required_info: Dict[str, str] = Field(default_factory=dict) + expected_tool_calls: Optional[List[ExpectedToolCall]] = None class ScenarioUpdate(BaseModel): @@ -822,6 +862,7 @@ class ScenarioUpdate(BaseModel): agent_id: Optional[UUID] = None description: Optional[str] = None required_info: Optional[Dict[str, str]] = None + expected_tool_calls: Optional[List[ExpectedToolCall]] = None class ScenarioResponse(BaseModel): @@ -831,12 +872,112 @@ class ScenarioResponse(BaseModel): agent_id: Optional[UUID] description: Optional[str] required_info: Dict[str, str] + expected_tool_calls: Optional[List[ExpectedToolCall]] = None created_at: datetime updated_at: datetime model_config = ConfigDict(from_attributes=True) +# Knowledge base schemas + +class KnowledgeBaseCreate(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + embedding_provider: Optional[str] = None + embedding_model: Optional[str] = None + embedding_credential_id: Optional[UUID] = None + + +class KnowledgeBaseUpdate(BaseModel): + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + description: Optional[str] = None + status: Optional[str] = None + embedding_provider: Optional[str] = None + embedding_model: Optional[str] = None + embedding_credential_id: Optional[UUID] = None + + +class EmbeddingModelOption(BaseModel): + id: str + label: str + + +class EmbeddingProviderOption(BaseModel): + id: str + label: str + suggested_models: List[EmbeddingModelOption] = Field(default_factory=list) + + +class EmbeddingConfigHelpResponse(BaseModel): + default_provider: str + default_model: str + required_dimensions: int + providers: List[EmbeddingProviderOption] = Field(default_factory=list) + + +class KnowledgeDocumentPagePreview(BaseModel): + page: Optional[int] = None + text: str + + +class KnowledgeDocumentViewResponse(BaseModel): + knowledge_base_id: UUID + knowledge_base_name: str + document_id: UUID + view_mode: str + original_filename: Optional[str] = None + content_type: Optional[str] = None + text_content: Optional[str] = None + pages: Optional[List[KnowledgeDocumentPagePreview]] = None + page_count: Optional[int] = None + + +class KnowledgeDocumentResponse(BaseModel): + id: UUID + knowledge_base_id: UUID + source_type: str + original_filename: Optional[str] = None + content_type: Optional[str] = None + byte_size: Optional[int] = None + status: str + error_text: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class KnowledgeBaseSummaryResponse(BaseModel): + id: UUID + name: str + description: Optional[str] = None + embedding_provider: str = "openai" + embedding_model: str = "text-embedding-3-small" + embedding_credential_id: Optional[UUID] = None + status: str + document_count: int = 0 + ready_document_count: int = 0 + failed_document_count: int = 0 + linked_agent_count: int = 0 + created_at: datetime + updated_at: datetime + + +class KnowledgeBaseDetailResponse(BaseModel): + id: UUID + name: str + description: Optional[str] = None + embedding_provider: str = "openai" + embedding_model: str = "text-embedding-3-small" + embedding_credential_id: Optional[UUID] = None + status: str + documents: List[KnowledgeDocumentResponse] = Field(default_factory=list) + linked_agent_count: int = 0 + created_at: datetime + updated_at: datetime + + # ============================================ # IAM & USER SCHEMAS # ============================================ @@ -2085,6 +2226,10 @@ class MetricCreate(BaseModel): # Ignored when ``parent_metric_id`` is set (children inherit the # parent's scope unconditionally). scope: MetricScope = "workspace" + # Empty or omitted = golden (scored on every scenario). Non-empty = only these scenarios. + scenario_ids: Optional[List[UUID]] = Field(default=None) + # Empty or omitted = golden (scored for every agent). Non-empty = only these agents. + agent_ids: Optional[List[UUID]] = Field(default=None) @model_validator(mode='after') def validate_compare_transcripts_exclusions(self): @@ -2193,6 +2338,8 @@ class MetricCreateWithChildren(BaseModel): # the whole category subtree is shared across every workspace in # the org. scope: MetricScope = "workspace" + scenario_ids: Optional[List[UUID]] = Field(default=None) + agent_ids: Optional[List[UUID]] = Field(default=None) class MetricUpdate(BaseModel): @@ -2221,6 +2368,9 @@ class MetricUpdate(BaseModel): # ``selection_mode`` when this is set to True, because the patch # body alone doesn't have enough context to validate cross-state. compare_transcripts: Optional[bool] = None + # ``None`` = leave links unchanged; ``[]`` = clear links (golden). + scenario_ids: Optional[List[UUID]] = None + agent_ids: Optional[List[UUID]] = None @model_validator(mode='after') def validate_compare_transcripts_exclusions(self): @@ -2310,10 +2460,13 @@ class MetricResponse(BaseModel): # render a "Compare transcripts" badge in the metric picker and # know to skip the run's transcript_source toggle for this metric. compare_transcripts: bool = False + scoring_method: str = "llm" lifecycle: str = "active" promoted_from_draft_at: Optional[datetime] = None studio_notes: Optional[str] = None children: List["MetricResponse"] = Field(default_factory=list) + scenario_ids: List[UUID] = Field(default_factory=list) + agent_ids: List[UUID] = Field(default_factory=list) created_at: datetime updated_at: datetime created_by: Optional[str] @@ -3712,6 +3865,13 @@ class CallImportResponse(BaseModel): total_rows: int completed_rows: int failed_rows: int + import_rows_processing: int = Field( + 0, + description=( + "Import rows currently in the processing status. Used to keep " + "recording-download progress polling after an evaluation exists." + ), + ) status: CallImportStatus error_message: Optional[str] = None latest_evaluation_status: Optional[str] = Field( @@ -3746,6 +3906,17 @@ class CallImportResponse(BaseModel): "unflushed Redis deltas." ), ) + latest_evaluation_bulk_operation: Optional[str] = Field( + None, + description="In-flight bulk op on the latest evaluation (retry, abort, etc.).", + ) + any_evaluation_poll_active: bool = Field( + False, + description=( + "True when any evaluation on this import still has in-flight rows, " + "a bulk operation, or pending/running status (not only the latest run)." + ), + ) created_at: datetime updated_at: datetime created_by_email: Optional[str] = None @@ -3862,6 +4033,82 @@ class CallImportDispatchDiagnosticsResponse(BaseModel): generated_at: datetime +class CallImportManualAudioFilePresignItem(BaseModel): + filename: str = Field(..., max_length=512) + size_bytes: int = Field(..., ge=1) + content_type: Optional[str] = Field(default=None, max_length=200) + + +class CallImportManualAudioPresignRequest(BaseModel): + files: List[CallImportManualAudioFilePresignItem] = Field(..., min_length=1) + dataset: Optional[str] = Field( + default=None, + description="Required when creating a new manual-audio batch.", + ) + tag_ids: Optional[List[UUID]] = None + batch_name: Optional[str] = Field(default=None, max_length=255) + + +class CallImportManualAudioPresignUploadItem(BaseModel): + row_id: UUID + recording_s3_key: str + conversation_id: str + upload_url: str + content_type: str + filename: str + + +class CallImportManualAudioPresignResponse(BaseModel): + call_import_id: UUID + uploads: List[CallImportManualAudioPresignUploadItem] + total_rows: int + status: CallImportStatus + dataset: Optional[str] = None + tags: List[CallImportTagResponse] = Field(default_factory=list) + message: str + + +class CallImportManualAudioCompleteRequest(BaseModel): + row_ids: List[UUID] = Field(..., min_length=1) + + +class CallImportManualAudioCompleteResponse(BaseModel): + verified: int + call_import_id: UUID + total_rows: int + + +class CallImportSourceStagePresignRequest(BaseModel): + filename: str = Field(..., max_length=512) + size_bytes: int = Field(..., ge=1) + content_type: Optional[str] = Field(default=None, max_length=200) + dataset: str = Field(..., min_length=1) + tag_ids: Optional[List[UUID]] = None + schema_id: Optional[UUID] = None + content_modality: Optional[str] = Field( + default=None, + description="Optional 'chat' for transcript-only imports.", + ) + + +class CallImportSourceStagePresignResponse(BaseModel): + call_import_id: UUID + source_s3_key: str + source_format: str + upload_url: str + content_type: str + + +class CallImportSourceStageCompleteRequest(BaseModel): + call_import_id: UUID + filename: str = Field(..., max_length=512) + size_bytes: int = Field(..., ge=1) + dataset: str = Field(..., min_length=1) + tag_ids: Optional[List[UUID]] = None + schema_id: Optional[UUID] = None + content_modality: Optional[str] = None + + class CallImportUploadResponse(BaseModel): """Response returned right after a CSV is accepted.""" @@ -4263,6 +4510,24 @@ class CallImportEvaluationBulkDelete(BaseModel): ) +class CallImportEvaluationRowRetryFilter(BaseModel): + """Retry matching evaluation rows in keyset pages instead of by id list.""" + + q: Optional[str] = Field( + default=None, + max_length=500, + description="Search conversation_id on the linked import row.", + ) + status: Optional[str] = Field( + default=None, + max_length=32, + description=( + "When set, only rows with this evaluation status are retried. " + "Omit to use the default failed-only or metric-subset statuses." + ), + ) + + class CallImportEvaluationRetryRequest(BaseModel): """Body for retrying a subset (or all failed rows) of an evaluation run. @@ -4285,12 +4550,25 @@ class CallImportEvaluationRetryRequest(BaseModel): eval_row_ids: Optional[List[UUID]] = Field( default=None, description=( - "Restrict the retry to a specific subset of evaluation rows. " - "When omitted, every row with status='failed' in this run is " - "re-enqueued." + "Restrict the retry to a small explicit subset of evaluation rows. " + "Prefer ``row_filter`` for select-all. When omitted, every matching " + "failed row in this run is re-enqueued in pages." + ), + ) + row_filter: Optional[CallImportEvaluationRowRetryFilter] = Field( + default=None, + description=( + "Server-side filter for bulk retry (search / status) without " + "posting every eval_row id." ), ) + @model_validator(mode="after") + def _retry_ids_or_filter(self) -> "CallImportEvaluationRetryRequest": + if self.eval_row_ids and self.row_filter is not None: + raise ValueError("Pass either eval_row_ids or row_filter, not both.") + return self + # --- Metric-subset re-run --- # When ``metric_ids`` is set, the retry recomputes ONLY those # metrics instead of the whole row, and the new scores are merged @@ -4531,6 +4809,13 @@ class CallImportEvaluationBulkActionResponse(BaseModel): ..., description="How many rows the background worker will process.", ) + worker_enqueued: bool = Field( + default=False, + description=( + "True when a background worker was queued (even if target_count is 0, " + "e.g. diarisation-only abort)." + ), + ) evaluation_id: UUID @@ -4695,14 +4980,40 @@ class CallImportEvaluationRowListResponse(BaseModel): page_size: int +class CallImportRowSelectionFilter(BaseModel): + """Server-side row selection for bulk import operations (no id list).""" + + q: Optional[str] = Field( + default=None, + max_length=500, + description="Case-insensitive substring match on conversation_id.", + ) + diarised_status: Optional[str] = Field( + default=None, + max_length=32, + description="Filter on diarised_transcript_status (pending, running, etc.).", + ) + + class CallImportRowBulkDelete(BaseModel): """Request body for deleting multiple rows from a call-import batch.""" - row_ids: List[UUID] = Field( - ..., - min_length=1, - description="Row ids to delete (must belong to the same call import).", + row_ids: Optional[List[UUID]] = Field( + default=None, + description="Explicit row ids for a small on-page selection.", ) + row_filter: Optional[CallImportRowSelectionFilter] = Field( + default=None, + description="Delete every row matching the import list filters.", + ) + + @model_validator(mode="after") + def _require_rows_or_filter(self) -> "CallImportRowBulkDelete": + if self.row_filter is not None: + return self + if not self.row_ids: + raise ValueError("Provide row_ids or row_filter.") + return self class CallImportRowBulkDeleteResponse(BaseModel): @@ -4799,8 +5110,9 @@ class CallImportTranscribeRequest(BaseModel): ``diarization_prompt``. The model both transcribes and diarises in a single pass. The STT fields are ignored (and must be omitted / null). Only providers whose chat API accepts audio input (OpenAI - ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``) are usable; other - providers will surface a typed error on the row. + ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``, OpenRouter models + with audio input) are usable; other providers will surface a typed + error on the row. """ mode: Literal["stt_llm", "llm_only"] = Field( @@ -4855,8 +5167,16 @@ class CallImportTranscribeRequest(BaseModel): row_ids: Optional[List[UUID]] = Field( default=None, description=( - "Restrict the run to a specific subset of rows. NULL = every " - "row in the import (subject to only_missing)." + "Restrict the run to a small explicit subset of rows. Prefer " + "``row_filter`` for cross-page select-all. NULL = every row " + "in the import (subject to only_missing), walked in pages." + ), + ) + row_filter: Optional[CallImportRowSelectionFilter] = Field( + default=None, + description=( + "Retry/diarise/delete rows matching these list filters without " + "sending every row id from the browser." ), ) # --- LLM diariser config --- @@ -4924,6 +5244,8 @@ def _validate_mode_fields(self) -> "CallImportTranscribeRequest": "mode='llm_only'; the LLM consumes the audio " "directly." ) + if self.row_ids and self.row_filter is not None: + raise ValueError("Pass either row_ids or row_filter, not both.") return self @@ -5012,6 +5334,16 @@ class CallImportCancelDiarisationRequest(BaseModel): "every pending / running diarisation in the import." ), ) + row_filter: Optional[CallImportRowSelectionFilter] = Field( + default=None, + description="Cancel diarisation for rows matching import list filters.", + ) + + @model_validator(mode="after") + def _cancel_ids_or_filter(self) -> "CallImportCancelDiarisationRequest": + if self.row_ids is not None and self.row_filter is not None: + raise ValueError("Pass either row_ids or row_filter, not both.") + return self class CallImportCancelDiarisationResponse(BaseModel): @@ -5118,6 +5450,10 @@ class CallImportMetricAggregate(BaseModel): histogram_buckets: List[CallImportMetricHistogramBucket] = Field( default_factory=list ) + # Micro-averaged WER / CER across scored rows (User WER / User CER). + # Denominator is production user turns. Agent turns are ignored. + corpus_wer: Optional[float] = None + corpus_cer: Optional[float] = None value_counts: List[CallImportMetricValueCount] = Field(default_factory=list) # Pairwise label intersections for multi-label parent metrics. # Empty for everything else. The frontend reconstructs a square @@ -5213,10 +5549,14 @@ class EvaluationInsightsRequest(BaseModel): ge=20, le=500, description=( - "Max LLM calls for user-insights sampling (extraction + synthesis). " + "Legacy LLM budget field (ignored for full-corpus batched insights). " "Defaults to 200 when omitted." ), ) + transcript_source: Optional[Literal["diarised", "production"]] = Field( + default="diarised", + description="Which transcript text to send to the user-insights LLM.", + ) class UserInsightCategory(BaseModel): @@ -5247,7 +5587,7 @@ class EvaluationUserInsightItem(BaseModel): class EvaluationUserInsightsState(BaseModel): """Cached map-reduce LLM user insights for an evaluation run.""" - status: Literal["idle", "running", "completed", "failed"] = "idle" + status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" insights: List[EvaluationUserInsightItem] = Field(default_factory=list) overview: Optional[str] = None generated_at: Optional[datetime] = None @@ -5259,6 +5599,14 @@ class EvaluationUserInsightsState(BaseModel): max_llm_calls: Optional[int] = None error_message: Optional[str] = None is_stale: bool = False + heartbeat_stale: bool = Field( + default=False, + description=( + "True when a ``running`` job has not been updated by a worker for " + "long enough that it is almost certainly abandoned." + ), + ) + transcript_source: Optional[Literal["diarised", "production"]] = None class EvaluationUserInsightsRequest(BaseModel): @@ -5266,10 +5614,15 @@ class EvaluationUserInsightsRequest(BaseModel): regenerate: bool = False force: bool = False + resume: bool = False provider: Optional[str] = None model: Optional[str] = Field(default=None, min_length=1) credential_id: Optional[UUID] = None max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + transcript_source: Optional[Literal["diarised", "production"]] = Field( + default="diarised", + description="Which transcript text to send to the insights LLM.", + ) MetricClusterGapLabel = Literal[ @@ -5463,6 +5816,14 @@ class EvaluationMetricClustersState(BaseModel): max_llm_calls: Optional[int] = None error_message: Optional[str] = None is_stale: bool = False + heartbeat_stale: bool = Field( + default=False, + description=( + "True when a ``running`` job has not been updated by a worker for " + "long enough that it is almost certainly abandoned. The UI offers " + "Resume instead of waiting on Stop." + ), + ) selected_evaluation_row_ids: List[str] = Field( default_factory=list, description="Evaluation row IDs included in the last clustering run.", @@ -5514,7 +5875,7 @@ class PromptImprovementSuggestion(BaseModel): class EvaluationPromptImprovementsState(BaseModel): """Cached prompt improvement suggestions for an evaluation run.""" - status: Literal["idle", "running", "completed", "failed"] = "idle" + status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" imported_agent_id: Optional[str] = None imported_agent_name: Optional[str] = None suggestions: List[PromptImprovementSuggestion] = Field(default_factory=list) @@ -5525,7 +5886,15 @@ class EvaluationPromptImprovementsState(BaseModel): model: Optional[str] = None error_message: Optional[str] = None is_stale: bool = False + heartbeat_stale: bool = Field( + default=False, + description=( + "True when a ``running`` job has not been updated by a worker for " + "long enough that it is almost certainly abandoned." + ), + ) require_review: bool = True + progress: Optional[Dict[str, Any]] = None class EvaluationPromptImprovementsRequest(BaseModel): @@ -5534,6 +5903,7 @@ class EvaluationPromptImprovementsRequest(BaseModel): imported_agent_id: Optional[UUID] = None regenerate: bool = False force: bool = False + resume: bool = False provider: Optional[str] = None model: Optional[str] = None credential_id: Optional[UUID] = None @@ -5591,10 +5961,15 @@ class EvaluationMetricClustersRequest(BaseModel): regenerate: bool = False force: bool = False + resume: bool = False provider: Optional[str] = None model: Optional[str] = Field(default=None, min_length=1) credential_id: Optional[UUID] = None max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + transcript_source: Optional[Literal["diarised", "production"]] = Field( + default="diarised", + description="Which transcript text to send to the clustering LLM.", + ) evaluation_row_ids: Optional[List[UUID]] = Field( default=None, description=( @@ -5978,3 +6353,285 @@ class CapabilityDomainResponse(BaseModel): key: str label: str capabilities: List[CapabilityInfoResponse] + + +# ============================================ +# Harbor simulation runs +# ============================================ + +SimulationSuccessCriteriaKind = Literal[ + "boolean", + "rating", + "number", + "enum", + "category_single", + "category_multi", + "classification", + "text", +] +SimulationSuccessWhen = Literal["yes", "no"] +SimulationNumericOp = Literal["lt", "lte", "gt", "gte"] + + +class MetricSuccessCriterionLabelRule(BaseModel): + child_id: UUID + expect: SimulationSuccessWhen + + +class MetricSuccessCriterionClassificationNoul(BaseModel): + success_when: SimulationSuccessWhen + + +class MetricSuccessCriterionClassificationChoice(BaseModel): + success_values: List[str] = Field(..., min_length=1) + + +class MetricSuccessCriterionClassificationScore(BaseModel): + success_values: List[str] = Field(..., min_length=1) + + +class MetricSuccessCriterionClassificationQuestions(BaseModel): + noul: Optional[MetricSuccessCriterionClassificationNoul] = None + choice: Optional[MetricSuccessCriterionClassificationChoice] = None + score: Optional[MetricSuccessCriterionClassificationScore] = None + + +class MetricSuccessCriterion(BaseModel): + metric_id: UUID + kind: SimulationSuccessCriteriaKind + success_when: Optional[SimulationSuccessWhen] = None + op: Optional[SimulationNumericOp] = None + threshold: Optional[float] = None + success_values: Optional[List[str]] = None + label_rules: Optional[List[MetricSuccessCriterionLabelRule]] = None + questions: Optional[MetricSuccessCriterionClassificationQuestions] = None + + +class SimulationRunCreate(BaseModel): + agent_id: UUID + name: Optional[str] = None + scenario_ids: List[UUID] = Field(..., min_length=1) + persona_ids: List[UUID] = Field(..., min_length=1) + metric_ids: Optional[List[UUID]] = Field(None, min_length=1) + metric_success_criteria: Optional[List[MetricSuccessCriterion]] = None + n_attempts: int = Field(1, ge=1, le=10) + n_concurrent: int = Field(4, ge=1, le=32) + pass_threshold: float = Field(0.7, ge=0.0, le=1.0) + production_llm_provider: Optional[str] = None + production_llm_model: Optional[str] = None + test_llm_provider: Optional[str] = None + test_llm_model: Optional[str] = None + + @model_validator(mode="after") + def _llm_overrides_paired(self): + for leg in ("production", "test"): + provider = getattr(self, f"{leg}_llm_provider") + model = getattr(self, f"{leg}_llm_model") + if bool((provider or "").strip()) != bool((model or "").strip()): + raise ValueError(f"{leg}_llm_provider and {leg}_llm_model must be set together") + return self + + @model_validator(mode="after") + def _criteria_requires_metrics(self): + if self.metric_success_criteria: + if not self.metric_ids: + raise ValueError("metric_ids is required when metric_success_criteria is set") + expected = {str(m) for m in self.metric_ids} + seen = {str(c.metric_id) for c in self.metric_success_criteria} + if seen != expected: + raise ValueError( + "metric_success_criteria must include exactly one rule per selected metric_id" + ) + return self + + +class SimulationRunRename(BaseModel): + """New display name. Blank clears it back to the default run title.""" + + name: Optional[str] = Field(None, max_length=200) + + @field_validator("name") + @classmethod + def _strip_name(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + stripped = value.strip() + return stripped or None + + +class SimulationRunResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: UUID + run_id: str + agent_id: UUID + name: Optional[str] = None + scenario_ids: List[UUID] = [] + persona_ids: List[UUID] = [] + metric_ids: Optional[List[UUID]] = None + metric_success_criteria: Optional[List[MetricSuccessCriterion]] = None + n_attempts: int + n_concurrent: int + pass_threshold: float + production_llm_provider: Optional[str] = None + production_llm_model: Optional[str] = None + test_llm_provider: Optional[str] = None + test_llm_model: Optional[str] = None + resolved_llms: Optional[Dict[str, Any]] = None + status: str + expected_trials: int + completed_trials: int + failed_trials: int + summary: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + harbor_job_name: Optional[str] = None + has_artifacts: bool = False + cancel_requested: bool = False + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: Optional[datetime] = None + created_by: Optional[str] = None + + +class SimulationTrialResponse(BaseModel): + id: UUID + result_id: str + harbor_trial_name: Optional[str] = None + scenario_id: Optional[UUID] = None + scenario_name: Optional[str] = None + persona_id: Optional[UUID] = None + persona_name: Optional[str] = None + attempt_index: Optional[int] = None + status: str + reward: Optional[float] = None + passed: Optional[bool] = None + turns: int = 0 + metric_scores: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + tool_call_count: int = 0 + tool_error_count: int = 0 + tool_expectations_met: Optional[bool] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class SimulationTrialDetailResponse(SimulationTrialResponse): + speaker_segments: Optional[List[Dict[str, Any]]] = None + call_analysis: Optional[Dict[str, Any]] = None + trajectory: Optional[Dict[str, Any]] = None + tool_calls: Optional[List[Dict[str, Any]]] = None + tool_evaluation: Optional[Dict[str, Any]] = None + llms: Optional[Dict[str, Any]] = None + + +class SimulationArtifactsResponse(BaseModel): + url: str + + +class SimulationFailureAnalysisRequest(BaseModel): + """Body for ``POST /simulation-runs/{run_ref}/failure-analysis``.""" + + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + + +class SimulationFailureAnalysisResponse(BaseModel): + """Failure clusters of trials that missed their success rules, plus prompt suggestions.""" + + failed_trials: int = 0 + success_rules: Dict[str, str] = Field( + default_factory=dict, description="metric_id -> human-readable success rule" + ) + metric_clusters: Optional[EvaluationMetricClustersState] = None + prompt_improvements: Optional[EvaluationPromptImprovementsState] = None + + +class SimulationComposedPromptResponse(BaseModel): + agent_id: UUID + agent_name: Optional[str] = None + base_prompt: str + composed_prompt: str + accepted_count: int = 0 + + +class SimulationRunPdfReportSections(BaseModel): + run_summary: bool = True + failure_diagnostics: bool = True + prompt_improvements: bool = True + methodology: bool = True + + +class SimulationRunPdfReportRequest(BaseModel): + sections: SimulationRunPdfReportSections = Field( + default_factory=SimulationRunPdfReportSections + ) + platform_base_url: Optional[str] = None + + +# Tool connections (MCP servers, HTTP tools, Postgres) for production agents + +ToolConnectionType = Literal["mcp", "http", "postgres"] +ToolSimulationMode = Literal["live", "mock_writes", "mock_all"] + + +class ToolConnectionCreate(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + connection_type: ToolConnectionType + preset: Optional[str] = Field(None, max_length=64) + config: Dict[str, Any] = Field(default_factory=dict) + # Write-only: {"headers": {...}, "env": {...}} or {"dsn": "..."}; never returned. + secrets: Optional[Dict[str, Any]] = None + simulation_mode: ToolSimulationMode = "mock_writes" + + +class ToolConnectionUpdate(BaseModel): + name: Optional[str] = Field(None, min_length=1, max_length=255) + config: Optional[Dict[str, Any]] = None + # Merged into stored secrets; an empty string removes a key. + secrets: Optional[Dict[str, Any]] = None + simulation_mode: Optional[ToolSimulationMode] = None + + +class ToolConnectionTool(BaseModel): + name: str + exposed_name: str + description: Optional[str] = None + side_effect: Literal["read", "write"] + input_schema: Optional[Dict[str, Any]] = None + + +class ToolConnectionResponse(BaseModel): + id: UUID + name: str + slug: str + connection_type: str + preset: Optional[str] = None + config: Dict[str, Any] = Field(default_factory=dict) + simulation_mode: str + secret_keys: List[str] = Field(default_factory=list) + tools: List[ToolConnectionTool] = Field(default_factory=list) + last_tested_at: Optional[datetime] = None + last_error: Optional[str] = None + agent_count: int = 0 + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class AgentToolConnectionLink(BaseModel): + tool_connection_id: UUID + # Raw tool names from the connection; null = every tool. + enabled_tools: Optional[List[str]] = None + + +class AgentToolConnectionsUpdate(BaseModel): + links: List[AgentToolConnectionLink] = Field(default_factory=list) + + +class AgentToolCatalogEntry(BaseModel): + name: str + description: Optional[str] = None + source: str + side_effect: Literal["read", "write"] + simulation_behavior: Literal["live", "mocked"] diff --git a/app/services/agents/agent_serializers.py b/app/services/agents/agent_serializers.py new file mode 100644 index 00000000..e512196b --- /dev/null +++ b/app/services/agents/agent_serializers.py @@ -0,0 +1,26 @@ +"""Agent API response helpers.""" + +from __future__ import annotations + +from typing import Iterable, List + +from sqlalchemy.orm import Session + +from app.models.database import Agent +from app.models.schemas import AgentResponse +from app.services.knowledge_bases.agent_links import KB_ROLE_TEST, get_agent_knowledge_base_ids + + +def agent_to_response(db: Session, agent: Agent) -> AgentResponse: + return AgentResponse.model_validate(agent).model_copy( + update={ + "knowledge_base_ids": get_agent_knowledge_base_ids(db, agent_id=agent.id), + "test_knowledge_base_ids": get_agent_knowledge_base_ids( + db, agent_id=agent.id, role=KB_ROLE_TEST + ), + } + ) + + +def agents_to_responses(db: Session, agents: Iterable[Agent]) -> List[AgentResponse]: + return [agent_to_response(db, agent) for agent in agents] diff --git a/app/services/agents/chat_production_leg.py b/app/services/agents/chat_production_leg.py index 3816924b..80112415 100644 --- a/app/services/agents/chat_production_leg.py +++ b/app/services/agents/chat_production_leg.py @@ -13,7 +13,7 @@ chat_connection_config, normalized_chat_connection_type, ) -from app.services.agents.chat_llm_config import resolve_simulation_llm +from app.services.agents.chat_llm_config import ResolvedSimulationLlm, resolve_simulation_llm from app.services.agents.chat_outbound_urls import assert_chat_connection_urls_safe from app.services.agents.customer_api_chat import call_customer_chat_api from app.services.agents.provider_platform_chat import ProviderChatState, generate_provider_platform_reply @@ -22,20 +22,55 @@ from app.services.testing.test_agent_simulation_prompt import is_chat_agent -def _agent_messages(system_prompt: str, transcript: list[dict[str, str]]) -> list[dict[str, str]]: - messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] +def _agent_messages( + system_prompt: str, + transcript: list[dict[str, str]], + *, + tool_sidecars: Optional[list[list[dict[str, Any]]]] = None, +) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [{"role": "system", "content": system_prompt}] + sidecars = tool_sidecars or [] + assistant_turn = 0 for entry in transcript: speaker = entry.get("speaker") text = (entry.get("text") or "").strip() if not text: continue if speaker == "Speaker 2": + if assistant_turn < len(sidecars) and sidecars[assistant_turn]: + messages.extend(sidecars[assistant_turn]) + assistant_turn += 1 messages.append({"role": "assistant", "content": text}) else: messages.append({"role": "user", "content": text}) return messages +def _assistant_turn_index(transcript: list[dict[str, str]]) -> int: + """Position of the reply being generated, counted as ``_agent_messages`` counts.""" + return sum( + 1 + for entry in transcript + if entry.get("speaker") == "Speaker 2" and (entry.get("text") or "").strip() + ) + + +def _store_tool_sidecar( + sidecars: list[list[dict[str, Any]]], + turn_index: int, + tool_message_tail: list[dict[str, Any]], +) -> list[list[dict[str, Any]]]: + """Put this turn's tool messages at its own index; turns without tools hold ``[]``. + + ``_agent_messages`` reads sidecars by assistant-turn position, so appending + only for tool-using turns would attach results to the wrong reply. + """ + out = list(sidecars[:turn_index]) + out.extend([] for _ in range(turn_index - len(out))) + out.append(list(tool_message_tail)) + return out + + def normalized_chat_eval_mode(agent: Agent) -> str: """Use stored mode, or connection-type default (platform/API = live, internal LLM = sim).""" from app.services.agents.chat_preprod_scope import ( @@ -152,8 +187,17 @@ def generate_production_chat_reply( transcript: list[dict[str, str]], provider_state: ProviderChatState, evaluator_result: Optional[Any] = None, + llm_override: Optional[ResolvedSimulationLlm] = None, + toolset: Optional[Any] = None, ) -> tuple[str, dict[str, Any]]: - """Return assistant text and metadata to merge into evaluator call_data.""" + """Return assistant text and metadata to merge into evaluator call_data. + + ``toolset`` (``app.services.tools.toolset.Toolset``) is built from the agent when + not given; multi-turn callers pass one in to reuse it across turns. + + ``llm_override`` replaces the agent's main LLM on the internal-LLM leg only + (live connections always use the customer's own endpoint). + """ conn = normalized_chat_connection_type(agent) mode = normalized_chat_eval_mode(agent) meta: dict[str, Any] = { @@ -319,19 +363,70 @@ def _apply_run_messaging_overrides(cfg: dict[str, Any]) -> dict[str, Any]: "Check agent connection settings and platform credentials." ) + main_llm = llm_override or resolve_simulation_llm( + db, agent=agent, organization_id=organization_id, leg="main" + ) + agent_system = build_agent_system_prompt(agent) + tool_sidecars_key = "internal_llm_tool_sidecars" + tool_sidecars: list[list[dict[str, Any]]] = list( + provider_state.extra.get(tool_sidecars_key) or [] + ) + messages = _agent_messages(agent_system, transcript, tool_sidecars=tool_sidecars) + turn_index = _assistant_turn_index(transcript) + tool_message_tail: list[dict[str, Any]] = [] + tool_records: list = [] + owns_toolset = toolset is None try: - text = generate_internal_agent_reply( - db, - agent=agent, - organization_id=organization_id, - transcript=transcript, - ) + from app.services.knowledge_bases.chat_tools import with_knowledge_base_instructions + from app.services.tools.loop import run_tool_loop + from app.services.tools.toolset import build_agent_toolset + + if owns_toolset: + toolset = build_agent_toolset( + db, agent=agent, organization_id=organization_id, llm_model=main_llm.model + ) + if toolset: + text, tool_records, tool_message_tail = run_tool_loop( + db, + toolset=toolset, + messages=with_knowledge_base_instructions(messages), + organization_id=organization_id, + llm_provider=main_llm.provider, + llm_model=main_llm.model, + llm_config=main_llm.llm_config, + credential_id=main_llm.credential_id, + task_defaults={"temperature": 0.7, "max_tokens": 1024}, + ) + else: + result = llm_service.generate_response( + messages=messages, + llm_provider=main_llm.provider, + llm_model=main_llm.model, + organization_id=organization_id, + db=db, + llm_config=main_llm.llm_config, + credential_id=main_llm.credential_id, + task_defaults={"temperature": 0.7, "max_tokens": 400}, + ) + text = (result.get("text") or "").strip() except RuntimeError as exc: - main_llm = resolve_simulation_llm(db, agent=agent, organization_id=organization_id, leg="main") raise RuntimeError( f"Production-side LLM simulation failed ({main_llm.source}): {exc}" ) from exc - main_llm = resolve_simulation_llm(db, agent=agent, organization_id=organization_id, leg="main") + finally: + if owns_toolset and toolset is not None: + toolset.close() + text = (text or "").strip() + if not text: + raise ValueError("Production LLM returned an empty message") + if tool_message_tail or len(tool_sidecars) > turn_index: + # Keep every turn's slot (empty when no tools ran) so later requests + # replay tool results next to the reply that produced them. + provider_state.extra[tool_sidecars_key] = _store_tool_sidecar( + tool_sidecars, turn_index, tool_message_tail or [] + ) meta["production_leg"] = "internal_llm_sim" meta["main_llm_source"] = main_llm.source + # This turn's calls only; the simulation loop accumulates them across turns. + meta["tool_calls"] = [record.as_dict() for record in tool_records] return text, meta diff --git a/app/services/agents/favorites.py b/app/services/agents/favorites.py new file mode 100644 index 00000000..6c307616 --- /dev/null +++ b/app/services/agents/favorites.py @@ -0,0 +1,56 @@ +"""Per-user agent stars. A favorite belongs to one user, not the workspace.""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import UserAgentFavorite +from app.models.schemas import AgentResponse + + +def favorite_agent_ids(db: Session, *, user_id: Optional[UUID], agent_ids: Iterable[UUID]) -> set[UUID]: + ids = list(agent_ids) + if user_id is None or not ids: + return set() + rows = ( + db.query(UserAgentFavorite.agent_id) + .filter(UserAgentFavorite.user_id == user_id, UserAgentFavorite.agent_id.in_(ids)) + .all() + ) + return {row[0] for row in rows} + + +def apply_favorite(response: AgentResponse, *, is_favorite: bool) -> AgentResponse: + return response.model_copy(update={"is_favorite": is_favorite}) + + +def set_favorite( + db: Session, + *, + user_id: UUID, + agent_id: UUID, + organization_id: UUID, + workspace_id: UUID, + favorite: bool, +) -> bool: + existing = ( + db.query(UserAgentFavorite) + .filter(UserAgentFavorite.user_id == user_id, UserAgentFavorite.agent_id == agent_id) + .first() + ) + if favorite and existing is None: + db.add( + UserAgentFavorite( + user_id=user_id, + agent_id=agent_id, + organization_id=organization_id, + workspace_id=workspace_id, + ) + ) + elif not favorite and existing is not None: + db.delete(existing) + db.commit() + return favorite diff --git a/app/services/ai/llm_call_scope.py b/app/services/ai/llm_call_scope.py new file mode 100644 index 00000000..e95e7434 --- /dev/null +++ b/app/services/ai/llm_call_scope.py @@ -0,0 +1,59 @@ +"""Per-LLM-call memo for catalog lookups. + +One ``generate_response`` call used to read the same ``organizations`` row +three times (key resolution, routing, gateway kwargs) and could resolve the +same credential twice. Inside ``llm_call_scope()`` each lookup runs once per +call; outside a scope, ``scoped`` just calls the loader. + +The scope lives in a ``ContextVar``, so it is per thread and per asyncio task +and never leaks between concurrent calls. +""" + +from __future__ import annotations + +import functools +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Callable, Dict, Hashable, Iterator, Optional + +_scope: ContextVar[Optional[Dict[Hashable, Any]]] = ContextVar( + "llm_call_scope", default=None +) + + +@contextmanager +def llm_call_scope() -> Iterator[None]: + """Memoize catalog lookups for the duration of one LLM call. + + Nested scopes reuse the outer one. + """ + if _scope.get() is not None: + yield + return + token = _scope.set({}) + try: + yield + finally: + _scope.reset(token) + + +def scoped(key: Hashable, loader: Callable[[], Any]) -> Any: + memo = _scope.get() + if memo is None: + return loader() + if key in memo: + return memo[key] + value = loader() + memo[key] = value + return value + + +def with_llm_call_scope(fn: Callable[..., Any]) -> Callable[..., Any]: + """Run ``fn`` inside ``llm_call_scope()`` (for plain, non-generator calls).""" + + @functools.wraps(fn) + def _wrapped(*args: Any, **kwargs: Any) -> Any: + with llm_call_scope(): + return fn(*args, **kwargs) + + return _wrapped diff --git a/app/services/ai/llm_credential_version.py b/app/services/ai/llm_credential_version.py new file mode 100644 index 00000000..f5820de5 --- /dev/null +++ b/app/services/ai/llm_credential_version.py @@ -0,0 +1,185 @@ +"""Redis version counter for an organization's LLM credentials and gateway settings. + +The LLM call path caches, per worker process, the org's gateway settings and +the resolved ``AIProvider`` / ``Integration`` rows, keyed by +``llmcfg:ver:{organization_id}``. Any committed change to those rows (or to +``organizations.llm_gateway_settings`` / ``is_active``) bumps the version, so +every process re-resolves on its next call. + +Bumps happen automatically through an ORM hook (``after_flush`` collects, +``after_commit`` bumps, ``after_rollback`` discards). Core / bulk updates +bypass the ORM: call ``mark_llm_config_changed(session, org_id)`` next to +them (``clear_other_defaults`` does). Decrypted secrets are never cached. + +Reads are memoized in-process for ``_VERSION_MEMO_SECONDS`` so Redis sees +about one GET per org per second per process. A bump made in this process +clears its own memo immediately; other processes see it within that window. +""" + +from __future__ import annotations + +import threading +import time +from typing import Dict, Iterable, Optional, Tuple +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy import event, inspect +from sqlalchemy.orm import Session + +from app.config import settings + +_KEY_TTL_SECONDS = 14 * 24 * 3600 +_SESSION_INFO_KEY = "llm_config_bump" +_ORG_WATCHED_ATTRIBUTES = ("llm_gateway_settings", "is_active") + +_memo: Dict[str, Tuple[float, int]] = {} +_memo_lock = threading.Lock() + + +def _version_memo_seconds() -> float: + try: + return max(0.0, float(getattr(settings, "LLM_CONFIG_VERSION_MEMO_SECONDS", 1.0))) + except (TypeError, ValueError): + return 1.0 + + +def cache_ttl_seconds() -> float: + try: + return max(1.0, float(getattr(settings, "LLM_CONFIG_CACHE_TTL_SECONDS", 120.0))) + except (TypeError, ValueError): + return 120.0 + + +def version_key(organization_id: UUID | str) -> str: + return f"llmcfg:ver:{organization_id}" + + +def _client() -> redis.Redis: + # Same fail-fast client (and test isolation) as the eval progress buffer. + from app.services.call_imports.eval_progress_buffer import _client as buffer_client + + return buffer_client() + + +def read_llm_config_version(organization_id: UUID | str) -> Optional[int]: + """Current version (0 if never bumped). ``None`` when Redis is down.""" + org = str(organization_id) + now = time.monotonic() + window = _version_memo_seconds() + if window > 0: + with _memo_lock: + hit = _memo.get(org) + if hit is not None and now - hit[0] < window: + return hit[1] + try: + raw = _client().get(version_key(org)) + except Exception as exc: # noqa: BLE001 - any client failure means "unknown version" + logger.debug("llm config version read failed for org {}: {}", org, exc) + return None + try: + value = int(raw) if raw is not None else 0 + except (TypeError, ValueError): + value = 0 + if window > 0: + with _memo_lock: + _memo[org] = (now, value) + return value + + +def bump_llm_config_version(organization_ids: Iterable[UUID | str]) -> None: + orgs = {str(org) for org in organization_ids if org is not None} + if not orgs: + return + with _memo_lock: + for org in orgs: + _memo.pop(org, None) + try: + pipe = _client().pipeline(transaction=False) + for org in orgs: + key = version_key(org) + pipe.incr(key) + pipe.expire(key, _KEY_TTL_SECONDS) + pipe.execute() + except Exception as exc: # noqa: BLE001 + # Runs from SQLAlchemy's after_commit hook: the commit already + # succeeded, so a cache-invalidation failure must never surface + # as an error to the caller. Catch beyond RedisError (e.g. socket + # setup errors) for that reason. + logger.warning( + "llm config version bump failed for orgs {} (stale for up to {}s): {}", + sorted(orgs), + int(cache_ttl_seconds()), + exc, + ) + + +def clear_version_memo() -> None: + with _memo_lock: + _memo.clear() + + +def mark_llm_config_changed(session: Session, organization_id: UUID | str | None) -> None: + """Bump this org's version when ``session`` commits (for Core/bulk writes).""" + if organization_id is None: + return + pending: set = session.info.setdefault(_SESSION_INFO_KEY, set()) + pending.add(str(organization_id)) + + +def _credential_models(): + from app.models.database import AIProvider, Integration + + return (AIProvider, Integration) + + +def _org_watched_change(obj) -> bool: + state = inspect(obj) + for name in _ORG_WATCHED_ATTRIBUTES: + attr = state.attrs.get(name) + if attr is not None and attr.history.has_changes(): + return True + return False + + +def _after_flush(session: Session, _flush_context) -> None: + from app.models.database import Organization + + credential_models = _credential_models() + pending: set = session.info.get(_SESSION_INFO_KEY) or set() + for obj in list(session.new) + list(session.dirty) + list(session.deleted): + if isinstance(obj, credential_models): + if obj in session.dirty and not session.is_modified(obj): + continue + org_id = getattr(obj, "organization_id", None) + if org_id is not None: + pending.add(str(org_id)) + elif isinstance(obj, Organization) and obj.id is not None: + if obj in session.deleted or _org_watched_change(obj): + pending.add(str(obj.id)) + if pending: + session.info[_SESSION_INFO_KEY] = pending + + +def _after_commit(session: Session) -> None: + pending = session.info.pop(_SESSION_INFO_KEY, None) + if pending: + bump_llm_config_version(pending) + + +def _after_rollback(session: Session) -> None: + session.info.pop(_SESSION_INFO_KEY, None) + + +_registered = False + + +def register_llm_config_version_hooks() -> None: + global _registered + if _registered: + return + event.listen(Session, "after_flush", _after_flush) + event.listen(Session, "after_commit", _after_commit) + event.listen(Session, "after_rollback", _after_rollback) + _registered = True diff --git a/app/services/ai/llm_gateway.py b/app/services/ai/llm_gateway.py index 90824ca7..2b1a4574 100644 --- a/app/services/ai/llm_gateway.py +++ b/app/services/ai/llm_gateway.py @@ -340,6 +340,38 @@ def _get_org_raw_settings(organization_id: UUID, db: Session) -> Dict[str, Any]: return dict(raw) if isinstance(raw, dict) else {} +def org_gateway_settings_for_call(organization_id: UUID, db: Session) -> Dict[str, Any]: + """``llm_gateway_settings`` for the LLM call path, without a DB read per call. + + Read once per LLM call (``llm_call_scope``) and cached per process across + calls, keyed by the org's LLM-config version (bumped on any committed + gateway-settings change). Secrets stay encrypted in the cached dict. + Redis down: version unknown, read the database (once per call). + Each caller gets its own copy. + """ + import copy + + from app.services.ai.llm_call_scope import scoped + from app.services.ai.llm_credential_version import ( + cache_ttl_seconds, + read_llm_config_version, + ) + from app.workers.catalog_cache import cached_catalog_fetch + + def _cached() -> Dict[str, Any]: + version = read_llm_config_version(organization_id) + if version is None: + return _get_org_raw_settings(organization_id, db) + return cached_catalog_fetch( + "llmcfg_org", + f"{organization_id}:{version}", + lambda: _get_org_raw_settings(organization_id, db), + ttl_seconds=cache_ttl_seconds(), + ) + + return copy.deepcopy(scoped(("org", str(organization_id)), _cached)) + + def _decrypt_org_secret(raw: Dict[str, Any], field: str) -> Optional[str]: encrypted = raw.get(field) if not encrypted: @@ -562,7 +594,7 @@ def resolve_effective_routing( ) -> Tuple[Optional[LLMGatewayConfig], EffectiveRouting]: """Merge org/platform gateway config with per-credential override.""" platform = _platform_config() - org = _get_org_raw_settings(organization_id, db) + org = org_gateway_settings_for_call(organization_id, db) credential_mode = credential.routing_mode if credential else "inherit" if credential_mode == "direct": diff --git a/app/services/ai/llm_resolver.py b/app/services/ai/llm_resolver.py index dc83fd52..99f7db7d 100644 --- a/app/services/ai/llm_resolver.py +++ b/app/services/ai/llm_resolver.py @@ -225,14 +225,6 @@ def get_llm_provider_and_model( ) if ai_prov: provider_enum = _provider_enum(ai_prov.provider) - if ( - explicit_provider - and explicit_provider.lower() != ai_prov.provider.lower() - ): - raise HTTPException( - status_code=400, - detail="provider does not match the selected credential.", - ) model_str = _resolved_model_for_row( organization_id, db, ai_prov, explicit_model ) @@ -244,15 +236,7 @@ def get_llm_provider_and_model( credential_id=credential_id, ) if integration_match: - provider_enum, integration = integration_match - if ( - explicit_provider - and explicit_provider.lower() != integration.platform.lower() - ): - raise HTTPException( - status_code=400, - detail="provider does not match the selected credential.", - ) + provider_enum, _integration = integration_match return provider_enum, _resolved_model_for_integration( provider_enum, explicit_model ) diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index abf6540b..1a72499f 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -20,7 +20,11 @@ from app.core.logging import log_exception_safely, redact_log_secrets from app.models.database import ModelProvider, AIProvider -from app.services.credentials import resolve_ai_provider, resolve_integration +from app.services.ai.llm_call_scope import with_llm_call_scope +from app.services.credentials import ( + resolve_ai_provider_for_call, + resolve_integration_for_call, +) from app.services.ai.llm_generation_config import build_litellm_kwargs from app.services.ai.openrouter_jev import ( call_openrouter_systemone, @@ -429,7 +433,7 @@ def _get_ai_provider( specific credential row when multiple keys exist for the same provider. """ - return resolve_ai_provider( + return resolve_ai_provider_for_call( provider, db, organization_id, credential_id=credential_id ) @@ -450,7 +454,7 @@ def _resolve_credential_context( provider_value = ( llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) ) - integration = resolve_integration( + integration = resolve_integration_for_call( provider_value, db, organization_id, credential_id=credential_id ) if integration: @@ -479,7 +483,7 @@ def _resolve_api_key( f"Failed to decrypt API key for provider {provider}: {e}" ) - integration = resolve_integration( + integration = resolve_integration_for_call( provider, db, organization_id, credential_id=credential_id ) if integration: @@ -517,6 +521,7 @@ def _litellm_model_name(provider: ModelProvider, model: str) -> str: return f"openrouter/{normalized}" return f"{prefix}/{model}" + @with_llm_call_scope def generate_response( self, messages: List[Dict[str, str]], @@ -805,6 +810,150 @@ def generate_response( ) return result + @with_llm_call_scope + def generate_response_with_tools( + self, + messages: List[Dict[str, Any]], + tools: List[Dict[str, Any]], + llm_provider: ModelProvider, + llm_model: str, + organization_id: UUID, + db: Session, + temperature: Optional[float] = 0.7, + max_tokens: Optional[int] = None, + llm_config: Optional[Dict[str, Any]] = None, + credential_id: Optional[UUID] = None, + task_defaults: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Single LiteLLM completion with optional function tools.""" + gen_kwargs = build_litellm_kwargs( + llm_config=llm_config, + override_llm_config=None, + legacy_temperature=temperature, + legacy_max_tokens=max_tokens, + task_defaults=task_defaults, + ) + temperature = gen_kwargs["temperature"] + max_tokens = gen_kwargs["max_tokens"] + config = gen_kwargs["config"] + + ai_provider, credential_ctx = self._resolve_credential_context( + llm_provider, db, organization_id, credential_id=credential_id + ) + if ai_provider: + api_key = resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) + else: + api_key = self._resolve_api_key( + llm_provider, db, organization_id, credential_id=credential_id + ) + + provider_value = ( + llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) + ).lower() + workload_model_str = self._litellm_model_name(llm_provider, llm_model) + _, effective_routing = resolve_effective_routing( + organization_id, db, credential_ctx + ) + model_str = canonical_litellm_model_id( + resolve_litellm_model( + workload_model_str=workload_model_str, + gateway_active=effective_routing != "direct", + credential=credential_ctx, + ) + ) + + call_kwargs: Dict[str, Any] = { + "model": model_str, + "messages": messages, + "temperature": temperature, + "tools": tools, + "tool_choice": "auto", + } + if api_key is not None: + call_kwargs["api_key"] = api_key + # Same thinking budget as generate_response. A 400-token cap on + # Gemini 2.5/3 is spent on reasoning, so the visible reply (or the + # tool call) comes back empty or cut off mid-sentence. + for key, value in _gemini_thinking_kwargs(llm_model).items(): + call_kwargs.setdefault(key, value) + if max_tokens: + effective_max_tokens = max_tokens + if _gemini_family(llm_model) is not None and effective_max_tokens < 4096: + effective_max_tokens = 4096 + call_kwargs["max_tokens"] = effective_max_tokens + if config: + call_kwargs.update(config) + + call_kwargs = apply_llm_gateway( + call_kwargs, + organization_id=organization_id, + db=db, + model=model_str, + credential=credential_ctx, + ) + + try: + response = litellm.completion(**call_kwargs) + except Exception as e: + logger.exception("[LLMService] LiteLLM tool call failed (%s)", model_str) + raise RuntimeError(f"LLM tool generation failed for {model_str}: {e}") from e + + message = response.choices[0].message if response.choices else None + text = getattr(message, "content", None) if message else "" + finish_reason = response.choices[0].finish_reason if response.choices else None + raw_tool_calls = getattr(message, "tool_calls", None) if message else None + tool_calls = None + if raw_tool_calls: + tool_calls = [] + for tc in raw_tool_calls: + fn = getattr(tc, "function", None) + tool_calls.append( + { + "id": getattr(tc, "id", None), + "type": getattr(tc, "type", "function"), + "function": { + "name": getattr(fn, "name", None) if fn else None, + "arguments": getattr(fn, "arguments", "{}") if fn else "{}", + }, + } + ) + names = [ + (tc.get("function") or {}).get("name") + for tc in tool_calls + if isinstance(tc, dict) + ] + logger.info( + "[LLMService] model={} LLM requested tool call(s): {}", + model_str, + names, + ) + + usage = getattr(response, "usage", None) + result: Dict[str, Any] = { + "text": text or "", + "finish_reason": finish_reason, + "tool_calls": tool_calls, + "model": llm_model, + "usage": { + "prompt_tokens": getattr(usage, "prompt_tokens", 0) if usage else 0, + "completion_tokens": getattr(usage, "completion_tokens", 0) if usage else 0, + "total_tokens": getattr(usage, "total_tokens", 0) if usage else 0, + }, + "raw_response": response, + } + _record_generate_response_usage( + result, + llm_model=llm_model, + organization_id=organization_id, + raw_response=response, + ) + return result + def _stream_completion( call_kwargs: Dict[str, Any], diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index d37d2bed..2128e39d 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -48,7 +48,10 @@ from app.models.database import ModelProvider, AIProvider, Integration from app.core.encryption import decrypt_api_key -from app.services.credentials import resolve_ai_provider, resolve_integration +from app.services.credentials import ( + resolve_ai_provider_for_call, + resolve_integration_for_call, +) from app.services.storage.s3_service import s3_service from app.core.exceptions import StorageError from sqlalchemy.orm import Session @@ -70,8 +73,12 @@ def _get_ai_provider( organization_id: UUID, credential_id: Optional[UUID] = None, ) -> Optional[AIProvider]: - """Resolve the AIProvider row to use, honoring ``credential_id``.""" - return resolve_ai_provider( + """Resolve the AIProvider row to use, honoring ``credential_id``. + + Read-only cached snapshot (per-process, invalidated by the org's + LLM-config version) so per-row STT does not read ``aiproviders``. + """ + return resolve_ai_provider_for_call( provider, db, organization_id, credential_id=credential_id ) @@ -107,7 +114,7 @@ def _get_api_key_for_provider( credential=credential_ctx, ) - integration = resolve_integration( + integration = resolve_integration_for_call( provider, db, organization_id, credential_id=credential_id ) if integration: diff --git a/app/services/billing/flexprice_buffer.py b/app/services/billing/flexprice_buffer.py new file mode 100644 index 00000000..a7de7a3a --- /dev/null +++ b/app/services/billing/flexprice_buffer.py @@ -0,0 +1,169 @@ +"""Redis queue for deferred Flexprice ingest (flushed by the usage worker).""" + +from __future__ import annotations + +import json +from typing import Any, Optional, Union +from uuid import UUID + +import redis +from loguru import logger + +from app.config import settings + +_QUEUE_KEY = "flexprice:pending:events" +_INFLIGHT_KEY = "flexprice:pending:inflight" +_FLUSH_LOCK_KEY = "flexprice:pending:flush_lock" +_FLUSH_LOCK_SECONDS = 300 +_DEFAULT_FLUSH_BATCH = 100 + +_redis: redis.Redis | None = None + + +def _client() -> redis.Redis: + global _redis + if _redis is None: + _redis = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis + + +def pending_event_count() -> int: + try: + client = _client() + queued = int(client.llen(_QUEUE_KEY) or 0) + inflight = int(client.llen(_INFLIGHT_KEY) or 0) + return queued + inflight + except redis.RedisError: + return 0 + + +def enqueue_flexprice_event( + event_name: str, + organization_id: UUID, + event_id: Union[str, UUID], + *, + properties: Optional[dict[str, Any]] = None, +) -> bool: + from app.services.billing.flexprice_service import disabled_reason + + if disabled_reason(): + return False + + payload = json.dumps( + { + "event_name": event_name, + "organization_id": str(organization_id), + "event_id": str(event_id), + "properties": properties or {}, + }, + separators=(",", ":"), + default=str, + ) + try: + client = _client() + client.rpush(_QUEUE_KEY, payload) + return True + except redis.RedisError as exc: + logger.warning( + "Flexprice buffer enqueue failed org={} event={} error={}", + organization_id, + event_name, + exc, + ) + return False + + +def flush_buffered_flexprice_events( + *, + max_events: int = _DEFAULT_FLUSH_BATCH, +) -> dict[str, int]: + """Send queued events. An item leaves Redis only after Flexprice accepts it. + + Claim moves the payload onto an inflight list (still in Redis). ``LREM`` + runs only after ``record_event`` returns true. A crash retries the same + ``event_id`` on the next flush. + """ + from app.services.billing.flexprice_service import disabled_reason, record_event + + if disabled_reason(): + return {"flushed": 0, "failed": 0, "pending": pending_event_count()} + + client = _client() + try: + locked = client.set( + _FLUSH_LOCK_KEY, + "1", + nx=True, + ex=_FLUSH_LOCK_SECONDS, + ) + except redis.RedisError as exc: + logger.warning("Flexprice flush lock failed: {}", exc) + return {"flushed": 0, "failed": 0, "pending": pending_event_count()} + if not locked: + return { + "flushed": 0, + "failed": 0, + "pending": pending_event_count(), + "skipped_locked": 1, + } + + flushed = 0 + failed = 0 + limit = max(1, int(max_events)) + attempted: set[str] = set() + try: + for _ in range(limit): + try: + raw = client.lindex(_INFLIGHT_KEY, 0) + if not raw: + raw = client.lmove( + _QUEUE_KEY, + _INFLIGHT_KEY, + "LEFT", + "LEFT", + ) + except redis.RedisError as exc: + logger.warning("Flexprice flush claim failed: {}", exc) + failed += 1 + break + if not raw: + break + if raw in attempted: + break + attempted.add(raw) + try: + item = json.loads(raw) + ok = record_event( + item["event_name"], + UUID(item["organization_id"]), + item["event_id"], + properties=item.get("properties") or {}, + ) + except Exception as exc: + logger.warning("Flexprice buffer flush parse/send failed: {}", exc) + ok = False + + if not ok: + failed += 1 + try: + client.lmove(_INFLIGHT_KEY, _QUEUE_KEY, "LEFT", "RIGHT") + except redis.RedisError as exc: + logger.warning("Flexprice inflight rotate failed: {}", exc) + break + continue + + try: + client.lrem(_INFLIGHT_KEY, 1, raw) + except redis.RedisError as exc: + logger.warning( + "Flexprice inflight remove failed after ingest: {}", + exc, + ) + flushed += 1 + finally: + try: + client.delete(_FLUSH_LOCK_KEY) + except redis.RedisError as exc: + logger.warning("Flexprice flush lock release failed: {}", exc) + + return {"flushed": flushed, "failed": failed, "pending": pending_event_count()} diff --git a/app/services/billing/flexprice_service.py b/app/services/billing/flexprice_service.py index a9471f56..66538fb9 100644 --- a/app/services/billing/flexprice_service.py +++ b/app/services/billing/flexprice_service.py @@ -709,6 +709,39 @@ def record_call_import_evaluation_completed( ) +def buffer_call_import_evaluation_completed( + organization_id: UUID, + evaluation_id: UUID, + *, + workspace_id: UUID, + call_import_id: UUID, + rows_billed: int, + completed_total: int, + total_rows: int = 0, + metric_count: int = 0, +) -> bool: + if rows_billed <= 0: + return False + from app.services.billing.flexprice_buffer import enqueue_flexprice_event + + return enqueue_flexprice_event( + CALL_IMPORT_EVALUATION_COMPLETED, + organization_id, + f"{evaluation_id}:{completed_total}", + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=rows_billed, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + completed_total=completed_total, + total_rows=total_rows, + metric_count=metric_count, + rows_billed=rows_billed, + ), + ) + + def record_call_import_recording_minutes_billed( organization_id: UUID, evaluation_row_id: UUID, @@ -739,6 +772,99 @@ def record_call_import_recording_minutes_billed( ) +def buffer_call_import_recording_minutes_batch( + organization_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, + rows_in_batch: int, + batch_index: int, + batch_size: int = 0, + evaluation_total_rows: int = 0, +) -> bool: + """One Flexprice event for many completed rows (summed minutes).""" + if billable_minutes <= 0: + return False + from app.services.billing.flexprice_buffer import enqueue_flexprice_event + from app.services.call_imports.eval_flexprice_billing import ( + RECORDING_MINUTES_TARGET_EVENTS, + ) + + target = RECORDING_MINUTES_TARGET_EVENTS + planned = batch_size or 0 + total = evaluation_total_rows or 0 + charge_note = ( + f"Recording-minutes batch {batch_index} for evaluation {evaluation_id}. " + f"{rows_in_batch} completed audio rows" + + (f" of {total} evaluation rows" if total else "") + + (f", planned {planned} rows per event (~{target} events per run)" if planned else "") + + f". Summed audio {audio_seconds}s, billed {billable_minutes} minutes " + "(each row rounded up to the next minute, then added). " + "quantity is that minute total." + ) + return enqueue_flexprice_event( + CALL_IMPORT_RECORDING_MINUTES_BILLED, + organization_id, + f"{evaluation_id}:audio-batch:{batch_index}", + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=billable_minutes, + billable_minutes=billable_minutes, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + rows_in_batch=rows_in_batch, + batch_index=batch_index, + batch_size=planned or None, + evaluation_total_rows=total or None, + target_events=target, + billing_mode="batch", + charge_note=charge_note, + ), + ) + + +def buffer_call_import_recording_minutes_billed( + organization_id: UUID, + evaluation_row_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, +) -> bool: + if billable_minutes <= 0: + return False + from app.services.billing.flexprice_buffer import enqueue_flexprice_event + + return enqueue_flexprice_event( + CALL_IMPORT_RECORDING_MINUTES_BILLED, + organization_id, + evaluation_row_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=billable_minutes, + billable_minutes=billable_minutes, + evaluation_row_id=evaluation_row_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + billing_mode="single_row", + charge_note=( + f"One completed audio row {evaluation_row_id} on evaluation " + f"{evaluation_id}. {audio_seconds}s billed as {billable_minutes} " + "minutes (rounded up to the next minute). quantity is that minute total." + ), + ), + ) + + def record_call_import_audio_minutes_billed( organization_id: UUID, evaluation_row_id: UUID, diff --git a/app/services/call_import_metric_clusters.py b/app/services/call_import_metric_clusters.py index 4865d11f..0645f600 100644 --- a/app/services/call_import_metric_clusters.py +++ b/app/services/call_import_metric_clusters.py @@ -37,6 +37,7 @@ compute_rca_summary, enrich_metric_cluster_groups, ) +from app.services.evaluation_analysis_checkpoints import job_heartbeat_is_stale from app.services.metric_cluster_rows import ( MetricClusterSourceRow, call_import_pair_to_cluster_row, @@ -58,6 +59,10 @@ from app.services.reporting.call_import_evaluation_pdf_report import ( call_import_evaluation_pdf_report_service, ) +from app.services.import_eval_llm_output_budgets import ( + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, +) MAX_LLM_CALLS_DEFAULT = 200 DISCOVERY_CALLS = 1 @@ -66,10 +71,10 @@ ) ROW_TRANSCRIPT_CHAR_CAP = 3000 RATIONALE_CHAR_CAP = 600 -EXTRACTION_MAX_TOKENS = 8000 -CLUSTER_SYNTHESIS_MAX_TOKENS = 8000 -DISCOVERY_MAX_TOKENS = 6000 -CLUSTER_TOKEN_CEILING = 16000 +EXTRACTION_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST +CLUSTER_SYNTHESIS_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST +DISCOVERY_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST +CLUSTER_TOKEN_CEILING = IMPORT_EVAL_LLM_OUTPUT_RETRY CLUSTER_CALL_TIMEOUT_SECONDS = 180 # GLM-5.3 rejects reasoning_effort="none". A small thinking budget leaves # the rest of max_tokens for the JSON answer. @@ -104,6 +109,7 @@ def build_metric_cluster_progress( payload["current_metric_name"] = current_metric_name return payload CancelCheck = Callable[[], bool] +FailurePredicate = Callable[[MetricClusterSourceRow, Metric], bool] def metric_clusters_raw_is_cancelled(raw: Any) -> bool: @@ -205,6 +211,18 @@ def _metric_is_quality(metric: Metric) -> bool: return not call_import_evaluation_pdf_report_service._is_user_insight_metric(metric) +def _clusterable_metrics( + metrics: Sequence[Metric], + failure_predicate: Optional[FailurePredicate], +) -> List[Metric]: + """Metrics to cluster: quality metrics, or every supplied metric when the + caller's ``failure_predicate`` decides which ones failed (simulation runs + gate on success rules that may target any metric category).""" + if failure_predicate is not None: + return list(metrics) + return [m for m in metrics if _metric_is_quality(m)] + + def _normalize_gap_label(raw: Any) -> MetricClusterGapLabel: text = str(raw or "").strip().upper().replace(" ", "_") if text in _GAP_LABELS: @@ -349,7 +367,9 @@ def _build_flagged_row_payload( def _build_flagged_row_payload_from_source( source_row: MetricClusterSourceRow, metric: Metric, - policy: MetricFailurePolicy, + policy: Optional[MetricFailurePolicy], + *, + failure_predicate: Optional[FailurePredicate] = None, ) -> Optional[Dict[str, Any]]: scores = source_row.metric_scores if isinstance(source_row.metric_scores, dict) else {} entry = scores.get(str(metric.id)) @@ -360,7 +380,10 @@ def _build_flagged_row_payload_from_source( if isinstance(entry, dict) and (entry.get("skipped") or entry.get("error")): return None - if not is_metric_failure_from_scores(scores, metric, policy): + if failure_predicate is not None: + if not failure_predicate(source_row, metric): + return None + elif policy is None or not is_metric_failure_from_scores(scores, metric, policy): return None value = _score_value(entry, metric) if isinstance(entry, dict) else None @@ -449,11 +472,10 @@ def _cluster_llm_attempts(model: str, max_tokens: int) -> List[Dict[str, Any]]: min(2048, max(CLUSTER_THINKING_BUDGET_TOKENS, max_tokens // 4)), ) attempts: List[Dict[str, Any]] = [] - for idx, thinking_budget in enumerate(thinking_budgets): - token_budget = min(max_tokens, 4096) if idx == 0 else max_tokens + for thinking_budget in thinking_budgets: attempts.append( { - "max_tokens": token_budget, + "max_tokens": max_tokens, "llm_config": {"timeout": timeout}, "completion_extra": { "thinking": { @@ -464,7 +486,7 @@ def _cluster_llm_attempts(model: str, max_tokens: int) -> List[Dict[str, Any]]: } ) return attempts - retry_budget = min(CLUSTER_TOKEN_CEILING, max(max_tokens * 2, 8000)) + retry_budget = min(CLUSTER_TOKEN_CEILING, max_tokens * 2) budgets = [max_tokens] if retry_budget <= max_tokens else [max_tokens, retry_budget] return [ {"max_tokens": budget, "llm_config": {"timeout": timeout}} @@ -481,6 +503,7 @@ def _call_cluster_llm( *, temperature: float, max_tokens: int, + credential_id: Optional[UUID] = None, ) -> str: """Call the cluster model. Cap reasoning so the JSON is not cut off.""" from app.services.ai.llm_service import llm_service @@ -510,6 +533,7 @@ def _run( max_tokens=token_budget, llm_config=config, completion_extra=extra, + credential_id=credential_id, ) try: @@ -557,6 +581,8 @@ def _run_extraction( model: str, metric_name: str, batch: Sequence[Dict[str, Any]], + *, + credential_id: Optional[UUID] = None, ) -> List[Dict[str, Any]]: if not batch: return [] @@ -582,6 +608,7 @@ def _run_extraction( ], temperature=0.3, max_tokens=EXTRACTION_MAX_TOKENS, + credential_id=credential_id, ) parsed = _parse_json_object(text) rows_raw = parsed.get("rows") @@ -696,6 +723,7 @@ def _synthesize_metric_clusters( signature_counts: Dict[str, int], sample_rows: Sequence[Dict[str, Any]], flagged_count: int, + credential_id: Optional[UUID] = None, ) -> List[MetricCluster]: if not signature_counts: return [] @@ -719,6 +747,7 @@ def _synthesize_metric_clusters( ], temperature=0.25, max_tokens=CLUSTER_SYNTHESIS_MAX_TOKENS, + credential_id=credential_id, ) parsed = _parse_json_object(text) clusters_raw = parsed.get("clusters") @@ -824,6 +853,7 @@ def _run_discovery( sample_payloads: Sequence[Dict[str, Any]], *, total_flagged: int, + credential_id: Optional[UUID] = None, ) -> List[DiscoveredProblemCluster]: if not sample_payloads or total_flagged <= 0: return [] @@ -845,6 +875,7 @@ def _run_discovery( ], temperature=0.35, max_tokens=DISCOVERY_MAX_TOKENS, + credential_id=credential_id, ) parsed = _parse_json_object(text) discovered_raw = parsed.get("discovered") @@ -911,10 +942,14 @@ def estimate_metric_clusters_llm_calls_for_source_rows( policies: Dict[str, MetricFailurePolicy], *, max_llm_calls: Optional[int] = None, + failure_predicate: Optional[FailurePredicate] = None, ) -> Tuple[int, int]: - """Return ``(flagged_metric_count, total_estimated_llm_calls)``.""" + """Return ``(flagged_metric_count, total_estimated_llm_calls)``. + + ``failure_predicate`` replaces the per-metric failure policies when given. + """ llm_budget = normalize_max_llm_calls(max_llm_calls) - quality_metrics = [m for m in metrics if _metric_is_quality(m)] + quality_metrics = _clusterable_metrics(metrics, failure_predicate) flagged_metric_count = 0 extraction_calls = 0 @@ -922,13 +957,17 @@ def estimate_metric_clusters_llm_calls_for_source_rows( for metric in quality_metrics: policy = policies.get(str(metric.id)) - if policy is None or not policy_has_failure_criteria(policy, metric): + if failure_predicate is None and ( + policy is None or not policy_has_failure_criteria(policy, metric) + ): continue flagged_count = 0 for source_row in source_rows: if source_row.status != "completed": continue - if _build_flagged_row_payload_from_source(source_row, metric, policy): + if _build_flagged_row_payload_from_source( + source_row, metric, policy, failure_predicate=failure_predicate + ): flagged_count += 1 if flagged_count <= 0: continue @@ -959,10 +998,16 @@ def generate_metric_clusters_for_source_rows( on_progress: Optional[ProgressCallback] = None, max_llm_calls: Optional[int] = None, is_cancelled: Optional[CancelCheck] = None, + failure_predicate: Optional[FailurePredicate] = None, + credential_id: Optional[UUID] = None, ) -> EvaluationMetricClustersState: - """Run per-metric clustering + proactive discovery for internal diagnostics.""" + """Run per-metric clustering + proactive discovery for internal diagnostics. + + ``failure_predicate`` replaces the per-metric failure policies when given + (e.g. simulation runs flag metrics that missed their success rule). + """ llm_budget = normalize_max_llm_calls(max_llm_calls) - quality_metrics = [m for m in metrics if _metric_is_quality(m)] + quality_metrics = _clusterable_metrics(metrics, failure_predicate) selected_row_ids = _selected_evaluation_row_ids_from_raw(metric_clusters_raw) stored_policies, policy_source = policies_from_evaluation_raw(metric_clusters_raw) failure_policies = policies or stored_policies @@ -990,6 +1035,7 @@ def generate_metric_clusters_for_source_rows( source_rows, policies, max_llm_calls=llm_budget, + failure_predicate=failure_predicate, ) if flagged_metric_count <= 0: return EvaluationMetricClustersState( @@ -1037,13 +1083,17 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: if cancelled_state is not None: return cancelled_state policy = policies.get(str(metric.id)) - if policy is None or not policy_has_failure_criteria(policy, metric): + if failure_predicate is None and ( + policy is None or not policy_has_failure_criteria(policy, metric) + ): continue flagged_payloads: List[Dict[str, Any]] = [] for source_row in source_rows: if source_row.status != "completed": continue - payload = _build_flagged_row_payload_from_source(source_row, metric, policy) + payload = _build_flagged_row_payload_from_source( + source_row, metric, policy, failure_predicate=failure_predicate + ) if payload: flagged_payloads.append(payload) all_flagged_samples.append(payload) @@ -1090,6 +1140,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: model, metric.name, batch, + credential_id=credential_id, ) extractions.extend(extracted) except ClusterOutputTruncated: @@ -1134,6 +1185,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: signature_counts=signature_counts, sample_rows=extractions, flagged_count=flagged_count, + credential_id=credential_id, ) except ClusterOutputTruncated: raise @@ -1186,6 +1238,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: model, all_flagged_samples, total_flagged=max(total_flagged, 1), + credential_id=credential_id, ) except ClusterOutputTruncated: raise @@ -1338,16 +1391,12 @@ def metric_clusters_state_from_raw( if isinstance(progress_raw, dict): progress = {} for key, value in progress_raw.items(): - if key == "current_metric_name" and isinstance(value, str): - progress[key] = value - elif isinstance(value, (int, float)): + if isinstance(value, (int, float)) and not isinstance(value, bool): progress[key] = int(value) - elif isinstance(value, str) and key not in progress: + elif isinstance(value, str): progress[key] = value - if "completed_llm_calls" not in progress: - progress["completed_llm_calls"] = 0 - if "total_llm_calls" not in progress: - progress["total_llm_calls"] = 0 + progress.setdefault("completed_llm_calls", 0) + progress.setdefault("total_llm_calls", 0) rca_summary = None rca_raw = raw.get("rca_summary") @@ -1390,6 +1439,7 @@ def metric_clusters_state_from_raw( raw.get("error_message") if isinstance(raw.get("error_message"), str) else None ), is_stale=completed_rows > snapshot_int and status == "completed", + heartbeat_stale=job_heartbeat_is_stale(raw), selected_evaluation_row_ids=[ str(rid) for rid in (raw.get("selected_evaluation_row_ids") or []) diff --git a/app/services/call_import_metric_clusters_wave.py b/app/services/call_import_metric_clusters_wave.py new file mode 100644 index 00000000..4fd81882 --- /dev/null +++ b/app/services/call_import_metric_clusters_wave.py @@ -0,0 +1,827 @@ +"""Paged metric-cluster generation with durable batch checkpoints (call-import evals).""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.db_sharding.scatter_gather import ( + EVALUATION_ANALYSIS_BATCH_SIZE, + EvaluationRowPageCursor, + count_completed_evaluation_row_pairs, + evaluation_row_page_cursor_from_pair, + load_completed_evaluation_row_pairs_for_eval_row_ids, + load_completed_evaluation_row_pairs_page, +) +from app.models.database import CallImportEvaluation, CallImportEvaluationRow, CallImportRow, Metric +from app.models.schemas import ( + DiscoveredProblemCluster, + EvaluationMetricClustersState, + MetricClusterGroup, + MetricFailurePolicy, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + ClusterOutputTruncated, + _aggregate_signatures, + _build_flagged_row_payload_from_source, + _metric_is_quality, + _run_discovery, + _run_extraction, + _selected_evaluation_row_ids_from_raw, + _synthesize_metric_clusters, + metric_clusters_raw_is_cancelled, + policies_from_evaluation_raw, + policy_has_failure_criteria, +) +from app.services.call_import_user_insights_wave import pick_transcript_for_source +from app.services.evaluation_analysis_checkpoints import ( + JOB_KIND_METRIC_CLUSTERS, + batch_count_for_rows, + claim_still_owns_header, + cursor_from_header, + cursor_to_dict, + keep_owned_header_write, + init_job_header, + list_analysis_batches, + merge_cluster_extractions_from_batches, + merge_signature_counts_from_cluster_batches, + normalize_transcript_source, + progress_header, + rows_completed_for_batches, + stamp_job_heartbeat, + upsert_analysis_batch, +) +from app.services.metric_cluster_rows import call_import_pair_to_cluster_row +from app.services.metric_clusters_rca_summary import compute_rca_summary, enrich_metric_cluster_groups + +CancelCheck = Callable[[], bool] +ProgressCallback = Callable[[Dict[str, Any]], None] + +EXTRACTION_SUB_BATCH_ROWS = 25 +SYNTHESIS_BATCH_INDEX_OFFSET = 100_000 +DISCOVERY_BATCH_INDEX_OFFSET = 200_000 + + +def _cluster_row_for_pair( + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow, + transcript_source: str, +): + base = call_import_pair_to_cluster_row(evaluation, eval_row, source_row) + transcript = pick_transcript_for_source(evaluation, source_row, transcript_source) + return replace(base, transcript=transcript) + + +def _sub_batch_payloads(rows: Sequence[Dict[str, Any]], *, size: int = EXTRACTION_SUB_BATCH_ROWS): + chunk = max(1, size) + return [list(rows[i : i + chunk]) for i in range(0, len(rows), chunk)] + + +def _merge_sig_maps( + target: Dict[str, Dict[str, int]], + metric_id: str, + delta: Dict[str, int], +) -> None: + bucket = target.setdefault(metric_id, {}) + for label, count in delta.items(): + bucket[label] = bucket.get(label, 0) + int(count) + + +def _count_row_scope( + db: Session, + evaluation: CallImportEvaluation, + metric_clusters_raw: Any, +) -> Tuple[int, Optional[List[UUID]]]: + selected = _selected_evaluation_row_ids_from_raw(metric_clusters_raw) + if selected: + return len(selected), [UUID(rid) for rid in selected] + total = count_completed_evaluation_row_pairs(db, evaluation.id) + return total, None + + +def _metrics_eligible_for_synthesis( + quality_metrics: Sequence[Metric], + failure_policies: Dict[str, MetricFailurePolicy], + sig_by_metric: Dict[str, Dict[str, int]], +) -> List[Metric]: + eligible: List[Metric] = [] + for metric in quality_metrics: + policy = failure_policies.get(str(metric.id)) + if policy is None or not policy_has_failure_criteria(policy, metric): + continue + if not sig_by_metric.get(str(metric.id)): + continue + eligible.append(metric) + return eligible + + +def start_metric_clusters_job_header( + evaluation: CallImportEvaluation, + db: Session, + *, + job_run_id: UUID, + transcript_source: str, + provider: Optional[str], + model: Optional[str], + metric_clusters_raw: Any, +) -> Dict[str, Any]: + total_rows, _ = _count_row_scope(db, evaluation, metric_clusters_raw) + total_batches = batch_count_for_rows(total_rows) + prior = metric_clusters_raw if isinstance(metric_clusters_raw, dict) else {} + header = init_job_header( + job_run_id=job_run_id, + transcript_source=normalize_transcript_source(transcript_source), + total_rows=total_rows, + total_batches=total_batches, + provider=provider, + model=model, + extra={ + "generated_at_completed_rows": evaluation.completed_rows, + "llm_calls_used": 0, + "selected_evaluation_row_ids": prior.get("selected_evaluation_row_ids") or [], + "failure_policies": prior.get("failure_policies") or {}, + "failure_policies_source": prior.get("failure_policies_source"), + "failure_policies_updated_at": prior.get("failure_policies_updated_at"), + "synthesis_completed_metric_ids": [], + "groups": [], + "discovered_problems": [], + }, + ) + header["progress"]["phase"] = "rows" + return header + + +def resume_metric_clusters_job_header( + db: Session, + evaluation: CallImportEvaluation, + prior: Dict[str, Any], + *, + transcript_source: str, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> Dict[str, Any]: + """Re-open a checkpointed clustering job without losing finished pages.""" + header = dict(prior) + header["status"] = "running" + header["error_message"] = None + header["celery_task_id"] = None + header["transcript_source"] = normalize_transcript_source(transcript_source) + if provider: + header["provider"] = provider + if model: + header["model"] = model + stamp_job_heartbeat(header) + + total_rows, _ = _count_row_scope(db, evaluation, header) + total_batches = batch_count_for_rows(total_rows) + progress = header.get("progress") if isinstance(header.get("progress"), dict) else {} + stored_batches = int(progress.get("completed_batches") or 0) + completed_batches = min(stored_batches, total_batches) if total_batches else stored_batches + phase = progress.get("phase") if isinstance(progress.get("phase"), str) else None + if phase not in {"rows", "synthesis", "discovery"}: + phase = "synthesis" if completed_batches >= total_batches else "rows" + header["progress"] = progress_header( + completed_rows=rows_completed_for_batches(completed_batches, total_rows) + if total_rows + else int(progress.get("completed_rows") or 0), + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=int(header.get("llm_calls_used") or 0), + total_llm_calls=int(progress.get("total_llm_calls") or 0), + phase=phase, + ) + return header + + +def _load_row_page( + db: Session, + evaluation_id: UUID, + *, + after_cursor: Optional[EvaluationRowPageCursor], + selected_ids: Optional[List[UUID]], + batch_index: int, +) -> List[tuple[CallImportEvaluationRow, CallImportRow]]: + if selected_ids: + pairs = load_completed_evaluation_row_pairs_for_eval_row_ids( + db, evaluation_id, selected_ids + ) + start = batch_index * EVALUATION_ANALYSIS_BATCH_SIZE + end = start + EVALUATION_ANALYSIS_BATCH_SIZE + return pairs[start:end] + return load_completed_evaluation_row_pairs_page( + db, + evaluation_id, + limit=EVALUATION_ANALYSIS_BATCH_SIZE, + after_cursor=after_cursor, + ) + + +def process_metric_clusters_wave_step( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + provider, + model: str, + *, + metrics: Sequence[Metric], + policies: Dict[str, MetricFailurePolicy], + job_run_id: UUID, + on_progress: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCheck] = None, +) -> Tuple[str, Optional[EvaluationMetricClustersState]]: + raw = evaluation.metric_clusters if isinstance(evaluation.metric_clusters, dict) else {} + if metric_clusters_raw_is_cancelled(raw): + return "idle", EvaluationMetricClustersState( + status="cancelled", + error_message=METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + phase = str(progress.get("phase") or "rows") + total_rows = int(progress.get("total_rows") or 0) + total_batches = int(progress.get("total_batches") or 0) + completed_batches = int(progress.get("completed_batches") or 0) + llm_calls_used = int(raw.get("llm_calls_used") or 0) + total_llm_calls = int(progress.get("total_llm_calls") or 0) + transcript_source = normalize_transcript_source(raw.get("transcript_source")) + selected_ids_raw = raw.get("selected_evaluation_row_ids") or [] + selected_ids = ( + [UUID(str(rid)) for rid in selected_ids_raw] if selected_ids_raw else None + ) + + stored_policies, policy_source = policies_from_evaluation_raw(raw) + failure_policies = policies or stored_policies + quality_metrics = [m for m in metrics if _metric_is_quality(m)] + + if phase == "rows" and completed_batches < total_batches: + cursor = cursor_from_header(raw) + pairs = _load_row_page( + db, + evaluation.id, + after_cursor=cursor, + selected_ids=selected_ids, + batch_index=completed_batches, + ) + if not pairs: + # Commit, not just assign: the worker expires and refreshes this + # column right after the wave step, which would discard the phase + # change and re-enter the row phase forever. + evaluation.metric_clusters = stamp_job_heartbeat( + { + **raw, + "status": "running", + "progress": { + **progress, + "phase": "synthesis", + "completed_batches": completed_batches, + }, + } + ) + flag_modified(evaluation, "metric_clusters") + if not keep_owned_header_write( + db, evaluation, "metric_clusters", job_run_id + ): + return "idle", None + db.commit() + return "continue", None + + rows_before_this_batch = rows_completed_for_batches(completed_batches, total_rows) + metrics_sig: Dict[str, Dict[str, int]] = {} + metric_extractions: Dict[str, List[Dict[str, Any]]] = {} + flagged_samples: List[Dict[str, Any]] = [] + + # Resume inside a page. A full checkpoint page can need dozens of extraction + # calls; a task that hits the 25 minute soft limit must not re-pay for + # sub-batches that already checkpointed, otherwise a large page can + # never finish. + done_sub: set[str] = set() + for batch in list_analysis_batches( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + ): + if batch.batch_index != completed_batches or (batch.status or "") != "in_progress": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + raw_done = payload.get("sub_batches_done") + if isinstance(raw_done, list): + done_sub = {str(item) for item in raw_done if item} + restored_metrics = payload.get("metrics") + if isinstance(restored_metrics, dict): + for mid, sig_map in restored_metrics.items(): + if isinstance(sig_map, dict): + _merge_sig_maps(metrics_sig, str(mid), sig_map) + restored_extractions = payload.get("metric_extractions") + if isinstance(restored_extractions, dict): + for mid, samples in restored_extractions.items(): + if isinstance(samples, list): + metric_extractions[str(mid)] = [ + item for item in samples if isinstance(item, dict) + ] + restored_samples = payload.get("flagged_samples") + if isinstance(restored_samples, list): + flagged_samples = [ + item for item in restored_samples if isinstance(item, dict) + ] + break + + metrics_for_page_progress = [ + m + for m in quality_metrics + if failure_policies.get(str(m.id)) is not None + and policy_has_failure_criteria(failure_policies[str(m.id)], m) + ] + total_metrics_in_page = len(metrics_for_page_progress) + processed_evaluation_row_ids: set[str] = set() + + for metric_idx_one_based, metric in enumerate(metrics_for_page_progress, start=1): + policy = failure_policies[str(metric.id)] + mid = str(metric.id) + flagged_payloads: List[Dict[str, Any]] = [] + for eval_row, source_row in pairs: + if eval_row.status != "completed": + continue + cluster_row = _cluster_row_for_pair( + evaluation, eval_row, source_row, transcript_source + ) + payload = _build_flagged_row_payload_from_source( + cluster_row, metric, policy + ) + if payload: + flagged_payloads.append(payload) + if len(flagged_samples) < 80: + flagged_samples.append(payload) + + sub_batches = _sub_batch_payloads(flagged_payloads) + for sub_idx, sub_batch in enumerate(sub_batches): + sub_key = f"{mid}:{sub_idx}" + if sub_key in done_sub: + continue + if is_cancelled and is_cancelled(): + return "idle", EvaluationMetricClustersState( + status="cancelled", + error_message=METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + try: + extracted = _run_extraction( + db, + organization_id, + provider, + model, + metric.name, + sub_batch, + ) + except ClusterOutputTruncated: + raise + except Exception as exc: # noqa: BLE001 + logger.warning( + "Metric cluster extraction failed for {} / {}: {}", + evaluation.id, + metric.id, + exc, + ) + extracted = [] + llm_calls_used += 1 + if extracted: + _merge_sig_maps( + metrics_sig, mid, _aggregate_signatures(extracted) + ) + reservoir = metric_extractions.setdefault(mid, []) + reservoir.extend(extracted) + metric_extractions[mid] = reservoir[:40] + done_sub.add(sub_key) + for item in sub_batch: + if isinstance(item, dict): + row_id = item.get("evaluation_row_id") + if row_id: + processed_evaluation_row_ids.add(str(row_id)) + completed_rows_partial = rows_before_this_batch + len( + processed_evaluation_row_ids + ) + header_progress_rows = progress_header( + completed_rows=completed_rows_partial, + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=llm_calls_used, + total_llm_calls=total_llm_calls, + phase="rows", + ) + header_progress_rows["current_metric_name"] = metric.name + header_progress_rows["current_metric_index"] = metric_idx_one_based + header_progress_rows["total_metrics"] = total_metrics_in_page + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + batch_index=completed_batches, + payload={ + "metrics": metrics_sig, + "metric_extractions": metric_extractions, + "flagged_samples": flagged_samples, + "rows_processed": len(pairs), + "sub_batches_done": sorted(done_sub), + }, + transcript_source=transcript_source, + status="in_progress", + ) + # Keep the header heartbeat fresh so a long page is never + # mistaken for an abandoned job. + evaluation.metric_clusters = stamp_job_heartbeat( + { + **( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ), + "status": "running", + "progress": header_progress_rows, + "llm_calls_used": llm_calls_used, + } + ) + flag_modified(evaluation, "metric_clusters") + if not keep_owned_header_write( + db, evaluation, "metric_clusters", job_run_id + ): + return "idle", None + db.commit() + if on_progress: + on_progress(header_progress_rows) + + page_cursor = evaluation_row_page_cursor_from_pair(pairs[-1]) + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + batch_index=completed_batches, + payload={ + "metrics": metrics_sig, + "metric_extractions": metric_extractions, + "flagged_samples": flagged_samples, + "rows_processed": len(pairs), + "sub_batches_done": sorted(done_sub), + }, + cursor=page_cursor if not selected_ids else None, + transcript_source=transcript_source, + status="completed", + ) + + completed_batches += 1 + new_progress = progress_header( + completed_rows=rows_completed_for_batches(completed_batches, total_rows) + if total_rows + else rows_before_this_batch + len(pairs), + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=llm_calls_used, + total_llm_calls=total_llm_calls, + phase="rows" if completed_batches < total_batches else "synthesis", + ) + raw["progress"] = new_progress + raw["llm_calls_used"] = llm_calls_used + if not selected_ids: + raw["cursor"] = cursor_to_dict(page_cursor) + + db.expire(evaluation, ["metric_clusters"]) + db.refresh(evaluation) + live = evaluation.metric_clusters if isinstance(evaluation.metric_clusters, dict) else {} + if metric_clusters_raw_is_cancelled(live): + return "idle", EvaluationMetricClustersState( + status="cancelled", + error_message=METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + if not claim_still_owns_header(live, job_run_id): + logger.info( + "Metric clusters page write skipped; run {} no longer owns the header ({})", + job_run_id, + live.get("job_run_id"), + ) + return "idle", None + evaluation.metric_clusters = stamp_job_heartbeat( + { + **live, + **raw, + "job_run_id": str(job_run_id), + "progress": new_progress, + } + ) + flag_modified(evaluation, "metric_clusters") + db.commit() + if on_progress: + on_progress(new_progress) + return "continue", None + + # Synthesis phase — one metric per wave step so each finish is checkpointed. + row_batches = list_analysis_batches( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + ) + row_batches = [b for b in row_batches if b.batch_index < SYNTHESIS_BATCH_INDEX_OFFSET] + sig_by_metric = merge_signature_counts_from_cluster_batches(row_batches) + synthesis_done = { + str(x) for x in (raw.get("synthesis_completed_metric_ids") or []) if x + } + + groups: List[MetricClusterGroup] = [] + groups_raw = raw.get("groups") + if isinstance(groups_raw, list): + for item in groups_raw: + if isinstance(item, dict): + try: + groups.append(MetricClusterGroup.model_validate(item)) + except Exception: # noqa: BLE001 + continue + + metrics_by_id = {str(m.id): m for m in metrics} + metrics_to_synth = _metrics_eligible_for_synthesis( + quality_metrics, failure_policies, sig_by_metric + ) + + for metric_pos, metric in enumerate(metrics_to_synth): + mid = str(metric.id) + if mid in synthesis_done: + continue + if is_cancelled and is_cancelled(): + return "idle", EvaluationMetricClustersState( + status="cancelled", + error_message=METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + + signature_counts = sig_by_metric.get(mid) or {} + sample_rows = merge_cluster_extractions_from_batches(row_batches, mid) + flagged_count = sum(signature_counts.values()) + # Publish the in-flight metric BEFORE the model call. Synthesis is one + # call per metric (up to the provider timeout, plus a larger-budget + # retry) and neither the row nor the batch counter moves, so without + # this the card looks frozen on the last batch until the call returns. + if on_progress: + on_progress( + { + **progress, + "phase": "synthesis", + "completed_batches": completed_batches, + "total_batches": total_batches, + "completed_llm_calls": llm_calls_used, + "total_llm_calls": total_llm_calls, + "current_metric_name": metric.name, + "current_metric_index": len(synthesis_done) + 1, + "total_metrics": len(metrics_to_synth), + } + ) + try: + clusters = _synthesize_metric_clusters( + db, + organization_id, + provider, + model, + metric_name=metric.name, + signature_counts=signature_counts, + sample_rows=sample_rows, + flagged_count=flagged_count, + ) + except ClusterOutputTruncated: + raise + except Exception as exc: # noqa: BLE001 + logger.error( + "Metric cluster synthesis failed for {} / {}: {}", + evaluation.id, + metric.id, + exc, + ) + clusters = [] + llm_calls_used += 1 + + if clusters: + group = MetricClusterGroup( + metric_id=mid, + metric_name=metric.name, + flagged_count=flagged_count, + clusters=clusters, + ) + groups = [g for g in groups if g.metric_id != mid] + [group] + + synthesis_done.add(mid) + batch_index = SYNTHESIS_BATCH_INDEX_OFFSET + metric_pos + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + batch_index=batch_index, + payload={ + "metric_id": mid, + "groups": [groups[-1].model_dump(mode="json")] if clusters else [], + }, + ) + + synth_index = len([m for m in metrics_to_synth if str(m.id) in synthesis_done]) + raw["synthesis_completed_metric_ids"] = sorted(synthesis_done) + raw["groups"] = [g.model_dump(mode="json") for g in groups] + raw["llm_calls_used"] = llm_calls_used + raw["progress"] = { + **progress, + "phase": "synthesis", + "completed_rows": progress.get("completed_rows"), + "total_rows": progress.get("total_rows"), + "completed_batches": completed_batches, + "total_batches": total_batches, + "completed_llm_calls": llm_calls_used, + "total_llm_calls": total_llm_calls, + "current_metric_name": metric.name, + "current_metric_index": synth_index, + "total_metrics": len(metrics_to_synth), + } + evaluation.metric_clusters = stamp_job_heartbeat(raw) + flag_modified(evaluation, "metric_clusters") + if not keep_owned_header_write( + db, evaluation, "metric_clusters", job_run_id + ): + return "idle", None + db.commit() + if on_progress: + on_progress(raw["progress"]) + return "continue", None + + # Discovery + final assembly + all_samples: List[Dict[str, Any]] = [] + total_flagged = 0 + for batch in row_batches: + payload = batch.payload if isinstance(batch.payload, dict) else {} + samples = payload.get("flagged_samples") + if isinstance(samples, list): + all_samples.extend(samples) + for sig_map in (payload.get("metrics") or {}).values(): + if isinstance(sig_map, dict): + total_flagged += sum(int(v) for v in sig_map.values()) + + discovered_raw = raw.get("discovered_problems") + discovered: List[DiscoveredProblemCluster] = [] + if isinstance(discovered_raw, list) and raw.get("discovery_completed"): + for item in discovered_raw: + if isinstance(item, dict): + try: + discovered.append(DiscoveredProblemCluster.model_validate(item)) + except Exception: # noqa: BLE001 + continue + else: + # Announce discovery before the call for the same reason as synthesis: + # rows and batches are already final, so the card would otherwise sit + # on the last synthesis step for the whole LLM round-trip. + if on_progress: + on_progress( + { + **progress, + "phase": "discovery", + "completed_batches": completed_batches, + "total_batches": total_batches, + "completed_llm_calls": llm_calls_used, + "total_llm_calls": total_llm_calls, + } + ) + try: + discovered = _run_discovery( + db, + organization_id, + provider, + model, + all_samples, + total_flagged=max(total_flagged, 1), + ) + except ClusterOutputTruncated: + raise + except Exception as exc: # noqa: BLE001 + logger.warning( + "Proactive discovery failed for evaluation {}: {}", evaluation.id, exc + ) + llm_calls_used += 1 + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_METRIC_CLUSTERS, + job_run_id=job_run_id, + batch_index=DISCOVERY_BATCH_INDEX_OFFSET, + payload={ + "discovered_problems": [d.model_dump(mode="json") for d in discovered] + }, + ) + raw["discovered_problems"] = [d.model_dump(mode="json") for d in discovered] + raw["discovery_completed"] = True + raw["llm_calls_used"] = llm_calls_used + evaluation.metric_clusters = stamp_job_heartbeat(raw) + flag_modified(evaluation, "metric_clusters") + if not keep_owned_header_write( + db, evaluation, "metric_clusters", job_run_id + ): + return "idle", None + db.commit() + if on_progress: + on_progress( + { + **progress, + "phase": "discovery", + "completed_llm_calls": llm_calls_used, + "total_llm_calls": total_llm_calls, + } + ) + return "continue", None + + payloads_by_metric: Dict[str, List[Dict[str, Any]]] = {} + extractions_by_metric: Dict[str, List[Dict[str, Any]]] = {} + for batch in row_batches: + payload = batch.payload if isinstance(batch.payload, dict) else {} + for mid, samples in (payload.get("metric_extractions") or {}).items(): + extractions_by_metric.setdefault(str(mid), []).extend(samples or []) + + groups, discovered = enrich_metric_cluster_groups( + groups, + discovered, + policies=failure_policies, + metrics_by_id=metrics_by_id, + payloads_by_metric=payloads_by_metric, + extractions_by_metric=extractions_by_metric, + ) + rca_summary = compute_rca_summary( + groups, + discovered, + metrics_by_id=metrics_by_id, + analysed_calls=max(int(progress.get("total_rows") or 1), 1), + ) + + final_progress = { + **progress, + "phase": "completed", + "completed_rows": int(progress.get("total_rows") or 0), + "completed_batches": total_batches, + "completed_llm_calls": llm_calls_used, + "total_llm_calls": total_llm_calls, + } + + if not groups and not discovered: + return "done", EvaluationMetricClustersState( + status="failed", + error_message="No flagged quality metrics or clusters produced.", + generated_at=datetime.now(timezone.utc), + llm_calls_used=llm_calls_used, + progress=final_progress, + selected_evaluation_row_ids=selected_ids_raw, + ) + + # The card is polling this header, so publish the terminal step before the + # task persists the final state. Without it the last visible phase is + # "discovery" and the job looks unfinished for the final round-trip. + if on_progress: + on_progress(final_progress) + + overview_parts = [ + f"{g.metric_name}: {len(g.clusters)} cluster(s) across {g.flagged_count} flagged calls" + for g in groups + ] + if discovered: + overview_parts.append(f"{len(discovered)} proactively discovered theme(s)") + + policies_updated_raw = None + if raw.get("failure_policies_updated_at"): + try: + policies_updated_raw = datetime.fromisoformat( + str(raw["failure_policies_updated_at"]) + ) + except ValueError: + policies_updated_raw = None + + return "done", EvaluationMetricClustersState( + status="completed", + groups=groups, + discovered_problems=discovered, + rca_summary=rca_summary, + overview="; ".join(overview_parts) if overview_parts else None, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=evaluation.completed_rows, + progress=final_progress, + provider=provider.value if hasattr(provider, "value") else str(provider), + model=model, + llm_calls_used=llm_calls_used, + is_stale=False, + selected_evaluation_row_ids=selected_ids_raw, + failure_policies=failure_policies, + failure_policies_source=policy_source, # type: ignore[arg-type] + failure_policies_updated_at=policies_updated_raw, + ) diff --git a/app/services/call_import_prompt_improvements.py b/app/services/call_import_prompt_improvements.py index b0456b8e..4cdbc65e 100644 --- a/app/services/call_import_prompt_improvements.py +++ b/app/services/call_import_prompt_improvements.py @@ -23,9 +23,71 @@ from app.services.agent_flowchart import _extract_json_object from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request from app.services.ai.llm_service import llm_service +from app.services.evaluation_analysis_checkpoints import job_heartbeat_is_stale +from app.services.import_eval_llm_output_budgets import ( + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, +) from app.services.imported_agent_constants import IMPORTED_AGENT_TAG +PROMPT_SUGGESTION_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST + + +def _generate_prompt_improvement_json( + *, + messages: List[Dict[str, str]], + llm_provider: Any, + llm_model: str, + organization_id: UUID, + db: Session, +) -> Dict[str, Any]: + """Two-step max_tokens with retry on truncation or JSON extraction failure.""" + budgets = [ + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, + ] + last_error: Optional[Exception] = None + for budget_idx, max_tok in enumerate(budgets): + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=0.3 if budget_idx == 0 else 0.2, + max_tokens=max_tok, + ) + if result.get("truncated"): + logger.warning( + "Prompt improvements LLM truncated at max_tokens={} (model={})", + max_tok, + llm_model, + ) + last_error = RuntimeError("truncated") + if budget_idx < len(budgets) - 1: + continue + raise RuntimeError( + f"Prompt improvement output was truncated ({llm_model})." + ) from last_error + try: + return _extract_json_object(result["text"]) + except ValueError as exc: + logger.warning( + "Prompt improvements JSON extract failed at max_tokens={} (model={}): {}", + max_tok, + llm_model, + exc, + ) + last_error = exc + if budget_idx < len(budgets) - 1: + continue + raise + raise RuntimeError( + f"Prompt improvement LLM failed after retries ({llm_model}): {last_error}" + ) + + def flag_evaluation_prompt_improvements_modified(evaluation: CallImportEvaluation) -> None: from sqlalchemy.orm.attributes import flag_modified @@ -115,6 +177,18 @@ def prompt_improvements_state_from_raw( and completed_rows > generated_at_completed ) + progress_raw = raw.get("progress") + progress: Optional[Dict[str, Any]] = None + if isinstance(progress_raw, dict): + progress = {} + for key, value in progress_raw.items(): + if isinstance(value, bool): + continue + if isinstance(value, (int, float)): + progress[key] = int(value) + elif isinstance(value, str): + progress[key] = value + return EvaluationPromptImprovementsState( status=raw.get("status") or "idle", imported_agent_id=raw.get("imported_agent_id"), @@ -127,7 +201,9 @@ def prompt_improvements_state_from_raw( model=raw.get("model"), error_message=raw.get("error_message"), is_stale=is_stale, + heartbeat_stale=job_heartbeat_is_stale(raw), require_review=True, + progress=progress, ) @@ -334,6 +410,38 @@ def generate_prompt_improvements( credential_id: Optional[UUID] = None, period_deltas: Optional[Dict[str, Dict[str, str]]] = None, ) -> EvaluationPromptImprovementsState: + return generate_prompt_improvements_for_prompt( + agent_id=str(imported_agent.id), + agent_name=imported_agent.name, + agent_prompt=imported_agent.content or "", + agent_flowchart=imported_agent.agent_flowchart, + clusters_state=clusters_state, + completed_rows=evaluation.completed_rows, + organization_id=organization_id, + db=db, + provider=provider, + model=model, + credential_id=credential_id, + period_deltas=period_deltas, + ) + + +def generate_prompt_improvements_for_prompt( + *, + agent_id: str, + agent_name: Optional[str], + agent_prompt: str, + agent_flowchart: Any, + clusters_state: EvaluationMetricClustersState, + completed_rows: int, + organization_id: UUID, + db: Session, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + period_deltas: Optional[Dict[str, Dict[str, str]]] = None, +) -> EvaluationPromptImprovementsState: + """Suggest prompt edits for ``agent_prompt`` from completed failure clusters.""" if clusters_state.status != "completed" or not clusters_state.groups: raise ValueError("Metric clusters must be completed before generating improvements") @@ -356,15 +464,15 @@ def generate_prompt_improvements( for area in clusters_state.rca_summary.prompt_areas ] - flow_nodes = _extract_flow_nodes(imported_agent.agent_flowchart) + flow_nodes = _extract_flow_nodes(agent_flowchart) flow_node_context = [ {"id": node_id, "label": label} for node_id, label in flow_nodes.items() ] user_payload: Dict[str, Any] = { - "agent_name": imported_agent.name, - "agent_prompt": imported_agent.content, + "agent_name": agent_name, + "agent_prompt": agent_prompt, "failure_clusters": cluster_context, "rca_prompt_areas": rca_prompt_areas, "rca_overview": ( @@ -384,31 +492,28 @@ def generate_prompt_improvements( }, ] - result = llm_service.generate_response( + raw = _generate_prompt_improvement_json( messages=messages, llm_provider=provider_enum, llm_model=model_str, organization_id=organization_id, db=db, - temperature=0.3, - max_tokens=6000, ) - raw = _extract_json_object(result["text"]) overview, suggestions = _parse_suggestions( raw, cluster_context, - agent_prompt=imported_agent.content or "", + agent_prompt=agent_prompt, flow_nodes=flow_nodes, ) return EvaluationPromptImprovementsState( status="completed", - imported_agent_id=str(imported_agent.id), - imported_agent_name=imported_agent.name, + imported_agent_id=agent_id, + imported_agent_name=agent_name, suggestions=suggestions, overview=overview or None, generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=evaluation.completed_rows, + generated_at_completed_rows=completed_rows, provider=provider_enum.value, model=model_str, error_message=None, diff --git a/app/services/call_import_prompt_improvements_wave.py b/app/services/call_import_prompt_improvements_wave.py new file mode 100644 index 00000000..eabbff8d --- /dev/null +++ b/app/services/call_import_prompt_improvements_wave.py @@ -0,0 +1,275 @@ +"""Batched prompt-improvement generation with checkpoints and cancel support.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.models.database import CallImportEvaluation, PromptPartial +from app.models.schemas import ( + EvaluationMetricClustersState, + EvaluationPromptImprovementsState, + PromptImprovementSuggestion, +) +from app.services.call_import_prompt_improvements import ( + PROMPT_SUGGESTION_MAX_TOKENS, + _IMPROVEMENTS_SYSTEM_PROMPT, + _build_cluster_context, + _extract_flow_nodes, + _generate_prompt_improvement_json, + _parse_suggestions, + prompt_improvements_state_to_db, +) +from app.services.evaluation_analysis_checkpoints import ( + JOB_KIND_PROMPT_IMPROVEMENTS, + PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR, + keep_owned_header_write, + list_analysis_batches, + progress_header, + prompt_improvements_raw_is_cancelled, + stamp_job_heartbeat, + upsert_analysis_batch, +) +from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + +CancelCheck = Callable[[], bool] +ProgressCallback = Callable[[Dict[str, Any]], None] + + +def _suggestions_from_batches( + batches: Sequence[Any], +) -> Tuple[Optional[str], List[PromptImprovementSuggestion]]: + overview: Optional[str] = None + suggestions: List[PromptImprovementSuggestion] = [] + seen: set[str] = set() + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + if overview is None and isinstance(payload.get("overview"), str): + overview = payload.get("overview") + raw_items = payload.get("suggestions") + if not isinstance(raw_items, list): + continue + for item in raw_items: + if not isinstance(item, dict): + continue + try: + suggestion = PromptImprovementSuggestion(**item) + except Exception: # noqa: BLE001 + continue + key = suggestion.id or f"{suggestion.metric_id}:{suggestion.target_section}:{suggestion.cluster_id}" + if key in seen: + continue + seen.add(key) + suggestions.append(suggestion) + return overview, suggestions + + +def _progress_for_groups( + groups: Sequence[Any], + completed_group_ids: set[str], + *, + total_metrics: int, +) -> Dict[str, Any]: + total = max(int(total_metrics), len(groups), 1) + done = len([g for g in groups if str(g.metric_id) in completed_group_ids]) + current = next((g for g in groups if str(g.metric_id) not in completed_group_ids), None) + progress = progress_header( + completed_rows=done, + total_rows=total, + completed_batches=done, + total_batches=total, + completed_llm_calls=done, + total_llm_calls=total, + phase="rows" if done < total else "synthesis", + ) + progress["completed_groups"] = done + progress["total_groups"] = total + if current is not None: + progress["current_metric_name"] = current.metric_name + progress["current_metric_index"] = done + 1 + progress["total_metrics"] = total + return progress + + +def process_prompt_improvements_wave_step( + db: Session, + evaluation: CallImportEvaluation, + imported_agent: PromptPartial, + clusters_state: EvaluationMetricClustersState, + organization_id: UUID, + *, + job_run_id: UUID, + provider: Optional[str], + model: Optional[str], + credential_id: Optional[UUID], + period_deltas: Optional[Dict[str, Dict[str, str]]] = None, + on_progress: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCheck] = None, +) -> Tuple[str, Optional[EvaluationPromptImprovementsState]]: + raw = evaluation.prompt_improvements if isinstance(evaluation.prompt_improvements, dict) else {} + if prompt_improvements_raw_is_cancelled(raw): + return "idle", EvaluationPromptImprovementsState( + status="cancelled", + imported_agent_id=str(imported_agent.id), + imported_agent_name=imported_agent.name, + error_message=PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + + completed_groups = { + str(x) for x in (raw.get("completed_group_metric_ids") or []) if x + } + groups = list(clusters_state.groups or []) + batch_index_base = int(raw.get("completed_group_count") or 0) + pending = [g for g in groups if str(g.metric_id) not in completed_groups] + total_metrics = max(int(raw.get("total_groups") or 0), len(groups), 1) + + if not pending: + batches = list_analysis_batches( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_PROMPT_IMPROVEMENTS, + job_run_id=job_run_id, + ) + overview, suggestions = _suggestions_from_batches(batches) + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, + db, + provider, + model, + credential_id, + ) + return "done", EvaluationPromptImprovementsState( + status="completed", + imported_agent_id=str(imported_agent.id), + imported_agent_name=imported_agent.name, + suggestions=suggestions, + overview=overview, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=evaluation.completed_rows, + provider=provider_enum.value, + model=model_str, + error_message=None, + is_stale=False, + ) + + if is_cancelled and is_cancelled(): + return "idle", EvaluationPromptImprovementsState( + status="cancelled", + imported_agent_id=str(imported_agent.id), + imported_agent_name=imported_agent.name, + error_message=PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR, + generated_at=datetime.now(timezone.utc), + ) + + group = pending[0] + partial_state = EvaluationMetricClustersState( + status="completed", + groups=[group], + ) + cluster_context = _build_cluster_context(partial_state, period_deltas) + if not cluster_context: + completed_groups.add(str(group.metric_id)) + raw["completed_group_metric_ids"] = sorted(completed_groups) + raw["completed_group_count"] = batch_index_base + 1 + raw["progress"] = _progress_for_groups( + groups, completed_groups, total_metrics=total_metrics + ) + evaluation.prompt_improvements = stamp_job_heartbeat(raw) + flag_modified(evaluation, "prompt_improvements") + if not keep_owned_header_write( + db, evaluation, "prompt_improvements", job_run_id + ): + return "idle", None + db.commit() + if on_progress: + on_progress(raw["progress"]) + return "continue", None + + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, + db, + provider, + model, + credential_id, + ) + flow_nodes = _extract_flow_nodes(imported_agent.agent_flowchart) + flow_node_context = [{"id": node_id, "label": label} for node_id, label in flow_nodes.items()] + rca_prompt_areas = [] + if clusters_state.rca_summary and clusters_state.rca_summary.prompt_areas: + rca_prompt_areas = [ + area.model_dump(mode="json") + for area in clusters_state.rca_summary.prompt_areas + ] + + user_payload: Dict[str, Any] = { + "agent_name": imported_agent.name, + "agent_prompt": imported_agent.content, + "failure_clusters": cluster_context, + "rca_prompt_areas": rca_prompt_areas, + "rca_overview": ( + clusters_state.rca_summary.model_dump(mode="json") + if clusters_state.rca_summary + else None + ), + } + if flow_node_context: + user_payload["agent_flow_nodes"] = flow_node_context + + messages = [ + {"role": "system", "content": _IMPROVEMENTS_SYSTEM_PROMPT}, + {"role": "user", "content": json.dumps(user_payload, ensure_ascii=False)}, + ] + + parsed = _generate_prompt_improvement_json( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + ) + overview, suggestions = _parse_suggestions( + parsed, + cluster_context, + agent_prompt=imported_agent.content or "", + flow_nodes=flow_nodes, + ) + + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_PROMPT_IMPROVEMENTS, + job_run_id=job_run_id, + batch_index=batch_index_base, + payload={ + "metric_id": str(group.metric_id), + "overview": overview, + "suggestions": [s.model_dump(mode="json") for s in suggestions], + }, + ) + + completed_groups.add(str(group.metric_id)) + raw["completed_group_metric_ids"] = sorted(completed_groups) + raw["completed_group_count"] = batch_index_base + 1 + raw["status"] = "running" + raw["progress"] = _progress_for_groups( + groups, completed_groups, total_metrics=total_metrics + ) + evaluation.prompt_improvements = stamp_job_heartbeat(raw) + flag_modified(evaluation, "prompt_improvements") + if not keep_owned_header_write( + db, evaluation, "prompt_improvements", job_run_id + ): + return "idle", None + db.commit() + if on_progress: + on_progress(raw["progress"]) + return "continue", None diff --git a/app/services/call_import_user_insights.py b/app/services/call_import_user_insights.py index 7484a34b..4a582645 100644 --- a/app/services/call_import_user_insights.py +++ b/app/services/call_import_user_insights.py @@ -31,6 +31,11 @@ UserInsightEvidenceTurn, ) from app.services.ai.llm_service import llm_service +from app.services.evaluation_analysis_checkpoints import job_heartbeat_is_stale +from app.services.import_eval_llm_output_budgets import ( + IMPORT_EVAL_LLM_OUTPUT_FIRST, + IMPORT_EVAL_LLM_OUTPUT_RETRY, +) MAX_LLM_CALLS_DEFAULT = 200 MAX_LLM_CALLS_MIN = 20 @@ -38,8 +43,8 @@ SYNTHESIS_CALLS = 1 ROW_TRANSCRIPT_CHAR_CAP = 3000 RATIONALE_CHAR_CAP = 600 -EXTRACTION_MAX_TOKENS = 1800 -SYNTHESIS_MAX_TOKENS = 2400 +EXTRACTION_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST +SYNTHESIS_MAX_TOKENS = IMPORT_EVAL_LLM_OUTPUT_FIRST ProgressCallback = Callable[[int, int], None] @@ -226,21 +231,40 @@ def _call_llm( max_tokens: int, usage_ctx: Optional[LLMUsageContext] = None, ) -> str: + """Two-step output budget: ``max_tokens`` then double on truncation.""" from app.services.usage.context import get_usage_context, llm_usage_context effective_ctx = usage_ctx or get_usage_context() + retry_tokens = min(IMPORT_EVAL_LLM_OUTPUT_RETRY, max_tokens * 2) + budgets = [max_tokens] if retry_tokens <= max_tokens else [max_tokens, retry_tokens] def _run() -> str: - result = llm_service.generate_response( - messages=messages, - llm_provider=provider, - llm_model=model, - organization_id=organization_id, - db=db, - temperature=temperature, - max_tokens=max_tokens, - ) - return str(result.get("text") or "") + last_text = "" + for attempt_idx, token_budget in enumerate(budgets): + result = llm_service.generate_response( + messages=messages, + llm_provider=provider, + llm_model=model, + organization_id=organization_id, + db=db, + temperature=temperature, + max_tokens=token_budget, + ) + last_text = str(result.get("text") or "") + if not result.get("truncated"): + return last_text + if attempt_idx < len(budgets) - 1: + logger.warning( + "User insights LLM output truncated at max_tokens={} (model={}); retrying at {}.", + token_budget, + model, + budgets[attempt_idx + 1], + ) + continue + raise RuntimeError( + f"User insights LLM still truncated after max_tokens={token_budget} (model={model})." + ) + return last_text if effective_ctx is not None: with llm_usage_context(effective_ctx): @@ -590,7 +614,7 @@ def user_insights_state_from_raw( if not isinstance(raw, dict): return None status = raw.get("status") - if status not in {"idle", "running", "completed", "failed"}: + if status not in {"idle", "running", "completed", "failed", "cancelled"}: status = "idle" insights: List[EvaluationUserInsightItem] = [] @@ -615,12 +639,47 @@ def user_insights_state_from_raw( snapshot = raw.get("generated_at_completed_rows") snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 progress_raw = raw.get("progress") - progress = ( - { - "completed_llm_calls": int(progress_raw.get("completed_llm_calls") or 0), - "total_llm_calls": int(progress_raw.get("total_llm_calls") or 0), - } - if isinstance(progress_raw, dict) + progress: Optional[Dict[str, Any]] = None + if isinstance(progress_raw, dict): + progress = {} + for key, value in progress_raw.items(): + if isinstance(value, bool): + continue + if isinstance(value, (int, float)): + progress[key] = int(value) + elif isinstance(value, str): + progress[key] = value + if progress: + if "completed_llm_calls" not in progress: + progress["completed_llm_calls"] = 0 + if "total_llm_calls" not in progress: + progress["total_llm_calls"] = 0 + from app.services.evaluation_analysis_checkpoints import batch_count_for_rows + from app.services.call_import_user_insights_wave import ( + estimate_user_insights_total_llm_calls, + ) + + effective_total_rows = int(progress.get("total_rows") or 0) or completed_rows + if effective_total_rows > 0: + progress["total_rows"] = effective_total_rows + if not progress.get("total_batches"): + progress["total_batches"] = batch_count_for_rows( + effective_total_rows + ) + progress["total_llm_calls"] = estimate_user_insights_total_llm_calls( + effective_total_rows + ) + llm_used = max( + int(raw.get("llm_calls_used") or 0), + int(progress.get("completed_llm_calls") or 0), + ) + progress["completed_llm_calls"] = llm_used + + transcript_source_raw = raw.get("transcript_source") + transcript_source = ( + str(transcript_source_raw).strip().lower() + if isinstance(transcript_source_raw, str) + and str(transcript_source_raw).strip().lower() in {"diarised", "production"} else None ) @@ -643,12 +702,14 @@ def user_insights_state_from_raw( raw.get("error_message") if isinstance(raw.get("error_message"), str) else None ), is_stale=completed_rows > snapshot_int and status == "completed", + heartbeat_stale=job_heartbeat_is_stale(raw), + transcript_source=transcript_source, ) def user_insights_state_to_db(state: EvaluationUserInsightsState) -> Dict[str, Any]: """Serialize state for the JSON column.""" - return { + payload: Dict[str, Any] = { "status": state.status, "insights": [item.model_dump(mode="json") for item in state.insights], "generated_at": ( @@ -663,3 +724,6 @@ def user_insights_state_to_db(state: EvaluationUserInsightsState) -> Dict[str, A "overview": state.overview, "error_message": state.error_message, } + if state.transcript_source: + payload["transcript_source"] = state.transcript_source + return payload diff --git a/app/services/call_import_user_insights_wave.py b/app/services/call_import_user_insights_wave.py new file mode 100644 index 00000000..308f0f8f --- /dev/null +++ b/app/services/call_import_user_insights_wave.py @@ -0,0 +1,679 @@ +"""Paged user-insights generation with durable batch checkpoints.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.db_sharding.scatter_gather import ( + EVALUATION_ANALYSIS_BATCH_SIZE, + EvaluationRowPageCursor, + count_completed_evaluation_row_pairs, + evaluation_row_page_cursor_from_pair, + load_completed_evaluation_row_pairs_page, +) +from app.models.database import CallImportEvaluation, CallImportEvaluationRow, CallImportRow, Metric +from app.models.schemas import EvaluationUserInsightsState +from app.services.call_import_user_insights import ( + SYNTHESIS_CALLS, + build_row_payload, + run_extraction_batch, + run_synthesis, + _format_aggregate_summary, + _aggregate_themes, +) +from app.services.evaluation_analysis_checkpoints import ( + JOB_KIND_USER_INSIGHTS, + USER_INSIGHTS_CANCELLED_BY_USER_ERROR, + batch_count_for_rows, + completed_rows_from_batches, + claim_still_owns_header, + cursor_from_header, + cursor_to_dict, + init_job_header, + list_analysis_batches, + merge_extractions_from_batches, + merge_theme_counts_from_batches, + normalize_transcript_source, + progress_header, + rows_completed_for_batches, + stamp_job_heartbeat, + upsert_analysis_batch, + user_insights_raw_is_cancelled, + new_job_run_id, +) + +CancelCheck = Callable[[], bool] +ProgressCallback = Callable[[Dict[str, Any]], None] + +EXTRACTION_SUB_BATCH_ROWS = 50 +# Rows small enough to fit in a single extraction pass (with headroom for +# per-row transcript + metric rationale tokens). +EXTRACTION_SINGLE_PASS_MAX_ROWS = 40 +# Checkpoint index used for the final synthesis payload (kept above any row +# page index so row batching can never collide with it). +SYNTHESIS_BATCH_INDEX_OFFSET = 100_000 + + +def extraction_sub_batch_size(row_count: int) -> int: + """Size each extraction LLM call; fewer round-trips for small evals.""" + if row_count <= 0: + return EXTRACTION_SUB_BATCH_ROWS + if row_count <= EXTRACTION_SINGLE_PASS_MAX_ROWS: + return row_count + return EXTRACTION_SUB_BATCH_ROWS + + +def extraction_pass_count(row_count: int) -> int: + if row_count <= 0: + return 0 + return (row_count + extraction_sub_batch_size(row_count) - 1) // extraction_sub_batch_size( + row_count + ) + + +def estimate_user_insights_total_llm_calls(total_rows: int) -> int: + """Extraction sub-passes across all row pages plus synthesis.""" + if total_rows <= 0: + return 0 + total_batches = batch_count_for_rows(total_rows) + extraction = 0 + remaining = total_rows + for _ in range(total_batches): + page_rows = min(EVALUATION_ANALYSIS_BATCH_SIZE, remaining) + if page_rows <= 0: + break + extraction += extraction_pass_count(page_rows) + remaining -= page_rows + return extraction + SYNTHESIS_CALLS + + +def reconcile_user_insights_job_header( + db: Session, + evaluation: CallImportEvaluation, + header: Dict[str, Any], +) -> Dict[str, Any]: + """Align row/batch totals with the DB and repair legacy progress.""" + total_rows = count_completed_evaluation_row_pairs(db, evaluation.id) + total_batches = batch_count_for_rows(total_rows) + progress = header.get("progress") if isinstance(header.get("progress"), dict) else {} + stored_batches = int(progress.get("completed_batches") or 0) + completed_batches = min(stored_batches, total_batches) if total_batches else stored_batches + completed_rows = rows_completed_for_batches( + completed_batches, total_rows + ) if total_rows else int(progress.get("completed_rows") or 0) + llm_used = max( + int(header.get("llm_calls_used") or 0), + int(progress.get("completed_llm_calls") or 0), + ) + phase = progress.get("phase") if isinstance(progress.get("phase"), str) else None + if phase not in {"rows", "synthesis"}: + phase = "synthesis" if total_batches > 0 and completed_batches >= total_batches else "rows" + new_progress = progress_header( + completed_rows=completed_rows, + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=llm_used, + total_llm_calls=estimate_user_insights_total_llm_calls(total_rows), + phase=phase, + ) + return { + **header, + "progress": new_progress, + "llm_calls_used": llm_used, + } + + +def pick_transcript_for_source( + evaluation: CallImportEvaluation, + source_row: CallImportRow, + transcript_source: str, +) -> str: + from app.services.call_import_user_insights import ROW_TRANSCRIPT_CHAR_CAP, _pick_transcript + + source = normalize_transcript_source(transcript_source) + if source == "production": + text = (source_row.transcript or "").strip() + else: + text = (source_row.diarised_transcript or "").strip() + if not text: + text = _pick_transcript(evaluation, source_row) + return text[:ROW_TRANSCRIPT_CHAR_CAP] + + +def build_row_payload_for_source( + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow, + metric_names: Dict[str, str], + transcript_source: str, +) -> Dict[str, Any]: + payload = build_row_payload(evaluation, eval_row, source_row, metric_names) + transcript = pick_transcript_for_source(evaluation, source_row, transcript_source) + payload["transcript"] = transcript + return payload + + +def _sub_batch_payloads( + rows: Sequence[Dict[str, Any]], + *, + size: int | None = None, +) -> List[List[Dict[str, Any]]]: + chunk = max(1, size if size is not None else extraction_sub_batch_size(len(rows))) + return [list(rows[i : i + chunk]) for i in range(0, len(rows), chunk)] + + +def merge_theme_counts_into_header( + header: Dict[str, Any], + theme_delta: Dict[str, int], +) -> Dict[str, int]: + existing = header.get("theme_counts") + merged: Dict[str, int] = dict(existing) if isinstance(existing, dict) else {} + for label, count in theme_delta.items(): + merged[label] = merged.get(label, 0) + int(count) + header["theme_counts"] = merged + return merged + + +def _cancelled_state( + evaluation: CallImportEvaluation, + raw: Dict[str, Any], + *, + llm_calls_used: int = 0, +) -> EvaluationUserInsightsState: + progress_raw = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + return EvaluationUserInsightsState( + status="cancelled", + insights=[], + overview=None, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=evaluation.completed_rows, + progress=progress_raw, + provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, + model=raw.get("model") if isinstance(raw.get("model"), str) else None, + llm_calls_used=llm_calls_used, + max_llm_calls=None, + error_message=USER_INSIGHTS_CANCELLED_BY_USER_ERROR, + is_stale=False, + ) + + +def _finalize_synthesis( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + provider, + model: str, + *, + job_run_id: UUID, + aggregate: Sequence[Any], + raw_header: Dict[str, Any], + on_progress: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCheck] = None, +) -> EvaluationUserInsightsState: + """Run synthesis once over every durable batch checkpoint.""" + if is_cancelled and is_cancelled(): + return _cancelled_state( + evaluation, + raw_header, + llm_calls_used=int(raw_header.get("llm_calls_used") or 0), + ) + + progress = raw_header.get("progress") if isinstance(raw_header.get("progress"), dict) else {} + provider_label = provider.value if hasattr(provider, "value") else str(provider) + + # Synthesis is idempotent for a job run: if it already produced batches of + # insight blocks we reuse them instead of paying for another LLM call. + existing_batches = list_analysis_batches( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_USER_INSIGHTS, + job_run_id=job_run_id, + ) + for batch in existing_batches: + if (batch.status or "") != "synthesis_completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + if payload.get("insights") is not None: + from app.models.schemas import EvaluationUserInsightItem + + insights = [ + EvaluationUserInsightItem.model_validate(item) + for item in payload.get("insights") or [] + if isinstance(item, dict) + ] + if insights: + return EvaluationUserInsightsState( + status="completed", + insights=insights, + overview=payload.get("overview"), + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=evaluation.completed_rows, + progress={**progress, "phase": "completed"}, + provider=provider_label, + model=model, + llm_calls_used=int(raw_header.get("llm_calls_used") or 0), + is_stale=False, + ) + + batches = [ + b + for b in existing_batches + if (b.status or "") == "completed" + and b.batch_index < SYNTHESIS_BATCH_INDEX_OFFSET + ] + theme_counts = merge_theme_counts_from_batches(batches) + if not theme_counts and isinstance(raw_header.get("theme_counts"), dict): + theme_counts = { + str(k): int(v) + for k, v in raw_header["theme_counts"].items() + if str(k).strip() + } + + sample_quotes = merge_extractions_from_batches(batches) + aggregate_summary = _format_aggregate_summary(aggregate) + total_sampled = completed_rows_from_batches(batches) + completed_calls = int(raw_header.get("llm_calls_used") or 0) + + # Announce the step before the LLM call. Synthesis is a single long + # round-trip after rows/batches are final; without this the card sits on + # the last extraction batch and looks wedged when a worker dies mid-call. + if on_progress: + on_progress( + { + **progress, + "phase": "synthesis", + "completed_llm_calls": completed_calls, + "total_llm_calls": int(progress.get("total_llm_calls") or 0), + } + ) + + try: + insights, overview = run_synthesis( + db, + organization_id, + provider, + model, + theme_counts=theme_counts, + sample_quotes=sample_quotes, + aggregate_summary=aggregate_summary, + total_sampled_rows=max(total_sampled, 1), + ) + except Exception as exc: # noqa: BLE001 + logger.error( + "User insights synthesis failed for evaluation {}: {}", + evaluation.id, + exc, + ) + return EvaluationUserInsightsState( + status="failed", + error_message=f"Synthesis failed: {exc}", + generated_at=datetime.now(timezone.utc), + llm_calls_used=completed_calls, + progress=progress, + provider=provider_label, + model=model, + ) + + completed_calls += SYNTHESIS_CALLS + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_USER_INSIGHTS, + job_run_id=job_run_id, + batch_index=SYNTHESIS_BATCH_INDEX_OFFSET, + payload={ + "insights": [item.model_dump(mode="json") for item in insights], + "overview": overview, + }, + status="synthesis_completed", + ) + + final_progress = { + **progress, + "phase": "completed", + "completed_llm_calls": completed_calls, + "completed_rows": int(progress.get("total_rows") or total_sampled or 0), + "completed_batches": int(progress.get("total_batches") or 0), + } + if on_progress: + on_progress(final_progress) + + if not insights: + return EvaluationUserInsightsState( + status="failed", + error_message="LLM returned no insight blocks.", + generated_at=datetime.now(timezone.utc), + llm_calls_used=completed_calls, + progress=final_progress, + provider=provider_label, + model=model, + ) + + return EvaluationUserInsightsState( + status="completed", + insights=insights, + overview=overview, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=evaluation.completed_rows, + progress=final_progress, + provider=provider_label, + model=model, + llm_calls_used=completed_calls, + is_stale=False, + ) + + +def process_user_insights_wave_step( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + provider, + model: str, + *, + metrics: Sequence[Metric], + aggregate: Sequence[Any], + transcript_source: str, + job_run_id: UUID, + on_progress: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCheck] = None, +) -> Tuple[str, Optional[EvaluationUserInsightsState]]: + """Run one wave step. Returns ``(next_action, state_if_terminal)``. + + ``next_action`` is ``continue`` (caller should re-enqueue), ``done``, or ``idle``. + """ + from app.services.call_import_user_insights import _metric_name_map + + raw = evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + if user_insights_raw_is_cancelled(raw): + return "idle", _cancelled_state(evaluation, raw) + + raw = reconcile_user_insights_job_header(db, evaluation, raw) + evaluation.user_insights = raw + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + phase = str(progress.get("phase") or "rows") + total_rows = int(progress.get("total_rows") or 0) + total_batches = int(progress.get("total_batches") or 0) + completed_batches = int(progress.get("completed_batches") or 0) + llm_calls_used = int(raw.get("llm_calls_used") or 0) + total_llm_calls = int(progress.get("total_llm_calls") or 0) or ( + estimate_user_insights_total_llm_calls(total_rows) + ) + + if phase == "synthesis" or ( + total_batches > 0 and completed_batches >= total_batches + ): + return "done", _finalize_synthesis( + db, + evaluation, + organization_id, + provider, + model, + job_run_id=job_run_id, + aggregate=aggregate, + raw_header=raw, + on_progress=on_progress, + is_cancelled=is_cancelled, + ) + + cursor = cursor_from_header(raw) + pairs = load_completed_evaluation_row_pairs_page( + db, + evaluation.id, + limit=EVALUATION_ANALYSIS_BATCH_SIZE, + after_cursor=cursor, + ) + if not pairs: + if completed_batches == 0: + return "done", EvaluationUserInsightsState( + status="failed", + error_message="No completed rows with metric scores to analyze.", + generated_at=datetime.now(timezone.utc), + progress=progress, + ) + raw["progress"] = {**progress, "phase": "synthesis"} + evaluation.user_insights = raw + return "done", _finalize_synthesis( + db, + evaluation, + organization_id, + provider, + model, + job_run_id=job_run_id, + aggregate=aggregate, + raw_header=raw, + on_progress=on_progress, + is_cancelled=is_cancelled, + ) + + metric_names = _metric_name_map(metrics) + row_payloads: List[Dict[str, Any]] = [] + skipped = 0 + for eval_row, source_row in pairs: + if eval_row.status != "completed": + continue + payload = build_row_payload_for_source( + evaluation, + eval_row, + source_row, + metric_names, + transcript_source, + ) + if not (payload.get("transcript") or "").strip(): + skipped += 1 + continue + row_payloads.append(payload) + + batch_index = completed_batches + rows_before_this_batch = rows_completed_for_batches(completed_batches, total_rows) + sub_batches = _sub_batch_payloads(row_payloads) + chunk_size = extraction_sub_batch_size(len(row_payloads)) if row_payloads else 0 + + # Resume inside a page: a worker killed by a time limit or redeploy mid-page + # must not re-pay for sub-batches that already checkpointed. + done_sub_batches: set[int] = set() + batch_extractions: List[Dict[str, Any]] = [] + for batch in list_analysis_batches( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_USER_INSIGHTS, + job_run_id=job_run_id, + ): + if batch.batch_index != batch_index or (batch.status or "") != "in_progress": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + raw_done = payload.get("sub_batches_done") + if isinstance(raw_done, list): + done_sub_batches = { + int(idx) for idx in raw_done if isinstance(idx, (int, float)) + } + restored = payload.get("extractions") + if isinstance(restored, list): + batch_extractions = [item for item in restored if isinstance(item, dict)] + break + batch_theme: Dict[str, int] = _aggregate_themes(batch_extractions) + + for sub_idx, sub_batch in enumerate(sub_batches): + if sub_idx in done_sub_batches: + continue + if is_cancelled and is_cancelled(): + return "idle", _cancelled_state( + evaluation, raw, llm_calls_used=llm_calls_used + ) + try: + extracted = run_extraction_batch( + db, organization_id, provider, model, sub_batch + ) + batch_extractions.extend(extracted) + batch_theme = _aggregate_themes(batch_extractions) + done_sub_batches.add(sub_idx) + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_USER_INSIGHTS, + job_run_id=job_run_id, + batch_index=batch_index, + payload={ + "extractions": batch_extractions, + "theme_counts": batch_theme, + "rows_processed": len(pairs), + "rows_skipped_empty_transcript": skipped, + "sub_batches_done": sorted(done_sub_batches), + }, + transcript_source=transcript_source, + status="in_progress", + ) + db.commit() + except Exception as exc: # noqa: BLE001 + logger.warning( + "User insights extraction sub-batch failed for evaluation {}: {}", + evaluation.id, + exc, + ) + llm_calls_used += 1 + if on_progress: + processed_in_batch = min((sub_idx + 1) * chunk_size, len(row_payloads)) + on_progress( + progress_header( + completed_rows=rows_before_this_batch + processed_in_batch, + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=llm_calls_used, + total_llm_calls=total_llm_calls, + phase="rows", + ) + ) + + if is_cancelled and is_cancelled(): + return "idle", _cancelled_state( + evaluation, raw, llm_calls_used=llm_calls_used + ) + + page_cursor = evaluation_row_page_cursor_from_pair(pairs[-1]) + upsert_analysis_batch( + db, + evaluation_id=evaluation.id, + job_kind=JOB_KIND_USER_INSIGHTS, + job_run_id=job_run_id, + batch_index=batch_index, + payload={ + "extractions": batch_extractions, + "theme_counts": batch_theme, + "rows_processed": len(pairs), + "rows_skipped_empty_transcript": skipped, + "sub_batches_done": list(range(len(sub_batches))), + }, + cursor=page_cursor, + transcript_source=transcript_source, + status="completed", + ) + + completed_batches += 1 + new_progress = progress_header( + completed_rows=rows_completed_for_batches(completed_batches, total_rows), + total_rows=total_rows, + completed_batches=completed_batches, + total_batches=total_batches, + completed_llm_calls=llm_calls_used, + total_llm_calls=total_llm_calls, + phase="rows" if completed_batches < total_batches else "synthesis", + ) + merge_theme_counts_into_header(raw, batch_theme) + + db.expire(evaluation, ["user_insights"]) + db.refresh(evaluation) + live = evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + if user_insights_raw_is_cancelled(live): + return "idle", _cancelled_state( + evaluation, live, llm_calls_used=llm_calls_used + ) + if not claim_still_owns_header(live, job_run_id): + logger.info( + "User insights page write skipped; run {} no longer owns the header ({})", + job_run_id, + live.get("job_run_id"), + ) + return "idle", None + + evaluation.user_insights = stamp_job_heartbeat({ + **live, + "status": "running", + "progress": new_progress, + "cursor": cursor_to_dict(page_cursor), + "llm_calls_used": llm_calls_used, + "theme_counts": raw.get("theme_counts", live.get("theme_counts")), + "job_run_id": str(job_run_id), + "transcript_source": live.get("transcript_source") or raw.get("transcript_source"), + "provider": live.get("provider") or raw.get("provider"), + "model": live.get("model") or raw.get("model"), + }) + flag_modified(evaluation, "user_insights") + db.commit() + + if on_progress: + on_progress(new_progress) + + return "continue", None + + +def start_user_insights_job_header( + evaluation: CallImportEvaluation, + db: Session, + *, + job_run_id: UUID, + transcript_source: str, + provider: Optional[str], + model: Optional[str], + prior_insights: Optional[List[Any]] = None, +) -> Dict[str, Any]: + total_rows = count_completed_evaluation_row_pairs(db, evaluation.id) + total_batches = batch_count_for_rows(total_rows) + header = init_job_header( + job_run_id=job_run_id, + transcript_source=normalize_transcript_source(transcript_source), + total_rows=total_rows, + total_batches=total_batches, + provider=provider, + model=model, + extra={ + "insights": prior_insights or [], + "generated_at_completed_rows": evaluation.completed_rows, + "llm_calls_used": 0, + "theme_counts": {}, + "cursor": None, + }, + ) + header["progress"]["total_llm_calls"] = estimate_user_insights_total_llm_calls( + total_rows + ) + return header + + +def resume_user_insights_job_header( + db: Session, + evaluation: CallImportEvaluation, + prior: Dict[str, Any], + *, + transcript_source: str, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> Dict[str, Any]: + """Re-open a checkpointed job without resetting batch row progress.""" + header = dict(prior) + header["status"] = "running" + header["error_message"] = None + header["celery_task_id"] = None + header["transcript_source"] = normalize_transcript_source(transcript_source) + if provider: + header["provider"] = provider + if model: + header["model"] = model + stamp_job_heartbeat(header) + return reconcile_user_insights_job_header(db, evaluation, header) diff --git a/app/services/call_imports/bulk_ops.py b/app/services/call_imports/bulk_ops.py index 8930a2ea..ab0146b2 100644 --- a/app/services/call_imports/bulk_ops.py +++ b/app/services/call_imports/bulk_ops.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional, Tuple +from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple from uuid import UUID, uuid4 from loguru import logger @@ -29,6 +29,8 @@ _BULK_INSERT_CHUNK = 1000 _REVOKE_TASK_BATCH = 500 +_IMPORT_ROW_FILTER_PAGE_SIZE = 500 +_MAX_EXPLICIT_EVAL_RETRY_IDS = 500 _ROW_TRANSCRIBE_COLUMNS = ( CallImportRow.id, @@ -107,13 +109,34 @@ def select_rows_for_transcription( return selected, skip_counts -def execute_bulk_diarization( +def _apply_transcription_selection( + rows: List[CallImportRow], + payload: CallImportTranscribeRequest, + skip_counts: Dict[str, int], +) -> List[CallImportRow]: + selected: List[CallImportRow] = [] + for row in rows: + recording = (row.recording_s3_key or "").strip() + if not recording: + skip_counts["no_recording"] = skip_counts.get("no_recording", 0) + 1 + continue + existing = (row.diarised_transcript or "").strip() + if existing and payload.only_missing and not payload.overwrite_existing: + skip_counts["transcript_present"] = ( + skip_counts.get("transcript_present", 0) + 1 + ) + continue + selected.append(row) + return selected + + +def _enqueue_diarization_for_rows( db: Session, call_import: CallImport, payload: CallImportTranscribeRequest, - requested_row_ids: Optional[List[UUID]] = None, + rows: List[CallImportRow], + skip_counts: Dict[str, int], ) -> BulkDiarizationResult: - """Store Redis params, then mark rows pending so dispatch never races ahead.""" from app.workers.concurrency.diarization_dispatch import ( _REDIS_PARAMS_STORE_ERROR, build_diarization_params_from_request, @@ -123,10 +146,6 @@ def execute_bulk_diarization( schedule_fair_diarization_dispatch, ) - rows, skip_counts = select_rows_for_transcription( - db, call_import, payload, requested_row_ids=requested_row_ids - ) - if not rows: return BulkDiarizationResult( queued=0, @@ -184,7 +203,8 @@ def execute_bulk_diarization( update_call_import_rows_on_shards(db, call_import.id, updates) if queued > 0: - schedule_fair_diarization_dispatch(max_workspace_turns=999) + # Delayed backup tick (bulk task also drains synchronously after commit). + schedule_fair_diarization_dispatch(max_workspace_turns=999, countdown=5) return BulkDiarizationResult( queued=queued, @@ -193,6 +213,178 @@ def execute_bulk_diarization( ) +def _id_chunks(ids: List[UUID]) -> Iterator[List[UUID]]: + size = _IMPORT_ROW_FILTER_PAGE_SIZE + for start in range(0, len(ids), size): + yield ids[start : start + size] + + +def execute_bulk_diarization( + db: Session, + call_import: CallImport, + payload: CallImportTranscribeRequest, + requested_row_ids: Optional[List[UUID]] = None, +) -> BulkDiarizationResult: + """Store Redis params, then mark rows pending so dispatch never races ahead.""" + if requested_row_ids: + rows, skip_counts = select_rows_for_transcription( + db, call_import, payload, requested_row_ids=requested_row_ids + ) + return _enqueue_diarization_for_rows( + db, call_import, payload, rows, skip_counts + ) + + from app.services.call_imports.row_selection import normalize_row_selection_filter + from app.db_sharding.scatter_gather import ( + list_call_import_row_ids_filtered, + load_call_import_rows_for_transcription, + ) + + search_term, diarised_status = normalize_row_selection_filter(payload.row_filter) + skip_counts: Dict[str, int] = {} + queued_total = 0 + + # Snapshot matching ids first: enqueueing moves rows to ``pending``, which + # drops them out of a status filter (e.g. retrying ``failed``), so offset + # paging over the live filter would skip every other page. + for chunk in _id_chunks( + list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status, + ) + ): + page = sorted( + load_call_import_rows_for_transcription( + db, call_import.id, requested_row_ids=chunk + ), + key=lambda row: int(row.row_index or 0), + ) + if not page: + continue + selected = _apply_transcription_selection(page, payload, skip_counts) + result = _enqueue_diarization_for_rows( + db, call_import, payload, selected, skip_counts + ) + queued_total += result.queued + skip_counts = dict(result.skipped_reason_counts) + + return BulkDiarizationResult( + queued=queued_total, + skipped_rows=sum(skip_counts.values()), + skipped_reason_counts=skip_counts, + ) + + +def execute_bulk_diarisation_cancel( + db: Session, + call_import_id: UUID, + *, + row_ids: Optional[List[UUID]] = None, + search_term: str = "", + diarised_status_filter: Optional[str] = None, + cancel_all_inflight: bool = False, +) -> Tuple[int, int]: + """Stop diarisation for many import rows (shard-aware when enabled). + + Returns ``(cancelled, skipped)``. ``skipped`` includes rows that were not + in a cancellable state and row ids that were not found. + """ + from app.db_sharding.row_ops import update_call_import_rows_on_shards + from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_filtered_page, + load_call_import_rows_for_cancel, + ) + from app.services.call_imports.diarisation_route_helpers import ( + CANCELLED_BY_USER_ERROR, + _cancellable_diarisation_states, + ) + + cancellable = {s.lower() for s in _cancellable_diarisation_states()} + cancelled = 0 + skipped = 0 + + def _cancel_loaded_rows(rows: List[CallImportRow]) -> None: + nonlocal cancelled, skipped + if not rows: + return + updates: List[dict] = [] + task_ids: List[str] = [] + seen_tasks: set[str] = set() + for row in rows: + status = (row.diarised_transcript_status or "").strip().lower() + if status not in cancellable: + skipped += 1 + continue + tid = (row.celery_task_id or "").strip() + if tid and tid not in seen_tasks: + seen_tasks.add(tid) + task_ids.append(tid) + updates.append( + { + "id": row.id, + "row_index": int(row.row_index or 0), + "diarised_transcript_status": "failed", + "diarised_transcript_error": CANCELLED_BY_USER_ERROR, + "celery_task_id": None, + } + ) + cancelled += 1 + if task_ids: + _batch_revoke_celery_task_ids(task_ids, terminate=True) + if updates: + update_call_import_rows_on_shards(db, call_import_id, updates) + + if cancel_all_inflight: + for status in _cancellable_diarisation_states(): + # Always offset=0: cancelled rows drop out of the status filter, so + # advancing offset would skip the next pending page (P1 bulk stop-all). + while True: + page = fetch_call_import_rows_filtered_page( + db, + call_import_id, + search_term="", + diarised_status_filter=status, + offset=0, + limit=_IMPORT_ROW_FILTER_PAGE_SIZE, + ) + if not page: + break + _cancel_loaded_rows(page) + return cancelled, skipped + + if row_ids is not None: + if not row_ids: + return 0, 0 + rows = load_call_import_rows_for_cancel(db, call_import_id, row_ids) + found_ids = {row.id for row in rows} + skipped += sum(1 for rid in row_ids if rid not in found_ids) + _cancel_loaded_rows(rows) + return cancelled, skipped + + drain_at_zero = (diarised_status_filter or "").strip().lower() in cancellable + offset = 0 + while True: + page = fetch_call_import_rows_filtered_page( + db, + call_import_id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + offset=0 if drain_at_zero else offset, + limit=_IMPORT_ROW_FILTER_PAGE_SIZE, + ) + if not page: + break + _cancel_loaded_rows(page) + if drain_at_zero: + continue + if len(page) < _IMPORT_ROW_FILTER_PAGE_SIZE: + break + offset += _IMPORT_ROW_FILTER_PAGE_SIZE + return cancelled, skipped + + def count_completed_source_rows(db: Session, call_import_id: UUID) -> int: from app.db_sharding.scatter_gather import count_completed_call_import_rows @@ -370,7 +562,20 @@ def materialize_and_enqueue_evaluation( db.commit() commit_pending_shard_sessions(pending_shard_sessions) evaluation.total_rows = committed + # Stamp the run start before any row is dispatched: the UI shows it, + # the stale-run sweep ages from it, and overwrite runs compare each + # row's ``diarised_at`` with it to tell fresh transcripts from old ones. + evaluation.started_at = evaluation.started_at or datetime.now(timezone.utc) db.commit() + from app.services.call_imports.eval_progress_buffer import seed_best_effort + + # Seed before any row is dispatched so ``remaining`` starts at total. + seed_best_effort( + evaluation_id, + gen=int(evaluation.progress_counter_gen or 0), + seq=int(evaluation.progress_flush_seq or 0), + remaining=committed, + ) except Exception as exc: committed = getattr(exc, "committed", committed) if shard_sessions is not None: @@ -386,6 +591,11 @@ def materialize_and_enqueue_evaluation( if shard_sessions is not None: shard_sessions.close_all() + if evaluation_run_will_bulk_reset_diarisation( + db, evaluation, transcribe_overwrite=transcribe_overwrite + ): + bulk_reset_diarisation_for_overwrite(db, evaluation.call_import_id) + try: _enqueue_eval_rows_with_optional_transcribe( db, @@ -410,7 +620,6 @@ def execute_bulk_row_delete( call_import: CallImport, row_ids: List[UUID], ) -> int: - """Revoke tasks, batch-delete blob recordings, bulk-delete DB rows.""" from app.api.v1.routes.call_imports import ( _delete_s3_objects, _recompute_call_import_counters, @@ -444,6 +653,31 @@ def execute_bulk_row_delete( return int(deleted or 0) +def execute_bulk_row_delete_by_filter( + db: Session, + organization_id: UUID, + call_import: CallImport, + *, + search_term: str = "", + diarised_status_filter: Optional[str] = None, +) -> int: + from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered + + # Snapshot ids first: each delete removes its page from the live results, + # so offset paging would skip every other page. + deleted_total = 0 + for chunk in _id_chunks( + list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + ) + ): + deleted_total += execute_bulk_row_delete(db, organization_id, call_import, chunk) + return deleted_total + + def _revoke_call_import_task_ids(db: Session, call_import_id: UUID) -> int: """Revoke in-flight Celery tasks for an import without loading full rows.""" task_id_rows = [ @@ -662,6 +896,18 @@ def execute_call_import_delete( } +def pending_source_row_indices(existing: set[int], row_count: int) -> list[int]: + """Source indexes in ``0 .. row_count-1`` that are not saved yet. + + A resumed import must fill holes below the highest saved index. Shard + commits are not one contiguous prefix: one shard can commit rows 0–249 + and 500–749 while the next shard's 250–499 never land. + """ + if row_count <= 0: + return [] + return [index for index in range(row_count) if index not in existing] + + def bulk_materialize_call_import_rows( db: Session, call_import: CallImport, @@ -670,6 +916,8 @@ def bulk_materialize_call_import_rows( *, defer_shard_commit: bool = False, shard_sessions=None, + row_index_offset: int = 0, + row_indices: List[int] | None = None, ) -> tuple[int, List[Session]]: """Insert import rows in chunks without per-row ORM overhead. @@ -681,14 +929,22 @@ def bulk_materialize_call_import_rows( if not parsed_rows: return 0, [] + if row_indices is not None and len(row_indices) != len(parsed_rows): + raise ValueError("row_indices length must match parsed rows") + if row_indices is not None: + indexed_rows = list(zip(row_indices, parsed_rows)) + else: + indexed_rows = [ + (int(row_index_offset) + offset, row) + for offset, row in enumerate(parsed_rows) + ] pending: List[Session] = [] committed = 0 - for start in range(0, len(parsed_rows), _BULK_INSERT_CHUNK): - chunk = parsed_rows[start : start + _BULK_INSERT_CHUNK] + for start in range(0, len(indexed_rows), _BULK_INSERT_CHUNK): + chunk = indexed_rows[start : start + _BULK_INSERT_CHUNK] mappings = [] - for offset, row in enumerate(chunk): - idx = start + offset + for idx, row in chunk: csv_transcript = row["transcript"] mappings.append( { @@ -793,9 +1049,7 @@ def execute_call_import_materialization( if not is_sharding_enabled() else None ) - if existing_rows is not None or ( - is_sharding_enabled() and int(call_import.total_rows or 0) > 0 - ): + if existing_rows is not None: if call_import.status == CallImportStatus.FAILED: return { "total_rows": int(call_import.total_rows or 0), @@ -897,26 +1151,56 @@ def execute_call_import_materialization( call_import.failed_rows = 0 row_count = len(parse_result.rows) + rows_payload = parse_result.rows + resume_indices: List[int] | None = None + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import load_call_import_row_indices + + existing_indices = load_call_import_row_indices(db, call_import.id) + missing_indices = pending_source_row_indices(existing_indices, row_count) + if not missing_indices: + call_import.total_rows = row_count + db.commit() + if schedule_import_dispatch: + _enqueue_row_tasks(db, call_import, []) + return { + "total_rows": row_count, + "status": "already_materialized", + "source_rows_skipped": skipped_count, + } + if len(missing_indices) < row_count: + rows_payload = [parse_result.rows[index] for index in missing_indices] + resume_indices = missing_indices + logger.info( + "execute_call_import_materialization: resuming import {} " + "for {} missing row index(es) starting at {}", + call_import_id, + len(missing_indices), + missing_indices[0], + ) + committed_rows = 0 shard_sessions = None try: if is_sharding_enabled(): from app.db_sharding.row_ops import ShardWriteSessions, register_shard_slices - register_shard_slices(db, call_import.id, row_count) - db.commit() - from app.db_sharding.registry import publish_slice_registry_change + if resume_indices is None: + register_shard_slices(db, call_import.id, row_count) + db.commit() + from app.db_sharding.registry import publish_slice_registry_change - publish_slice_registry_change(call_import.id) + publish_slice_registry_change(call_import.id) shard_sessions = ShardWriteSessions() committed_rows, _pending = bulk_materialize_call_import_rows( db, call_import, - parse_result.rows, + rows_payload, organization_id, shard_sessions=shard_sessions, + row_indices=resume_indices, ) - call_import.total_rows = committed_rows + call_import.total_rows = row_count db.commit() else: from app.db_sharding.row_ops import register_shard_slices @@ -925,7 +1209,7 @@ def execute_call_import_materialization( committed_rows, _pending = bulk_materialize_call_import_rows( db, call_import, - parse_result.rows, + rows_payload, organization_id, ) register_shard_slices(db, call_import.id, row_count) @@ -1079,7 +1363,7 @@ def _call_import_status_from_evaluation_status(eval_status: str) -> CallImportSt return CallImportStatus.COMPLETED if normalized == "partial": return CallImportStatus.PARTIAL - if normalized in ("failed", "cancelled"): + if normalized in ("failed", "cancelled", "auto_aborted"): return CallImportStatus.FAILED if normalized in ("pending", "running"): return CallImportStatus.PROCESSING @@ -1127,6 +1411,7 @@ def _latest_evaluation_progress_for_ui( "latest_evaluation_total_rows": None, "latest_evaluation_completed_rows": None, "latest_evaluation_failed_rows": None, + "latest_evaluation_bulk_operation": None, } ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) total = int(row.total_rows or 0) @@ -1136,15 +1421,57 @@ def _latest_evaluation_progress_for_ui( if hasattr(status, "value"): status = status.value transcript_source = row.transcript_source + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + return { "latest_evaluation_status": status, "latest_evaluation_transcript_source": transcript_source, "latest_evaluation_total_rows": total, "latest_evaluation_completed_rows": ui_completed, "latest_evaluation_failed_rows": ui_failed, + "latest_evaluation_bulk_operation": get_evaluation_bulk_operation(row.id), } +def _any_evaluation_needs_ui_poll(db: Session, call_import_id: UUID) -> bool: + """True when any evaluation on this import still needs UI polling.""" + from sqlalchemy.orm import load_only + + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + rows = ( + db.query(CallImportEvaluation) + .options( + load_only( + CallImportEvaluation.id, + CallImportEvaluation.status, + CallImportEvaluation.total_rows, + CallImportEvaluation.completed_rows, + CallImportEvaluation.failed_rows, + ) + ) + .filter(CallImportEvaluation.call_import_id == call_import_id) + .all() + ) + for row in rows: + if get_evaluation_bulk_operation(row.id): + return True + total = int(row.total_rows or 0) + accounted = int(row.completed_rows or 0) + int(row.failed_rows or 0) + if total > accounted: + return True + status = row.status + if hasattr(status, "value"): + status = status.value + if str(status) in ("pending", "running"): + return True + return False + + def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> None: """Recompute batch counters and terminal status on the parent import. @@ -1153,16 +1480,36 @@ def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> Non rows were never enqueued on the legacy import worker and must not keep the parent stuck in ``processing`` once every evaluation run is terminal. """ - + db.flush() + db.expire_all() total, completed, failed, pending, processing = _aggregate_import_row_status_counts( db, call_import.id ) pending_or_processing = pending + processing + from sqlalchemy import update + + db.execute( + update(CallImport) + .where(CallImport.id == call_import.id) + .values( + total_rows=total, + completed_rows=completed, + failed_rows=failed, + ) + ) + db.expire( + call_import, + ["total_rows", "completed_rows", "failed_rows"], + ) call_import.total_rows = total call_import.completed_rows = completed call_import.failed_rows = failed + from app.services.call_imports.progress_counters import clear_import_progress_redis + + clear_import_progress_redis(call_import.id) + if call_import.status == CallImportStatus.DELETING: return @@ -1207,8 +1554,14 @@ def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> Non from app.services.call_imports.debounced_rollup import ( platform_discovery_blocks_terminal_status, ) + from app.services.call_imports.platform_discovery_sync import ( + sync_platform_discovery_when_import_rows_idle, + ) + + sync_platform_discovery_when_import_rows_idle(db, call_import) + db.refresh(call_import) - if platform_discovery_blocks_terminal_status(call_import): + if platform_discovery_blocks_terminal_status(call_import, db=db): call_import.status = CallImportStatus.PROCESSING return @@ -1221,10 +1574,6 @@ def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> Non else: call_import.status = CallImportStatus.PARTIAL - from app.services.call_imports.progress_counters import clear_import_progress_redis - - clear_import_progress_redis(call_import.id) - def count_evaluation_cancel_targets( db: Session, @@ -1353,26 +1702,119 @@ def _batch_revoke_celery_task_ids( terminate: bool, ) -> int: """Revoke Celery tasks in batches; best-effort on control-plane errors.""" - cleaned = [tid for tid in task_ids if (tid or "").strip()] - if not cleaned: - return 0 - try: - from app.workers.celery_app import celery_app + from app.workers.celery_revoke import revoke_celery_tasks - revoked = 0 - for start in range(0, len(cleaned), _REVOKE_TASK_BATCH): - chunk = cleaned[start : start + _REVOKE_TASK_BATCH] - if terminate: - celery_app.control.revoke( - chunk, terminate=True, signal="SIGTERM" - ) - else: - celery_app.control.revoke(chunk, terminate=False) - revoked += len(chunk) - return revoked - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to batch-revoke {} Celery tasks: {}", len(cleaned), exc) - return 0 + return revoke_celery_tasks( + task_ids, + terminate=None if terminate else False, + ) + + +_OVERWRITE_DIA_RESET_VALUES = { + "diarised_transcript": None, + "diarised_segments": None, + "diarised_native_transcript": None, + # ``idle`` — not ``pending``. Standalone fair diarisation only claims + # ``pending`` rows with stored modal params; eval dispatch flips to + # ``pending`` when it reserves a slot and enqueues the eval-chain worker. + "diarised_transcript_status": "idle", + "diarised_transcript_error": None, + # Clear the stamp so the new pass writes its own. Dispatch compares it with + # the run's ``started_at`` to tell fresh text from text from an older run. + "diarised_at": None, + "celery_task_id": None, +} + + +def evaluation_run_will_bulk_reset_diarisation( + db: Session, + evaluation: CallImportEvaluation, + *, + transcribe_overwrite: bool, +) -> bool: + """True when an overwrite run should clear stored diarised transcripts up front.""" + if not transcribe_overwrite: + return False + transcript_source = ( + (getattr(evaluation, "transcript_source", None) or "diarised") + .strip() + .lower() + ) + if transcript_source == "production": + from app.workers.tasks.evaluate_call_import_row_core import ( + evaluation_needs_diarised_reference, + ) + + return evaluation_needs_diarised_reference(db, evaluation) + return True + + +def bulk_reset_diarisation_for_overwrite( + db: Session, + call_import_id: UUID, +) -> int: + """Clear diarised transcripts and mark rows idle before overwrite rediarise. + + Run Evaluation with ``transcribe_overwrite`` must reset every row that will + be re-diarised up front. Rows are left ``idle`` (not ``pending``) so + standalone fair diarisation — which only claims ``pending`` rows with modal + params in Redis — cannot steal eval-owned work. + """ + from app.db_sharding.row_ops import shard_row_write_context + from app.db_sharding.scatter_gather import scatter_gather_on_shards, shard_ids_for_import + + task_ids: List[str] = [] + seen_tasks: set[str] = set() + total = 0 + + def _reset_on_session(session: Session, _shard_id: str) -> int: + rows = ( + session.query(CallImportRow.id, CallImportRow.celery_task_id) + .filter( + CallImportRow.call_import_id == call_import_id, + CallImportRow.recording_s3_key.isnot(None), + CallImportRow.recording_s3_key != "", + ) + .all() + ) + for _row_id, raw_task in rows: + cleaned = (raw_task or "").strip() + if cleaned and cleaned not in seen_tasks: + seen_tasks.add(cleaned) + task_ids.append(cleaned) + if not rows: + return 0 + row_ids = [row[0] for row in rows] + with shard_row_write_context(session): + result = session.execute( + update(CallImportRow) + .where(CallImportRow.id.in_(row_ids)) + .values(**_OVERWRITE_DIA_RESET_VALUES), + execution_options={"synchronize_session": False}, + ) + session.commit() + count = int(result.rowcount or 0) + return count if count >= 0 else len(row_ids) + + if not is_sharding_enabled(): + total = _reset_on_session(db, "legacy") + else: + for count in scatter_gather_on_shards( + shard_ids_for_import(db, call_import_id), + _reset_on_session, + ): + total += count + + if task_ids: + _batch_revoke_celery_task_ids(task_ids, terminate=False) + if total: + logger.info( + "bulk_reset_diarisation_for_overwrite call_import={} rows={} revoked_tasks={}", + call_import_id, + total, + len(task_ids), + ) + return total def execute_evaluation_cancel( @@ -1380,9 +1822,14 @@ def execute_evaluation_cancel( evaluation_id: UUID, *, mode: Literal["abort", "force_fail_pending"], + row_error_message: str | None = None, + sticky_status: str | None = None, ) -> dict: """Cancel eval rows in chunks off the API thread.""" - from app.api.v1.routes.call_import_evaluations import _rollup_evaluation_status + from app.api.v1.routes.call_import_evaluations import ( + EVAL_CANCELLED_BY_USER_ERROR, + _rollup_evaluation_status, + ) evaluation = ( db.query(CallImportEvaluation) @@ -1396,18 +1843,29 @@ def execute_evaluation_cancel( ) return {"evaluation_id": str(evaluation_id), "cancelled": 0, "mode": mode} + eval_error_message = row_error_message or EVAL_CANCELLED_BY_USER_ERROR + try: + if sticky_status and (evaluation.status or "").strip().lower() != "running": + return { + "evaluation_id": str(evaluation_id), + "cancelled": 0, + "mode": mode, + } + cancelled_total = 0 if is_sharding_enabled(): cancelled_total = _cancel_evaluation_rows_all_shards( evaluation_id, mode=mode, + eval_error_message=eval_error_message, ) else: cancelled_total = _cancel_evaluation_rows_on_session( db, evaluation_id, mode=mode, + eval_error_message=eval_error_message, ) if mode == "abort": @@ -1419,6 +1877,13 @@ def execute_evaluation_cancel( db.refresh(evaluation) _rollup_evaluation_status(evaluation, db) + if ( + sticky_status + and cancelled_total > 0 + and (evaluation.status or "").strip().lower() + not in {"pending", "running"} + ): + evaluation.status = sticky_status db.commit() return { @@ -1450,9 +1915,9 @@ def _write_cancel_page( status: str, source_ids: List[UUID], now: datetime, + eval_error_message: str, ) -> int: """One UPDATE per table for this page. Eval rowcount, or source rowcount when the page is diarisation-only.""" - from app.api.v1.routes.call_import_evaluations import EVAL_CANCELLED_BY_USER_ERROR from app.api.v1.routes.call_imports import CANCELLED_BY_USER_ERROR from app.db_sharding.row_ops import shard_row_write_context @@ -1469,7 +1934,7 @@ def _write_cancel_page( ) .values( status="failed", - error_message=EVAL_CANCELLED_BY_USER_ERROR, + error_message=eval_error_message, finished_at=now, celery_task_id=None, ), @@ -1542,6 +2007,7 @@ def _sweep_diarisation_on_session( status=status, source_ids=page_ids, now=datetime.now(timezone.utc), + eval_error_message="", ) _batch_revoke_celery_task_ids(task_ids, terminate=True) if updated == 0: @@ -1592,6 +2058,7 @@ def _cancel_evaluation_rows_on_session( evaluation_id: UUID, *, mode: Literal["abort", "force_fail_pending"], + eval_error_message: str, ) -> int: statuses = ("pending", "running") if mode == "abort" else ("pending",) cascade_diarisation = mode == "abort" @@ -1633,6 +2100,7 @@ def _cancel_evaluation_rows_on_session( status=status, source_ids=source_ids, now=datetime.now(timezone.utc), + eval_error_message=eval_error_message, ) _batch_revoke_celery_task_ids(task_ids, terminate=True) if updated == 0: @@ -1650,6 +2118,7 @@ def _cancel_evaluation_rows_all_shards( evaluation_id: UUID, *, mode: Literal["abort", "force_fail_pending"], + eval_error_message: str, ) -> int: from app.db_sharding.pool_manager import db_pool_manager @@ -1664,6 +2133,7 @@ def _cancel_evaluation_rows_all_shards( shard_db, evaluation_id, mode=mode, + eval_error_message=eval_error_message, ) finally: shard_db.close() @@ -2001,6 +2471,8 @@ def execute_evaluation_retry( metric_ids: Optional[List[UUID]] = None, include_completed: bool = False, transcribe_overwrite: bool = False, + retry_search_term: str = "", + retry_status_filter: Optional[str] = None, ) -> dict: """Page failed rows, bulk-reset them, then start throttled dispatch.""" from app.api.v1.routes.call_import_evaluations import _rollup_evaluation_status @@ -2027,14 +2499,35 @@ def execute_evaluation_retry( ) return {"evaluation_id": str(evaluation_id), "requeued": 0} + # Retry overwrite is per retried row only (``_apply_source_retry_updates``). + # Do not call ``bulk_reset_diarisation_for_overwrite`` here — that resets + # every row on the import and belongs on a full Run Evaluation only. + include = include_completed or bool(metric_ids) statuses = ["failed", "completed"] if include else ["failed"] + if retry_status_filter: + statuses = [retry_status_filter.strip().lower()] + search_term = (retry_search_term or "").strip() restricted = [str(mid) for mid in metric_ids] if metric_ids else None eval_pk = evaluation.id import_id = evaluation.call_import_id eval_only = 0 transcribe = 0 scheduled_early = False + + reopened_status: Optional[str] = None + if (getattr(evaluation, "status", None) or "").strip().lower() in { + "completed", + "failed", + "partial", + }: + # Rows reset below can be dispatched (and finish) before the final + # recount. Mark the parent running first so finalize triggered by + # those rows treats the run as open rather than already closed. + reopened_status = evaluation.status + evaluation.status = "running" + evaluation.finished_at = None + db.commit() seen: set[UUID] = set() orphans: List[UUID] = [] shard_reset: List[UUID] = [] @@ -2116,23 +2609,36 @@ def _drain_session(session: Session, shard_id: Optional[str]) -> int: _reset_loaded_page(session, raw) else: for status in statuses: + after_id: Optional[UUID] = None while True: raw = load_retry_target_page( session, eval_pk, status=status, - exclude_ids=orphans or None, + after_id=after_id, include_metric_scores=bool(metric_ids), - ordered=False, + ordered=True, limit=RETRY_ROW_PAGE_SIZE, + search_term=search_term, ) + if raw: + page_last_id = raw[-1][0].id + else: + page_last_id = None action = _reset_loaded_page(session, raw) if action in {"empty", "stop"}: break + if page_last_id is not None: + after_id = page_last_id if shard_reset: reset_groups.append((shard_id, shard_reset)) return len(shard_reset) + # Dispatch skips evaluations that hold a bulk-operation lock, and the + # ``retry`` lock is only released in ``finally`` below. Dispatch requested + # while it is held can run first, skip this run and never be re-triggered + # (nothing else is in flight after an abort), so ask again once released. + dispatch_after_release = False try: saw_rows = 0 for session, shard_id in iter_evaluation_row_sessions(db, import_id): @@ -2142,6 +2648,9 @@ def _drain_session(session: Session, shard_id: Optional[str]) -> int: requeued = eval_only + transcribe if requeued == 0: + if reopened_status is not None: + _rollup_evaluation_status(evaluation, db) + db.commit() return { "evaluation_id": str(evaluation_id), "requeued": 0, @@ -2173,6 +2682,7 @@ def _drain_session(session: Session, shard_id: Optional[str]) -> int: evaluation.finished_at = None _rollup_evaluation_status(evaluation, db) db.commit() + dispatch_after_release = True return { "evaluation_id": str(evaluation_id), "requeued": requeued, @@ -2185,6 +2695,8 @@ def _drain_session(session: Session, shard_id: Optional[str]) -> int: ) clear_evaluation_bulk_operation(evaluation_id) + if dispatch_after_release: + schedule_fair_dispatch(max_workspace_turns=999) _IMPORT_RETRY_COLUMNS = ( diff --git a/app/services/call_imports/debounced_rollup.py b/app/services/call_imports/debounced_rollup.py index 757e6e9d..cd26d7f4 100644 --- a/app/services/call_imports/debounced_rollup.py +++ b/app/services/call_imports/debounced_rollup.py @@ -55,11 +55,24 @@ def _schedule_trailing_import_rollup(call_import_id: UUID) -> None: logger.debug("import trailing rollup enqueue failed: {}", exc) -def platform_discovery_blocks_terminal_status(call_import) -> bool: +def platform_discovery_blocks_terminal_status( + call_import, + db: Session | None = None, +) -> bool: if (getattr(call_import, "source_format", None) or "").lower() != "platform": return False discovery = (getattr(call_import, "discovery_status", None) or "").strip().lower() - return discovery in {"queued", "running"} + if discovery not in {"queued", "running"}: + return False + if db is not None: + from app.services.call_imports.bulk_ops import _aggregate_import_row_status_counts + + total, _completed, _failed, pending, processing = ( + _aggregate_import_row_status_counts(db, call_import.id) + ) + if total > 0 and pending == 0 and processing == 0: + return False + return True def maybe_record_import_row_transition( @@ -67,6 +80,7 @@ def maybe_record_import_row_transition( *, previous_status: Optional[str], new_status: Optional[str], + db: Session | None = None, ) -> None: if previous_status is None or new_status is None: return @@ -82,6 +96,28 @@ def maybe_record_import_row_transition( call_import_id, previous_status=prev, new_status=new, + db=db, + ) + + +def record_import_progress_before_row_commit( + call_import_id: UUID, + *, + previous_status, + new_status, + db: Session | None = None, +) -> None: + """Record Redis progress delta before the row terminal status is committed. + + Debounced rollup recounts only see committed rows. Recording after commit + lets another worker rollup (SQL includes the row, then clear Redis) race + ahead of this worker's +1 and inflate ``merge_import_counters_for_ui``. + """ + maybe_record_import_row_transition( + call_import_id, + previous_status=previous_status, + new_status=new_status, + db=db, ) @@ -91,22 +127,30 @@ def schedule_debounced_import_rollup( *, force: bool = False, ) -> None: - """Flush Redis deltas and optionally run full status aggregate.""" + """Run debounced full status aggregate (Redis deltas flush on recount only).""" from app.models.database import CallImport from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - from app.services.call_imports.progress_counters import flush_import_progress_to_catalog cid = call_import.id if hasattr(call_import, "id") else call_import if not isinstance(cid, UUID): cid = UUID(str(cid)) - flush_import_progress_to_catalog(db, cid) + parent = db.query(CallImport).filter(CallImport.id == cid).first() + if parent is None: + return + + from app.services.call_imports.platform_discovery_sync import ( + sync_platform_discovery_when_import_rows_idle, + ) + + sync_platform_discovery_when_import_rows_idle(db, parent) + db.refresh(parent) - if platform_discovery_blocks_terminal_status(call_import): + if platform_discovery_blocks_terminal_status(parent, db=db): from app.models.enums import CallImportStatus - if call_import.status != CallImportStatus.DELETING: - call_import.status = CallImportStatus.PROCESSING + if parent.status != CallImportStatus.DELETING: + parent.status = CallImportStatus.PROCESSING return should_aggregate = force @@ -128,7 +172,4 @@ def schedule_debounced_import_rollup( _schedule_trailing_import_rollup(cid) return - parent = db.query(CallImport).filter(CallImport.id == cid).first() - if parent is None: - return rollup_call_import_batch_status(db, parent) diff --git a/app/services/call_imports/diarisation_route_helpers.py b/app/services/call_imports/diarisation_route_helpers.py new file mode 100644 index 00000000..ce20d095 --- /dev/null +++ b/app/services/call_imports/diarisation_route_helpers.py @@ -0,0 +1,84 @@ +"""Diarisation cancel/render helpers (used by call-import API routes).""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from loguru import logger + +from app.models.database import CallImportRow + +CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" + + +def _cancellable_diarisation_states() -> Tuple[str, ...]: + """States that a diarisation row can be cancelled from.""" + return ("pending", "running") + + +def _revoke_diarisation_task(row: CallImportRow) -> None: + """Best-effort revoke of a single row's diarisation Celery task.""" + task_id = (row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([task_id]) + logger.info( + "Revoked diarisation task {} for call-import row {}", + task_id, + row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke diarisation task {} for row {}: {}", + task_id, + row.id, + exc, + ) + + +def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: + """Cancel diarisation on every cancellable row in ``rows``. + + Returns ``(cancelled, skipped)``. Caller owns ``db.commit()``. + """ + cancellable_states = _cancellable_diarisation_states() + cancelled = 0 + skipped = 0 + for row in rows: + if (row.diarised_transcript_status or "").lower() not in cancellable_states: + skipped += 1 + continue + row.diarised_transcript_status = "failed" + row.diarised_transcript_error = CANCELLED_BY_USER_ERROR + _revoke_diarisation_task(row) + row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +def _render_diarised_segments_text( + segments: Optional[List[Dict[str, Any]]], + *, + swap: bool = False, +) -> str: + """Render ``CallImportRow.diarised_segments`` as ``: `` lines.""" + if not segments: + return "" + out: List[str] = [] + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = (turn.get("speaker") or "").strip() + text = (turn.get("text") or "").strip() + if not speaker or not text: + continue + if swap: + if speaker == "agent": + speaker = "user" + elif speaker == "user": + speaker = "agent" + out.append(f"{speaker}: {text}") + return "\n".join(out) diff --git a/app/services/call_imports/eval_flexprice_billing.py b/app/services/call_imports/eval_flexprice_billing.py new file mode 100644 index 00000000..5386338d --- /dev/null +++ b/app/services/call_imports/eval_flexprice_billing.py @@ -0,0 +1,531 @@ +"""Deferred Flexprice metering for call-import evaluation workers. + +Recording minutes are summed in Redis and sent as about 50 events per +evaluation, not one event per row. Each event's ``quantity`` is the sum of +per-row billable minutes, so the meter total matches per-row billing. + +A full batch is copied to a sealed Redis key in the same script that clears +the counters. The sealed payload is deleted only after it is on the Flexprice +queue, so a crash between those steps retries the same batch. +""" + +from __future__ import annotations + +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + CallImportRow, +) + +# Split every evaluation into this many recording-minute events (runs smaller +# than the target still send one event per audio row). +RECORDING_MINUTES_TARGET_EVENTS = 50 +_BATCH_TTL_SECONDS = 14 * 24 * 60 * 60 +_ROW_BILLED_TTL_SECONDS = 90 * 24 * 60 * 60 + +_redis: redis.Redis | None = None + +_ACCUMULATE_LUA = """ +if redis.call('EXISTS', KEYS[2]) == 1 then + return {'pending_sealed'} +end +if redis.call('SET', KEYS[3], '1', 'NX', 'EX', tonumber(ARGV[9])) == false then + return {'duplicate'} +end +local seconds = tonumber(ARGV[1]) +local minutes = tonumber(ARGV[2]) +local batch_size = tonumber(ARGV[3]) +local ttl = tonumber(ARGV[7]) +redis.call('HSET', KEYS[1], + 'organization_id', ARGV[4], + 'workspace_id', ARGV[5], + 'call_import_id', ARGV[6], + 'total_rows', ARGV[8], + 'batch_size', ARGV[3]) +redis.call('HINCRBY', KEYS[1], 'audio_seconds', seconds) +redis.call('HINCRBY', KEYS[1], 'billable_minutes', minutes) +local rows = redis.call('HINCRBY', KEYS[1], 'rows', 1) +redis.call('EXPIRE', KEYS[1], ttl) +if rows < batch_size then + return {} +end +local audio = redis.call('HGET', KEYS[1], 'audio_seconds') +local mins = redis.call('HGET', KEYS[1], 'billable_minutes') +local counted = redis.call('HGET', KEYS[1], 'rows') +local idx = redis.call('HINCRBY', KEYS[1], 'batch_index', 1) +local org = redis.call('HGET', KEYS[1], 'organization_id') +local ws = redis.call('HGET', KEYS[1], 'workspace_id') +local imp = redis.call('HGET', KEYS[1], 'call_import_id') +local total_rows = redis.call('HGET', KEYS[1], 'total_rows') or '0' +local planned_batch = redis.call('HGET', KEYS[1], 'batch_size') or '0' +local payload = table.concat({audio, mins, counted, idx, org, ws, imp, total_rows, planned_batch}, '|') +redis.call('SET', KEYS[2], payload, 'EX', ttl) +redis.call('HSET', KEYS[1], 'audio_seconds', 0, 'billable_minutes', 0, 'rows', 0) +return {'sealed'} +""" + +_FLUSH_LUA = """ +if redis.call('EXISTS', KEYS[2]) == 1 then + return {'pending_sealed'} +end +local rows = tonumber(redis.call('HGET', KEYS[1], 'rows') or '0') +if rows <= 0 then + return {} +end +local ttl = tonumber(ARGV[1]) +local audio = redis.call('HGET', KEYS[1], 'audio_seconds') +local mins = redis.call('HGET', KEYS[1], 'billable_minutes') +local counted = redis.call('HGET', KEYS[1], 'rows') +local idx = redis.call('HINCRBY', KEYS[1], 'batch_index', 1) +local org = redis.call('HGET', KEYS[1], 'organization_id') +local ws = redis.call('HGET', KEYS[1], 'workspace_id') +local imp = redis.call('HGET', KEYS[1], 'call_import_id') +local total_rows = redis.call('HGET', KEYS[1], 'total_rows') or '0' +local planned_batch = redis.call('HGET', KEYS[1], 'batch_size') or '0' +local payload = table.concat({audio, mins, counted, idx, org, ws, imp, total_rows, planned_batch}, '|') +redis.call('SET', KEYS[2], payload, 'EX', ttl) +redis.call('HSET', KEYS[1], 'audio_seconds', 0, 'billable_minutes', 0, 'rows', 0) +return {'sealed'} +""" + + +def recording_minutes_batch_size( + total_rows: int, + *, + target_events: int = RECORDING_MINUTES_TARGET_EVENTS, +) -> int: + """Rows of audio to fold into one Flexprice event. + + ``ceil(total_rows / target_events)`` so a run of at least ``target_events`` + rows produces about that many events (1e6 → 20k rows/event → 50 events, + 50k → 1k rows/event → 50 events). Fewer rows than the target cannot fill + 50 events, so those stay one row per event. + """ + rows = max(0, int(total_rows or 0)) + target = max(1, int(target_events)) + if rows <= 0: + return 1 + return max(1, (rows + target - 1) // target) + + +def _client() -> redis.Redis: + global _redis + if _redis is None: + _redis = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis + + +def _batch_key(evaluation_id: UUID) -> str: + return f"flexprice:eval_audio_batch:{evaluation_id}" + + +def _sealed_key(evaluation_id: UUID) -> str: + return f"flexprice:eval_audio_sealed:{evaluation_id}" + + +def _row_billed_key(eval_row_id: UUID) -> str: + return f"flexprice:eval_audio_row:{eval_row_id}" + + +def _claim_row_for_billing(eval_row_id: UUID) -> bool: + """True only the first time this evaluation row is counted.""" + try: + claimed = _client().set( + _row_billed_key(eval_row_id), + "1", + nx=True, + ex=_ROW_BILLED_TTL_SECONDS, + ) + except redis.RedisError: + return False + return bool(claimed) + + +def _release_row_billing_claim(eval_row_id: UUID) -> None: + try: + _client().delete(_row_billed_key(eval_row_id)) + except redis.RedisError: + pass + + +def _fallback_row_event( + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + *, + workspace_id, + audio_seconds: int, + billable_minutes: int, +) -> None: + if not _claim_row_for_billing(eval_row.id): + return + from app.services.billing.flexprice_service import ( + buffer_call_import_recording_minutes_billed, + ) + + accepted = buffer_call_import_recording_minutes_billed( + evaluation.organization_id, + eval_row.id, + workspace_id=workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + if not accepted: + _release_row_billing_claim(eval_row.id) + + +def _enqueue_sealed_payload(evaluation_id: UUID, payload: str) -> bool: + parts = str(payload).split("|") + if len(parts) < 7: + logger.warning( + "Flexprice sealed batch unreadable evaluation={}", + evaluation_id, + ) + return True + audio_seconds = int(parts[0] or 0) + billable_minutes = int(parts[1] or 0) + rows_in_batch = int(parts[2] or 0) + batch_index = int(parts[3] or 0) + organization_id = parts[4] + workspace_id = parts[5] + call_import_id = parts[6] + total_rows = int(parts[7] or 0) if len(parts) > 7 else 0 + batch_size = int(parts[8] or 0) if len(parts) > 8 else 0 + if billable_minutes <= 0 or not organization_id or not workspace_id: + logger.warning( + "Flexprice sealed batch dropped evaluation={} payload={}", + evaluation_id, + payload[:120], + ) + return True + from app.services.billing.flexprice_service import ( + buffer_call_import_recording_minutes_batch, + ) + + return buffer_call_import_recording_minutes_batch( + UUID(str(organization_id)), + workspace_id=UUID(str(workspace_id)), + evaluation_id=evaluation_id, + call_import_id=UUID(str(call_import_id)) if call_import_id else evaluation_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + rows_in_batch=rows_in_batch, + batch_index=batch_index, + batch_size=batch_size, + evaluation_total_rows=total_rows, + ) + + +_DELETE_SEAL_IF_MATCH_LUA = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" + + +def _publish_sealed(evaluation_id: UUID) -> bool: + """Move a sealed batch onto the Flexprice queue, then delete the seal.""" + key = _sealed_key(evaluation_id) + try: + payload = _client().get(key) + except redis.RedisError as exc: + logger.warning( + "Flexprice sealed batch read failed evaluation={}: {}", + evaluation_id, + exc, + ) + return False + if not payload: + return True + accepted = _enqueue_sealed_payload(evaluation_id, payload) + if not accepted: + try: + _client().expire(key, _BATCH_TTL_SECONDS) + except redis.RedisError: + pass + return False + try: + deleted = _client().eval( + _DELETE_SEAL_IF_MATCH_LUA, + 1, + key, + payload, + ) + if not deleted: + logger.warning( + "Flexprice sealed batch delete skipped evaluation={} (payload changed)", + evaluation_id, + ) + except redis.RedisError as exc: + logger.warning( + "Flexprice sealed batch delete failed evaluation={}: {}", + evaluation_id, + exc, + ) + return True + + +def republish_sealed_recording_batches(*, max_keys: int = 50) -> int: + """Queue sealed batches left behind by a crash. Safe to run on a timer.""" + published = 0 + try: + client = _client() + cursor = 0 + seen = 0 + while seen < max_keys: + cursor, keys = client.scan( + cursor=cursor, + match="flexprice:eval_audio_sealed:*", + count=max_keys, + ) + for key in keys: + if seen >= max_keys: + break + seen += 1 + raw_id = str(key).rsplit(":", 1)[-1] + try: + evaluation_id = UUID(raw_id) + except ValueError: + continue + if _publish_sealed(evaluation_id): + published += 1 + if cursor in (0, "0", b"0"): + break + except redis.RedisError as exc: + logger.warning("Flexprice sealed batch scan failed: {}", exc) + return published + + +def enqueue_recording_minutes_for_completed_row( + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + *, + audio_seconds: int, + billable_minutes: int, +) -> None: + """Add this row's minutes to the evaluation batch; emit when the batch fills.""" + if billable_minutes <= 0: + return + batch_size = recording_minutes_batch_size(int(evaluation.total_rows or 0)) + workspace_id = eval_row.workspace_id or evaluation.workspace_id + + def _accumulate(): + return _client().eval( + _ACCUMULATE_LUA, + 3, + _batch_key(evaluation.id), + _sealed_key(evaluation.id), + _row_billed_key(eval_row.id), + str(max(0, int(audio_seconds or 0))), + str(int(billable_minutes)), + str(batch_size), + str(evaluation.organization_id), + str(workspace_id), + str(evaluation.call_import_id), + str(_BATCH_TTL_SECONDS), + str(int(evaluation.total_rows or 0)), + str(_ROW_BILLED_TTL_SECONDS), + ) + + if not _publish_sealed(evaluation.id): + _fallback_row_event( + evaluation, + eval_row, + workspace_id=workspace_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + return + try: + raw = _accumulate() + except redis.RedisError as exc: + logger.warning( + "Flexprice recording batch accumulate failed evaluation={}: {}", + evaluation.id, + exc, + ) + _fallback_row_event( + evaluation, + eval_row, + workspace_id=workspace_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + return + if not raw: + return + marker = str(raw[0]) + if marker == "duplicate": + return + if marker == "pending_sealed": + if not _publish_sealed(evaluation.id): + _fallback_row_event( + evaluation, + eval_row, + workspace_id=workspace_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + return + try: + raw = _accumulate() + except redis.RedisError as exc: + logger.warning( + "Flexprice recording batch accumulate retry failed evaluation={}: {}", + evaluation.id, + exc, + ) + _fallback_row_event( + evaluation, + eval_row, + workspace_id=workspace_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + return + if not raw: + return + marker = str(raw[0]) + if marker == "duplicate": + return + if marker == "pending_sealed": + _fallback_row_event( + evaluation, + eval_row, + workspace_id=workspace_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + return + if marker == "sealed": + _publish_sealed(evaluation.id) + + +def flush_recording_minutes_remainder(evaluation_id: UUID) -> bool: + """Send any partial batch once the evaluation has no rows left in flight.""" + if not _publish_sealed(evaluation_id): + return False + try: + raw = _client().eval( + _FLUSH_LUA, + 2, + _batch_key(evaluation_id), + _sealed_key(evaluation_id), + str(_BATCH_TTL_SECONDS), + ) + except redis.RedisError as exc: + logger.warning( + "Flexprice recording batch flush failed evaluation={}: {}", + evaluation_id, + exc, + ) + return False + if not raw: + return False + return _publish_sealed(evaluation_id) + + +def maybe_enqueue_recording_minutes_for_completed_row( + db: Session, + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + *, + source_row: CallImportRow | None = None, + resolve_audio_seconds, + billable_minutes_fn, +) -> None: + if (eval_row.status or "").lower() != "completed": + return + if source_row is None: + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + if source_row is None: + return + if not (source_row.recording_s3_key or "").strip(): + return + + audio_seconds = resolve_audio_seconds(db, eval_row, source_row) + minutes = billable_minutes_fn(audio_seconds) + enqueue_recording_minutes_for_completed_row( + evaluation, + eval_row, + audio_seconds=audio_seconds, + billable_minutes=minutes, + ) + + +def buffer_call_import_evaluation_pass( + evaluation: CallImportEvaluation, + *, + rows_billed: int, + completed_total: int, +) -> bool: + if rows_billed <= 0: + return False + from app.services.billing.flexprice_service import ( + buffer_call_import_evaluation_completed, + ) + + metric_count = len(evaluation.selected_metric_ids or []) + return buffer_call_import_evaluation_completed( + evaluation.organization_id, + evaluation.id, + workspace_id=evaluation.workspace_id, + call_import_id=evaluation.call_import_id, + rows_billed=rows_billed, + completed_total=completed_total, + total_rows=int(evaluation.total_rows or 0), + metric_count=metric_count, + ) + + +def maybe_finalize_evaluation_flexprice_billing( + db: Session, + evaluation: CallImportEvaluation, +) -> None: + """Flush partial recording-minute batches once no eval rows are in flight. + + Called from the per-row worker rollup and from evaluation-wide reconciles + (abort/cancel, counter repair) so remainder batches are not left in Redis. + """ + total = int(evaluation.total_rows or 0) + completed = int(evaluation.completed_rows or 0) + failed = int(evaluation.failed_rows or 0) + in_progress = total - completed - failed + if in_progress > 0: + return + + evaluation_id = evaluation.id + flush_recording_minutes_remainder(evaluation_id) + + already_billed = int(getattr(evaluation, "billed_completed_rows", 0) or 0) + delta = completed - already_billed + if delta <= 0: + return + + locked = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == evaluation_id) + .with_for_update() + .one() + ) + billing_accepted = buffer_call_import_evaluation_pass( + locked, + rows_billed=delta, + completed_total=completed, + ) + if billing_accepted: + locked.billed_completed_rows = completed + evaluation.billed_completed_rows = completed diff --git a/app/services/call_imports/eval_progress_buffer.py b/app/services/call_imports/eval_progress_buffer.py new file mode 100644 index 00000000..c6fc66c1 --- /dev/null +++ b/app/services/call_imports/eval_progress_buffer.py @@ -0,0 +1,448 @@ +"""Redis-buffered evaluation progress counters. + +One hash per evaluation, ``eval:ctr:{evaluation_id}``: + +* ``c`` / ``f``: completed / failed deltas not yet drained (may be negative). +* ``pc`` / ``pf`` / ``pgen`` / ``pseq``: the pending slot, a drained batch that + has not been acknowledged yet. At most one per evaluation. A failed catalog + write leaves it in place and the next flush re-sends the same payload with + the same ``seq``, so an ambiguous commit is never applied twice. +* ``remaining``: rows still in flight. Only used to trigger finalize; reset + from the shard pending+running count on every recount. +* ``gen``: fence version, mirrors ``call_import_evaluations.progress_counter_gen``. +* ``seq``: last flush sequence number issued. +* ``uninit``: the hash was (re)created by a row transition, so gen/seq are not + known. Nothing is flushed until a fenced recount seeds it from the catalog. +* ``recount_at``: unix time of the last fenced recount. + +``eval:ctr:dirty`` holds evaluations with unflushed deltas or a pending slot. + +Every mutation is a Lua script, so each is atomic. +""" + +from __future__ import annotations + +import time +import uuid as uuid_lib +from dataclasses import dataclass +from typing import Optional +from uuid import UUID + +import redis +from loguru import logger + +from app.config import settings + +DIRTY_SET_KEY = "eval:ctr:dirty" +# Fence parks ``remaining`` at this offset until the shard count is known, so +# rows finishing during the recount never see ``remaining <= 0``. +_REMAINING_FENCE_OFFSET = 10**12 +# Keys are refreshed on every write; this only reclaims abandoned evaluations. +_KEY_TTL_SECONDS = 14 * 24 * 3600 +_FINALIZE_DEDUP_SECONDS = 15 +_LOCK_TTL_MS = 60_000 + +_redis: redis.Redis | None = None + + +def _client() -> redis.Redis: + """Fail fast: callers fall back (direct catalog write, catalog-only UI) + when Redis is down, so a short timeout and a single quick retry beat + redis-py's default multi-second backoff on every row.""" + global _redis + if _redis is None: + from redis.backoff import ExponentialBackoff + from redis.retry import Retry + + _redis = redis.from_url( + settings.REDIS_URL, + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + retry=Retry(ExponentialBackoff(cap=0.2, base=0.05), 1), + ) + return _redis + + +def counter_key(evaluation_id: UUID | str) -> str: + return f"eval:ctr:{evaluation_id}" + + +def _lock_key(evaluation_id: UUID | str) -> str: + return f"eval:ctr:lock:{evaluation_id}" + + +def _finalize_key(evaluation_id: UUID | str) -> str: + return f"eval:finalize:{evaluation_id}" + + +# KEYS[1]=hash KEYS[2]=dirty ARGV: dc, df, dremaining, eval_id, ttl +_RECORD_LUA = """ +local existed = redis.call('EXISTS', KEYS[1]) +if existed == 0 then + redis.call('HSET', KEYS[1], 'uninit', 1) +end +local dc = tonumber(ARGV[1]) +local df = tonumber(ARGV[2]) +if dc ~= 0 then redis.call('HINCRBY', KEYS[1], 'c', dc) end +if df ~= 0 then redis.call('HINCRBY', KEYS[1], 'f', df) end +local rem = redis.call('HINCRBY', KEYS[1], 'remaining', tonumber(ARGV[3])) +if dc ~= 0 or df ~= 0 or existed == 0 then + redis.call('SADD', KEYS[2], ARGV[4]) +end +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[5])) +return {rem, existed} +""" + +# KEYS[1]=hash KEYS[2]=dirty ARGV: eval_id +# Returns {kind, pc, pf, pgen, pseq, remaining}; kind is +# 'missing' | 'uninit' | 'empty' | 'pending'. +_DRAIN_LUA = """ +if redis.call('EXISTS', KEYS[1]) == 0 then + redis.call('SREM', KEYS[2], ARGV[1]) + return {'missing', 0, 0, 0, 0, 0} +end +local h = redis.call('HMGET', KEYS[1], 'pc', 'pf', 'pgen', 'pseq', 'uninit', 'c', 'f', 'gen', 'remaining') +local remaining = tonumber(h[9]) or 0 +if h[4] then + return {'pending', tonumber(h[1]) or 0, tonumber(h[2]) or 0, tonumber(h[3]) or 0, tonumber(h[4]), remaining} +end +if h[5] then + return {'uninit', 0, 0, 0, 0, remaining} +end +local c = tonumber(h[6]) or 0 +local f = tonumber(h[7]) or 0 +if c == 0 and f == 0 then + redis.call('SREM', KEYS[2], ARGV[1]) + return {'empty', 0, 0, 0, 0, remaining} +end +local gen = tonumber(h[8]) or 0 +local seq = redis.call('HINCRBY', KEYS[1], 'seq', 1) +redis.call('HSET', KEYS[1], 'pc', c, 'pf', f, 'pgen', gen, 'pseq', seq, 'c', 0, 'f', 0) +return {'pending', c, f, gen, seq, remaining} +""" + +# KEYS[1]=hash KEYS[2]=dirty ARGV: eval_id, pseq +_ACK_LUA = """ +local pseq = redis.call('HGET', KEYS[1], 'pseq') +if pseq and pseq == ARGV[2] then + redis.call('HDEL', KEYS[1], 'pc', 'pf', 'pgen', 'pseq') +end +local h = redis.call('HMGET', KEYS[1], 'c', 'f', 'pseq', 'uninit') +if (tonumber(h[1]) or 0) == 0 and (tonumber(h[2]) or 0) == 0 and not h[3] and not h[4] then + redis.call('SREM', KEYS[2], ARGV[1]) +end +return 1 +""" + +# KEYS[1]=hash KEYS[2]=dirty ARGV: eval_id, gen, catalog_seq, now, ttl, offset +_FENCE_LUA = """ +local seq = tonumber(redis.call('HGET', KEYS[1], 'seq')) or 0 +local catalog_seq = tonumber(ARGV[3]) +if catalog_seq > seq then seq = catalog_seq end +redis.call('HDEL', KEYS[1], 'pc', 'pf', 'pgen', 'pseq', 'uninit') +redis.call('HSET', KEYS[1], 'c', 0, 'f', 0, 'gen', ARGV[2], 'seq', seq, + 'remaining', ARGV[6], 'recount_at', ARGV[4]) +redis.call('SREM', KEYS[2], ARGV[1]) +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[5])) +return seq +""" + +# KEYS[1]=hash ARGV: gen, in_progress, offset +# Applies the shard in-progress count on top of decrements that landed since +# the fence. A stale gen means a newer recount owns ``remaining``. +_SET_REMAINING_LUA = """ +local gen = redis.call('HGET', KEYS[1], 'gen') +if not gen or gen ~= ARGV[1] then return nil end +return redis.call('HINCRBY', KEYS[1], 'remaining', tonumber(ARGV[2]) - tonumber(ARGV[3])) +""" + +# KEYS[1]=hash KEYS[2]=dirty ARGV: eval_id, gen, seq, remaining, ttl +_SEED_LUA = """ +redis.call('DEL', KEYS[1]) +redis.call('HSET', KEYS[1], 'c', 0, 'f', 0, 'gen', ARGV[2], 'seq', ARGV[3], 'remaining', ARGV[4]) +redis.call('SREM', KEYS[2], ARGV[1]) +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[5])) +return 1 +""" + +_RELEASE_LOCK_LUA = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" + +_scripts: dict[str, object] = {} + + +def _script(name: str, source: str): + client = _client() + cached = _scripts.get(name) + if cached is None or getattr(cached, "registered_client", None) is not client: + cached = client.register_script(source) + _scripts[name] = cached + return cached + + +def _as_int(value) -> int: + if value is None: + return 0 + if isinstance(value, bytes): + value = value.decode() + return int(value) + + +@dataclass(frozen=True) +class TransitionResult: + remaining: int + created: bool + + +@dataclass(frozen=True) +class DrainResult: + kind: str # missing | uninit | empty | pending + completed_delta: int = 0 + failed_delta: int = 0 + gen: int = 0 + seq: int = 0 + remaining: int = 0 + + +def remaining_delta_for_transition(previous_status: str | None, new_status: str) -> int: + """-1 when a row leaves pending/running, +1 when it re-enters it.""" + active = {"pending", "running"} + was_active = (previous_status or "").strip().lower() in active + is_active = (new_status or "").strip().lower() in active + if was_active and not is_active: + return -1 + if is_active and not was_active: + return 1 + return 0 + + +def record_transition( + evaluation_id: UUID | str, + *, + completed_delta: int, + failed_delta: int, + remaining_delta: int, +) -> TransitionResult: + """Apply one row transition. Raises ``redis.RedisError`` when Redis is down.""" + rem, existed = _script("record", _RECORD_LUA)( + keys=[counter_key(evaluation_id), DIRTY_SET_KEY], + args=[ + int(completed_delta), + int(failed_delta), + int(remaining_delta), + str(evaluation_id), + _KEY_TTL_SECONDS, + ], + ) + return TransitionResult(remaining=_as_int(rem), created=not bool(_as_int(existed))) + + +def dirty_evaluation_ids(limit: int = 5000) -> list[str]: + client = _client() + members: list[str] = [] + for member in client.sscan_iter(DIRTY_SET_KEY, count=500): + members.append(member.decode() if isinstance(member, bytes) else member) + if len(members) >= limit: + break + return members + + +def drain(evaluation_id: UUID | str) -> DrainResult: + kind, pc, pf, pgen, pseq, remaining = _script("drain", _DRAIN_LUA)( + keys=[counter_key(evaluation_id), DIRTY_SET_KEY], + args=[str(evaluation_id)], + ) + if isinstance(kind, bytes): + kind = kind.decode() + return DrainResult( + kind=str(kind), + completed_delta=_as_int(pc), + failed_delta=_as_int(pf), + gen=_as_int(pgen), + seq=_as_int(pseq), + remaining=_as_int(remaining), + ) + + +def ack(evaluation_id: UUID | str, seq: int) -> None: + _script("ack", _ACK_LUA)( + keys=[counter_key(evaluation_id), DIRTY_SET_KEY], + args=[str(evaluation_id), str(int(seq))], + ) + + +def fence(evaluation_id: UUID | str, *, gen: int, catalog_seq: int) -> None: + """Discard buffered/pending deltas and adopt the catalog's gen. + + Must run *before* the shard count, so any batch drained afterwards carries + the new gen and only holds transitions that landed after the fence. + """ + _script("fence", _FENCE_LUA)( + keys=[counter_key(evaluation_id), DIRTY_SET_KEY], + args=[ + str(evaluation_id), + int(gen), + int(catalog_seq), + int(time.time()), + _KEY_TTL_SECONDS, + _REMAINING_FENCE_OFFSET, + ], + ) + + +def set_remaining_after_count( + evaluation_id: UUID | str, *, gen: int, in_progress: int +) -> Optional[int]: + result = _script("set_remaining", _SET_REMAINING_LUA)( + keys=[counter_key(evaluation_id)], + args=[int(gen), int(in_progress), _REMAINING_FENCE_OFFSET], + ) + return None if result is None else _as_int(result) + + +def seed(evaluation_id: UUID | str, *, gen: int, seq: int, remaining: int) -> None: + """Initialise the hash at materialize time from catalog gen/seq.""" + _script("seed", _SEED_LUA)( + keys=[counter_key(evaluation_id), DIRTY_SET_KEY], + args=[str(evaluation_id), int(gen), int(seq), int(remaining), _KEY_TTL_SECONDS], + ) + + +def seed_best_effort(evaluation_id: UUID | str, *, gen: int, seq: int, remaining: int) -> None: + try: + seed(evaluation_id, gen=gen, seq=seq, remaining=remaining) + except redis.RedisError as exc: + # Transitions will recreate the key as ``uninit`` and force a recount. + logger.warning("eval counter seed skipped for {}: {}", evaluation_id, exc) + + +def delete(evaluation_id: UUID | str) -> None: + try: + client = _client() + pipe = client.pipeline() + pipe.delete(counter_key(evaluation_id)) + pipe.srem(DIRTY_SET_KEY, str(evaluation_id)) + pipe.execute() + except redis.RedisError as exc: + logger.debug("eval counter delete skipped for {}: {}", evaluation_id, exc) + + +@dataclass(frozen=True) +class BufferedCounters: + completed_delta: int = 0 + failed_delta: int = 0 + pending_completed: int = 0 + pending_failed: int = 0 + pending_gen: Optional[int] = None + pending_seq: Optional[int] = None + gen: Optional[int] = None + recount_at: Optional[int] = None + + +def read_buffered(evaluation_id: UUID | str) -> BufferedCounters: + """Snapshot for readers. Returns zeros when Redis is unavailable.""" + try: + values = _client().hmget( + counter_key(evaluation_id), + "c", "f", "pc", "pf", "pgen", "pseq", "gen", "recount_at", + ) + except redis.RedisError: + return BufferedCounters() + c, f, pc, pf, pgen, pseq, gen, recount_at = values + return BufferedCounters( + completed_delta=_as_int(c), + failed_delta=_as_int(f), + pending_completed=_as_int(pc), + pending_failed=_as_int(pf), + pending_gen=None if pgen is None else _as_int(pgen), + pending_seq=None if pseq is None else _as_int(pseq), + gen=None if gen is None else _as_int(gen), + recount_at=None if recount_at is None else _as_int(recount_at), + ) + + +def unapplied_deltas( + buffered: BufferedCounters, + *, + catalog_gen: int, + catalog_seq: int, +) -> tuple[int, int]: + """Deltas not yet reflected in the catalog row. + + Buffered ``c/f`` count only when they belong to the catalog's gen (after + a recount the catalog is ahead of a stale Redis view). The pending slot + counts only while the catalog has not applied its ``seq``. + """ + completed = failed = 0 + if buffered.gen is None or buffered.gen == catalog_gen: + completed += buffered.completed_delta + failed += buffered.failed_delta + if ( + buffered.pending_seq is not None + and buffered.pending_gen == catalog_gen + and catalog_seq < buffered.pending_seq + ): + completed += buffered.pending_completed + failed += buffered.pending_failed + return completed, failed + + +# --- locks / finalize dedupe ------------------------------------------------- + + +def acquire_lock(evaluation_id: UUID | str, *, ttl_ms: int = _LOCK_TTL_MS) -> Optional[str]: + token = uuid_lib.uuid4().hex + try: + if _client().set(_lock_key(evaluation_id), token, nx=True, px=ttl_ms): + return token + except redis.RedisError as exc: + logger.debug("eval counter lock skipped for {}: {}", evaluation_id, exc) + return None + + +def release_lock(evaluation_id: UUID | str, token: Optional[str]) -> None: + if not token: + return + try: + _script("release_lock", _RELEASE_LOCK_LUA)( + keys=[_lock_key(evaluation_id)], args=[token] + ) + except redis.RedisError as exc: + logger.debug("eval counter lock release skipped for {}: {}", evaluation_id, exc) + + +def claim_finalize_slot(evaluation_id: UUID | str) -> bool: + """``SET NX`` dedupe so concurrent closers enqueue one finalize.""" + try: + return bool( + _client().set( + _finalize_key(evaluation_id), "1", nx=True, ex=_FINALIZE_DEDUP_SECONDS + ) + ) + except redis.RedisError: + return True + + +def request_finalize(evaluation_id: UUID | str) -> bool: + """Enqueue ``finalize_call_import_evaluation`` unless one was just queued.""" + if not claim_finalize_slot(evaluation_id): + return False + try: + from app.workers.tasks.finalize_call_import_evaluation import ( + finalize_call_import_evaluation_task, + ) + + finalize_call_import_evaluation_task.apply_async(args=[str(evaluation_id)]) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("finalize enqueue failed for evaluation {}: {}", evaluation_id, exc) + try: + _client().delete(_finalize_key(evaluation_id)) + except redis.RedisError: + pass + return False diff --git a/app/services/call_imports/eval_settings_version.py b/app/services/call_imports/eval_settings_version.py new file mode 100644 index 00000000..e71c5911 --- /dev/null +++ b/app/services/call_imports/eval_settings_version.py @@ -0,0 +1,142 @@ +"""Redis settings-version counter for call-import evaluations. + +Eval workers cache the parent evaluation's settings per process (see +``load_evaluation_snapshot``) keyed by ``eval:settings_ver:{evaluation_id}``. +Any change to a column the worker reads bumps the counter, so the next row in +every worker process reloads once from the catalog. + +Bumps happen automatically through an ORM hook: when a session commits after +flushing a ``CallImportEvaluation`` whose watched columns changed (or that was +deleted), the ids are bumped after the commit. Core ``update()`` statements +bypass the ORM, so any new Core writer of a watched column must call +``bump_settings_version`` itself. Today's Core writers only touch counters and +the progress fence, which the worker does not depend on. +""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy import event, inspect +from sqlalchemy.orm import Session + +_KEY_TTL_SECONDS = 14 * 24 * 3600 +_SESSION_INFO_KEY = "eval_settings_bump" + +# Columns the eval worker reads that are not progress counters. Counter +# columns (completed/failed/total/billed rows) are excluded on purpose: the +# Redis-down rollup writes them per row and must not invalidate every cache. +WATCHED_ATTRIBUTES = frozenset( + { + "status", + "started_at", + "finished_at", + "error_message", + "selected_metric_ids", + "discover_new_metrics", + "discovered_metric_aliases", + "discovered_label_aliases", + "metric_llm_overrides", + "llm_provider", + "llm_model", + "llm_credential_id", + "llm_config", + "transcript_source", + "organization_id", + "workspace_id", + "call_import_id", + } +) + + +def settings_version_key(evaluation_id: UUID | str) -> str: + return f"eval:settings_ver:{evaluation_id}" + + +def _client() -> redis.Redis: + # Same fail-fast client (and test isolation) as the progress buffer. + from app.services.call_imports.eval_progress_buffer import _client as buffer_client + + return buffer_client() + + +def read_settings_version(evaluation_id: UUID | str) -> Optional[int]: + """Current version (0 when never bumped). ``None`` when Redis is down.""" + try: + raw = _client().get(settings_version_key(evaluation_id)) + except redis.RedisError as exc: + logger.debug("eval settings version read failed for {}: {}", evaluation_id, exc) + return None + try: + return int(raw) if raw is not None else 0 + except (TypeError, ValueError): + return 0 + + +def bump_settings_version(evaluation_ids: Iterable[UUID | str]) -> None: + ids = {str(eid) for eid in evaluation_ids if eid is not None} + if not ids: + return + try: + pipe = _client().pipeline(transaction=False) + for eid in ids: + key = settings_version_key(eid) + pipe.incr(key) + pipe.expire(key, _KEY_TTL_SECONDS) + pipe.execute() + except redis.RedisError as exc: + # Workers fall back to a short cache TTL; nothing else to do here. + logger.warning("eval settings version bump failed for {}: {}", sorted(ids), exc) + + +def _is_evaluation(obj) -> bool: + from app.models.database import CallImportEvaluation + + return isinstance(obj, CallImportEvaluation) + + +def _watched_change(obj) -> bool: + state = inspect(obj) + for name in WATCHED_ATTRIBUTES: + attr = state.attrs.get(name) + if attr is not None and attr.history.has_changes(): + return True + return False + + +def _after_flush(session: Session, _flush_context) -> None: + pending: set = session.info.get(_SESSION_INFO_KEY) or set() + for obj in session.deleted: + if _is_evaluation(obj) and obj.id is not None: + pending.add(obj.id) + for obj in session.dirty: + if _is_evaluation(obj) and obj.id is not None and _watched_change(obj): + pending.add(obj.id) + if pending: + session.info[_SESSION_INFO_KEY] = pending + + +def _after_commit(session: Session) -> None: + pending = session.info.pop(_SESSION_INFO_KEY, None) + if pending: + bump_settings_version(pending) + + +def _after_rollback(session: Session) -> None: + session.info.pop(_SESSION_INFO_KEY, None) + + +_registered = False + + +def register_settings_version_hooks() -> None: + global _registered + if _registered: + return + event.listen(Session, "after_flush", _after_flush) + event.listen(Session, "after_commit", _after_commit) + event.listen(Session, "after_rollback", _after_rollback) + _registered = True diff --git a/app/services/call_imports/evaluation_create.py b/app/services/call_imports/evaluation_create.py new file mode 100644 index 00000000..ab8d3aad --- /dev/null +++ b/app/services/call_imports/evaluation_create.py @@ -0,0 +1,690 @@ +"""Create call-import evaluation runs (validation, persistence, enqueue).""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.core.auth import Principal +from app.models.database import AIProvider, CallImportEvaluation, Metric +from app.models.enums import ModelProvider +from app.models.schemas import CallImportEvaluationCreate, CallImportEvaluationResponse +from app.services.call_imports.audit import stamp_call_import_actor, stamp_evaluation_actor +from app.api.v1.routes.call_import_evaluations._shared import ( + _expand_metric_selection, + _normalize_name, + _require_import, + _serialize_eval, +) + + +def create_call_import_evaluations( + db: Session, + *, + organization_id: UUID, + principal: Principal, + call_import_id: UUID, + payload: CallImportEvaluationCreate, +) -> CallImportEvaluationResponse: + call_import = _require_import(db, call_import_id, organization_id) + + metric_ids = payload.metric_ids + if not metric_ids: + raise HTTPException( + status_code=400, + detail="Select at least one metric to run the evaluation against.", + ) + + org_metrics = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(metric_ids), + ) + .all() + ) + by_id = {metric.id: metric for metric in org_metrics} + unknown_ids = [mid for mid in metric_ids if mid not in by_id] + if unknown_ids: + raise HTTPException( + status_code=400, + detail=( + "These metric ids do not exist in your organization: " + f"{', '.join(str(mid) for mid in unknown_ids)}. " + "Refresh the metrics list and try again." + ), + ) + draft_metrics = [ + metric + for metric in org_metrics + if (getattr(metric, "lifecycle", None) or "active") == "draft" + ] + if draft_metrics: + names = ", ".join(metric.name for metric in draft_metrics) + raise HTTPException( + status_code=400, + detail=( + f"Draft metrics cannot be used in call import evaluations: {names}. " + "Promote them in Metrics Studio first." + ), + ) + # Parents themselves are containers, not scored rows, so a disabled + # parent shouldn't block the run as long as it has enabled children. + # We only reject disabled rows that the worker will actually try to + # evaluate (children + standalone leaves). + disabled_leaves = [ + metric + for metric in org_metrics + if not metric.enabled + and not (metric.selection_mode and not metric.parent_metric_id) + ] + if disabled_leaves: + names = ", ".join(metric.name for metric in disabled_leaves) + raise HTTPException( + status_code=400, + detail=( + f"These metrics are disabled and cannot be evaluated: {names}. " + "Enable them on the Metrics page (or pick different ones) and " + "try again." + ), + ) + + # Expand hierarchical selection: parents auto-include their enabled + # children, mixed parent+child selections respect the user's subset. + effective_metrics, parent_to_children = _expand_metric_selection( + db, organization_id, metric_ids + ) + if not effective_metrics: + raise HTTPException( + status_code=400, + detail=( + "None of the selected metrics yielded an enabled leaf to " + "evaluate. Check that parent categories have enabled " + "children, then try again." + ), + ) + + # The effective list (children + standalone leaves) is what gets + # persisted to ``selected_metric_ids`` and scored by the worker. + # The original parents are preserved in ``selected_metric_groups`` + # so the UI can rebuild the tree later. + leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] + selected_metric_groups: Dict[str, List[str]] = { + str(pid): [str(c.id) for c in children] + for pid, children in parent_to_children.items() + } + metric_rows = effective_metrics + valid_metric_id_strs = {str(m.id) for m in metric_rows} + + from app.services.metrics.computed_metrics import ( + metrics_require_transcript_agreement, + ) + + requires_transcript_agreement = metrics_require_transcript_agreement(metric_rows) + + if requires_transcript_agreement: + from app.models.enums import CallImportParameterType + from app.api.v1.routes.call_imports import ( + _is_manual_audio_call_import, + _is_platform_call_import, + ) + + has_production_transcript = False + if _is_platform_call_import(call_import) or _is_manual_audio_call_import( + call_import + ): + has_production_transcript = True + elif call_import.schema_id: + from app.api.v1.routes.call_imports import _resolve_schema + + schema = _resolve_schema( + db, + organization_id, + call_import.workspace_id, + call_import.schema_id, + ) + has_production_transcript = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in schema.parameters + ) + else: + legacy_transcript = ( + (call_import.column_mapping or {}).get("transcript") or "" + ).strip() + has_production_transcript = bool(legacy_transcript) + + if not has_production_transcript: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="WER/CER requires a production transcript column", + ) + + # ----- Validate run-level + per-metric LLM config ----- + llm_provider_norm: Optional[str] = None + llm_model_norm: Optional[str] = None + if payload.llm_provider or payload.llm_model: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail="Both llm_provider and llm_model are required when overriding the run LLM.", + ) + try: + llm_provider_norm = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + llm_model_norm = payload.llm_model.strip() or None + if not llm_model_norm: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in this " + "organization." + ), + ) + + # Per-metric overrides: keys can be either a leaf metric id (applies + # to that metric only) or a parent metric id (applies to every + # child of that parent). Parent keys are expanded to their + # children so the worker only sees concrete leaf ids. + metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None + if payload.metric_llm_overrides: + metric_overrides_payload = {} + for metric_id, override in payload.metric_llm_overrides.items(): + target_leaf_ids: List[str] = [] + if metric_id in valid_metric_id_strs: + target_leaf_ids = [metric_id] + else: + # Maybe it's a parent id — expand to the children that + # are part of THIS run. + try: + parent_uuid = UUID(metric_id) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a valid UUID." + ), + ) + children_for_parent = parent_to_children.get(parent_uuid) + if not children_for_parent: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not in metric_ids." + ), + ) + target_leaf_ids = [str(c.id) for c in children_for_parent] + + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a provider " + "but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses unknown " + f"provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + # Model without provider doesn't make sense — treat as 400 + # so the UI can fix it instead of silently falling back. + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model but " + "no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + for leaf_id in target_leaf_ids: + metric_overrides_payload[leaf_id] = override_dict + + # ----- Validate auto-transcribe settings ----- + # Diarised runs auto-diarise rows missing a diarised transcript and + # require STT + diariser LLM config. Production runs score the CSV + # transcript directly and skip diarisation entirely. + use_diarised = payload.transcript_sources[0] == "diarised" + auto_transcribe = use_diarised or requires_transcript_agreement + + transcribe_mode_norm: Optional[str] = None + stt_provider_norm: Optional[str] = None + stt_model_norm: Optional[str] = None + diarisation_llm_provider_norm: Optional[str] = None + diarisation_llm_model_norm: Optional[str] = None + diarisation_prompt_norm: Optional[str] = None + + if auto_transcribe: + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) + + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: + raise HTTPException( + status_code=400, + detail=( + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." + ), + ) + if not payload.stt_model: + raise HTTPException( + status_code=400, + detail=( + "stt_model is required when transcribe_mode='stt_llm'." + ), + ) + try: + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): + raise HTTPException( + status_code=400, + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), + ) + + # --- Validate LLM diariser settings ----- + if not payload.diarization_llm_provider: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + if not payload.diarization_llm_model: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + try: + diarisation_llm_provider_norm = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + diarisation_llm_model_norm = ( + payload.diarization_llm_model.strip() or None + ) + if not diarisation_llm_model_norm: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + diarisation_prompt_norm = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None + + from app.models.enums import CallImportParameterType, CallImportStatus + from app.services.call_imports.bulk_ops import ( + count_all_source_rows, + count_source_rows_with_production_transcript, + ) + + if auto_transcribe: + from app.api.v1.routes.call_imports import ( + _is_manual_audio_call_import, + _is_platform_call_import, + platform_import_hydration_complete, + ) + + if not _is_manual_audio_call_import(call_import) and not _is_platform_call_import( + call_import + ): + if call_import.schema_id: + from app.api.v1.routes.call_imports import ( + _resolve_schema, + _validate_diarised_eval_recording_ready, + ) + + diarised_schema = _resolve_schema( + db, organization_id, call_import.workspace_id, call_import.schema_id + ) + _validate_diarised_eval_recording_ready( + list(diarised_schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + legacy_recording_column = ( + (call_import.column_mapping or {}).get("recording_url") or "" + ).strip() + if not legacy_recording_column: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires a recording URL column " + "to be mapped." + ), + ) + + starting_from_mapped = False + from app.api.v1.routes.call_imports import ( + _is_platform_call_import, + platform_import_hydration_complete, + ) + + if _is_platform_call_import(call_import) and not platform_import_hydration_complete( + call_import + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Platform import is still discovering or fetching recordings. " + "Wait until the batch is ready before running evaluation." + ), + ) + + if call_import.status == CallImportStatus.MAPPED: + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file. Upload and map " + "a CSV/Excel file before running evaluation." + ), + ) + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot run evaluation without a mapped schema.", + ) + from app.api.v1.routes.call_imports import ( + _ensure_blob_storage_enabled, + _resolve_schema, + _resolve_telephony_integration, + _validate_direct_url_import_ready, + _validate_telephony_credentials_live, + ) + + workspace_id = call_import.workspace_id + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + if not use_diarised: + transcript_mapped = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in parameters + ) + if not transcript_mapped: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No transcript column is mapped in this batch. " + "Map a schema transcript parameter to a CSV column, " + "or choose 'Diarize then evaluate'." + ), + ) + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + starting_from_mapped = True + + all_source_row_count = count_all_source_rows(db, call_import.id) + + if auto_transcribe: + # Diarized runs materialize every import row; recording fetch may + # still be pending when switching from a production eval-primary run. + total_row_count = all_source_row_count + else: + # Production runs score CSV text — rows need not wait for + # recording fetch to finish before they are evaluable. + total_row_count = count_source_rows_with_production_transcript( + db, call_import.id + ) + + if ( + (use_diarised or requires_transcript_agreement) + and not starting_from_mapped + and all_source_row_count > 0 + and call_import.status != CallImportStatus.DELETING + ): + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + requested_sources: List[str] = list(payload.transcript_sources) + + if ( + not use_diarised + and not starting_from_mapped + and count_all_source_rows(db, call_import.id) > 0 + and total_row_count == 0 + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No rows have a production transcript. " + "Choose 'Diarize then evaluate' or import rows with " + "a transcript column." + ), + ) + + base_name = _normalize_name(payload.name) + + def _name_for_source(source: str) -> Optional[str]: + # Single-source runs preserve the user's chosen name verbatim. + del source + return base_name + + created_evaluations: List[CallImportEvaluation] = [] + + for source in requested_sources: + evaluation = CallImportEvaluation( + call_import_id=call_import.id, + organization_id=organization_id, + # Mirror the parent CallImport's workspace so listings can + # filter on workspace_id directly without joining. + workspace_id=call_import.workspace_id, + name=_name_for_source(source), + selected_metric_ids=[ + str(metric_id) for metric_id in leaf_metric_ids + ], + selected_metric_groups=selected_metric_groups or None, + status="pending", + total_rows=total_row_count, + completed_rows=0, + failed_rows=0, + llm_provider=llm_provider_norm, + llm_model=llm_model_norm, + llm_credential_id=payload.llm_credential_id, + llm_config=payload.llm_config, + metric_llm_overrides=metric_overrides_payload, + stt_provider=stt_provider_norm, + stt_model=stt_model_norm, + stt_credential_id=( + payload.stt_credential_id if auto_transcribe else None + ), + diarisation_llm_provider=diarisation_llm_provider_norm, + diarisation_llm_model=diarisation_llm_model_norm, + diarisation_llm_credential_id=( + payload.diarization_llm_credential_id if auto_transcribe else None + ), + diarisation_prompt=diarisation_prompt_norm, + transcribe_mode=transcribe_mode_norm, + transcript_source=source, + discover_new_metrics=bool( + getattr(payload, "discover_new_metrics", False) + ), + ) + stamp_evaluation_actor(evaluation, principal, creating=True) + db.add(evaluation) + db.flush() + created_evaluations.append(evaluation) + + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + + primary_evaluation = created_evaluations[0] + sibling_ids = [e.id for e in created_evaluations[1:]] + + if not all_source_row_count and not starting_from_mapped: + for evaluation in created_evaluations: + evaluation.status = "completed" + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + if starting_from_mapped: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_mapped_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_mapped_call_import_evaluation_task.delay( + str(call_import.id), + str(organization_id), + str(call_import.workspace_id), + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + else: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_call_import_evaluation_task.delay( + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + + from app.services.billing.flexprice_service import ( + record_call_import_evaluation_started, + ) + + for evaluation in created_evaluations: + record_call_import_evaluation_started( + organization_id, + evaluation.id, + workspace_id=evaluation.workspace_id, + call_import_id=call_import.id, + total_rows=int(evaluation.total_rows or 0), + metric_count=len(leaf_metric_ids), + ) + + for evaluation in created_evaluations: + db.refresh(evaluation) + + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) diff --git a/app/services/call_imports/evaluation_export.py b/app/services/call_imports/evaluation_export.py index 60801e60..ac92267e 100644 --- a/app/services/call_imports/evaluation_export.py +++ b/app/services/call_imports/evaluation_export.py @@ -236,6 +236,23 @@ def _add_metric(metric: Metric) -> None: seen.add(mid) header = metric_names[mid] metric_headers.append(header) + from app.services.metrics.computed_metrics import ( + COMPUTED_TRANSCRIPT_SCORING_METHODS, + metric_scoring_method_key, + ) + + if metric_scoring_method_key(metric) in COMPUTED_TRANSCRIPT_SCORING_METHODS: + metric_headers.extend( + [ + f"{header} cer", + f"{header} wer", + f"{header} error_rate_mode", + f"{header} script_mismatch", + f"{header} ref chars", + f"{header} ref words", + f"{header} hyp words", + ] + ) if include_transcripts and bool(getattr(metric, "capture_rationale", False)): rationale_header = f"{header} - LLM Rationale" metric_headers.append(rationale_header) @@ -324,6 +341,34 @@ def project_export_row( else: value = metric_score.get("chosen_child_name") or metric_score.get("value") row_out[metric.name] = "" if value is None else str(value) + from app.services.metrics.computed_metrics import ( + COMPUTED_TRANSCRIPT_SCORING_METHODS, + metric_scoring_method_key, + ) + + if ( + metric_scoring_method_key(metric) in COMPUTED_TRANSCRIPT_SCORING_METHODS + and isinstance(metric_score, dict) + ): + details = metric_score.get("details") + if isinstance(details, dict): + row_out[f"{metric.name} cer"] = str(details.get("cer", "")) + row_out[f"{metric.name} wer"] = str(details.get("wer", "")) + row_out[f"{metric.name} error_rate_mode"] = str( + details.get("mode", "") + ) + row_out[f"{metric.name} script_mismatch"] = str( + details.get("script_mismatch", "") + ) + row_out[f"{metric.name} ref chars"] = str( + details.get("ref_chars", "") + ) + row_out[f"{metric.name} ref words"] = str( + details.get("ref_words", details.get("ref_word_count", "")) + ) + row_out[f"{metric.name} hyp words"] = str( + details.get("hyp_words", details.get("hyp_word_count", "")) + ) rationale_header = layout.rationale_headers.get(str(metric.id)) if rationale_header is not None: rationale = ( diff --git a/app/services/call_imports/evaluation_period_delta_cache.py b/app/services/call_imports/evaluation_period_delta_cache.py new file mode 100644 index 00000000..c7e4c982 --- /dev/null +++ b/app/services/call_imports/evaluation_period_delta_cache.py @@ -0,0 +1,94 @@ +"""Persist week-over-week failure-rate deltas without recomputing on every GET.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any, Dict, Optional +from uuid import UUID + +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.models.database import CallImportEvaluation + +_MAX_CACHED_PERIOD_DELTA_KEYS = 8 + + +def failure_policy_cache_version(metric_clusters: Any) -> str: + """Token for the failure rules a cached comparison was computed with. + + Inferred rules (nothing stored) share one token. Saved policies include + their content, so a later policy edit misses the previous cache entry. + """ + if not isinstance(metric_clusters, dict): + return "inferred" + policies = metric_clusters.get("failure_policies") + if not isinstance(policies, dict) or not policies: + return "inferred" + payload = { + "source": metric_clusters.get("failure_policies_source") or "inferred", + "updated_at": metric_clusters.get("failure_policies_updated_at") or "", + "policies": policies, + } + blob = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":")) + return hashlib.sha256(blob.encode()).hexdigest()[:16] + + +def period_delta_numeric_cache_key( + baseline_evaluation_id: UUID, + *, + completed_rows: int, + baseline_completed_rows: int, + metric_clusters: Any = None, + policy_version: str | None = None, +) -> str: + version = ( + policy_version + if policy_version is not None + else failure_policy_cache_version(metric_clusters) + ) + return ( + f"{baseline_evaluation_id}:{completed_rows}:" + f"{baseline_completed_rows}:{version}:reconciled-v2" + ) + + +def load_cached_period_deltas( + evaluation: CallImportEvaluation, + cache_key: str, +) -> Optional[dict[str, dict[str, str]]]: + raw = getattr(evaluation, "cached_period_deltas", None) + if not isinstance(raw, dict): + return None + entry = raw.get(cache_key) + if not isinstance(entry, dict): + return None + deltas = entry.get("deltas") + if not isinstance(deltas, dict): + return None + return { + str(metric_id): dict(delta) + for metric_id, delta in deltas.items() + if isinstance(delta, dict) + } + + +def save_cached_period_deltas( + db: Session, + evaluation: CallImportEvaluation, + cache_key: str, + deltas: dict[str, dict[str, str]], +) -> None: + raw = evaluation.cached_period_deltas + if not isinstance(raw, dict): + raw = {} + updated: Dict[str, Any] = dict(raw) + updated[cache_key] = {"deltas": deltas} + if len(updated) > _MAX_CACHED_PERIOD_DELTA_KEYS: + for stale_key in list(updated.keys())[:-_MAX_CACHED_PERIOD_DELTA_KEYS]: + if stale_key != cache_key: + updated.pop(stale_key, None) + evaluation.cached_period_deltas = updated + flag_modified(evaluation, "cached_period_deltas") + db.flush() diff --git a/app/services/call_imports/evaluation_report_period.py b/app/services/call_imports/evaluation_report_period.py new file mode 100644 index 00000000..84c3d271 --- /dev/null +++ b/app/services/call_imports/evaluation_report_period.py @@ -0,0 +1,90 @@ +"""Compute and persist evaluation report period (ISO week) without full row scans.""" + +from __future__ import annotations + +from datetime import date, timedelta +from typing import Optional, Tuple + +from sqlalchemy.orm import Session + +from app.db_sharding.scatter_gather import query_completed_recording_date_bounds +from app.models.database import CallImportEvaluation + + +def report_period_from_recording_bounds( + min_date: date, + max_date: date, +) -> Tuple[date, date, str, str]: + """Match ``_report_period_from_rows`` week labeling (anchor = latest date).""" + start = min_date + end = max_date + week_anchor = max_date + week_start = week_anchor - timedelta(days=week_anchor.weekday()) + week_end = week_start + timedelta(days=6) + iso_year, iso_week, _ = week_anchor.isocalendar() + label = f"{iso_year}-W{iso_week:02d}" + if week_start.year == week_end.year: + week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" + else: + week_range = ( + f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" + ) + display = f"W{iso_week:02d} · {week_range}" + return start, end, label, display + + +def evaluation_report_period_is_current(evaluation: CallImportEvaluation) -> bool: + completed = int(evaluation.completed_rows or 0) + cached_at = evaluation.report_period_at_count + if cached_at is None: + return False + if int(cached_at) != completed: + return False + return evaluation.report_period_start is not None or completed == 0 + + +def refresh_evaluation_report_period( + db: Session, + evaluation: CallImportEvaluation, +) -> None: + """Recompute period from SQL min/max recording dates for completed rows.""" + completed = int(evaluation.completed_rows or 0) + min_date, max_date = query_completed_recording_date_bounds(db, evaluation.id) + if min_date is None or max_date is None: + evaluation.report_period_start = None + evaluation.report_period_end = None + evaluation.report_period_label = None + evaluation.report_period_display = None + else: + start, end, label, display = report_period_from_recording_bounds( + min_date, max_date + ) + evaluation.report_period_start = start + evaluation.report_period_end = end + evaluation.report_period_label = label + evaluation.report_period_display = display + evaluation.report_period_at_count = completed + + +def ensure_evaluation_report_period( + db: Session, + evaluation: CallImportEvaluation, + *, + force: bool = False, +) -> None: + if not force and evaluation_report_period_is_current(evaluation): + return + refresh_evaluation_report_period(db, evaluation) + + +def report_period_tuple( + evaluation: CallImportEvaluation, +) -> tuple[Optional[date], Optional[date], Optional[str], str]: + if evaluation.report_period_start is None: + return None, None, None, evaluation.report_period_display or "Not specified" + return ( + evaluation.report_period_start, + evaluation.report_period_end, + evaluation.report_period_label, + evaluation.report_period_display or "Not specified", + ) diff --git a/app/services/call_imports/import_batch_helpers.py b/app/services/call_imports/import_batch_helpers.py new file mode 100644 index 00000000..035ead92 --- /dev/null +++ b/app/services/call_imports/import_batch_helpers.py @@ -0,0 +1,304 @@ +"""Validation and row materialization helpers for call-import batches.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.models.database import CallImport, CallImportRow, CallImportSchemaParameter, TelephonyIntegration +from app.models.enums import CallImportParameterType, CallImportRowStatus +from app.services.call_imports.source_parse import ( + _parameter_is_required, + _parse_recording_date_cell, +) + +STUCK_PENDING_ERROR_MARKERS = ( + "rejected credentials", + "telephony credentials rejected", + "auth failed", + "recording fetch failed after", +) + + +def normalize_content_modality(content_modality: Optional[str]) -> Optional[str]: + if content_modality is None or not content_modality.strip(): + return None + mod = content_modality.strip().lower() + if mod not in ("chat", "voice"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="content_modality must be 'chat' or 'voice' when provided.", + ) + return mod + + +def validate_direct_url_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + return + if not _parameter_is_required(rec_url_param): + return + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def validate_credentialed_import_ready( + provider: str, + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + provider_key = (provider or "").lower() + provider_label = provider_key.capitalize() + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{provider_label} import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{provider_label} import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def validate_exotel_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + validate_credentialed_import_ready("exotel", parameters, parameter_mapping) + + +def is_manual_audio_call_import(call_import: CallImport) -> bool: + return (call_import.source_format or "").lower() == "audio" + + +def is_platform_call_import(call_import: CallImport) -> bool: + return (call_import.source_format or "").lower() == "platform" + + +def platform_import_hydration_complete(call_import: CallImport) -> bool: + """True when evaluation may start on the rows discovered so far.""" + if not is_platform_call_import(call_import): + return True + from sqlalchemy.orm import object_session + + from app.services.call_imports.bulk_ops import _aggregate_import_row_status_counts + + db = object_session(call_import) + if db is None: + return False + total, _completed, _failed, pending, processing = _aggregate_import_row_status_counts( + db, call_import.id + ) + if pending > 0 or processing > 0: + return False + discovery = (call_import.discovery_status or "").lower() + if discovery in {"completed", "capped", "failed", "partial"}: + return True + # Recording hydrate is done for every listed row; do not block eval on a + # stuck or slow last discovery window (listing can lag row downloads). + return total > 0 + + +def validate_diarised_eval_recording_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires a schema parameter of type " + "'recording_url'. Add one to the schema and map it to a " + "source column." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires the " + f"'{rec_url_param.name}' recording URL parameter to be " + "mapped to a source column." + ), + ) + + +def resolve_telephony_integration( + db: Session, + organization_id: UUID, + telephony_integration_id: UUID, + provider: str, +) -> TelephonyIntegration: + integration = ( + db.query(TelephonyIntegration) + .filter( + TelephonyIntegration.id == telephony_integration_id, + TelephonyIntegration.organization_id == organization_id, + ) + .first() + ) + if not integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Telephony credential not found for this organization.", + ) + if (integration.provider or "").lower() != provider.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Selected credential is for provider '{integration.provider}', " + f"but request specified '{provider}'." + ), + ) + if not integration.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected telephony credential is inactive.", + ) + return integration + + +def is_stuck_pending_import_row(row: CallImportRow) -> bool: + if row.status != CallImportRowStatus.PENDING: + return False + if (row.attempts or 0) > 0: + return True + message = (row.error_message or "").lower() + return any(marker in message for marker in STUCK_PENDING_ERROR_MARKERS) + + +def validate_telephony_credentials_live( + db: Session, + organization_id: UUID, + integration: TelephonyIntegration, +) -> None: + from app.services.telephony.telephony_service import telephony_service + + provider_label = (integration.provider or "telephony").title() + try: + client = telephony_service.get_provider_client( + organization_id, + db, + provider=integration.provider, + credential_id=integration.id, + ) + client.test_connection() + except HTTPException: + raise + except Exception as exc: + detail = str(exc).strip() or "Unknown error" + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{provider_label} credentials could not be verified: {detail}. " + "Update credentials in Integrations and retry." + ), + ) from exc + + +def materialize_rows( + db: Session, + call_import: CallImport, + parsed_rows: List[Dict[str, Any]], + organization_id: UUID, +) -> List[CallImportRow]: + row_models: List[CallImportRow] = [] + for idx, row in enumerate(parsed_rows): + csv_transcript = row["transcript"] + row_model = CallImportRow( + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + row_index=idx, + conversation_id=row["conversation_id"], + recording_date=( + _parse_recording_date_cell(row["recording_date"]) + if row.get("recording_date") + else None + ), + recording_url=row["recording_url"], + transcript=csv_transcript, + transcript_source=( + "csv" if csv_transcript and csv_transcript.strip() else None + ), + raw_columns=row["parameter_values"] or None, + status=CallImportRowStatus.PENDING, + ) + db.add(row_model) + row_models.append(row_model) + return row_models + + +def enqueue_row_tasks( + db: Session, + call_import: CallImport, + row_models: List[CallImportRow], +) -> None: + del db, call_import, row_models + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + schedule_fair_import_dispatch(max_workspace_turns=999) + + +def ensure_blob_storage_enabled() -> None: + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + err = ( + s3_service.get_status_message() + or "Cloud blob storage is not enabled or not configured" + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Call uploads require cloud blob storage so the file can be " + f"persisted between stages: {err}" + ), + ) diff --git a/app/services/call_imports/import_queries.py b/app/services/call_imports/import_queries.py new file mode 100644 index 00000000..ac40e396 --- /dev/null +++ b/app/services/call_imports/import_queries.py @@ -0,0 +1,526 @@ +"""Read paths for call-import batches and rows.""" + +from __future__ import annotations + +from typing import Dict, List, Optional, Tuple +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.db_sharding.sessions import is_sharding_enabled +from app.models.database import CallImport, CallImportRow +from app.models.enums import CallImportStatus +from app.models.schemas import ( + CallImportDetailResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMetricAggregate, + CallImportRowIdsResponse, + CallImportRowResponse, +) +from app.services.call_imports.audit import ( + emails_for_user_ids, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.services.call_imports.present import ( + _call_import_row_list_response, + _imported_agent_partial_id, + _remember_platform_agent_names, + _serialize_call_import, +) + + +def get_call_import_for_org_or_404( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImport: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + return call_import + + +def locate_call_import_row_or_404( + catalog_db: Session, + *, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> Tuple[Session, CallImportRow, Optional[Session]]: + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + except LookupError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) from None + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + close_row_sessions( + row_db, + located_catalog if located_catalog is not row_db else None, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + extra_catalog = located_catalog if located_catalog is not row_db else None + return row_db, row, extra_catalog + + +def list_call_imports_page( + db: Session, + organization_id: UUID, + workspace_id: UUID, + *, + page: int, + page_size: int, + status_filter: Optional[CallImportStatus] = None, + dataset: Optional[str] = None, + tag_id: Optional[List[UUID]] = None, + source_format: Optional[str] = None, + content_modality: Optional[str] = None, +) -> CallImportListResponse: + query = ( + db.query(CallImport) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + ) + if status_filter is not None: + query = query.filter(CallImport.status == status_filter) + + source_filter = (source_format or "").strip().lower() + if source_filter == "__non_audio__": + query = query.filter( + or_( + CallImport.source_format.is_(None), + CallImport.source_format.notin_(["audio", "platform"]), + ) + ) + elif source_filter: + query = query.filter(func.lower(CallImport.source_format) == source_filter) + + modality_filter = (content_modality or "").strip().lower() + if modality_filter == "chat": + query = query.filter(func.lower(CallImport.content_modality) == "chat") + elif modality_filter in ("__voice__", "voice"): + query = query.filter( + or_( + CallImport.content_modality.is_(None), + func.lower(CallImport.content_modality) != "chat", + ) + ) + + if dataset is not None: + if dataset == "__none__": + query = query.filter(CallImport.dataset.is_(None)) + elif dataset.strip(): + query = query.filter( + func.lower(CallImport.dataset) == dataset.strip().lower() + ) + + if tag_id: + from app.models.database import CallImportTagAssignment + + for single_tag_id in tag_id: + sub = ( + db.query(CallImportTagAssignment.call_import_id) + .filter(CallImportTagAssignment.tag_id == single_tag_id) + .subquery() + ) + query = query.filter(CallImport.id.in_(sub)) + + total = query.count() + items = ( + query.order_by(desc(CallImport.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) + return CallImportListResponse( + items=[ + _serialize_call_import(db, item, user_emails=email_map) + for item in items + ], + total=total, + page=page, + page_size=page_size, + ) + + +def get_call_import_dispatch_diagnostics_payload( + db: Session, + organization_id: UUID, + *, + workspace_id: Optional[UUID] = None, + include_idle_workspaces: bool = False, +) -> dict: + return build_call_import_dispatch_diagnostics( + db, + organization_id, + workspace_id=workspace_id, + include_idle_workspaces=include_idle_workspaces, + ) + + +def list_call_import_dataset_labels( + db: Session, + organization_id: UUID, + workspace_id: UUID, +) -> List[str]: + rows = ( + db.query(CallImport.dataset) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + .distinct() + .order_by(CallImport.dataset.asc()) + .all() + ) + return [row[0] for row in rows if row[0]] + + +def get_call_import_detail( + db: Session, + call_import_id: UUID, + organization_id: UUID, + *, + row_limit: int, + row_offset: int, + q: Optional[str] = None, + diarised_status: Optional[str] = None, +) -> CallImportDetailResponse: + call_import = get_call_import_for_org_or_404(db, call_import_id, organization_id) + + if ( + (call_import.source_format or "") == "platform" + and _imported_agent_partial_id(call_import) is None + ): + from app.services.platform_import_imported_agent import ( + ensure_imported_agent_for_platform_import, + ) + + try: + if ensure_imported_agent_for_platform_import(db, call_import, None): + db.commit() + db.refresh(call_import) + except Exception: + db.rollback() + + search_term = (q or "").strip() + diarised_status_filter = (diarised_status or "").strip() or None + filtered_total_rows: Optional[int] = None + has_row_filters = bool(search_term or diarised_status_filter) + + if has_row_filters: + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import count_call_import_rows_filtered + + filtered_total_rows = count_call_import_rows_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + from app.services.call_imports.row_text_search import ( + apply_conversation_id_search, + ) + + rows_query = apply_conversation_id_search(rows_query, search_term) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + filtered_total_rows = rows_query.count() + + if row_limit == 0: + rows: List[CallImportRow] = [] + elif is_sharding_enabled(): + from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_filtered_page, + fetch_call_import_rows_page, + ) + + if has_row_filters: + rows = fetch_call_import_rows_filtered_page( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + offset=row_offset, + limit=row_limit, + ) + else: + rows = fetch_call_import_rows_page( + db, + call_import.id, + offset=row_offset, + limit=row_limit, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + from app.services.call_imports.row_text_search import apply_conversation_id_search + + rows_query = apply_conversation_id_search(rows_query, search_term) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + from app.db_sharding.scatter_gather import defer_call_import_row_text + + rows = ( + rows_query.options(*defer_call_import_row_text()) + .order_by(CallImportRow.row_index) + .offset(row_offset) + .limit(row_limit) + .all() + ) + + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts + + diarised_status_counts = aggregate_diarised_transcript_counts( + db, call_import.id + ) + else: + diarised_status_counts: Dict[str, int] = {} + for status_value, count in ( + db.query(CallImportRow.diarised_transcript_status, func.count()) + .filter(CallImportRow.call_import_id == call_import.id) + .group_by(CallImportRow.diarised_transcript_status) + .all() + ): + if isinstance(status_value, str): + diarised_status_counts[status_value] = int(count or 0) + + if (call_import.source_format or "").lower() == "platform": + _remember_platform_agent_names(db, call_import) + detail = CallImportDetailResponse.model_validate( + _serialize_call_import( + db, + call_import, + refresh_import_progress_from_rows=True, + ).model_dump() + ) + detail.rows = [_call_import_row_list_response(r) for r in rows] + detail.filtered_total_rows = filtered_total_rows + detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) + detail.diarised_running_rows = diarised_status_counts.get("running", 0) + detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) + detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) + if (call_import.source_format or "").lower() == "platform": + from app.services.call_imports.platform_import_progress import ( + build_platform_import_progress, + ) + + detail.platform_progress = build_platform_import_progress(db, call_import) + return detail + + +def list_call_import_row_ids( + db: Session, + call_import_id: UUID, + organization_id: UUID, + *, + q: Optional[str] = None, + diarised_status: Optional[str] = None, +) -> CallImportRowIdsResponse: + call_import = get_call_import_for_org_or_404(db, call_import_id, organization_id) + + search_term = (q or "").strip() + status_filter = (diarised_status or "").strip() or None + + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered + + ids = list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=status_filter, + ) + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + rows_query = db.query(CallImportRow.id).filter( + CallImportRow.call_import_id == call_import.id + ) + from app.services.call_imports.row_text_search import apply_conversation_id_search + + rows_query = apply_conversation_id_search(rows_query, search_term) + if status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == status_filter + ) + + ids = [ + row_id + for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() + ] + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + +def get_call_import_row( + db: Session, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> CallImportRowResponse: + if not is_sharding_enabled(): + row = ( + db.query(CallImportRow) + .filter( + CallImportRow.id == row_id, + CallImportRow.call_import_id == call_import_id, + CallImportRow.organization_id == organization_id, + ) + .first() + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + return CallImportRowResponse.model_validate(row) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +def build_call_import_insights( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImportInsightsResponse: + get_call_import_for_org_or_404(db, call_import_id, organization_id) + + from app.models.database import CallImportEvaluation, Metric + + from app.db_sharding.scatter_gather import ( + aggregate_transcript_coverage, + load_evaluation_score_rows, + ) + + ( + total_rows, + rows_with_transcript, + rows_without_transcript, + source_counts, + ) = aggregate_transcript_coverage(db, call_import_id) + + evaluations = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(CallImportEvaluation.created_at.asc()) + .all() + ) + + from app.api.v1.routes.call_import_evaluations import ( + _compute_metric_aggregates, + ) + + metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} + metric_meta: Dict[str, Metric] = {} + metric_latest: Dict[str, CallImportMetricAggregate] = {} + + for evaluation in evaluations: + eval_rows = load_evaluation_score_rows(db, evaluation.id) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + for agg in aggregates: + if agg.metric_id not in metric_meta: + try: + metric_uuid = UUID(agg.metric_id) + except (ValueError, AttributeError, TypeError): + metric_uuid = None + if metric_uuid is not None: + metric_obj = ( + db.query(Metric) + .filter( + Metric.id == metric_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if metric_obj is not None: + metric_meta[agg.metric_id] = metric_obj + history = metric_history.setdefault(agg.metric_id, []) + history.append( + CallImportInsightsRunPoint( + evaluation_id=evaluation.id, + name=evaluation.name, + created_at=evaluation.created_at, + mean=agg.mean, + completed_rows=agg.count, + ) + ) + metric_latest[agg.metric_id] = agg + + metrics_payload: List[CallImportInsightsMetric] = [] + for metric_id, latest in metric_latest.items(): + meta = metric_meta.get(metric_id) + metrics_payload.append( + CallImportInsightsMetric( + metric_id=metric_id, + metric_name=(meta.name if meta else latest.metric_name), + metric_type=(meta.metric_type if meta else latest.metric_type), + latest=latest, + trend=metric_history.get(metric_id, []), + ) + ) + + return CallImportInsightsResponse( + call_import_id=call_import_id, + total_rows=total_rows, + rows_with_transcript=rows_with_transcript, + rows_without_transcript=rows_without_transcript, + transcript_source_counts=source_counts, + evaluation_count=len(evaluations), + metrics=metrics_payload, + ) diff --git a/app/services/call_imports/import_row_mutations.py b/app/services/call_imports/import_row_mutations.py new file mode 100644 index 00000000..63df3924 --- /dev/null +++ b/app/services/call_imports/import_row_mutations.py @@ -0,0 +1,444 @@ +"""Write paths and orchestration for call-import batches and rows.""" + +from __future__ import annotations + +from typing import List, Optional +from uuid import UUID + +from fastapi import HTTPException, status +from loguru import logger +from sqlalchemy.orm import Session + +from app.core.auth import Principal +from app.models.database import CallImport, CallImportRow +from app.models.enums import CallImportRowStatus, CallImportStatus +from app.models.schemas import ( + CallImportDeleteResponse, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportStartRequest, + CallImportUploadResponse, +) +from app.services.call_imports.audit import stamp_call_import_actor +from app.services.call_imports.import_batch_helpers import ( + ensure_blob_storage_enabled, + resolve_telephony_integration, + validate_credentialed_import_ready, + validate_direct_url_import_ready, + validate_telephony_credentials_live, +) +from app.services.call_imports.import_queries import ( + get_call_import_for_org_or_404, + locate_call_import_row_or_404, +) +from app.services.call_imports.present import _tag_response_payload +from app.services.call_imports.source_parse import _resolve_schema + + +def recompute_call_import_counters(db: Session, call_import: CallImport) -> None: + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + rollup_call_import_batch_status(db, call_import) + + +def revoke_pending_tasks(rows: List[CallImportRow]) -> None: + task_ids = [ + r.celery_task_id + for r in rows + if r.celery_task_id + and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) + ] + if not task_ids: + return + + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_ids, terminate=False) + logger.info("Revoked {} pending call-import tasks", len(task_ids)) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to revoke pending call-import tasks: {}", exc) + + +def delete_s3_objects( + organization_id: UUID, + call_import_id: UUID, + rows: List[CallImportRow], +) -> tuple[int, int]: + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + return 0, 0 + + keys = [r.recording_s3_key for r in rows if r.recording_s3_key] + deleted = 0 + errors = 0 + + if keys: + try: + d, errs = s3_service.delete_keys(keys) + deleted += d + errors += len(errs) + if errs: + logger.warning( + "S3 bulk-delete reported {} errors for call_import {}", + len(errs), + call_import_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc + ) + errors += len(keys) + + sweep_prefix = ( + f"{s3_service.prefix}organizations/{organization_id}/" + f"call_imports/{call_import_id}/" + ) + try: + d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) + deleted += d + errors += len(errs) + except Exception as exc: # noqa: BLE001 + logger.exception( + "S3 prefix sweep failed for {}: {}", sweep_prefix, exc + ) + + return deleted, errors + + +def delete_call_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, + principal: Principal, +) -> CallImportDeleteResponse: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + return CallImportDeleteResponse( + id=call_import_id, + status="completed", + ) + + if call_import.status == CallImportStatus.DELETING: + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + call_import.status = CallImportStatus.DELETING + call_import.error_message = None + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import delete_call_import_task + + delete_call_import_task.delay( + str(call_import_id), + str(organization_id), + ) + + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + +def delete_call_import_row( + db: Session, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, + principal: Principal, +) -> None: + from app.services.storage.s3_service import s3_service + + call_import = get_call_import_for_org_or_404(db, call_import_id, organization_id) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = locate_call_import_row_or_404( + db, + call_import_id=call_import.id, + row_id=row_id, + organization_id=organization_id, + ) + try: + revoke_pending_tasks([row]) + + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) + + row_db.delete(row) + row_db.commit() + + recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + finally: + close_row_sessions(row_db, extra_catalog) + + logger.info( + "Deleted call_import_row {} (call_import={}, org={})", + row_id, + call_import.id, + organization_id, + ) + + +def retry_failed_call_import_rows( + db: Session, + call_import_id: UUID, + organization_id: UUID, + principal: Principal, + payload: Optional[CallImportRetryFailedRowsRequest] = None, +) -> CallImportRetryFailedRowsResponse: + call_import = get_call_import_for_org_or_404(db, call_import_id, organization_id) + + if payload is not None: + from app.services.call_imports.import_batch_helpers import ( + resolve_telephony_integration, + validate_credentialed_import_ready, + validate_telephony_credentials_live, + ) + from app.services.call_imports.source_parse import _resolve_schema + + if payload.telephony_integration_id is not None: + integration = resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + validate_telephony_credentials_live(db, organization_id, integration) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + if (integration.provider or "").lower() in {"exotel", "plivo"}: + schema = _resolve_schema( + db, + organization_id, + call_import.workspace_id, + call_import.schema_id, + ) + validate_credentialed_import_ready( + integration.provider, + list(schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + call_import.provider = None + call_import.telephony_integration_id = None + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.services.call_imports.bulk_ops import ( + mark_import_rows_enqueue_failed, + requeue_failed_import_rows, + ) + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + reset_ids = requeue_failed_import_rows(db, call_import.id) + if not reset_ids: + return CallImportRetryFailedRowsResponse( + requeued=0, + enqueue_failed=0, + skipped=0, + ) + + recompute_call_import_counters(db, call_import) + # Rollup leaves a finished evaluation as partial/completed while the + # requeued rows are still pending. Mark the batch processing so the UI + # keeps polling until those downloads finish. + call_import.status = CallImportStatus.PROCESSING + if payload is None: + stamp_call_import_actor(call_import, principal) + db.commit() + + try: + schedule_fair_import_dispatch(max_workspace_turns=999) + requeued = len(reset_ids) + enqueue_failed = 0 + skipped = 0 + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to schedule fair import dispatch for import {}", + call_import.id, + ) + requeued = 0 + enqueue_failed, skipped = mark_import_rows_enqueue_failed( + db, + call_import.id, + reset_ids, + f"Failed to enqueue retry: {exc}", + ) + recompute_call_import_counters(db, call_import) + db.commit() + + return CallImportRetryFailedRowsResponse( + requeued=requeued, + enqueue_failed=enqueue_failed, + skipped=skipped, + ) + + +def bulk_delete_call_import_rows( + db: Session, + call_import_id: UUID, + organization_id: UUID, + principal: Principal, + payload: CallImportRowBulkDelete, +) -> CallImportRowBulkDeleteResponse: + call_import = get_call_import_for_org_or_404(db, call_import_id, organization_id) + + if not payload.row_ids and payload.row_filter is None: + return CallImportRowBulkDeleteResponse(deleted=0, status="completed") + + from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_delete_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + [str(rid) for rid in payload.row_ids] if payload.row_ids else None, + payload.row_filter.model_dump(mode="json") if payload.row_filter else None, + ) + + return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") + + +def start_legacy_call_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, + workspace_id: UUID, + principal: Principal, + payload: CallImportStartRequest, +) -> CallImportUploadResponse: + from sqlalchemy.orm import selectinload as _selectinload + + call_import = ( + db.query(CallImport) + .options(_selectinload(CallImport.tags)) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status != CallImportStatus.MAPPED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot start import for a batch in status " + f"'{call_import.status.value}'. Map the columns first." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file and cannot be imported " + "through the staged flow." + ), + ) + + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot start import without a mapped schema.", + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + + if payload.telephony_integration_id is not None: + integration = resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + validate_telephony_credentials_live(db, organization_id, integration) + if (integration.provider or "").lower() in {"exotel", "plivo"}: + validate_credentialed_import_ready( + integration.provider, + parameters, + dict(call_import.parameter_mapping or {}), + ) + else: + validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_rows_task, + ) + + materialize_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + str(workspace_id), + schedule_import_dispatch=True, + ) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=0, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + "Import accepted. Rows are being materialized in the background; " + "recordings will be fetched asynchronously." + ), + ) diff --git a/app/services/call_imports/manual_audio_presign.py b/app/services/call_imports/manual_audio_presign.py new file mode 100644 index 00000000..d2b0e562 --- /dev/null +++ b/app/services/call_imports/manual_audio_presign.py @@ -0,0 +1,262 @@ +"""Presigned PUT flow for manual call-import audio (no bytes through the API).""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.db_sharding.sessions import is_sharding_enabled +from app.models.database import CallImport, CallImportRow +from app.models.enums import CallImportRowStatus, CallImportStatus +from app.models.schemas import CallImportManualAudioFilePresignItem + + +def prepare_manual_audio_metadata( + files: List[CallImportManualAudioFilePresignItem], + conversation_counts: Dict[str, int], + *, + validate_extension, + validate_size, + audio_content_type, + sanitize_conversation_id, + dedupe_conversation_id, + filename_stem, +) -> List[Dict[str, Any]]: + if not files: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one audio file is required.", + ) + prepared: List[Dict[str, Any]] = [] + for idx, item in enumerate(files): + filename = (item.filename or "").strip() or f"recording-{idx + 1}" + ext = validate_extension(filename) + if not ext: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported audio file '{filename}'.", + ) + size_bytes = int(item.size_bytes or 0) + validate_size(filename, size_bytes) + base_conversation_id = sanitize_conversation_id(filename_stem(filename)) + conversation_id = dedupe_conversation_id(base_conversation_id, conversation_counts) + content_type = audio_content_type(ext, item.content_type) + prepared.append( + { + "filename": filename, + "extension": ext, + "content_type": content_type, + "size_bytes": size_bytes, + "conversation_id": conversation_id, + } + ) + return prepared + + +def build_manual_audio_presign_uploads( + *, + organization_id: UUID, + call_import_id: UUID, + prepared: List[Dict[str, Any]], + audio_s3_key, + generate_presigned_put_url_by_key, + expiration: int = 3600, +) -> Tuple[List[Dict[str, Any]], List[str]]: + uploads: List[Dict[str, Any]] = [] + keys: List[str] = [] + for item in prepared: + row_id = uuid4() + key = audio_s3_key(organization_id, call_import_id, row_id, item["extension"]) + keys.append(key) + upload_url = generate_presigned_put_url_by_key( + key, + content_type=item["content_type"], + expiration=expiration, + ) + uploads.append( + { + "row_id": row_id, + "recording_s3_key": key, + "conversation_id": item["conversation_id"], + "upload_url": upload_url, + "content_type": item["content_type"], + "filename": item["filename"], + "size_bytes": item["size_bytes"], + } + ) + return uploads, keys + + +def persist_manual_audio_row_mappings( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + workspace_id: UUID, + uploads: List[Dict[str, Any]], + start_row_index: int, +) -> None: + row_mappings: List[Dict[str, Any]] = [] + added_size = 0 + for offset, item in enumerate(uploads): + size_bytes = int(item.get("size_bytes") or 0) + added_size += size_bytes + row_mappings.append( + { + "id": item["row_id"], + "call_import_id": call_import.id, + "organization_id": organization_id, + "workspace_id": workspace_id, + "row_index": start_row_index + offset, + "conversation_id": item["conversation_id"], + "recording_url": None, + "transcript": None, + "transcript_source": None, + "raw_columns": {"conversation_id": item["conversation_id"]}, + "status": CallImportRowStatus.COMPLETED, + "recording_s3_key": item["recording_s3_key"], + "recording_content_type": item["content_type"], + "recording_size_bytes": size_bytes, + } + ) + + new_total_rows = (call_import.total_rows or 0) + len(uploads) + if is_sharding_enabled(): + from app.db_sharding.row_ops import ( + bulk_insert_mappings_on_shards, + register_shard_slices, + ) + + bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) + register_shard_slices(db, call_import.id, new_total_rows) + else: + for mapping in row_mappings: + db.add(CallImportRow(**mapping)) + + call_import.total_rows = new_total_rows + call_import.completed_rows = (call_import.completed_rows or 0) + len(uploads) + call_import.source_size_bytes = (call_import.source_size_bytes or 0) + added_size + + +def _load_rows_for_verify( + db: Session, + *, + call_import_id: UUID, + organization_id: UUID, + row_ids: List[UUID], +) -> List[Tuple[Session, CallImportRow]]: + """Return (session, row) for each id; session must be closed by caller.""" + unique_ids = list(dict.fromkeys(row_ids)) + if not is_sharding_enabled(): + rows = ( + db.query(CallImportRow) + .filter( + CallImportRow.call_import_id == call_import_id, + CallImportRow.organization_id == organization_id, + CallImportRow.id.in_(unique_ids), + ) + .all() + ) + if len(rows) != len(unique_ids): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="One or more row ids are invalid for this import.", + ) + return [(db, row) for row in rows] + + from app.db_sharding.row_ops import locate_call_import_row + + open_shard_sessions: dict[str, Session] = {} + loaded: List[Tuple[Session, CallImportRow]] = [] + try: + for rid in unique_ids: + row_db, catalog_db, row, shard_id = locate_call_import_row(rid) + if catalog_db is not row_db: + catalog_db.close() + shard_key = str(shard_id or "legacy") + if shard_key in open_shard_sessions: + row_db.close() + row_db = open_shard_sessions[shard_key] + row = ( + row_db.query(CallImportRow) + .filter(CallImportRow.id == rid) + .one() + ) + else: + open_shard_sessions[shard_key] = row_db + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="One or more row ids are invalid for this import.", + ) + loaded.append((row_db, row)) + except LookupError: + for row_db in open_shard_sessions.values(): + row_db.close() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="One or more row ids are invalid for this import.", + ) from None + except HTTPException: + for row_db in open_shard_sessions.values(): + row_db.close() + raise + return loaded + + +def verify_manual_audio_uploads( + db: Session, + *, + call_import_id: UUID, + organization_id: UUID, + row_ids: List[UUID], + head_object_by_key, +) -> int: + if not row_ids: + return 0 + sessions_rows = _load_rows_for_verify( + db, + call_import_id=call_import_id, + organization_id=organization_id, + row_ids=row_ids, + ) + verified = 0 + modified_sessions: set[Session] = set() + try: + for row_db, row in sessions_rows: + key = (row.recording_s3_key or "").strip() + if not key: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Row {row.id} has no recording key.", + ) + meta = head_object_by_key(key) + if not meta: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Recording not found in storage for row {row.id}.", + ) + size = int(meta.get("ContentLength") or 0) + if size > 0: + row.recording_size_bytes = size + modified_sessions.add(row_db) + verified += 1 + for sess in modified_sessions: + sess.commit() + finally: + closed: set[int] = set() + for row_db, _row in sessions_rows: + if row_db is db: + continue + token = id(row_db) + if token in closed: + continue + closed.add(token) + row_db.close() + return verified diff --git a/app/services/call_imports/manual_audio_route_helpers.py b/app/services/call_imports/manual_audio_route_helpers.py new file mode 100644 index 00000000..3f99b571 --- /dev/null +++ b/app/services/call_imports/manual_audio_route_helpers.py @@ -0,0 +1,310 @@ +"""Manual call-import audio upload helpers (used by API routes).""" + +from __future__ import annotations + +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import HTTPException, UploadFile, status +from sqlalchemy.orm import Session + +from app.config import settings +from app.db_sharding.sessions import is_sharding_enabled +from app.models.database import CallImport, CallImportRow +from app.models.enums import CallImportRowStatus + +MAX_AUDIO_FILES_PER_REQUEST = 100 +MANUAL_AUDIO_BLOB_UPLOAD_WORKERS = 10 + +AUDIO_CONTENT_TYPES = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "m4a": "audio/mp4", +} + + +def _audio_extension(filename: Optional[str]) -> Optional[str]: + """Return the validated lower-case extension for a manual recording.""" + if not filename or "." not in filename: + return None + ext = filename.rsplit(".", 1)[-1].lower().strip() + allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} + return ext if ext in allowed else None + + +def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: + """Prefer the browser-supplied audio content type, with a safe fallback.""" + supplied = (upload_content_type or "").strip() + if supplied and supplied != "application/octet-stream": + return supplied + return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") + + +def _audio_s3_key( + organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str +) -> str: + """Build the canonical S3 key for a manually uploaded recording.""" + from app.services.storage.s3_service import s3_service + + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/{row_id}.{ext}" + ) + + +def _filename_stem(filename: Optional[str]) -> str: + """Extract a cross-platform filename stem from an UploadFile name.""" + raw = (filename or "").strip() + basename = re.split(r"[\\/]", raw)[-1] if raw else "" + if "." in basename: + basename = basename.rsplit(".", 1)[0] + return basename.strip() + + +def _sanitize_conversation_id(raw: str) -> str: + """Turn a filename stem into a stable conversation_id.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("._-") + return (cleaned or "recording")[:255] + + +def _dedupe_conversation_id( + base: str, counts: Dict[str, int] +) -> str: + """Make conversation ids unique within one manual upload batch.""" + count = counts.get(base, 0) + 1 + counts[base] = count + if count == 1: + return base + suffix = f"-{count}" + return f"{base[: 255 - len(suffix)]}{suffix}" + + +def _conversation_id_base(conversation_id: str) -> str: + match = re.match(r"^(.*)-(\d+)$", (conversation_id or "").strip()) + if match: + return match.group(1) + return (conversation_id or "").strip() + + +def _manual_audio_append_start_state( + db: Session, + call_import: CallImport, +) -> tuple[int, Dict[str, int]]: + """Next row_index and conversation-id suffix counts for audio append.""" + from app.db_sharding.scatter_gather import ( + load_call_import_conversation_ids, + max_call_import_row_index, + ) + + max_index = max_call_import_row_index(db, call_import.id) + start_row_index = max_index + 1 + if start_row_index == 0 and (call_import.total_rows or 0) > 0: + start_row_index = call_import.total_rows + conversation_counts = _seed_conversation_counts( + cid for cid in load_call_import_conversation_ids(db, call_import.id) if cid + ) + return start_row_index, conversation_counts + + +def _seed_conversation_counts(existing_conversation_ids: Iterable[str]) -> Dict[str, int]: + counts: Dict[str, int] = {} + for conversation_id in existing_conversation_ids: + base = _conversation_id_base(conversation_id) + if not base: + continue + counts[base] = counts.get(base, 0) + 1 + return counts + + +def _enforce_audio_upload_file_limit(file_count: int) -> None: + if file_count > MAX_AUDIO_FILES_PER_REQUEST: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"At most {MAX_AUDIO_FILES_PER_REQUEST} audio files may be uploaded " + "per request. Split large batches into multiple requests." + ), + ) + + +async def _prepare_audio_upload_files( + files: List[UploadFile], + conversation_counts: Dict[str, int], +) -> List[Dict[str, Any]]: + if not files: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one audio file is required.", + ) + _enforce_audio_upload_file_limit(len(files)) + + max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 + prepared: List[Dict[str, Any]] = [] + + for idx, upload in enumerate(files): + filename = upload.filename or f"recording-{idx + 1}" + ext = _audio_extension(filename) + if not ext: + allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", + ) + + contents = await upload.read() + if not contents: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Audio file '{filename}' is empty.", + ) + if len(contents) > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"Audio file '{filename}' exceeds " + f"{settings.MAX_FILE_SIZE_MB} MB." + ), + ) + + base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) + conversation_id = _dedupe_conversation_id( + base_conversation_id, + conversation_counts, + ) + prepared.append( + { + "filename": filename, + "extension": ext, + "content_type": _audio_content_type(ext, upload.content_type), + "contents": contents, + "conversation_id": conversation_id, + } + ) + return prepared + + +def _validate_audio_file_size(filename: str, size_bytes: int) -> None: + max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 + if size_bytes > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"Audio file '{filename}' exceeds " + f"{settings.MAX_FILE_SIZE_MB} MB." + ), + ) + + +def _prepare_audio_presign_items( + files: List, + conversation_counts: Dict[str, int], +) -> List[Dict[str, Any]]: + from app.services.call_imports import manual_audio_presign as presign_mod + + return presign_mod.prepare_manual_audio_metadata( + files, + conversation_counts, + validate_extension=_audio_extension, + validate_size=_validate_audio_file_size, + audio_content_type=_audio_content_type, + sanitize_conversation_id=_sanitize_conversation_id, + dedupe_conversation_id=_dedupe_conversation_id, + filename_stem=_filename_stem, + ) + + +def _upload_manual_audio_blobs_parallel( + upload_specs: List[Tuple[str, bytes, str]], +) -> None: + """Upload prepared manual-audio blobs concurrently (key, body, content_type).""" + if not upload_specs: + return + + from app.services.storage.s3_service import s3_service + + def _upload_one(spec: Tuple[str, bytes, str]) -> None: + key, contents, content_type = spec + s3_service.upload_file_by_key( + contents, + key, + content_type=content_type, + ) + + if len(upload_specs) == 1: + _upload_one(upload_specs[0]) + return + + workers = min(MANUAL_AUDIO_BLOB_UPLOAD_WORKERS, len(upload_specs)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(_upload_one, spec) for spec in upload_specs] + for future in as_completed(futures): + future.result() + + +def _persist_prepared_audio_rows( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + workspace_id: UUID, + prepared: List[Dict[str, Any]], + start_row_index: int, +) -> List[str]: + uploaded_keys: List[str] = [] + row_mappings: List[Dict[str, Any]] = [] + upload_specs: List[Tuple[str, bytes, str]] = [] + + for offset, item in enumerate(prepared): + row_id = uuid4() + key = _audio_s3_key( + organization_id, + call_import.id, + row_id, + item["extension"], + ) + uploaded_keys.append(key) + upload_specs.append((key, item["contents"], item["content_type"])) + row_mappings.append( + { + "id": row_id, + "call_import_id": call_import.id, + "organization_id": organization_id, + "workspace_id": workspace_id, + "row_index": start_row_index + offset, + "conversation_id": item["conversation_id"], + "recording_url": None, + "transcript": None, + "transcript_source": None, + "raw_columns": {"conversation_id": item["conversation_id"]}, + "status": CallImportRowStatus.COMPLETED, + "recording_s3_key": key, + "recording_content_type": item["content_type"], + "recording_size_bytes": len(item["contents"]), + } + ) + + _upload_manual_audio_blobs_parallel(upload_specs) + + added_size = sum(len(item["contents"]) for item in prepared) + new_total_rows = (call_import.total_rows or 0) + len(prepared) + + if is_sharding_enabled(): + from app.db_sharding.row_ops import ( + bulk_insert_mappings_on_shards, + register_shard_slices, + ) + + bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) + register_shard_slices(db, call_import.id, new_total_rows) + else: + for mapping in row_mappings: + db.add(CallImportRow(**mapping)) + + call_import.total_rows = new_total_rows + call_import.completed_rows = (call_import.completed_rows or 0) + len(prepared) + call_import.source_size_bytes = (call_import.source_size_bytes or 0) + added_size + return uploaded_keys diff --git a/app/services/call_imports/metric_aggregate_cache.py b/app/services/call_imports/metric_aggregate_cache.py new file mode 100644 index 00000000..a57fb7dc --- /dev/null +++ b/app/services/call_imports/metric_aggregate_cache.py @@ -0,0 +1,146 @@ +"""Persist and serve evaluation metric aggregates without full-table rescans on every poll.""" + +from __future__ import annotations + +from typing import List, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.models.database import CallImportEvaluation +from app.models.schemas import CallImportMetricAggregate + +_TERMINAL_EVAL_STATUSES = frozenset( + {"completed", "failed", "partial", "cancelled", "aborted", "auto_aborted"} +) + + +def aggregates_from_cache( + evaluation: CallImportEvaluation, +) -> Optional[List[CallImportMetricAggregate]]: + completed = int(evaluation.completed_rows or 0) + cached_at = evaluation.cached_metric_aggregates_at_count + raw = evaluation.cached_metric_aggregates + if cached_at is None or raw is None or not isinstance(raw, list): + return None + if int(cached_at) != completed: + return None + try: + return [CallImportMetricAggregate.model_validate(item) for item in raw] + except Exception: + return None + + +def save_aggregate_cache( + db: Session, + evaluation: CallImportEvaluation, + aggregates: List[CallImportMetricAggregate], +) -> None: + from app.api.v1.routes.call_import_evaluations import _aggregate_to_dict + + evaluation.cached_metric_aggregates = [ + _aggregate_to_dict(item) for item in aggregates + ] + evaluation.cached_metric_aggregates_at_count = int( + evaluation.completed_rows or 0 + ) + flag_modified(evaluation, "cached_metric_aggregates") + db.flush() + + +def rebuild_evaluation_metric_aggregate_cache( + db: Session, + evaluation_id: UUID, +) -> List[CallImportMetricAggregate]: + from app.api.v1.routes.call_import_evaluations import _compute_metric_aggregates + from app.db_sharding.scatter_gather import load_evaluation_score_rows_for_aggregate + + evaluation = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == evaluation_id) + .first() + ) + if evaluation is None: + return [] + eval_rows = load_evaluation_score_rows_for_aggregate(db, evaluation_id) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + save_aggregate_cache(db, evaluation, aggregates) + from app.services.call_imports.evaluation_report_period import ( + ensure_evaluation_report_period, + ) + + ensure_evaluation_report_period(db, evaluation, force=True) + return aggregates + + +def metrics_for_aggregate_endpoint( + db: Session, + evaluation: CallImportEvaluation, +) -> List[CallImportMetricAggregate]: + cached = aggregates_from_cache(evaluation) + if cached is not None: + return cached + from app.api.v1.routes.call_import_evaluations import _compute_metric_aggregates + from app.db_sharding.scatter_gather import load_evaluation_score_rows_for_aggregate + + eval_rows = load_evaluation_score_rows_for_aggregate(db, evaluation.id) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + try: + save_aggregate_cache(db, evaluation, aggregates) + from app.services.call_imports.evaluation_report_period import ( + ensure_evaluation_report_period, + ) + + ensure_evaluation_report_period(db, evaluation, force=True) + db.commit() + except Exception as exc: + logger.warning( + "aggregate cache save failed for evaluation {}: {}", + evaluation.id, + exc, + ) + db.rollback() + return aggregates + + +def maybe_refresh_aggregate_cache_after_row( + db: Session, + evaluation: CallImportEvaluation, + *, + new_row_status: str | None, +) -> None: + """Rebuild cache only when the evaluation run reaches a terminal status. + + While a run is active, charts refresh on demand via + ``metrics_for_aggregate_endpoint`` (stale ``cached_metric_aggregates_at_count``). + """ + status = (new_row_status or "").lower() + if status != "completed": + return + parent_status = (evaluation.status or "").lower() + if parent_status not in _TERMINAL_EVAL_STATUSES: + return + completed = int(evaluation.completed_rows or 0) + cached_at = int(evaluation.cached_metric_aggregates_at_count or -1) + if completed == cached_at: + return + try: + rebuild_evaluation_metric_aggregate_cache(db, evaluation.id) + except Exception as exc: + logger.warning( + "aggregate cache refresh failed for evaluation {}: {}", + evaluation.id, + exc, + ) + + +def evaluation_metric_aggregates_for_analysis( + db: Session, + evaluation: CallImportEvaluation, +) -> List[CallImportMetricAggregate]: + cached = aggregates_from_cache(evaluation) + if cached is not None: + return cached + return rebuild_evaluation_metric_aggregate_cache(db, evaluation.id) diff --git a/app/services/call_imports/platform_discovery_sync.py b/app/services/call_imports/platform_discovery_sync.py new file mode 100644 index 00000000..33f656e1 --- /dev/null +++ b/app/services/call_imports/platform_discovery_sync.py @@ -0,0 +1,89 @@ +"""Finalize platform discovery when import rows are idle (stale slice recovery).""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from loguru import logger +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import CallImport, CallImportDiscoverySlice +from app.models.enums import CallImportStatus +from app.services.call_imports.import_batch_helpers import is_platform_call_import + + +def reclaim_stale_platform_discovery_slices( + db: Session, + *, + limit: int = 8, +) -> int: + """Reset ``queued``/``running`` slices with no heartbeat back to ``pending``.""" + stale_cutoff = datetime.now(timezone.utc) - timedelta( + minutes=settings.PLATFORM_IMPORT_DISCOVERY_SLICE_STALE_MINUTES + ) + from app.workers.concurrency.platform_import_limits import force_release_slice_lock + + stale_slices = ( + db.query(CallImportDiscoverySlice) + .join(CallImport, CallImport.id == CallImportDiscoverySlice.call_import_id) + .filter( + CallImport.source_format == "platform", + CallImport.status != CallImportStatus.DELETING, + CallImportDiscoverySlice.status.in_(("queued", "running")), + CallImportDiscoverySlice.updated_at < stale_cutoff, + ) + .limit(max(1, int(limit))) + .all() + ) + if not stale_slices: + return 0 + for slice_row in stale_slices: + force_release_slice_lock(slice_row.call_import_id, slice_row.slice_id) + slice_row.status = "pending" + db.commit() + return len(stale_slices) + + +def sync_platform_discovery_when_import_rows_idle( + db: Session, + call_import: CallImport, +) -> None: + """Reclaim stale slices once row hydrate is idle. + + Unfinished slices stay eligible. Listing can still be waiting on a rate + limit or a later time window after the rows discovered so far have + finished. Stale ``queued``/``running`` slices are returned to ``pending`` + from their heartbeat, and discovery is finalized only when every slice + is already terminal. + """ + if not is_platform_call_import(call_import): + return + discovery = (call_import.discovery_status or "").strip().lower() + if discovery in {"completed", "capped", "failed", "partial"}: + return + + from app.services.call_imports.bulk_ops import _aggregate_import_row_status_counts + + total, _completed, _failed, pending, processing = _aggregate_import_row_status_counts( + db, call_import.id + ) + if pending > 0 or processing > 0: + return + if total <= 0: + return + + reclaimed = reclaim_stale_platform_discovery_slices(db, limit=8) + if reclaimed: + logger.info( + "platform discovery reclaimed {} stale slice(s) for import {}", + reclaimed, + call_import.id, + ) + + from app.services.call_imports.platform_discovery import ( + finalize_platform_discovery_if_complete, + ) + + finalize_platform_discovery_if_complete(db, call_import.id) + db.refresh(call_import) diff --git a/app/services/call_imports/present.py b/app/services/call_imports/present.py new file mode 100644 index 00000000..20f00105 --- /dev/null +++ b/app/services/call_imports/present.py @@ -0,0 +1,290 @@ +"""Serialization and display helpers for call import API responses.""" + +from __future__ import annotations + +import time +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.models.database import CallImport, CallImportRow, CallImportTag +from app.models.enums import CallImportStatus +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + user_ids_from_call_imports, +) +from app.models.schemas import ( + CallImportResponse, + CallImportRowResponse, + PlatformImportAgentRef, +) + +def _normalize_dataset(raw: Optional[str]) -> Optional[str]: + """Trim and treat empty strings as 'no dataset' (NULL).""" + if raw is None: + return None + cleaned = raw.strip() + return cleaned or None + + +def _normalize_import_display_name(raw: Optional[str]) -> Optional[str]: + """Trim a user-facing import/batch label; empty clears to NULL.""" + if raw is None: + return None + cleaned = raw.strip() + if not cleaned: + return None + return cleaned[:512] + + +def _imported_agent_partial_id(call_import: CallImport): + from app.services.platform_import_imported_agent import linked_partial_id + + return linked_partial_id(call_import) + + +def _platform_agent_refs(call_import: CallImport) -> list: + """Agent id and name saved with a platform import, for later eval and prompt work.""" + raw = getattr(call_import, "platform_filters", None) or {} + if not isinstance(raw, dict): + return [] + refs = [] + for agent in raw.get("agents") or []: + if not isinstance(agent, dict): + continue + agent_id = str(agent.get("id") or "").strip() + if not agent_id: + continue + medium = agent.get("medium") or raw.get("medium") + refs.append( + { + "id": agent_id, + "name": str(agent.get("name") or agent_id), + "medium": str(medium).strip().lower() if medium else None, + } + ) + if refs: + return refs + medium = raw.get("medium") + for agent_id in raw.get("agent_ids") or []: + text = str(agent_id or "").strip() + if not text: + continue + refs.append( + { + "id": text, + "name": text, + "medium": str(medium).strip().lower() if medium else None, + } + ) + return refs + + +_agent_name_lookup_after: Dict[str, float] = {} + + +def _remember_platform_agent_names(db: Session, call_import: CallImport) -> None: + """Fill in display names for imports that only stored the provider agent id.""" + import time + + refs = _platform_agent_refs(call_import) + if not refs or not call_import.voice_integration_id: + return + if all(ref["name"] and ref["name"] != ref["id"] for ref in refs): + return + lookup_key = str(call_import.id) + now = time.time() + if _agent_name_lookup_after.get(lookup_key, 0) > now: + return + from sqlalchemy.orm.attributes import flag_modified + + from app.core.encryption import decrypt_api_key + from app.models.database import Integration + from app.services.voice_providers import get_voice_provider + + integration = ( + db.query(Integration) + .filter(Integration.id == call_import.voice_integration_id) + .first() + ) + if integration is None or not integration.api_key: + return + try: + from app.api.v1.routes.platform_call_imports import _list_import_agents + + api_key = decrypt_api_key(integration.api_key) + provider_cls = get_voice_provider(integration.platform) + if (integration.platform or "").lower() == "elevenlabs": + client = provider_cls(api_key, base_url=integration.api_base_url) + else: + client = provider_cls(api_key) + listed = _list_import_agents(client, integration.platform) + except Exception: + _agent_name_lookup_after[lookup_key] = time.time() + 300 + return + by_id = {row.id: row for row in listed} + updated = [] + changed = False + for ref in refs: + match = by_id.get(ref["id"]) + name = ref["name"] + medium = ref.get("medium") + if match is not None and match.name and match.name != ref["id"]: + name = match.name + if match is not None and match.medium: + medium = match.medium + if name != ref["name"] or medium != ref.get("medium"): + changed = True + updated.append({"id": ref["id"], "name": name, "medium": medium}) + if not changed: + _agent_name_lookup_after[lookup_key] = time.time() + 300 + return + filters = dict(call_import.platform_filters or {}) + filters["agents"] = updated + call_import.platform_filters = filters + flag_modified(call_import, "platform_filters") + db.commit() + + +def _serialize_call_import( + db: Session, + call_import: CallImport, + *, + user_emails: Optional[Dict[UUID, str]] = None, + refresh_import_progress_from_rows: bool = False, +) -> CallImportResponse: + """Catalog parent fields; counters include unflushed Redis deltas for display.""" + from app.services.call_imports.bulk_ops import ( + _any_evaluation_needs_ui_poll, + _latest_evaluation_progress_for_ui, + ) + from app.services.call_imports.progress_counters import merge_import_counters_for_ui + + ui_completed, ui_failed = merge_import_counters_for_ui(call_import) + total = int(call_import.total_rows or 0) + completed = min(ui_completed, total) if total else ui_completed + failed = min(ui_failed, total) if total else ui_failed + batch_status = ( + call_import.status.value + if isinstance(call_import.status, CallImportStatus) + else str(call_import.status or "") + ) + # Active imports need the live processing count so a recording retry + # still polls after an evaluation has already finished. + import_rows_processing = 0 + if batch_status in {"pending", "processing"}: + from app.services.call_imports.bulk_ops import ( + _aggregate_import_row_status_counts, + ) + + agg_total, agg_completed, agg_failed, _, agg_processing = ( + _aggregate_import_row_status_counts(db, call_import.id) + ) + import_rows_processing = int(agg_processing or 0) + if refresh_import_progress_from_rows and agg_total > 0: + total = agg_total + completed = agg_completed + failed = agg_failed + if user_emails is None: + user_emails = emails_for_user_ids( + db, user_ids_from_call_imports([call_import]) + ) + created_email, updated_email = actor_emails_for_call_import( + call_import, user_emails + ) + base = CallImportResponse.model_validate(call_import) + return base.model_copy( + update={ + "total_rows": total, + "completed_rows": completed, + "failed_rows": failed, + "import_rows_processing": import_rows_processing, + "platform_agents": [ + PlatformImportAgentRef.model_validate(ref) + for ref in _platform_agent_refs(call_import) + ], + "imported_agent_partial_id": _imported_agent_partial_id(call_import), + **_latest_evaluation_progress_for_ui(db, call_import.id), + "any_evaluation_poll_active": _any_evaluation_needs_ui_poll( + db, call_import.id + ), + "created_by_email": created_email, + "last_updated_by_email": updated_email, + } + ) +def _resolve_tags( + db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] +) -> List[CallImportTag]: + """Look up tag rows by id, scoped to the organization. + + Raises HTTPException(400) if any id is unknown for the org. + """ + if not tag_ids: + return [] + rows = ( + db.query(CallImportTag) + .filter( + CallImportTag.organization_id == organization_id, + CallImportTag.id.in_(tag_ids), + ) + .all() + ) + found_ids = {row.id for row in rows} + missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown call_import_tag id(s): {missing}", + ) + return rows +def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: + """Shape a CallImport's tag relationship for the upload response.""" + return [ + { + "id": tag.id, + "name": tag.name, + "color": tag.color, + "created_at": tag.created_at, + "updated_at": tag.updated_at, + } + for tag in (tags or []) + ] +def _call_import_row_list_response(row: CallImportRow) -> CallImportRowResponse: + """List payload. Transcript text is loaded by the single-row GET.""" + return CallImportRowResponse( + id=row.id, + row_index=row.row_index, + conversation_id=row.conversation_id, + recording_url=row.recording_url, + recording_date=row.recording_date, + transcript=None, + transcript_source=row.transcript_source, + transcript_provider=row.transcript_provider, + transcript_model=row.transcript_model, + transcript_status=row.transcript_status, + transcript_error=row.transcript_error, + transcribed_at=row.transcribed_at, + diarised_transcript=None, + diarised_transcript_provider=row.diarised_transcript_provider, + diarised_transcript_model=row.diarised_transcript_model, + diarised_transcript_status=row.diarised_transcript_status, + diarised_transcript_error=row.diarised_transcript_error, + diarised_at=row.diarised_at, + diarised_llm_provider=row.diarised_llm_provider, + diarised_llm_model=row.diarised_llm_model, + diarised_prompt=None, + diarised_segments=None, + diarised_speaker_swap=bool(row.diarised_speaker_swap), + status=row.status, + recording_s3_key=row.recording_s3_key, + recording_content_type=row.recording_content_type, + recording_size_bytes=row.recording_size_bytes, + error_message=row.error_message, + attempts=row.attempts, + raw_columns=None, + created_at=row.created_at, + updated_at=row.updated_at, + ) diff --git a/app/services/call_imports/progress_counters.py b/app/services/call_imports/progress_counters.py index 538ce56d..052103c7 100644 --- a/app/services/call_imports/progress_counters.py +++ b/app/services/call_imports/progress_counters.py @@ -164,14 +164,60 @@ def clear_eval_progress_redis(evaluation_id: UUID | str) -> None: logger.debug("eval progress redis clear skipped: {}", exc) +_TERMINAL_EVAL_STATUSES = frozenset({"completed", "failed", "partial"}) + + +def _catalog_fence(evaluation) -> tuple[int, int]: + try: + gen = int(getattr(evaluation, "progress_counter_gen", 0) or 0) + seq = int(getattr(evaluation, "progress_flush_seq", 0) or 0) + except Exception: # noqa: BLE001 - detached/deferred column + return 0, 0 + return gen, seq + + def merge_eval_counters_for_ui( evaluation, ) -> tuple[int, int]: - """Catalog counters plus unflushed Redis deltas for progress display.""" + """Catalog counters plus unflushed Redis deltas for progress display. + + Adds the legacy ``eval:progress`` deltas and the buffered + ``eval:ctr:{id}`` deltas that the catalog has not applied yet (the pending + slot only while its seq is ahead of the catalog). Clamped to + ``[0, total_rows]``. + """ completed = int(getattr(evaluation, "completed_rows", 0) or 0) failed = int(getattr(evaluation, "failed_rows", 0) or 0) rc, rf = read_eval_progress(evaluation.id) - return completed + rc, failed + rf + completed += rc + failed += rf + + status = str(getattr(evaluation, "status", "") or "").strip().lower() + if status not in _TERMINAL_EVAL_STATUSES: + from app.services.call_imports.eval_progress_buffer import ( + read_buffered, + unapplied_deltas, + ) + + catalog_gen, catalog_seq = _catalog_fence(evaluation) + bc, bf = unapplied_deltas( + read_buffered(evaluation.id), + catalog_gen=catalog_gen, + catalog_seq=catalog_seq, + ) + completed += bc + failed += bf + + completed = max(0, completed) + failed = max(0, failed) + try: + total = int(getattr(evaluation, "total_rows", 0) or 0) + except (TypeError, ValueError): + total = 0 + if total > 0: + completed = min(completed, total) + failed = min(failed, total) + return completed, failed def read_import_progress(call_import_id: UUID | str) -> tuple[int, int]: @@ -195,11 +241,37 @@ def merge_import_counters_for_ui(call_import) -> tuple[int, int]: return completed + rc, failed + rf +def apply_import_counter_deltas_to_catalog( + db: Session, + call_import_id: UUID | str, + *, + completed_delta: int = 0, + failed_delta: int = 0, +) -> None: + if not completed_delta and not failed_delta: + return + from sqlalchemy import update + + from app.models.database import CallImport + + cid = call_import_id if isinstance(call_import_id, UUID) else UUID(str(call_import_id)) + db.execute( + update(CallImport) + .where(CallImport.id == cid) + .values( + completed_rows=CallImport.completed_rows + completed_delta, + failed_rows=CallImport.failed_rows + failed_delta, + ) + ) + db.flush() + + def record_import_row_status_transition( call_import_id: UUID | str, *, previous_status: str, new_status: str, + db: Session | None = None, ) -> None: from app.workers.tasks.evaluate_call_import_row_core import ( counter_deltas_for_status_transition, @@ -213,12 +285,31 @@ def _norm(value) -> str: _norm(previous_status), _norm(new_status), ) - if completed_delta or failed_delta: - record_import_row_terminal( - call_import_id, - completed_delta=completed_delta, - failed_delta=failed_delta, - ) + if not completed_delta and not failed_delta: + return + try: + client = _client() + if completed_delta: + client.hincrby( + "import:progress", + _import_completed_key(call_import_id), + completed_delta, + ) + if failed_delta: + client.hincrby( + "import:progress", + _import_failed_key(call_import_id), + failed_delta, + ) + except redis.RedisError as exc: + logger.debug("import progress counter skipped: {}", exc) + if db is not None: + apply_import_counter_deltas_to_catalog( + db, + call_import_id, + completed_delta=completed_delta, + failed_delta=failed_delta, + ) def flush_import_progress_to_catalog(db: Session, call_import_id: UUID) -> None: diff --git a/app/services/call_imports/recording_storage.py b/app/services/call_imports/recording_storage.py new file mode 100644 index 00000000..53acf789 --- /dev/null +++ b/app/services/call_imports/recording_storage.py @@ -0,0 +1,51 @@ +"""Upload call-import recordings via a local file path (one S3 object, duration at ingest).""" + +from __future__ import annotations + +import os +import tempfile + +from app.services.storage.s3_service import s3_service +from app.services.usage.llm_usage import probe_audio_seconds + + +def upload_local_recording_file( + path: str, + *, + key: str, + content_type: str, + size_bytes: int | None = None, +) -> tuple[int, int]: + """Upload from disk; return (size_bytes, duration_seconds).""" + if size_bytes is None: + size_bytes = os.path.getsize(path) + duration = max(0, int(probe_audio_seconds(path) or 0)) + s3_service.upload_file_by_key_from_path( + path, key, content_type=content_type or "audio/mpeg" + ) + return int(size_bytes), duration + + +def persist_recording_bytes( + audio_bytes: bytes, + *, + key: str, + content_type: str, +) -> tuple[int, int]: + """Write bytes to a temp file, probe duration, upload once.""" + tmp_fd, tmp_path = tempfile.mkstemp(suffix=".audio") + os.close(tmp_fd) + try: + with open(tmp_path, "wb") as handle: + handle.write(audio_bytes) + return upload_local_recording_file( + tmp_path, + key=key, + content_type=content_type, + size_bytes=len(audio_bytes), + ) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass diff --git a/app/services/call_imports/row_selection.py b/app/services/call_imports/row_selection.py new file mode 100644 index 00000000..c14b0b4d --- /dev/null +++ b/app/services/call_imports/row_selection.py @@ -0,0 +1,17 @@ +"""Normalize import row selection filters for bulk APIs.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +from app.models.schemas import CallImportRowSelectionFilter + + +def normalize_row_selection_filter( + row_filter: Optional[CallImportRowSelectionFilter], +) -> Tuple[str, Optional[str]]: + if row_filter is None: + return "", None + q = (row_filter.q or "").strip() + diarised = (row_filter.diarised_status or "").strip() or None + return q, diarised diff --git a/app/services/call_imports/row_text_search.py b/app/services/call_imports/row_text_search.py new file mode 100644 index 00000000..0e839ed5 --- /dev/null +++ b/app/services/call_imports/row_text_search.py @@ -0,0 +1,65 @@ +"""Substring search helpers for call-import rows (pg_trgm-backed on Postgres).""" + +from __future__ import annotations + +from typing import Optional + +from sqlalchemy import or_ + +from app.models.database import CallImportRow + + +def normalize_row_search_term(raw: Optional[str]) -> Optional[str]: + term = (raw or "").strip() + if not term: + return None + stripped = term.replace("\\", "").replace("%", "").replace("_", "") + if not stripped: + return None + return term + + +def _escape_ilike(term: str) -> str: + return ( + term.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + + +def ilike_substring_pattern(term: str) -> str: + return f"%{_escape_ilike(term)}%" + + +def conversation_id_substring_filter(term: Optional[str]): + needle = normalize_row_search_term(term) + if not needle: + return None + pattern = ilike_substring_pattern(needle) + return CallImportRow.conversation_id.ilike(pattern, escape="\\") + + +def evaluation_row_free_text_filter(term: Optional[str]): + needle = normalize_row_search_term(term) + if not needle: + return None + pattern = ilike_substring_pattern(needle) + return or_( + CallImportRow.conversation_id.ilike(pattern, escape="\\"), + CallImportRow.transcript.ilike(pattern, escape="\\"), + CallImportRow.diarised_transcript.ilike(pattern, escape="\\"), + ) + + +def apply_conversation_id_search(query, term: Optional[str]): + clause = conversation_id_substring_filter(term) + if clause is None: + return query + return query.filter(clause) + + +def apply_evaluation_row_search(query, term: Optional[str]): + clause = evaluation_row_free_text_filter(term) + if clause is None: + return query + return query.filter(clause) diff --git a/app/services/call_imports/source_parse.py b/app/services/call_imports/source_parse.py new file mode 100644 index 00000000..8f576d9d --- /dev/null +++ b/app/services/call_imports/source_parse.py @@ -0,0 +1,1069 @@ +"""CSV/Excel source parsing and staging helpers for call imports.""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.core.auth import Principal +from app.models.database import ( + CallImport, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, +) +from app.models.enums import CallImportParameterType, CallImportStatus +from app.models.schemas import CallImportPreviewSheet +from app.services.call_imports.audit import stamp_call_import_actor + +@dataclass(frozen=True) +class CallImportParseSkip: + """One source row excluded during CSV/Excel parse (identity / recording URL).""" + + source_row: int + reason: str + message: str + + +@dataclass +class CallImportParseResult: + rows: List[Dict[str, Any]] = field(default_factory=list) + skipped: List[CallImportParseSkip] = field(default_factory=list) + + +def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: + """Persistable JSON shape for ``CallImport.source_row_skips``.""" + return [ + { + "source_row": item.source_row, + "reason": item.reason, + "message": item.message, + } + for item in skips + ] +MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) +MAX_AUDIO_FILES_PER_REQUEST = 100 +MANUAL_AUDIO_BLOB_UPLOAD_WORKERS = 10 + +# File extensions accepted by the upload + preview endpoints. Keep in +# lockstep with the frontend ``accept`` attribute on the file picker. +CSV_EXTENSIONS = (".csv",) +XLSX_EXTENSIONS = (".xlsx", ".xlsm") +ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS +def _file_format(filename: Optional[str]) -> Optional[str]: + """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" + if not filename: + return None + name = filename.lower() + if name.endswith(CSV_EXTENSIONS): + return "csv" + if name.endswith(XLSX_EXTENSIONS): + return "xlsx" + return None + +def _normalize_header(name: str) -> str: + return (name or "").strip().lower() + + +def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: + """Map normalized header -> original header for case-insensitive lookup.""" + return {_normalize_header(h): h for h in fieldnames or []} + + +def _resolve_mapped_header( + mapping_value: Optional[str], header_lookup: Dict[str, str] +) -> Optional[str]: + """Translate a user-supplied CSV header into the actual column key. + + The frontend sends headers exactly as they appear in the source file, + but we still normalize on the server so trailing whitespace / casing + doesn't break matching. Returns the canonical fieldname or ``None`` + if not present in the file. + """ + if not mapping_value: + return None + return header_lookup.get(_normalize_header(mapping_value)) + + +def _xlsx_cell_to_str(value: Any) -> str: + """Coerce an openpyxl cell value to the string the rest of the + pipeline expects. + + openpyxl returns native Python types (int, float, datetime, bool, + None). The CSV path always works with strings, so we mirror that: + integers stringify cleanly (no ``.0`` suffix on whole-number floats), + datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. + """ + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer(): + return str(int(value)) + return str(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat() + if isinstance(value, timedelta): + return str(value) + return str(value) + + +def _parse_recording_date_cell(cell: str) -> date: + """Parse day-first dates with one/two digit day-month parts.""" + match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) + if match: + day, month, year = (int(part) for part in match.groups()) + return date(year, month, day) + + # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO + # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, + # while keeping plain ISO dates rejected for hand-entered text/CSV cells. + if "T" in cell: + return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() + + raise ValueError("expected D/M/YYYY or D-M-YYYY") + + +def _coerce_parameter_value( + raw: str, + param_type: CallImportParameterType, + *, + row_idx: int, + param_name: str, +) -> Any: + """Validate + coerce a single CSV cell against its declared type. + + Returns the typed Python value to surface in ``raw_columns``. Empty + strings are returned as ``None`` regardless of the parameter type so + optional cells stay null end-to-end. Coercion failures raise a + 400 with a row-anchored message. + """ + cell = (raw or "").strip() + if not cell: + return None + + if param_type == CallImportParameterType.CONVERSATION_ID: + return cell + if param_type == CallImportParameterType.RECORDING_URL: + # Recording URLs are exercised by the worker (which downloads + # them); we only do a light "starts with http" check here so a + # paste-error surfaces immediately at upload time. + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid recording URL (must start with http:// or https://)." + ), + ) + return cell + if param_type == CallImportParameterType.RECORDING_DATE: + try: + parsed_date = _parse_recording_date_cell(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid recording date ({cell!r}); expected day-first " + "D/M/YYYY or D-M-YYYY." + ), + ) + return parsed_date.strftime("%d/%m/%Y") + if param_type == CallImportParameterType.TRANSCRIPT: + return cell + if param_type == CallImportParameterType.TEXT: + return cell + if param_type == CallImportParameterType.NUMBER: + try: + value = float(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid number ({cell!r})." + ), + ) + if value.is_integer(): + return int(value) + return value + if param_type == CallImportParameterType.BOOLEAN: + truthy = {"true", "yes", "y", "1", "t"} + falsy = {"false", "no", "n", "0", "f"} + norm = cell.lower() + if norm in truthy: + return True + if norm in falsy: + return False + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid boolean ({cell!r})." + ), + ) + if param_type == CallImportParameterType.DATETIME: + try: + parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid ISO-8601 date/time ({cell!r})." + ), + ) + return parsed.isoformat() + if param_type == CallImportParameterType.URL: + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid URL (must start with http:// or https://)." + ), + ) + return cell + # Unknown types: store as text and let the next migration catch up. + return cell + + +def _recording_url_cell_is_valid_http(raw: str) -> bool: + cell = (raw or "").strip() + if not cell: + return False + lower = cell.lower() + return lower.startswith("http://") or lower.startswith("https://") + + +def _parameter_is_required(param: CallImportSchemaParameter) -> bool: + """Return whether a schema parameter must be mapped on every upload.""" + if param.is_required: + return True + try: + param_type = CallImportParameterType(param.type) + except ValueError: + return False + return param_type == CallImportParameterType.CONVERSATION_ID + + +def _apply_schema_mapping( + fieldnames: List[str], + rows_iter: Iterable[Dict[str, str]], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], + *, + source_label: str = "CSV", + validate_only: bool = False, +) -> CallImportParseResult: + """Schema-driven row projection: parameter -> CSV header -> typed value. + + Validates that every required schema parameter is mapped to a CSV + header that actually exists in the file, and that every CSV header + is either mapped to a parameter or explicitly listed in + ``skipped_columns``. Returns one dict per non-empty data row with: + + * ``conversation_id`` (str, mandatory) + * ``recording_date`` (Optional[str], DD/MM/YYYY date) + * ``recording_url`` (Optional[str]) + * ``transcript`` (Optional[str]) + * ``parameter_values`` (Dict[str, Any]) of typed values keyed by + parameter name (drives ``raw_columns`` so the export can + reproduce the source). + + ``validate_only=True`` runs the header / mapping / skipped-column + checks (every check that doesn't need to read row data) and then + returns an empty list — used by the MAP stage to validate a + mapping payload against the cached sheet snapshot without + re-fetching the source bytes from S3. + """ + header_lookup = _header_lookup(list(fieldnames)) + + # 1. Look up the conversation_id parameter so we can address it + # directly while building each row. + conv_param = next( + (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), + None, + ) + if conv_param is None: + # The schema invariant should have caught this on create/update, + # but a defensive 400 here keeps us safe against hand-rolled + # API callers that bypassed validation. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema is missing the mandatory conversation_id parameter.", + ) + # 2. Resolve every mapped parameter to a canonical fieldname. + # Required parameters MUST resolve; optional ones may resolve to + # None if the user left them blank (no mapping). + canonical_by_param: Dict[str, Optional[str]] = {} + recording_date_param_name: Optional[str] = None + rec_url_param_name: Optional[str] = None + transcript_param_name: Optional[str] = None + for param in parameters: + mapped_header = parameter_mapping.get(param.name) + canonical = ( + _resolve_mapped_header(mapped_header, header_lookup) + if mapped_header + else None + ) + if _parameter_is_required(param) and canonical is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} does not contain the column " + f"'{mapped_header or ''}' mapped to required parameter " + f"'{param.name}'." + ), + ) + canonical_by_param[param.name] = canonical + if param.type == CallImportParameterType.RECORDING_DATE: + recording_date_param_name = param.name + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_param_name = param.name + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_param_name = param.name + + # 3. Every CSV column must either be mapped to a parameter or + # explicitly skipped. Catches "I forgot to skip the email + # column" gracefully instead of dropping data silently. + mapped_canonicals = {c for c in canonical_by_param.values() if c} + skipped_canonicals = { + _resolve_mapped_header(h, header_lookup) + for h in skipped_columns + } + skipped_canonicals.discard(None) + unhandled = [ + h + for h in fieldnames + if h not in mapped_canonicals and h not in skipped_canonicals + ] + if unhandled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} columns must either be mapped to a schema " + f"parameter or explicitly skipped. Unhandled: {unhandled}." + ), + ) + + conv_canonical = canonical_by_param[conv_param.name] + rec_canonical = ( + canonical_by_param.get(rec_url_param_name) + if rec_url_param_name + else None + ) + recording_date_canonical = ( + canonical_by_param.get(recording_date_param_name) + if recording_date_param_name + else None + ) + transcript_canonical = ( + canonical_by_param.get(transcript_param_name) + if transcript_param_name + else None + ) + + if validate_only: + # MAP-stage validation: every header check above has already + # run; the row loop only matters at IMPORT time. Skip it (and + # the "no data rows" guard at the bottom of the function) so + # the caller gets a clean pass when the mapping is shaped right. + return CallImportParseResult() + + parsed: List[Dict[str, Any]] = [] + skipped: List[CallImportParseSkip] = [] + for idx, row in enumerate(rows_iter): + # Drop fully-blank lines - matches the legacy parser behavior so + # trailing-newline edge cases don't fail an otherwise-good upload. + non_blank = any( + (row.get(c) or "").strip() + for c in mapped_canonicals + if c + ) + if not non_blank: + continue + + source_row = idx + 1 + conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" + if not conv_value: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_conversation_id", + message=( + f"Row {source_row} is missing the '{conv_param.name}' " + "(conversation_id) value." + ), + ) + ) + continue + + if rec_canonical and rec_url_param_name: + rec_param = next( + (p for p in parameters if p.name == rec_url_param_name), + None, + ) + if rec_param is not None: + rec_raw = (row.get(rec_canonical) or "").strip() + if _parameter_is_required(rec_param) and not rec_raw: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{rec_url_param_name}' value." + ), + ) + ) + continue + if rec_raw and not _recording_url_cell_is_valid_http(rec_raw): + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="invalid_recording_url", + message=( + f"Row {source_row}: value for " + f"'{rec_url_param_name}' is not a valid recording " + "URL (must start with http:// or https://)." + ), + ) + ) + continue + + # Materialize every mapped parameter into the per-row snapshot, + # running per-type coercion so a bad cell aborts the upload + # rather than silently storing garbage. + parameter_values: Dict[str, Any] = {} + row_skipped = False + for param in parameters: + canonical = canonical_by_param[param.name] + if canonical is None: + continue + try: + param_type = CallImportParameterType(param.type) + except ValueError: + param_type = CallImportParameterType.TEXT + raw_cell = row.get(canonical) or "" + if ( + param_type == CallImportParameterType.RECORDING_URL + and (raw_cell or "").strip() + and not _recording_url_cell_is_valid_http(raw_cell) + ): + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="invalid_recording_url", + message=( + f"Row {source_row}: value for '{param.name}' is not a " + "valid recording URL (must start with http:// or https://)." + ), + ) + ) + row_skipped = True + break + coerced = _coerce_parameter_value( + raw_cell, + param_type, + row_idx=idx, + param_name=param.name, + ) + if _parameter_is_required(param) and coerced is None: + if param_type == CallImportParameterType.RECORDING_URL: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + ) + row_skipped = True + break + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + parameter_values[param.name] = coerced + if row_skipped: + continue + + rec_value = ( + (row.get(rec_canonical) or "").strip() if rec_canonical else "" + ) + transcript_value = ( + (row.get(transcript_canonical) or "").strip() + if transcript_canonical + else "" + ) + recording_date_value = ( + parameter_values.get(recording_date_param_name) + if recording_date_param_name + else None + ) + + parsed.append( + { + "conversation_id": conv_value, + "recording_date": recording_date_value, + "recording_url": rec_value or None, + "transcript": transcript_value or None, + "parameter_values": parameter_values, + } + ) + + if not parsed and not skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + return CallImportParseResult(rows=parsed, skipped=skipped) + + +def _raise_if_no_importable_rows( + result: CallImportParseResult, *, source_label: str = "CSV" +) -> None: + """Sync upload / API callers fail fast when every data row was skipped.""" + if result.rows: + return + if result.skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"No importable rows. {len(result.skipped)} row(s) skipped due " + "to missing or invalid conversation ID or recording URL." + ), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + +def _parse_csv( + file_bytes: bytes, + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a CSV file using the resolved schema parameters.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + + return _apply_schema_mapping( + list(reader.fieldnames), + reader, + parameters, + parameter_mapping, + skipped_columns, + source_label="CSV", + ) + + +def _open_xlsx_workbook(file_bytes: bytes): + """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). + + Imports openpyxl lazily so the module loads even in environments that + haven't installed the optional dep yet (e.g. lightweight tooling + images). Surfaces a clean 400 if openpyxl is missing or the file is + not a valid Office Open XML workbook. + """ + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded Excel file is empty.", + ) + try: + from openpyxl import load_workbook # type: ignore + from openpyxl.utils.exceptions import InvalidFileException # type: ignore + except ImportError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + "Excel uploads require the 'openpyxl' package which is " + "not installed in this environment." + ), + ) from exc + + try: + return load_workbook( + io.BytesIO(file_bytes), + read_only=True, + data_only=True, + ) + except InvalidFileException as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"File is not a valid .xlsx workbook: {exc}", + ) from exc + except Exception as exc: # zipfile.BadZipFile etc. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not open Excel workbook: {exc}", + ) from exc + + +def _xlsx_sheet_headers_and_rows( + worksheet, +) -> Tuple[List[str], List[Dict[str, str]]]: + """Read row 1 as headers and the rest as dicts of stringified cells. + + Empty trailing header cells are dropped. Duplicate headers preserve + the first occurrence (matches ``csv.DictReader`` behavior, which + silently drops duplicates). + """ + iterator = worksheet.iter_rows(values_only=True) + try: + header_row = next(iterator) + except StopIteration: + return [], [] + + headers: List[str] = [] + seen: set[str] = set() + for cell in header_row: + name = _xlsx_cell_to_str(cell).strip() + if not name: + # Stop at the first blank header — treats trailing empty + # columns as not part of the table (matches typical Excel + # workbook conventions). + break + norm = name.lower() + if norm in seen: + continue + seen.add(norm) + headers.append(name) + + rows: List[Dict[str, str]] = [] + for row in iterator: + if row is None: + continue + # Pad / truncate to the header length so dict construction is + # stable even when a row has fewer / extra cells than the header. + cells = list(row[: len(headers)]) + if len(cells) < len(headers): + cells.extend([None] * (len(headers) - len(cells))) + if not any(_xlsx_cell_to_str(c).strip() for c in cells): + # Skip fully-blank rows (openpyxl read_only routinely yields + # trailing empties when the worksheet's used range exceeds + # the actual data). + continue + rows.append( + { + header: _xlsx_cell_to_str(value) + for header, value in zip(headers, cells) + } + ) + + return headers, rows + + +def _parse_xlsx( + file_bytes: bytes, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a single worksheet from an xlsx/xlsm workbook. + + ``sheet_name`` must match one of the workbook's sheets (case + insensitive whitespace-trimmed match). Returns the same shape as + :func:`_parse_csv` so the upload handler can persist either format + through the same code path. + """ + if not sheet_name or not sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + workbook = _open_xlsx_workbook(file_bytes) + try: + sheet_names = list(workbook.sheetnames) + target_norm = sheet_name.strip().lower() + match = next( + (s for s in sheet_names if s.strip().lower() == target_norm), + None, + ) + if match is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{sheet_name}' not found in workbook. " + f"Available sheets: {sheet_names}" + ), + ) + worksheet = workbook[match] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + finally: + workbook.close() + + if not headers: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Sheet '{sheet_name}' is missing a header row.", + ) + + return _apply_schema_mapping( + headers, + rows, + parameters, + parameter_mapping, + skipped_columns, + source_label=f"Sheet '{sheet_name}'", + ) + + +def _csv_preview_sheets( + file_bytes: bytes, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Build the synthetic single-sheet preview entry for a CSV upload.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + headers = list(reader.fieldnames) + row_count = 0 + for row in reader: + # Match the parse-time skip: ignore fully blank rows so the + # count the user sees lines up with what /upload will ingest. + if any((v or "").strip() for v in row.values()): + row_count += 1 + + sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" + return [ + CallImportPreviewSheet( + name=sheet_label, + headers=headers, + row_count=row_count, + ) + ] + + +def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: + """List every worksheet in the workbook with its headers and row count.""" + workbook = _open_xlsx_workbook(file_bytes) + sheets: List[CallImportPreviewSheet] = [] + try: + for name in workbook.sheetnames: + worksheet = workbook[name] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + sheets.append( + CallImportPreviewSheet( + name=name, + headers=headers, + row_count=len(rows), + ) + ) + finally: + workbook.close() + return sheets +def _parse_json_form_field(name: str, raw: Optional[str], default): + """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" + if raw is None or raw == "": + return default + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{name} must be valid JSON: {exc}", + ) +def _source_content_type(fmt: str) -> str: + """Return the canonical ``Content-Type`` for a parsed file format.""" + if fmt == "csv": + return "text/csv" + if fmt == "xlsx": + return ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + return "application/octet-stream" + + +def _source_s3_key( + organization_id: UUID, call_import_id: UUID, fmt: str +) -> str: + """Build the canonical S3 key for an upload's source file. + + Mirrors the per-row recording key convention used by + ``process_call_import_row`` so a single prefix sweep on delete still + cleans up both the source artefact and every fetched recording. + """ + from app.services.storage.s3_service import s3_service + + ext = "xlsx" if fmt == "xlsx" else "csv" + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/source.{ext}" + ) + + +def _build_available_sheets( + file_bytes: bytes, fmt: str, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" + if fmt == "csv": + return _csv_preview_sheets(file_bytes, filename) + return _xlsx_preview_sheets(file_bytes) + +def _insert_staged_call_import_record( + db: Session, + *, + call_import_id: UUID, + organization_id: UUID, + workspace_id: UUID, + principal: Principal, + filename: str, + fmt: str, + s3_key: str, + content_type: str, + file_bytes: bytes, + normalized_dataset: str, + schema_id: Optional[UUID], + normalized_modality: Optional[str], + tag_rows: List[CallImportTag], +) -> CallImport: + sheets = _build_available_sheets(file_bytes, fmt, filename) + call_import = CallImport( + id=call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=filename, + sheet_name=None, + dataset=normalized_dataset, + schema_id=schema_id, + parameter_mapping={}, + skipped_columns=[], + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + source_s3_key=s3_key, + source_format=fmt, + source_size_bytes=len(file_bytes), + source_content_type=content_type, + available_sheets=[sheet.model_dump() for sheet in sheets], + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + content_modality=normalized_modality, + ) + if tag_rows: + call_import.tags = tag_rows + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + db.commit() + db.refresh(call_import) + return call_import +def _resolve_schema( + db: Session, + organization_id: UUID, + workspace_id: UUID, + schema_id: UUID, +) -> CallImportSchema: + """Fetch + validate a schema row in the active workspace. + + Eager-loads ``parameters`` so callers can iterate without re-querying. + """ + from sqlalchemy.orm import selectinload as _selectinload + + schema = ( + db.query(CallImportSchema) + .options(_selectinload(CallImportSchema.parameters)) + .filter( + CallImportSchema.id == schema_id, + CallImportSchema.organization_id == organization_id, + CallImportSchema.workspace_id == workspace_id, + ) + .first() + ) + if not schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call import schema not found in the active workspace.", + ) + if not list(schema.parameters): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema has no parameters defined.", + ) + return schema +def _clean_parameter_mapping( + mapping_payload: Any, + parameters: List[CallImportSchemaParameter], + schema_name: str, +) -> Dict[str, str]: + """Trim values and drop empties; reject unknown parameter names. + + Accepts an already-decoded value (dict-shaped) so the same helper + works for the JSON-form upload path and the JSON-body PATCH path. + """ + if not isinstance(mapping_payload, dict) or not all( + isinstance(k, str) and (v is None or isinstance(v, str)) + for k, v in mapping_payload.items() + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "parameter_mapping must be an object of " + "{parameter_name: csv_header}." + ), + ) + + valid_param_names = {p.name for p in parameters} + cleaned: Dict[str, str] = {} + for raw_name, raw_header in mapping_payload.items(): + if raw_name not in valid_param_names: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"parameter_mapping references unknown parameter " + f"'{raw_name}' on schema '{schema_name}'." + ), + ) + header = (raw_header or "").strip() + if header: + cleaned[raw_name] = header + return cleaned + + +def _clean_skipped_columns(skipped_payload: Any) -> List[str]: + """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" + if not isinstance(skipped_payload, list) or not all( + isinstance(item, str) for item in skipped_payload + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="skipped_columns must be a list of header strings.", + ) + cleaned: List[str] = [] + seen: set[str] = set() + for item in skipped_payload: + norm = _normalize_header(item) + if not norm or norm in seen: + continue + seen.add(norm) + cleaned.append(item) + return cleaned + + +def _parse_source_file( + file_bytes: bytes, + fmt: str, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + cleaned_mapping: Dict[str, str], + cleaned_skipped: List[str], +) -> CallImportParseResult: + """Run the format-appropriate parser against a buffer of file bytes.""" + if fmt == "csv": + return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) + return _parse_xlsx( + file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped + ) +def _validate_sheet_choice( + fmt: str, + sheet_name: Optional[str], + available_sheets: Optional[List[Dict[str, Any]]], +) -> Optional[str]: + """Normalize / validate ``sheet_name`` against the persisted snapshot. + + Returns the canonical sheet name (matching the workbook's casing) + so downstream parsing addresses the right worksheet. + """ + if fmt == "csv": + if sheet_name and sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + return None + + cleaned = (sheet_name or "").strip() or None + if cleaned is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when the source is an Excel workbook.", + ) + + if not available_sheets: + # Nothing to validate against (e.g. legacy batch without snapshot); + # let downstream parsing error out instead of silently importing. + return cleaned + + target = cleaned.strip().lower() + for entry in available_sheets: + name = entry.get("name") if isinstance(entry, dict) else None + if isinstance(name, str) and name.strip().lower() == target: + return name + sheet_names = [ + entry.get("name") for entry in available_sheets if isinstance(entry, dict) + ] + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{cleaned}' not found in the staged file. " + f"Available sheets: {sheet_names}" + ), + ) diff --git a/app/services/call_imports/staged_source_presign.py b/app/services/call_imports/staged_source_presign.py new file mode 100644 index 00000000..afed429e --- /dev/null +++ b/app/services/call_imports/staged_source_presign.py @@ -0,0 +1,94 @@ +"""Presigned PUT for staged CSV/Excel call-import sources.""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional, Tuple +from uuid import UUID + +from fastapi import HTTPException, status + + +def validate_staged_source_file( + filename: str, + size_bytes: int, + *, + file_format, + max_upload_bytes: int, +) -> str: + fmt = file_format(filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Unsupported file format for staged import.", + ) + if size_bytes <= 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="size_bytes must be positive.", + ) + if size_bytes > max_upload_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {max_upload_bytes} bytes", + ) + return fmt + + +def build_staged_source_presign( + *, + organization_id: UUID, + call_import_id: UUID, + fmt: str, + content_type: str, + source_s3_key: Callable[[UUID, UUID, str], str], + generate_presigned_put_url_by_key, + expiration: int = 3600, +) -> Tuple[str, str]: + key = source_s3_key(organization_id, call_import_id, fmt) + upload_url = generate_presigned_put_url_by_key( + key, + content_type=content_type, + expiration=expiration, + ) + return key, upload_url + + +def load_staged_source_bytes( + *, + source_s3_key: str, + expected_size: Optional[int], + head_object_by_key, + download_file_by_key, +) -> bytes: + meta = head_object_by_key(source_s3_key) + if not meta: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Staged source file was not found in storage.", + ) + stored_size = int(meta.get("ContentLength") or 0) + if stored_size <= 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Staged source file is empty.", + ) + if expected_size is not None and expected_size > 0 and stored_size != expected_size: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Staged file size does not match the declared upload size. " + "Retry the upload." + ), + ) + return download_file_by_key(source_s3_key) + + +def sheets_snapshot_from_bytes( + file_bytes: bytes, + fmt: str, + filename: Optional[str], + *, + build_available_sheets, +) -> List[Dict[str, Any]]: + sheets = build_available_sheets(file_bytes, fmt, filename) + return [sheet.model_dump() for sheet in sheets] diff --git a/app/services/credentials/__init__.py b/app/services/credentials/__init__.py index 479b862a..378f8b5f 100644 --- a/app/services/credentials/__init__.py +++ b/app/services/credentials/__init__.py @@ -5,15 +5,21 @@ """ from app.services.credentials.resolver import ( + CredentialSnapshot, clear_other_defaults, resolve_ai_provider, + resolve_ai_provider_for_call, resolve_integration, + resolve_integration_for_call, resolve_telephony_integration, ) __all__ = [ + "CredentialSnapshot", "clear_other_defaults", "resolve_ai_provider", + "resolve_ai_provider_for_call", "resolve_integration", + "resolve_integration_for_call", "resolve_telephony_integration", ] diff --git a/app/services/credentials/resolver.py b/app/services/credentials/resolver.py index 5840753b..34500c4a 100644 --- a/app/services/credentials/resolver.py +++ b/app/services/credentials/resolver.py @@ -115,6 +115,137 @@ def resolve_ai_provider( ) +class CredentialSnapshot: + """Read-only copy of a resolved ``AIProvider`` / ``Integration`` row. + + Used by the LLM call path cache: every column attribute is present (so + ``ai_provider.api_key`` / ``routing_mode`` / ``gateway_*`` reads work as + on the ORM row), nothing lazy-loads, and assignments raise so a shared + cached copy can never be mutated or flushed. ``api_key`` stays encrypted. + """ + + __slots__ = ("_values", "_kind") + + def __init__(self, kind: str, values: dict) -> None: + object.__setattr__(self, "_kind", kind) + object.__setattr__(self, "_values", dict(values)) + + @classmethod + def from_row(cls, row: Any) -> Optional["CredentialSnapshot"]: + if row is None: + return None + from sqlalchemy import inspect + + mapper = inspect(row).mapper + values = {attr.key: getattr(row, attr.key) for attr in mapper.column_attrs} + return cls(type(row).__name__, values) + + def __getattr__(self, name: str) -> Any: + values = object.__getattribute__(self, "_values") + if name in values: + return values[name] + raise AttributeError(name) + + def __setattr__(self, name: str, value: Any) -> None: + raise AttributeError(f"CredentialSnapshot is read-only (tried to set {name})") + + def __repr__(self) -> str: + return f"CredentialSnapshot({self._kind}, id={self._values.get('id')!r})" + + +def _provider_key(value: Any) -> str: + raw = getattr(value, "value", value) + return str(raw or "").strip().lower() + + +def _resolve_for_call( + model: Type[Any], + namespace: str, + *, + db: Session, + org_id: UUID, + provider_field: str, + provider_value: Any, + credential_id: Optional[UUID], +) -> Optional[CredentialSnapshot]: + """Resolve once per LLM call, and from a per-process cache across calls. + + Cache key includes the org's LLM-config version, which is bumped on any + committed credential/gateway change. When Redis is down the version is + unknown and the row is read from the database (per call, not cached). + """ + from app.services.ai.llm_call_scope import scoped + from app.services.ai.llm_credential_version import ( + cache_ttl_seconds, + read_llm_config_version, + ) + from app.workers.catalog_cache import cached_catalog_fetch + + provider_key = _provider_key(provider_value) + cred_key = str(credential_id) if credential_id is not None else "-" + + def _load() -> Optional[CredentialSnapshot]: + return CredentialSnapshot.from_row( + _resolve_for_org_provider( + model, + db=db, + org_id=org_id, + provider_field=provider_field, + provider_value=provider_key, + credential_id=credential_id, + ) + ) + + def _cached() -> Optional[CredentialSnapshot]: + version = read_llm_config_version(org_id) + if version is None: + return _load() + return cached_catalog_fetch( + f"llmcfg_{namespace}", + f"{org_id}:{version}:{provider_key}:{cred_key}", + _load, + ttl_seconds=cache_ttl_seconds(), + ) + + return scoped((namespace, str(org_id), provider_key, cred_key), _cached) + + +def resolve_ai_provider_for_call( + provider: Any, + db: Session, + org_id: UUID, + credential_id: Optional[UUID] = None, +) -> Optional[CredentialSnapshot]: + """LLM call path: cached, read-only ``AIProvider`` resolution.""" + return _resolve_for_call( + AIProvider, + "aip", + db=db, + org_id=org_id, + provider_field="provider", + provider_value=provider, + credential_id=credential_id, + ) + + +def resolve_integration_for_call( + platform: Any, + db: Session, + org_id: UUID, + credential_id: Optional[UUID] = None, +) -> Optional[CredentialSnapshot]: + """LLM call path: cached, read-only ``Integration`` resolution.""" + return _resolve_for_call( + Integration, + "int", + db=db, + org_id=org_id, + provider_field="platform", + provider_value=platform, + credential_id=credential_id, + ) + + def resolve_azure_endpoint_for_model_provider( provider: ModelProvider, db: Session, @@ -192,3 +323,8 @@ def clear_other_defaults( model.id != keep_id, model.is_default.is_(True), ).update({"is_default": False}, synchronize_session=False) + # Bulk update bypasses the ORM hook; bump the org's LLM-config version + # when this session commits so cached credential resolutions refresh. + from app.services.ai.llm_credential_version import mark_llm_config_changed + + mark_llm_config_changed(db, org_id) diff --git a/app/services/evaluation/metrics_service.py b/app/services/evaluation/metrics_service.py index b003843d..15b787ad 100644 --- a/app/services/evaluation/metrics_service.py +++ b/app/services/evaluation/metrics_service.py @@ -29,30 +29,9 @@ def calculate_wer(reference: str, hypothesis: str) -> float: if not ref_words: return 0.0 if not hyp_words else float("inf") - # Simple Levenshtein distance for WER calculation - # For production, consider using jiwer library for more accurate calculation - n = len(ref_words) - m = len(hyp_words) - - # Create DP table - dp = [[0] * (m + 1) for _ in range(n + 1)] - - # Initialize - for i in range(n + 1): - dp[i][0] = i - for j in range(m + 1): - dp[0][j] = j - - # Fill DP table - for i in range(1, n + 1): - for j in range(1, m + 1): - if ref_words[i - 1] == hyp_words[j - 1]: - dp[i][j] = dp[i - 1][j - 1] - else: - dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) - - errors = dp[n][m] - return errors / n + from rapidfuzz.distance import Levenshtein + + return Levenshtein.distance(ref_words, hyp_words) / len(ref_words) @staticmethod def calculate_cer(reference: str, hypothesis: str) -> float: @@ -75,26 +54,9 @@ def calculate_cer(reference: str, hypothesis: str) -> float: if not ref_chars: return 0.0 if not hyp_chars else float("inf") - # Character-level Levenshtein distance - n = len(ref_chars) - m = len(hyp_chars) - - dp = [[0] * (m + 1) for _ in range(n + 1)] - - for i in range(n + 1): - dp[i][0] = i - for j in range(m + 1): - dp[0][j] = j - - for i in range(1, n + 1): - for j in range(1, m + 1): - if ref_chars[i - 1] == hyp_chars[j - 1]: - dp[i][j] = dp[i - 1][j - 1] - else: - dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) + from rapidfuzz.distance import Levenshtein - errors = dp[n][m] - return errors / n + return Levenshtein.distance(ref_chars, hyp_chars) / len(ref_chars) @staticmethod def calculate_latency(start_time: float, end_time: float) -> Dict[str, float]: diff --git a/app/services/evaluation/transcript_error_rates.py b/app/services/evaluation/transcript_error_rates.py new file mode 100644 index 00000000..07041f5e --- /dev/null +++ b/app/services/evaluation/transcript_error_rates.py @@ -0,0 +1,864 @@ +"""Deterministic transcript WER/CER for call-import evaluations. + +User WER / User CER compare production user turns (reference, the +denominator) with diarised user turns (hypothesis). Agent turns are +ignored. ``diarised_speaker_swap`` decides which diarised speaker is +the user. + +``normalize_for_wer`` / ``compute_tts_wer_cer`` stay the English TTS path. +""" + +from __future__ import annotations + +import re +import string +import unicodedata +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from app.workers.tasks.helpers.script_enforcement import contains_indic_script + +NUMBER_WORDS = frozenset( + { + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + "twenty", + "thirty", + "forty", + "fifty", + "sixty", + "seventy", + "eighty", + "ninety", + "hundred", + "thousand", + "million", + "billion", + "trillion", + "point", + "and", + } +) +CURRENCY_WORDS = frozenset( + { + "dollar", + "dollars", + "usd", + "cent", + "cents", + "rupee", + "rupees", + "inr", + "euro", + "euros", + "eur", + "pound", + "pounds", + "gbp", + } +) + +DEFAULT_SPEAKER_ALIASES: Dict[str, Tuple[str, ...]] = { + "user": ( + "user", + "customer", + "caller", + "human", + "client", + "patient", + "speaker 2", + "speaker2", + ), + "agent": ( + "agent", + "assistant", + "bot", + "ai", + "representative", + "rep", + "speaker 1", + "speaker1", + ), +} + +_SPEAKER_LINE_RE = re.compile( + r"^\s*(?:\[?[\d:.\s\-]+\]?\s*)?(?P[A-Za-z][A-Za-z0-9 _]*)\s*[:\-–]\s*(?P.*)$", +) + +_CONTRACTIONS = { + "won't": "will not", + "can't": "cannot", + "n't": " not", + "'re": " are", + "'ve": " have", + "'ll": " will", + "'d": " would", + "'m": " am", +} + + +@dataclass(frozen=True) +class ParsedTurns: + user_text: str + agent_text: str + user_turn_count: int + agent_turn_count: int + unlabelled: bool + + +def _normalize_speaker_key(name: str) -> str: + return " ".join((name or "").strip().lower().split()) + + +def _resolve_role( + speaker_label: str, + aliases: Mapping[str, Tuple[str, ...]], +) -> Optional[str]: + key = _normalize_speaker_key(speaker_label) + for role, names in aliases.items(): + for alias in names: + if key == _normalize_speaker_key(alias): + return role + return None + + +def _is_numeric_token(token: str) -> bool: + return bool(re.fullmatch(r"\d+(?:\.\d+)?", token)) + + +def normalize_base(text: str) -> str: + punct_table = str.maketrans("", "", string.punctuation) + return text.lower().translate(punct_table).strip() + + +def normalize_entities(text: str) -> str: + """Collapse numeric/currency spans (English) for WER comparison.""" + tokens = normalize_base(text).split() + normalized_tokens: List[str] = [] + i = 0 + while i < len(tokens): + token = tokens[i] + is_entity_token = ( + _is_numeric_token(token) + or token in NUMBER_WORDS + or token in CURRENCY_WORDS + ) + if not is_entity_token: + normalized_tokens.append(token) + i += 1 + continue + + j = i + has_currency = token in CURRENCY_WORDS + while j < len(tokens): + t = tokens[j] + if _is_numeric_token(t) or t in NUMBER_WORDS or t in CURRENCY_WORDS: + if t in CURRENCY_WORDS: + has_currency = True + j += 1 + continue + break + + normalized_tokens.append("" if has_currency else "") + i = j + + return " ".join(normalized_tokens) + + +def normalize_for_wer(text: str) -> str: + """Primary normalized string for jiwer (lowercase, punctuation, entities).""" + lowered = text.lower() + for src, dst in _CONTRACTIONS.items(): + lowered = lowered.replace(src, dst) + return normalize_entities(lowered) + + +def parse_labeled_transcript( + text: str, + *, + aliases: Optional[Mapping[str, Tuple[str, ...]]] = None, +) -> ParsedTurns: + alias_map = aliases or DEFAULT_SPEAKER_ALIASES + raw = (text or "").strip() + if not raw: + return ParsedTurns( + user_text="", + agent_text="", + user_turn_count=0, + agent_turn_count=0, + unlabelled=True, + ) + + user_parts: List[str] = [] + agent_parts: List[str] = [] + user_turn_count = 0 + agent_turn_count = 0 + saw_label = False + current_role: Optional[str] = None + + for line in raw.splitlines(): + stripped = line.strip() + if not stripped: + continue + match = _SPEAKER_LINE_RE.match(stripped) + if match: + role = _resolve_role(match.group("spk"), alias_map) + turn_text = (match.group("text") or "").strip() + if role is None: + if current_role == "user" and turn_text: + user_parts.append(turn_text) + elif current_role == "agent" and turn_text: + agent_parts.append(turn_text) + continue + saw_label = True + current_role = role + if role == "user" and turn_text: + user_parts.append(turn_text) + user_turn_count += 1 + elif role == "agent" and turn_text: + agent_parts.append(turn_text) + agent_turn_count += 1 + continue + + if current_role == "user": + user_parts.append(stripped) + elif current_role == "agent": + agent_parts.append(stripped) + + return ParsedTurns( + user_text=" ".join(user_parts).strip(), + agent_text=" ".join(agent_parts).strip(), + user_turn_count=user_turn_count, + agent_turn_count=agent_turn_count, + unlabelled=not saw_label, + ) + + +def extract_user_text_from_production( + text: str, + *, + aliases: Optional[Mapping[str, Tuple[str, ...]]] = None, +) -> ParsedTurns: + return parse_labeled_transcript(text, aliases=aliases) + + +def _effective_diarised_speaker(speaker: str, *, swap: bool) -> str: + spk = (speaker or "").strip().lower() + if not swap: + return spk + if spk == "agent": + return "user" + if spk == "user": + return "agent" + return spk + + +def extract_user_text_from_diarised( + segments: Optional[Sequence[Mapping[str, Any]]], + *, + swap: bool = False, + diarised_transcript: Optional[str] = None, + aliases: Optional[Mapping[str, Tuple[str, ...]]] = None, +) -> str: + parts: List[str] = [] + if segments: + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = _effective_diarised_speaker( + str(turn.get("speaker") or ""), swap=swap + ) + text = (turn.get("text") or "").strip() + if speaker == "user" and text: + parts.append(text) + if parts: + return " ".join(parts).strip() + + if diarised_transcript: + parsed = parse_labeled_transcript(diarised_transcript, aliases=aliases) + if not parsed.unlabelled: + return parsed.user_text + return " ".join(parts).strip() + + +def extract_agent_text_from_diarised( + segments: Optional[Sequence[Mapping[str, Any]]], + *, + swap: bool = False, + diarised_transcript: Optional[str] = None, + aliases: Optional[Mapping[str, Tuple[str, ...]]] = None, +) -> str: + parts: List[str] = [] + if segments: + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = _effective_diarised_speaker( + str(turn.get("speaker") or ""), swap=swap + ) + text = (turn.get("text") or "").strip() + if speaker == "agent" and text: + parts.append(text) + if parts: + return " ".join(parts).strip() + + if diarised_transcript: + parsed = parse_labeled_transcript(diarised_transcript, aliases=aliases) + if not parsed.unlabelled: + return parsed.agent_text + return " ".join(parts).strip() + + +def _word_counts(text: str) -> int: + base = normalize_base(text) + if not base: + return 0 + return len(base.split()) + + +def compute_user_error_rates( + reference: str, + hypothesis: str, + *, + agent_reference_for_swap: Optional[str] = None, + production_user_for_swap: Optional[str] = None, +) -> Dict[str, Any]: + """Compute WER/CER with diarized user text as reference.""" + ref_raw = (reference or "").strip() + hyp_raw = (hypothesis or "").strip() + + if not ref_raw: + return {"skipped": "diarised_user_empty"} + + try: + from jiwer import cer, process_characters, process_words, wer + except ImportError: + return {"skipped": "jiwer_unavailable"} + + ref_base = normalize_base(ref_raw) + hyp_base = normalize_base(hyp_raw) + norm_ref = normalize_for_wer(ref_raw) + norm_hyp = normalize_for_wer(hyp_raw) + + raw_wer = float(wer(ref_base, hyp_base)) if ref_base else 0.0 + raw_cer = float(cer(ref_base, hyp_base)) if ref_base else 0.0 + norm_wer = float(wer(norm_ref, norm_hyp)) if norm_ref else 0.0 + norm_cer = float(cer(norm_ref, norm_hyp)) if norm_ref else 0.0 + + word_out = process_words(norm_ref, norm_hyp) + char_out = process_characters(norm_ref, norm_hyp) + + ref_word_count = _word_counts(norm_ref) + hyp_word_count = _word_counts(norm_hyp) + ref_char_count = len(norm_ref.replace(" ", "")) + + chart_wer = min(1.0, norm_wer) + chart_cer = min(1.0, norm_cer) + + details: Dict[str, Any] = { + "wer_raw": round(raw_wer, 6), + "cer_raw": round(raw_cer, 6), + "wer_true": round(norm_wer, 6), + "cer_true": round(norm_cer, 6), + "substitutions": int(word_out.substitutions), + "deletions": int(word_out.deletions), + "insertions": int(word_out.insertions), + "hits": int(word_out.hits), + "ref_word_count": ref_word_count, + "hyp_word_count": hyp_word_count, + "ref_char_count": ref_char_count, + "reference_preview": ref_raw[:500], + "hypothesis_preview": hyp_raw[:500], + } + + if agent_reference_for_swap is not None and production_user_for_swap is not None: + swap_hyp = (production_user_for_swap or "").strip() + if swap_hyp and agent_reference_for_swap.strip(): + agent_ref_norm = normalize_for_wer(agent_reference_for_swap) + swap_hyp_norm = normalize_for_wer(swap_hyp) + if agent_ref_norm: + agent_wer = float(wer(agent_ref_norm, swap_hyp_norm)) + if norm_wer > 0 and agent_wer < 0.5 * norm_wer: + details["roles_likely_swapped"] = True + + return { + "wer": round(chart_wer, 6), + "cer": round(chart_cer, 6), + "details": details, + } + + +def compute_tts_wer_cer(ground_truth: str, predicted: str) -> Dict[str, Any]: + """TTS/ASR comparison helper (reference = ground truth text).""" + ref = normalize_base(ground_truth) + if not ref: + return { + "raw_wer": None, + "raw_cer": None, + "normalized_wer": None, + "normalized_cer": None, + } + try: + from jiwer import cer, wer + except ImportError: + return { + "raw_wer": None, + "raw_cer": None, + "normalized_wer": None, + "normalized_cer": None, + } + + hyp = normalize_base(predicted) + raw_wer = round(float(wer(ref, hyp)), 4) + raw_cer = round(float(cer(ref, hyp)), 4) + norm_ref = normalize_entities(ground_truth) + norm_hyp = normalize_entities(predicted) + normalized_wer = round(float(wer(norm_ref, norm_hyp)), 4) if norm_ref else None + normalized_cer = round(float(cer(norm_ref, norm_hyp)), 4) if norm_ref else None + return { + "raw_wer": raw_wer, + "raw_cer": raw_cer, + "normalized_wer": normalized_wer, + "normalized_cer": normalized_cer, + } + + +_ZW_CHARS = "\u200c\u200d" +_INDIC_PUNCT = "।॥" +_FILLER_TOKENS = frozenset( + {"hmm", "haan", "umm", "uh", "achha", "हम्म", "हाँ", "हां"} +) +_SPEAKER_PREFIX_RE = re.compile( + r"^\s*(?:\[?\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\]?\s*)?" + r"(?P[A-Za-z][A-Za-z0-9 _]*?)\s*[:\-–]\s*(?P.*)$" +) +_LEADING_TIMESTAMP_RE = re.compile( + r"^\s*\[?\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\]?\s*" +) +_INDEPENDENT_VOWELS = { + "अ": "a", + "आ": "aa", + "इ": "i", + "ई": "ee", + "उ": "u", + "ऊ": "oo", + "ऋ": "ri", + "ए": "e", + "ऐ": "ai", + "ओ": "o", + "औ": "au", +} +_MATRAS = { + "ा": "aa", + "ि": "i", + "ी": "ee", + "ु": "u", + "ू": "oo", + "ृ": "ri", + "े": "e", + "ै": "ai", + "ो": "o", + "ौ": "au", +} +_CONSONANTS = { + "क": "k", + "ख": "kh", + "ग": "g", + "घ": "gh", + "ङ": "ng", + "च": "ch", + "छ": "chh", + "ज": "j", + "झ": "jh", + "ञ": "ny", + "ट": "t", + "ठ": "th", + "ड": "d", + "ढ": "dh", + "ण": "n", + "त": "t", + "थ": "th", + "द": "d", + "ध": "dh", + "न": "n", + "प": "p", + "फ": "ph", + "ब": "b", + "भ": "bh", + "म": "m", + "य": "y", + "र": "r", + "ल": "l", + "व": "v", + "श": "sh", + "ष": "sh", + "स": "s", + "ह": "h", + "ळ": "l", + "क़": "k", + "ख़": "kh", + "ग़": "g", + "ज़": "j", + "ड़": "d", + "ढ़": "dh", + "फ़": "ph", + "य़": "y", +} +_NUKTA_BASE = { + "क़": "क", + "ख़": "ख", + "ग़": "ग", + "ज़": "ज", + "ड़": "ड", + "ढ़": "ढ", + "फ़": "फ", + "य़": "य", +} +_DEVANAGARI_DIGITS = str.maketrans("०१२३४५६७८९", "0123456789") +_VIRAMA = "\u094d" +_ANUSVARA = "\u0902" +_NUKTA = "\u093c" + + +def _fold_devanagari(text: str) -> str: + folded = text + for src, dst in _NUKTA_BASE.items(): + folded = folded.replace(src, dst) + return folded.replace("ँ", "ं").translate(_DEVANAGARI_DIGITS) + + +def transliterate_devanagari_to_latin(text: str) -> str: + """Minimal Devanagari to Latin. Other characters pass through.""" + chars = list(_fold_devanagari(unicodedata.normalize("NFC", text or ""))) + out: List[str] = [] + index = 0 + while index < len(chars): + char = chars[index] + if char in _CONSONANTS: + consonant = _CONSONANTS[char] + index += 1 + if index < len(chars) and chars[index] == _NUKTA: + index += 1 + vowel = "a" + if index < len(chars) and chars[index] == _VIRAMA: + vowel = "" + index += 1 + elif index < len(chars) and chars[index] in _MATRAS: + vowel = _MATRAS[chars[index]] + index += 1 + out.append(consonant + vowel) + if index < len(chars) and chars[index] == _ANUSVARA: + out.append("n") + index += 1 + continue + if char in _INDEPENDENT_VOWELS: + out.append(_INDEPENDENT_VOWELS[char]) + index += 1 + if index < len(chars) and chars[index] == _ANUSVARA: + out.append("n") + index += 1 + continue + if char == _ANUSVARA: + out.append("n") + index += 1 + continue + out.append(char) + index += 1 + return "".join(out) + + +def _strip_speaker_prefixes(text: str) -> str: + lines: List[str] = [] + for line in (text or "").splitlines(): + match = _SPEAKER_PREFIX_RE.match(line.strip()) + if match: + kept = (match.group("text") or "").strip() + if kept: + lines.append(kept) + continue + stripped = _LEADING_TIMESTAMP_RE.sub("", line.strip()) + if stripped: + lines.append(stripped) + return " ".join(lines) + + +def _fold_romanised_token(token: str) -> str: + folded = token.replace("th", "t").replace("dh", "d") + previous = None + while folded != previous: + previous = folded + folded = folded.replace("aa", "a").replace("ee", "i").replace("oo", "u") + folded = folded.replace("w", "v") + folded = re.sub(r"(.)\1+", r"\1", folded) + if len(folded) > 1 and folded.endswith("h"): + folded = folded[:-1] + return folded + + +def normalise_for_error_rate(text: str, *, romanised: bool) -> str: + """Normalise one side of a production-vs-diarised comparison.""" + cleaned = _strip_speaker_prefixes(unicodedata.normalize("NFC", text or "")) + cleaned = _fold_devanagari(cleaned.lower()) + punct = string.punctuation + _INDIC_PUNCT + _ZW_CHARS + cleaned = cleaned.translate(str.maketrans("", "", punct)) + tokens = [token for token in cleaned.split() if token not in _FILLER_TOKENS] + if romanised: + tokens = [_fold_romanised_token(token) for token in tokens] + tokens = [token for token in tokens if token and token not in _FILLER_TOKENS] + return " ".join(tokens) + + +def _rate(distance: int, denominator: int) -> Optional[float]: + if denominator <= 0: + return None + return round(distance / denominator, 4) + + +def compute_transcript_error_rates( + reference: str, + hypothesis: str, + *, + mode: str, + script_mismatch: bool, +) -> Dict[str, Any]: + """Score diarised user text against production user text. + + ``reference`` is the production user turns (the denominator). + ``hypothesis`` is the diarised user turns. Rates are ``None`` when + production user text is empty after normalisation. A non-empty + reference with an empty hypothesis scores 1.0. + """ + romanised = mode == "romanised_fallback" + norm_ref = normalise_for_error_rate(reference or "", romanised=romanised) + norm_hyp = normalise_for_error_rate(hypothesis or "", romanised=romanised) + ref_words = norm_ref.split() if norm_ref else [] + hyp_words = norm_hyp.split() if norm_hyp else [] + ref_chars = norm_ref.replace(" ", "") + hyp_chars = norm_hyp.replace(" ", "") + if not ref_words and not ref_chars: + return { + "cer": None, + "wer": None, + "mode": mode, + "script_mismatch": script_mismatch, + "ref_chars": 0, + "ref_words": 0, + "hyp_words": len(hyp_words), + "char_distance": 0, + "word_distance": 0, + } + + from rapidfuzz.distance import Levenshtein + + word_distance = int(Levenshtein.distance(ref_words, hyp_words)) + char_distance = int(Levenshtein.distance(ref_chars, hyp_chars)) + return { + "cer": _rate(char_distance, len(ref_chars)), + "wer": _rate(word_distance, len(ref_words)), + "mode": mode, + "script_mismatch": script_mismatch, + "ref_chars": len(ref_chars), + "ref_words": len(ref_words), + "hyp_words": len(hyp_words), + "char_distance": char_distance, + "word_distance": word_distance, + } + + +def _aliases_for_user_turns( + custom: Optional[Mapping[str, Tuple[str, ...]]], +) -> Dict[str, Tuple[str, ...]]: + """Default user/agent names, plus any extra names stored on the import.""" + merged = { + role: tuple(names) for role, names in DEFAULT_SPEAKER_ALIASES.items() + } + if not isinstance(custom, dict): + return merged + for role, names in custom.items(): + key = str(role or "").strip().lower() + if key not in merged or isinstance(names, str) or not isinstance(names, Sequence): + continue + extra = tuple(str(name).strip() for name in names if str(name).strip()) + merged[key] = tuple(dict.fromkeys((*merged[key], *extra))) + return merged + + +def _user_hypothesis_from_diarised( + segments: Optional[Sequence[Mapping[str, Any]]], + diarised_transcript: str, + *, + swap: bool, + aliases: Mapping[str, Tuple[str, ...]], +) -> Tuple[str, bool]: + """Return diarised user text and whether any selected turn used ``text_native``. + + Segments win over the rendered transcript. The full STT string is not + used: it includes agent speech. A user turn without ``text_native`` + keeps that turn's romanised ``text``. + """ + if segments: + parts: List[str] = [] + used_native = False + for segment in segments: + if not isinstance(segment, dict): + continue + speaker = _effective_diarised_speaker( + str(segment.get("speaker") or ""), + swap=swap, + ) + if _resolve_role(speaker, aliases) != "user": + continue + native = str(segment.get("text_native") or "").strip() + text = str(segment.get("text") or "").strip() + if native: + used_native = True + parts.append(native) + elif text: + parts.append(text) + return " ".join(parts).strip(), used_native + + parsed = parse_labeled_transcript(diarised_transcript or "", aliases=aliases) + if parsed.unlabelled: + return "", False + return parsed.user_text, False + + +def select_comparison_texts( + *, + production_transcript: str, + diarised_transcript: str = "", + diarised_segments: Optional[Sequence[Mapping[str, Any]]] = None, + diarised_native_transcript: Optional[str] = None, + diarised_speaker_swap: bool = False, + speaker_aliases: Optional[Mapping[str, Tuple[str, ...]]] = None, +) -> Dict[str, Any]: + """Pick production user turns and diarised user turns. + + Agent turns are dropped on both sides. ``diarised_speaker_swap`` + flips which diarised speaker is the user. Native mode uses + ``text_native`` on those user turns. The full + ``diarised_native_transcript`` is ignored because it includes the + agent. The romanised fallback transliterates Indic production user + text when no native user hypothesis exists. + """ + del diarised_native_transcript # full-call STT; agent speech included + aliases = _aliases_for_user_turns(speaker_aliases) + production_user = extract_user_text_from_production( + production_transcript or "", + aliases=aliases, + ) + reference_raw = "" if production_user.unlabelled else production_user.user_text + hypothesis, has_native_hypothesis = _user_hypothesis_from_diarised( + diarised_segments, + diarised_transcript or "", + swap=diarised_speaker_swap, + aliases=aliases, + ) + script_mismatch = contains_indic_script(reference_raw) != contains_indic_script( + hypothesis + ) + reference = reference_raw + mode = "native" + # Romanise only when the hypothesis is not already in the reference + # script. A diariser can return the same Indic sentence in ``text`` + # with no ``text_native``; converting only the reference would score + # matching speech as a large error. + if ( + not has_native_hypothesis + and contains_indic_script(reference_raw) + and not contains_indic_script(hypothesis) + ): + reference = transliterate_devanagari_to_latin(reference_raw) + mode = "romanised_fallback" + return { + "reference": reference, + "hypothesis": hypothesis, + "mode": mode, + "script_mismatch": script_mismatch, + } + + +def build_word_alignment( + reference: str, + hypothesis: str, + *, + romanised: bool = False, +) -> Dict[str, Any]: + """On-demand word alignment for the production-vs-diarised diff.""" + norm_ref = normalise_for_error_rate(reference or "", romanised=romanised) + norm_hyp = normalise_for_error_rate(hypothesis or "", romanised=romanised) + ref_tokens = norm_ref.split() if norm_ref else [] + hyp_tokens = norm_hyp.split() if norm_hyp else [] + if not ref_tokens: + return { + "reference": [], + "hypothesis": hyp_tokens, + "chunks": [], + "substitutions": 0, + "deletions": 0, + "insertions": 0, + "hits": 0, + "wer": 0.0, + } + from rapidfuzz.distance import Levenshtein + + chunks = [] + substitutions = deletions = insertions = hits = 0 + for opcode in Levenshtein.opcodes(ref_tokens, hyp_tokens): + kind = { + "equal": "equal", + "replace": "substitute", + "delete": "delete", + "insert": "insert", + }.get(opcode.tag, opcode.tag) + if kind == "equal": + hits += opcode.src_end - opcode.src_start + elif kind == "substitute": + substitutions += max( + opcode.src_end - opcode.src_start, + opcode.dest_end - opcode.dest_start, + ) + elif kind == "delete": + deletions += opcode.src_end - opcode.src_start + elif kind == "insert": + insertions += opcode.dest_end - opcode.dest_start + chunks.append( + { + "type": kind, + "ref_start_idx": opcode.src_start, + "ref_end_idx": opcode.src_end, + "hyp_start_idx": opcode.dest_start, + "hyp_end_idx": opcode.dest_end, + } + ) + word_distance = int(Levenshtein.distance(ref_tokens, hyp_tokens)) + return { + "reference": ref_tokens, + "hypothesis": hyp_tokens, + "chunks": chunks, + "substitutions": substitutions, + "deletions": deletions, + "insertions": insertions, + "hits": hits, + "wer": round(word_distance / len(ref_tokens), 4), + } + + diff --git a/app/services/evaluation_analysis_checkpoints.py b/app/services/evaluation_analysis_checkpoints.py new file mode 100644 index 00000000..a017aae2 --- /dev/null +++ b/app/services/evaluation_analysis_checkpoints.py @@ -0,0 +1,448 @@ +"""Persist and load durable batch checkpoints for evaluation analysis jobs.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Literal, Optional, Tuple +from uuid import UUID, uuid4 + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.db_sharding.scatter_gather import ( + EVALUATION_ANALYSIS_BATCH_SIZE, + EvaluationRowPageCursor, + count_completed_evaluation_row_pairs, +) +from app.models.database import EvaluationAnalysisBatch + +JobKind = Literal["user_insights", "metric_clusters", "prompt_improvements"] + +JOB_KIND_USER_INSIGHTS: JobKind = "user_insights" +JOB_KIND_METRIC_CLUSTERS: JobKind = "metric_clusters" +JOB_KIND_PROMPT_IMPROVEMENTS: JobKind = "prompt_improvements" + +USER_INSIGHTS_CANCELLED_BY_USER_ERROR = "Cancelled by user" +PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR = "Cancelled by user" + +EVIDENCE_RESERVOIR_CAP = 80 + +# A running analysis header is stamped after every LLM call. Three cluster +# calls (180 s timeout + one larger-budget retry each) is the longest gap a +# live worker can legitimately produce, so allow generous headroom before +# treating the job as abandoned. +JOB_HEARTBEAT_STALE_SECONDS = 15 * 60 + + +def analysis_header_matches_run(raw: Any, job_run_id: Any) -> bool: + """True when *job_run_id* still owns the saved analysis header. + + A missing claim matches any header (the first enqueue, before a run + id exists). A claim against a different header means this task is + stale and must not write. + """ + if job_run_id is None or str(job_run_id).strip() == "": + return True + if not isinstance(raw, dict): + return False + current = raw.get("job_run_id") + if not current: + return False + return str(current) == str(job_run_id) + + +def classify_task_run(raw: Any, job_run_id: Any) -> tuple[str, Optional[UUID]]: + """How this task relates to the saved analysis header. + + ``owned`` — the task's claim still matches the header. + ``fresh`` — neither side has a run yet; the caller should mint one. + ``stale`` — the task names a run that no longer owns the header. + ``unclaimed`` — the task has no run id, and the header already belongs + to one. That task must not attach to the replacement run. + """ + claimed = str(job_run_id).strip() if job_run_id is not None else "" + header = "" + if isinstance(raw, dict) and raw.get("job_run_id"): + header = str(raw.get("job_run_id")).strip() + if claimed: + if header != claimed: + return "stale", None + try: + return "owned", UUID(claimed) + except ValueError: + return "stale", None + if header: + return "unclaimed", None + return "fresh", None + + +def claim_still_owns_header(raw: Any, job_run_id: Any) -> bool: + """False when a write would land on a header this run does not own.""" + if job_run_id is None or str(job_run_id).strip() == "": + return False + return analysis_header_matches_run(raw, job_run_id) + + +def keep_owned_header_write( + db: Session, + evaluation: Any, + attr: str, + job_run_id: Any, +) -> bool: + """Refresh *attr* and keep a pending write only while this run owns it. + + Call after assigning the next header and before commit. A newer run + is left in place. When the claim still matches, the pending payload is + restored with that run id so a merge cannot adopt the replacement. + """ + pending = getattr(evaluation, attr, None) + db.expire(evaluation, [attr]) + db.refresh(evaluation) + current = getattr(evaluation, attr, None) + if not claim_still_owns_header(current, job_run_id): + logger.info( + "Analysis write skipped for {}; run {} no longer owns the header ({})", + attr, + job_run_id, + current.get("job_run_id") if isinstance(current, dict) else None, + ) + return False + if isinstance(pending, dict): + setattr(evaluation, attr, {**pending, "job_run_id": str(job_run_id)}) + flag_modified(evaluation, attr) + return True + + +def new_job_run_id() -> UUID: + return uuid4() + + +def normalize_transcript_source(value: Optional[str]) -> str: + source = (value or "diarised").strip().lower() + return source if source in {"diarised", "production"} else "diarised" + + +def batch_count_for_rows(total_rows: int, batch_size: int = EVALUATION_ANALYSIS_BATCH_SIZE) -> int: + if total_rows <= 0: + return 0 + return (total_rows + batch_size - 1) // batch_size + + +def progress_header( + *, + completed_rows: int, + total_rows: int, + completed_batches: int, + total_batches: int, + completed_llm_calls: int = 0, + total_llm_calls: int = 0, + phase: Optional[str] = None, +) -> Dict[str, Any]: + if total_rows > 0: + completed_rows = min(int(completed_rows), int(total_rows)) + if total_batches > 0: + completed_batches = min(int(completed_batches), int(total_batches)) + payload: Dict[str, Any] = { + "completed_rows": completed_rows, + "total_rows": total_rows, + "completed_batches": completed_batches, + "total_batches": total_batches, + "completed_llm_calls": completed_llm_calls, + "total_llm_calls": total_llm_calls, + } + if phase: + payload["phase"] = phase + return payload + + +def rows_completed_for_batches( + completed_batches: int, + total_rows: int, + *, + batch_size: int = EVALUATION_ANALYSIS_BATCH_SIZE, +) -> int: + """Rows finished after ``completed_batches`` full pages (never exceeds total).""" + if total_rows <= 0 or completed_batches <= 0: + return 0 + return min(int(total_rows), int(completed_batches) * int(batch_size)) + + +def cursor_from_header(raw: Any) -> Optional[EvaluationRowPageCursor]: + if not isinstance(raw, dict): + return None + cursor = raw.get("cursor") + if not isinstance(cursor, dict): + return None + row_index = cursor.get("row_index") + eval_row_id = cursor.get("evaluation_row_id") + if not isinstance(row_index, (int, float)) or not eval_row_id: + return None + try: + return EvaluationRowPageCursor( + row_index=int(row_index), + evaluation_row_id=UUID(str(eval_row_id)), + ) + except (TypeError, ValueError): + return None + + +def cursor_to_dict(cursor: EvaluationRowPageCursor) -> Dict[str, Any]: + return { + "row_index": cursor.row_index, + "evaluation_row_id": str(cursor.evaluation_row_id), + } + + +def user_insights_raw_is_cancelled(raw: Any) -> bool: + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() == "cancelled": + return True + return raw.get("error_message") == USER_INSIGHTS_CANCELLED_BY_USER_ERROR + + +def prompt_improvements_raw_is_cancelled(raw: Any) -> bool: + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() == "cancelled": + return True + return raw.get("error_message") == PROMPT_IMPROVEMENTS_CANCELLED_BY_USER_ERROR + + +def upsert_analysis_batch( + db: Session, + *, + evaluation_id: UUID, + job_kind: JobKind, + job_run_id: UUID, + batch_index: int, + payload: Dict[str, Any], + cursor: Optional[EvaluationRowPageCursor] = None, + transcript_source: Optional[str] = None, + status: str = "completed", +) -> EvaluationAnalysisBatch: + row = ( + db.query(EvaluationAnalysisBatch) + .filter( + EvaluationAnalysisBatch.evaluation_id == evaluation_id, + EvaluationAnalysisBatch.job_kind == job_kind, + EvaluationAnalysisBatch.job_run_id == job_run_id, + EvaluationAnalysisBatch.batch_index == batch_index, + ) + .first() + ) + if row is None: + row = EvaluationAnalysisBatch( + evaluation_id=evaluation_id, + job_kind=job_kind, + job_run_id=job_run_id, + batch_index=batch_index, + ) + db.add(row) + row.payload = payload + row.status = status + row.transcript_source = transcript_source + if cursor is not None: + row.cursor_row_index = cursor.row_index + row.cursor_evaluation_row_id = cursor.evaluation_row_id + row.updated_at = datetime.now(timezone.utc) + db.flush() + return row + + +def list_analysis_batches( + db: Session, + *, + evaluation_id: UUID, + job_kind: JobKind, + job_run_id: UUID, +) -> List[EvaluationAnalysisBatch]: + return ( + db.query(EvaluationAnalysisBatch) + .filter( + EvaluationAnalysisBatch.evaluation_id == evaluation_id, + EvaluationAnalysisBatch.job_kind == job_kind, + EvaluationAnalysisBatch.job_run_id == job_run_id, + ) + .order_by(EvaluationAnalysisBatch.batch_index) + .all() + ) + + +def merge_theme_counts_from_batches(batches: List[EvaluationAnalysisBatch]) -> Dict[str, int]: + counts: Dict[str, int] = {} + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + partial = payload.get("theme_counts") + if isinstance(partial, dict): + for key, value in partial.items(): + label = str(key).strip() + if not label: + continue + try: + counts[label] = counts.get(label, 0) + int(value) + except (TypeError, ValueError): + continue + return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))) + + +def merge_extractions_from_batches( + batches: List[EvaluationAnalysisBatch], + *, + cap: int = EVIDENCE_RESERVOIR_CAP, +) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + samples = payload.get("extractions") + if not isinstance(samples, list): + continue + for item in samples: + if isinstance(item, dict): + out.append(item) + if len(out) >= cap: + return out + return out + + +def merge_signature_counts_from_cluster_batches( + batches: List[EvaluationAnalysisBatch], +) -> Dict[str, Dict[str, int]]: + """metric_id -> signature label -> count.""" + by_metric: Dict[str, Dict[str, int]] = {} + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + metrics_raw = payload.get("metrics") + if not isinstance(metrics_raw, dict): + continue + for metric_id, sig_map in metrics_raw.items(): + if not isinstance(sig_map, dict): + continue + bucket = by_metric.setdefault(str(metric_id), {}) + for label, count in sig_map.items(): + name = str(label).strip() + if not name: + continue + try: + bucket[name] = bucket.get(name, 0) + int(count) + except (TypeError, ValueError): + continue + return by_metric + + +def merge_cluster_extractions_from_batches( + batches: List[EvaluationAnalysisBatch], + metric_id: str, + *, + cap: int = 40, +) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + payload = batch.payload if isinstance(batch.payload, dict) else {} + metrics_raw = payload.get("metric_extractions") + if not isinstance(metrics_raw, dict): + continue + samples = metrics_raw.get(metric_id) + if not isinstance(samples, list): + continue + for item in samples: + if isinstance(item, dict): + out.append(item) + if len(out) >= cap: + return out + return out + + +def job_heartbeat_is_stale( + raw: Any, + *, + threshold_seconds: int = JOB_HEARTBEAT_STALE_SECONDS, + now: Optional[datetime] = None, +) -> bool: + """True when a ``running`` analysis header has stopped updating. + + Long LLM steps rewrite the header after every call, so a heartbeat older + than the threshold means no worker is driving this job any more (crash, + restart, revoke without terminate). Without this the card stays on + "Synthesizing…" forever and the only escape is Stop. + """ + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").strip().lower() != "running": + return False + beat_raw = raw.get("heartbeat_at") or raw.get("generated_at") + if not isinstance(beat_raw, str) or not beat_raw: + return True + try: + beat = datetime.fromisoformat(beat_raw) + except ValueError: + return True + if beat.tzinfo is None: + beat = beat.replace(tzinfo=timezone.utc) + moment = now or datetime.now(timezone.utc) + return (moment - beat).total_seconds() >= max(1, threshold_seconds) + + +def stamp_job_heartbeat(header: Dict[str, Any]) -> Dict[str, Any]: + """Record that a worker is still driving ``header``. Mutates and returns it.""" + header["heartbeat_at"] = datetime.now(timezone.utc).isoformat() + return header + + +def init_job_header( + *, + job_run_id: UUID, + transcript_source: str, + total_rows: int, + total_batches: int, + provider: Optional[str], + model: Optional[str], + extra: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + header: Dict[str, Any] = { + "status": "running", + "job_run_id": str(job_run_id), + "transcript_source": transcript_source, + "generated_at": datetime.now(timezone.utc).isoformat(), + "heartbeat_at": datetime.now(timezone.utc).isoformat(), + "progress": progress_header( + completed_rows=0, + total_rows=total_rows, + completed_batches=0, + total_batches=total_batches, + ), + "theme_counts": {}, + "provider": provider, + "model": model, + "error_message": None, + "celery_task_id": None, + } + if extra: + header.update(extra) + return header + + +def completed_rows_from_batches(batches: List[EvaluationAnalysisBatch]) -> int: + total = 0 + seen: set[int] = set() + for batch in batches: + if (getattr(batch, "status", None) or "completed") != "completed": + continue + if batch.batch_index in seen: + continue + seen.add(batch.batch_index) + payload = batch.payload if isinstance(batch.payload, dict) else {} + try: + total += int(payload.get("rows_processed") or 0) + except (TypeError, ValueError): + continue + return total diff --git a/app/services/harbor/__init__.py b/app/services/harbor/__init__.py new file mode 100644 index 00000000..33614c7b --- /dev/null +++ b/app/services/harbor/__init__.py @@ -0,0 +1,6 @@ +"""Harbor-powered simulation runs (test agent ↔ production agent). + +Only ``conversation_agent``, ``metric_verifier``, ``noop_environment`` and +``job_runner`` import ``harbor``; they run inside the worker-harbor process. +The remaining modules are dependency-free so the API can import them. +""" diff --git a/app/services/harbor/conversation_agent.py b/app/services/harbor/conversation_agent.py new file mode 100644 index 00000000..67a07672 --- /dev/null +++ b/app/services/harbor/conversation_agent.py @@ -0,0 +1,96 @@ +"""Harbor agent that runs a test-agent ↔ production-agent conversation. + +External ``BaseAgent``: the loop runs in the Harbor host process (no sandbox) and +reuses the platform's LLM-to-LLM simulation. Harbor's built-in simulated user +(``--user-agent --bridge acp``) only targets coding agents, so this agent drives +both legs itself. +""" + +from __future__ import annotations + +import asyncio +import tomllib +from pathlib import Path +from uuid import UUID + +from harbor.agents.base import BaseAgent +from harbor.agents.capabilities import AgentCapabilities +from harbor.agents.options import AgentOptions +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trial.config import TrialConfig + +from app.database import SessionLocal +from app.services.harbor.trial_store import ( + TRAJECTORY_AGENT_NAME, + TRAJECTORY_AGENT_VERSION, + TrialRefs, + llm_override, + run_trial_conversation, +) + + +class ConversationAgentOptions(AgentOptions): + simulation_run_id: str + production_llm_provider: str | None = None + production_llm_model: str | None = None + test_llm_provider: str | None = None + test_llm_model: str | None = None + + +def task_metadata_for_trial(trial_dir: Path) -> dict: + """Read our ``[metadata]`` from the trial's task.toml (Harbor only passes the instruction).""" + config = TrialConfig.model_validate_json((trial_dir / "config.json").read_text()) + if config.task.path is None: + raise ValueError("Simulation trials must use local task paths") + with open(Path(config.task.path) / "task.toml", "rb") as fh: + return tomllib.load(fh).get("metadata", {}) + + +class EfficientAIConversationAgent(BaseAgent): + capabilities = AgentCapabilities(atif=True) + options_model = ConversationAgentOptions + + @staticmethod + def name() -> str: + return TRAJECTORY_AGENT_NAME + + def version(self) -> str | None: + return TRAJECTORY_AGENT_VERSION + + async def setup(self, environment: BaseEnvironment) -> None: + pass + + async def run( + self, instruction: str, environment: BaseEnvironment, context: AgentContext + ) -> None: + options: ConversationAgentOptions = self.options # type: ignore[assignment] + trial_dir = self.logs_dir.parent + metadata = task_metadata_for_trial(trial_dir) + refs = TrialRefs( + simulation_run_id=UUID(options.simulation_run_id), + scenario_id=UUID(metadata["scenario_id"]), + persona_id=UUID(metadata["persona_id"]), + trial_name=trial_dir.name, + ) + # Sync DB + LLM calls run in a worker thread so Harbor keeps trials concurrent. + outcome = await asyncio.to_thread( + self._run_sync, + refs, + llm_override(options.production_llm_provider, options.production_llm_model), + llm_override(options.test_llm_provider, options.test_llm_model), + ) + context.metadata = outcome + + def _run_sync(self, refs, production_override, test_override) -> dict: + db = SessionLocal() + try: + return run_trial_conversation( + db, + refs=refs, + trajectory_path=self.logs_dir / "trajectory.json", + production_override=production_override, + test_override=test_override, + ) + finally: + db.close() diff --git a/app/services/harbor/failure_analysis.py b/app/services/harbor/failure_analysis.py new file mode 100644 index 00000000..4776880c --- /dev/null +++ b/app/services/harbor/failure_analysis.py @@ -0,0 +1,330 @@ +"""Failure clusters and prompt suggestions for a finished simulation run. + +Reuses the call-import clustering pipeline: a trial's metric counts as failed when it +missed the success rule the user set for the run (``metric_scores[mid]["passed"] is +False``). Runs without rules fall back to ``normalize_metric_score < pass_threshold``. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import ( + Agent, + EvaluatorResult, + EvaluatorResultStatus, + Metric, + ModelProvider, + Persona, + Scenario, + SimulationRun, +) +from app.models.schemas import ( + EvaluationMetricClustersState, + EvaluationPromptImprovementsState, +) +from app.services.call_import_metric_clusters import ( + CancelCheck, + FailurePredicate, + ProgressCallback, + generate_metric_clusters_for_source_rows, +) +from app.services.call_import_prompt_improvements import ( + generate_prompt_improvements_for_prompt, +) +from app.services.harbor.result_ingest import trial_results +from app.services.harbor.rewards import normalize_metric_score +from app.services.harbor.success_criteria import is_gating_criterion +from app.services.metric_cluster_rows import ( + MetricClusterSourceRow, + evaluator_result_to_cluster_row, +) +from app.services.metric_clusters_rca_summary import format_failure_policy_reason +from app.services.testing.test_agent_simulation_prompt import ( + get_agent_base_prompt, + is_chat_agent, +) + +_OP_SYMBOLS = {"lt": "<", "lte": "≤", "gt": ">", "gte": "≥"} + + +def gating_criteria_by_metric(run: SimulationRun) -> dict[str, dict[str, Any]]: + criteria = run.metric_success_criteria if isinstance(run.metric_success_criteria, list) else [] + return { + str(c.get("metric_id")): c + for c in criteria + if isinstance(c, dict) and c.get("metric_id") and is_gating_criterion(c) + } + + +def _child_names(db: Session, organization_id: UUID, parent_ids: list[UUID]) -> dict[str, str]: + if not parent_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(parent_ids), + ) + .all() + ) + return {str(c.id): c.name for c in children} + + +def describe_success_rule( + criterion: dict[str, Any], + child_names: Optional[dict[str, str]] = None, +) -> str: + """Human-readable success rule, e.g. ``score ≥ 0.8`` or ``one of: Resolved``.""" + kind = criterion.get("kind") + if kind == "boolean": + return f"answer is {criterion.get('success_when') or 'yes'}" + if kind in ("rating", "number"): + op = _OP_SYMBOLS.get(criterion.get("op") or "gte", criterion.get("op") or "") + return f"score {op} {criterion.get('threshold')}" + if kind in ("enum", "category_single"): + values = ", ".join(str(v) for v in criterion.get("success_values") or []) + return f"one of: {values}" if values else "allowed values" + if kind == "category_multi": + names = child_names or {} + rules = [ + f"{names.get(str(r.get('child_id')), 'label')} = {r.get('expect') or 'yes'}" + for r in criterion.get("label_rules") or [] + if isinstance(r, dict) + ] + return "; ".join(rules) if rules else "label rules" + if kind == "classification": + questions = criterion.get("questions") or {} + parts = [] + if questions.get("noul"): + parts.append(f"answer is {questions['noul'].get('success_when') or 'yes'}") + for key in ("choice", "score"): + if questions.get(key): + values = ", ".join(str(v) for v in questions[key].get("success_values") or []) + parts.append(f"{key} in: {values}") + return "; ".join(parts) if parts else "classification rules" + return "no rule" + + +def success_rule_labels(db: Session, run: SimulationRun) -> dict[str, str]: + """``metric_id -> rule description`` for the run's gating success rules.""" + criteria = gating_criteria_by_metric(run) + multi_parents = [ + UUID(mid) for mid, c in criteria.items() if c.get("kind") == "category_multi" + ] + child_names = _child_names(db, run.organization_id, multi_parents) + return {mid: describe_success_rule(c, child_names) for mid, c in criteria.items()} + + +def simulation_failure_predicate( + run: SimulationRun, + metrics_by_id: dict[str, Metric], +) -> FailurePredicate: + criteria = gating_criteria_by_metric(run) + + def _failed(row: MetricClusterSourceRow, metric: Metric) -> bool: + mid = str(metric.id) + entry = (row.metric_scores or {}).get(mid) + if criteria: + if mid not in criteria: + return False + # apply_success_criteria marks each gating metric entry with ``passed``. + if isinstance(entry, dict) and entry.get("passed") is not None: + return entry.get("passed") is False + # Multi-label parents may only have child entries. + return any( + isinstance(child, dict) + and str(child.get("parent_metric_id") or "") == mid + and child.get("passed") is False + for child in (row.metric_scores or {}).values() + ) + if not isinstance(entry, dict): + return False + score = normalize_metric_score(entry, metrics_by_id.get(mid, metric)) + return score is not None and score < run.pass_threshold + + return _failed + + +def run_metrics_for_analysis(db: Session, run: SimulationRun, agent: Agent) -> list[Metric]: + """Top-level metrics that can gate this run: rule metrics, else the run's scored metrics.""" + criteria = gating_criteria_by_metric(run) + if criteria: + return ( + db.query(Metric) + .filter( + Metric.organization_id == run.organization_id, + Metric.id.in_([UUID(mid) for mid in criteria]), + ) + .all() + ) + from app.workers.tasks.process_evaluator_result import select_metrics_for_evaluation + + metrics = select_metrics_for_evaluation( + db, + organization_id=run.organization_id, + agent=agent, + metric_ids=run.metric_ids, + workspace_id=run.workspace_id, + ) + return [m for m in metrics if not m.parent_metric_id] + + +def trial_label(result: EvaluatorResult, scenario_name: str, persona_name: str) -> str: + attempt = f" · attempt {result.attempt_index}" if result.attempt_index else "" + return f"{scenario_name} · {persona_name}{attempt} (#{result.result_id})" + + +def simulation_cluster_rows( + db: Session, + run: SimulationRun, +) -> list[MetricClusterSourceRow]: + """Completed trials as clustering rows, labelled by scenario · persona · attempt.""" + results = [ + r for r in trial_results(db, run.id) if r.status == EvaluatorResultStatus.COMPLETED.value + ] + scenario_ids = {r.scenario_id for r in results if r.scenario_id} + persona_ids = {r.persona_id for r in results if r.persona_id} + scenario_names = ( + {s.id: s.name for s in db.query(Scenario).filter(Scenario.id.in_(scenario_ids)).all()} + if scenario_ids + else {} + ) + persona_names = ( + {p.id: p.name for p in db.query(Persona).filter(Persona.id.in_(persona_ids)).all()} + if persona_ids + else {} + ) + rows = [] + for result in results: + row = evaluator_result_to_cluster_row(result) + rows.append( + MetricClusterSourceRow( + row_id=row.row_id, + conversation_id=trial_label( + result, + scenario_names.get(result.scenario_id) or "Scenario", + persona_names.get(result.persona_id) or "Persona", + ), + row_index=result.attempt_index, + metric_scores=row.metric_scores, + status=EvaluatorResultStatus.COMPLETED.value, + transcript=row.transcript, + ) + ) + return rows + + +def count_failed_trials(db: Session, run: SimulationRun) -> int: + return sum( + 1 + for r in trial_results(db, run.id) + if r.status == EvaluatorResultStatus.COMPLETED.value and r.passed is False + ) + + +def editable_agent_prompt(agent: Agent) -> str: + """The prompt the user maintains for the production agent (before simulation wrapping).""" + if is_chat_agent(agent): + provider = (agent.provider_prompt or "").strip() + if provider: + return provider + return get_agent_base_prompt(agent) + + +def _label_failure_reasons( + state: EvaluationMetricClustersState, + rule_labels: dict[str, str], +) -> None: + """Replace the generic policy text with the success rule each trial missed.""" + for group in state.groups: + rule = rule_labels.get(group.metric_id) + if not rule: + continue + group.failure_reason = f"Missed success rule: {rule}" + for cluster in group.clusters: + gap = format_failure_policy_reason(None, None, gap_label=cluster.gap_label) + cluster.failure_reason = f"Missed success rule: {rule}. {gap}" + + +def generate_simulation_metric_clusters( + db: Session, + run: SimulationRun, + *, + provider: ModelProvider, + model: str, + on_progress: Optional[ProgressCallback] = None, + max_llm_calls: Optional[int] = None, + is_cancelled: Optional[CancelCheck] = None, + credential_id: Optional[UUID] = None, +) -> EvaluationMetricClustersState: + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + metrics = run_metrics_for_analysis(db, run, agent) + metrics_by_id = {str(m.id): m for m in metrics} + source_rows = simulation_cluster_rows(db, run) + state = generate_metric_clusters_for_source_rows( + db, + job_key=run.id, + metric_clusters_raw=None, + completed_row_count=len(source_rows), + organization_id=run.organization_id, + provider=provider, + model=model, + source_rows=source_rows, + metrics=metrics, + policies={}, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + is_cancelled=is_cancelled, + failure_predicate=simulation_failure_predicate(run, metrics_by_id), + credential_id=credential_id, + ) + if state.status == "failed" and not state.groups and state.error_message: + if state.error_message.startswith("No flagged calls"): + state.error_message = ( + "No completed trial missed a success rule, so there is nothing to cluster." + ) + _label_failure_reasons(state, success_rule_labels(db, run)) + return state + + +def generate_simulation_prompt_improvements( + db: Session, + run: SimulationRun, + clusters_state: EvaluationMetricClustersState, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, +) -> dict[str, Any]: + """Prompt suggestions for the production agent, stored with the base prompt they target.""" + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + base_prompt = editable_agent_prompt(agent) + state = generate_prompt_improvements_for_prompt( + agent_id=str(agent.id), + agent_name=agent.name, + agent_prompt=base_prompt, + agent_flowchart=None, + clusters_state=clusters_state, + completed_rows=clusters_state.generated_at_completed_rows, + organization_id=run.organization_id, + db=db, + provider=provider, + model=model, + credential_id=credential_id, + ) + raw = state.model_dump(mode="json") + raw["compose_base_prompt"] = base_prompt + return raw + + +def failed_prompt_improvements(message: str) -> dict[str, Any]: + return EvaluationPromptImprovementsState( + status="failed", + error_message=message[:500], + generated_at=datetime.now(timezone.utc), + ).model_dump(mode="json") diff --git a/app/services/harbor/job_runner.py b/app/services/harbor/job_runner.py new file mode 100644 index 00000000..3909b785 --- /dev/null +++ b/app/services/harbor/job_runner.py @@ -0,0 +1,127 @@ +"""Harbor job entry point, run as a subprocess by ``run_harbor_simulation``. + + python -m app.services.harbor.job_runner + +A separate process keeps Harbor's asyncio runtime and logging handlers out of the +Celery worker, and lets the worker cancel a run with SIGINT. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any +from uuid import UUID + +from app.services.harbor.task_exporter import run_work_dir + +AGENT_IMPORT_PATH = "app.services.harbor.conversation_agent:EfficientAIConversationAgent" +ENVIRONMENT_IMPORT_PATH = "app.services.harbor.noop_environment:NoopEnvironment" +VERIFIER_IMPORT_PATH = "app.services.harbor.metric_verifier:EfficientAIMetricVerifier" +JOBS_DIR_NAME = "jobs" + + +def harbor_job_name(run) -> str: + return f"run-{run.run_id}" + + +def harbor_job_dir(run) -> Path: + return run_work_dir(run) / JOBS_DIR_NAME / harbor_job_name(run) + + +def build_job_config( + run, *, task_paths: list[Path], production_model: str | None +) -> dict[str, Any]: + agent_kwargs = { + "simulation_run_id": str(run.id), + "production_llm_provider": run.production_llm_provider, + "production_llm_model": run.production_llm_model, + "test_llm_provider": run.test_llm_provider, + "test_llm_model": run.test_llm_model, + } + return { + "job_name": harbor_job_name(run), + "jobs_dir": str(run_work_dir(run) / JOBS_DIR_NAME), + "n_attempts": run.n_attempts, + "n_concurrent_trials": run.n_concurrent, + "retry": {"max_retries": 1}, + "quiet": True, + "tasks": [{"path": str(path)} for path in task_paths], + "agents": [ + { + "import_path": AGENT_IMPORT_PATH, + "model_name": production_model, + "kwargs": {k: v for k, v in agent_kwargs.items() if v is not None}, + } + ], + "environment": {"import_path": ENVIRONMENT_IMPORT_PATH}, + "verifier": { + "import_path": VERIFIER_IMPORT_PATH, + "kwargs": {"simulation_run_id": str(run.id), "pass_threshold": run.pass_threshold}, + }, + } + + +def production_model_label(run, agent) -> str | None: + if run.production_llm_provider and run.production_llm_model: + return f"{run.production_llm_provider}/{run.production_llm_model}" + provider = getattr(agent.main_llm_provider, "value", agent.main_llm_provider) + if provider and agent.main_llm_model: + return f"{provider}/{agent.main_llm_model}" + return None + + +async def run_job(simulation_run_id: UUID) -> None: + from harbor.job import Job + from harbor.models.job.config import JobConfig + + from app.database import SessionLocal + from app.models.database import Agent, SimulationRun + + db = SessionLocal() + try: + run = db.query(SimulationRun).filter(SimulationRun.id == simulation_run_id).one() + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + tasks_root = run_work_dir(run) / "tasks" + task_paths = sorted(p for p in tasks_root.iterdir() if p.is_dir()) + config = build_job_config( + run, + task_paths=task_paths, + production_model=production_model_label(run, agent), + ) + finally: + db.close() + + job = await Job.create(JobConfig.model_validate(config)) + await job.run() + + +def load_runtime_config() -> None: + """Load config.yml like Celery workers do (``app/workers/config.py``). + + This process never imports the Celery app, so without this it would only see + ``.env`` (e.g. the docker-compose ``db`` host) instead of the configured database. + """ + import os + + from app.config import load_config_from_file + + config_path = Path(os.environ.get("EFFICIENTAI_CONFIG_PATH", "config.yml")) + if config_path.exists(): + load_config_from_file(str(config_path)) + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print( + "usage: python -m app.services.harbor.job_runner ", file=sys.stderr + ) + return 2 + load_runtime_config() + asyncio.run(run_job(UUID(argv[1]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/app/services/harbor/metric_verifier.py b/app/services/harbor/metric_verifier.py new file mode 100644 index 00000000..a953c22a --- /dev/null +++ b/app/services/harbor/metric_verifier.py @@ -0,0 +1,48 @@ +"""Harbor verifier that scores a trial with the platform's metric LLM judges.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any +from uuid import UUID + +from harbor.models.verifier.result import VerifierResult +from harbor.verifier.base import BaseVerifier + +from app.database import SessionLocal +from app.services.harbor.trial_store import score_trial + + +class EfficientAIMetricVerifier(BaseVerifier): + """Host-side verifier: never touches the (no-op) environment.""" + + def __init__( + self, *, simulation_run_id: str, pass_threshold: float = 0.7, **kwargs: Any + ) -> None: + super().__init__(**kwargs) + self.simulation_run_id = UUID(str(simulation_run_id)) + self.pass_threshold = float(pass_threshold) + + async def verify(self) -> VerifierResult: + trial_name = self.trial_paths.trial_dir.name + rewards, details = await asyncio.to_thread(self._score_sync, trial_name) + verifier_dir = self.trial_paths.trial_dir / "verifier" + verifier_dir.mkdir(parents=True, exist_ok=True) + (verifier_dir / "reward.json").write_text(json.dumps(rewards, indent=2) + "\n") + (verifier_dir / "reward-details.json").write_text( + json.dumps(details, indent=2, default=str) + "\n" + ) + return VerifierResult(rewards=rewards) + + def _score_sync(self, trial_name: str): + db = SessionLocal() + try: + return score_trial( + db, + simulation_run_id=self.simulation_run_id, + trial_name=trial_name, + pass_threshold=self.pass_threshold, + ) + finally: + db.close() diff --git a/app/services/harbor/noop_environment.py b/app/services/harbor/noop_environment.py new file mode 100644 index 00000000..4ed6d445 --- /dev/null +++ b/app/services/harbor/noop_environment.py @@ -0,0 +1,50 @@ +"""Harbor environment with no sandbox: conversations need no container. + +The conversation agent and metric verifier both run in the Harbor host process and +write straight into the trial's host log dirs, so ``mounted=True`` tells Harbor +there is nothing to download back. +""" + +from __future__ import annotations + +from pathlib import Path + +from harbor.environments.base import BaseEnvironment, ExecResult +from harbor.environments.capabilities import EnvironmentCapabilities + + +class NoopEnvironment(BaseEnvironment): + @staticmethod + def type() -> str: + return "efficientai-noop" + + @property + def capabilities(self) -> EnvironmentCapabilities: + return EnvironmentCapabilities(mounted=True) + + def _validate_definition(self): + pass + + async def start(self, force_build: bool) -> None: + pass + + async def stop(self, delete: bool): + pass + + async def exec( + self, command, cwd=None, env=None, timeout_sec=None, user=None, **kwargs + ) -> ExecResult: + # Harbor only issues housekeeping commands (mkdir/chmod of /logs dirs) here. + return ExecResult(stdout="", stderr="", return_code=0) + + async def upload_file(self, source_path: Path | str, target_path: str): + pass + + async def upload_dir(self, source_dir: Path | str, target_dir: str): + pass + + async def download_file(self, source_path: str, target_path: Path | str): + pass + + async def download_dir(self, source_dir: str, target_dir: Path | str): + pass diff --git a/app/services/harbor/result_ingest.py b/app/services/harbor/result_ingest.py new file mode 100644 index 00000000..ab6b9a52 --- /dev/null +++ b/app/services/harbor/result_ingest.py @@ -0,0 +1,401 @@ +"""Progress, summary and artifact handling for Harbor simulation runs.""" + +from __future__ import annotations + +import io +import json +import tarfile +from collections import defaultdict +from pathlib import Path +from statistics import mean +from typing import Any, Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import ( + EvaluatorResult, + EvaluatorResultStatus, + Metric, + Persona, + Scenario, + SimulationRun, +) +from app.services.harbor.rewards import normalize_metric_score +from app.services.metrics.evaluation_scope import ( + is_fully_golden, + is_golden_scope, + load_agent_link_map, + load_scenario_link_map, + scope_metric_id, +) + +LOCAL_ARTIFACT_PREFIX = "local:" +_UNFINISHED_STATUSES = frozenset( + { + EvaluatorResultStatus.QUEUED.value, + EvaluatorResultStatus.CALL_IN_PROGRESS.value, + EvaluatorResultStatus.EVALUATING.value, + } +) + + +def trial_results(db: Session, run_id: UUID) -> list[EvaluatorResult]: + from app.services.live_entity_storage import hydrate_evaluator_results + + results = ( + db.query(EvaluatorResult) + .filter(EvaluatorResult.simulation_run_id == run_id) + .order_by(EvaluatorResult.created_at.asc()) + .all() + ) + hydrate_evaluator_results(results) + return results + + +def refresh_run_progress(db: Session, run: SimulationRun) -> None: + results = trial_results(db, run.id) + run.completed_trials = sum( + 1 for r in results if r.status == EvaluatorResultStatus.COMPLETED.value + ) + run.failed_trials = sum(1 for r in results if r.status == EvaluatorResultStatus.FAILED.value) + + +def attempt_numbers(results: Iterable[EvaluatorResult]) -> dict[UUID, int]: + """Number attempts 1..k per scenario × persona in creation order.""" + counters: dict[tuple, int] = defaultdict(int) + numbers: dict[UUID, int] = {} + for result in sorted(results, key=lambda r: (r.created_at is None, r.created_at)): + key = (result.scenario_id, result.persona_id) + counters[key] += 1 + numbers[result.id] = counters[key] + return numbers + + +def assign_attempt_indexes(results: list[EvaluatorResult]) -> None: + numbers = attempt_numbers(results) + for result in results: + result.attempt_index = numbers[result.id] + + +def fail_unfinished_trials(results: Iterable[EvaluatorResult], reason: str) -> None: + for result in results: + if result.status in _UNFINISHED_STATUSES: + result.status = EvaluatorResultStatus.FAILED.value + call_data = result.call_data if isinstance(result.call_data, dict) else {} + trial_err = call_data.get("trial_error") + if isinstance(trial_err, str) and trial_err.strip(): + result.error_message = result.error_message or trial_err.strip() + result.error_message = result.error_message or reason + + +def _trial_tool_calls(result: EvaluatorResult) -> list[dict[str, Any]]: + data = result.call_data if isinstance(result.call_data, dict) else {} + return [c for c in data.get("tool_calls") or [] if isinstance(c, dict)] + + +def _tool_expectations_met(result: EvaluatorResult) -> Optional[bool]: + data = result.call_data if isinstance(result.call_data, dict) else {} + return (data.get("tool_evaluation") or {}).get("all_expectations_met") + + +def compute_tool_summary(results: list[EvaluatorResult]) -> Optional[dict[str, Any]]: + """Run-level tool-use KPIs plus a per-tool breakdown; ``None`` when no tools were used.""" + per_tool: dict[str, dict[str, Any]] = {} + for r in results: + for call in _trial_tool_calls(r): + tool = call.get("tool") or "unknown" + row = per_tool.setdefault( + tool, + { + "tool": tool, + "provider": call.get("provider"), + "calls": 0, + "errors": 0, + "mocked": 0, + "latencies": [], + "trials": set(), + }, + ) + row["calls"] += 1 + row["errors"] += call.get("status") == "error" + row["mocked"] += call.get("status") == "mocked" + if call.get("latency_ms") is not None: + row["latencies"].append(call["latency_ms"]) + row["trials"].add(r.id) + checked = [m for m in (_tool_expectations_met(r) for r in results) if m is not None] + if not per_tool and not checked: + return None + total = sum(row["calls"] for row in per_tool.values()) + errors = sum(row["errors"] for row in per_tool.values()) + tools = [ + { + "tool": row["tool"], + "provider": row["provider"], + "calls": row["calls"], + "errors": row["errors"], + "mocked": row["mocked"], + "error_rate": round(row["errors"] / row["calls"], 4), + "avg_latency_ms": round(mean(row["latencies"])) if row["latencies"] else None, + "trials": len(row["trials"]), + } + for row in per_tool.values() + ] + tools.sort(key=lambda t: (-t["error_rate"], -t["calls"])) + return { + "total_calls": total, + "errored_calls": errors, + "success_rate": round((total - errors) / total, 4) if total else None, + "trials_with_calls": sum(1 for r in results if _trial_tool_calls(r)), + "expectations_checked": len(checked), + "expectations_met_rate": ( + round(sum(1 for m in checked if m) / len(checked), 4) if checked else None + ), + "tools": tools, + } + + +def compute_summary( + db: Session, run: SimulationRun, results: list[EvaluatorResult] +) -> dict[str, Any]: + """KPIs for the run page; safe to call while the run is still in progress.""" + scored = [r for r in results if r.reward is not None] + completed = [r for r in results if r.status == EvaluatorResultStatus.COMPLETED.value] + failed = [r for r in results if r.status == EvaluatorResultStatus.FAILED.value] + + # Per scenario × persona consistency: pass^k (all attempts pass) and pass@k (any passes). + combos: dict[tuple, list[EvaluatorResult]] = defaultdict(list) + for r in results: + combos[(r.scenario_id, r.persona_id)].append(r) + finished_combos = [ + rs + for rs in combos.values() + if all(r.status == EvaluatorResultStatus.COMPLETED.value for r in rs) + ] + + metric_ids: set[str] = set() + for r in scored: + metric_ids.update((r.metric_scores or {}).keys()) + metrics = ( + db.query(Metric).filter(Metric.id.in_([UUID(m) for m in metric_ids])).all() + if metric_ids + else [] + ) + scope_ids = {scope_metric_id(m) for m in metrics} + agent_link_map = load_agent_link_map(db, scope_ids) + scenario_link_map = load_scenario_link_map(db, scope_ids) + uses_criteria = bool(run.metric_success_criteria) + criteria_metric_ids = ( + {str(c.get("metric_id")) for c in (run.metric_success_criteria or [])} + if uses_criteria + else set() + ) + metric_rows = [] + for metric in metrics: + mid = str(metric.id) + if uses_criteria and mid in criteria_metric_ids: + passes = [] + for r in scored: + entry = (r.metric_scores or {}).get(mid) + if isinstance(entry, dict) and entry.get("passed") is not None: + passes.append(bool(entry.get("passed"))) + if passes: + pass_rate = round(sum(1 for p in passes if p) / len(passes), 4) + scope_id = scope_metric_id(metric) + metric_rows.append( + { + "metric_id": mid, + "name": metric.name, + "mean": pass_rate, + "pass_rate": pass_rate, + "n": len(passes), + "agent_ids": ( + [] + if is_golden_scope(agent_link_map, scope_id) + else [str(a) for a in sorted(agent_link_map.get(scope_id, ()))] + ), + "scenario_ids": ( + [] + if is_golden_scope(scenario_link_map, scope_id) + else [str(s) for s in sorted(scenario_link_map.get(scope_id, ()))] + ), + } + ) + continue + values = [ + v + for v in ( + normalize_metric_score((r.metric_scores or {}).get(str(metric.id)), metric) + for r in scored + ) + if v is not None + ] + if values: + scope_id = scope_metric_id(metric) + metric_rows.append( + { + "metric_id": str(metric.id), + "name": metric.name, + "mean": round(mean(values), 4), + "n": len(values), + "agent_ids": ( + [] + if is_golden_scope(agent_link_map, scope_id) + else [str(a) for a in sorted(agent_link_map.get(scope_id, ()))] + ), + "scenario_ids": ( + [] + if is_golden_scope(scenario_link_map, scope_id) + else [str(s) for s in sorted(scenario_link_map.get(scope_id, ()))] + ), + } + ) + metric_rows.sort(key=lambda m: m["mean"]) + + golden_trial_means: list[float] = [] + for r in scored: + golden_values = [ + v + for v in ( + normalize_metric_score((r.metric_scores or {}).get(str(metric.id)), metric) + for metric in metrics + if is_fully_golden( + scope_metric_id(metric), + agent_link_map=agent_link_map, + scenario_link_map=scenario_link_map, + ) + ) + if v is not None + ] + if golden_values: + golden_trial_means.append(round(mean(golden_values), 4)) + + scenario_names = ( + { + s.id: s.name + for s in db.query(Scenario).filter(Scenario.id.in_([k[0] for k in combos])).all() + } + if combos + else {} + ) + persona_names = ( + { + p.id: p.name + for p in db.query(Persona).filter(Persona.id.in_([k[1] for k in combos])).all() + } + if combos + else {} + ) + combo_rows = [] + for (scenario_id, persona_id), rs in combos.items(): + rewards = [r.reward for r in rs if r.reward is not None] + combo_rows.append( + { + "scenario_id": str(scenario_id), + "scenario_name": scenario_names.get(scenario_id), + "persona_id": str(persona_id), + "persona_name": persona_names.get(persona_id), + "attempts": len(rs), + "passed": sum(1 for r in rs if r.passed), + "mean_reward": round(mean(rewards), 4) if rewards else None, + "tool_errors": sum( + 1 for r in rs for c in _trial_tool_calls(r) if c.get("status") == "error" + ), + "tool_expectations_failed": sum( + 1 for r in rs if _tool_expectations_met(r) is False + ), + } + ) + combo_rows.sort(key=lambda c: (c["mean_reward"] is None, c["mean_reward"] or 0)) + + turns = [len(r.speaker_segments or []) for r in completed] + failed_trial_errors = [ + { + "result_id": r.result_id, + "scenario_id": str(r.scenario_id) if r.scenario_id else None, + "scenario_name": scenario_names.get(r.scenario_id), + "persona_name": persona_names.get(r.persona_id), + "error_message": (r.error_message or "").strip(), + } + for r in failed + if (r.error_message or "").strip() + ] + return { + "n_trials": len(results), + "n_completed": len(completed), + "n_failed": len(failed), + "failed_trial_errors": failed_trial_errors[:10], + "n_passed": sum(1 for r in scored if r.passed), + "uses_success_criteria": uses_criteria, + "pass_rate": round(sum(1 for r in scored if r.passed) / len(scored), 4) if scored else None, + "mean_reward": round(mean(r.reward for r in scored), 4) if scored else None, + "golden_mean_reward": ( + round(mean(golden_trial_means), 4) if golden_trial_means else None + ), + "pass_all_rate": ( + round( + sum(1 for rs in finished_combos if all(r.passed for r in rs)) + / len(finished_combos), + 4, + ) + if finished_combos + else None + ), + "pass_any_rate": ( + round( + sum(1 for rs in finished_combos if any(r.passed for r in rs)) + / len(finished_combos), + 4, + ) + if finished_combos + else None + ), + "avg_turns": round(mean(turns), 2) if turns else None, + "metrics": metric_rows, + "combos": combo_rows, + "tools": compute_tool_summary(results), + } + + +def read_harbor_job_stats(job_dir: Path) -> Optional[dict[str, Any]]: + path = job_dir / "result.json" + if not path.exists(): + return None + stats = json.loads(path.read_text()).get("stats") or {} + return { + key: stats.get(key) + for key in ( + "n_completed_trials", + "n_errored_trials", + "n_cancelled_trials", + "n_retries", + "n_input_tokens", + "n_output_tokens", + "cost_usd", + ) + } | { + "exception_stats": { + name: list(ev.get("exception_stats", {}).keys()) + for name, ev in (stats.get("evals") or {}).items() + } + } + + +def archive_run_dir(run: SimulationRun, work_dir: Path) -> Optional[str]: + """Tar the exported tasks + Harbor job dir; upload to blob storage when enabled.""" + if not work_dir.exists(): + return None + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + tar.add(work_dir, arcname=f"simulation-run-{run.run_id}") + data = buf.getvalue() + + from app.services.storage.blob_storage_service import blob_storage_service + + if blob_storage_service.is_enabled(): + key = f"simulation_runs/{run.organization_id}/{run.id}/harbor-{run.run_id}.tar.gz" + blob_storage_service.upload_file_by_key(data, key, content_type="application/gzip") + return key + tarball = work_dir.parent / f"{run.id}.tar.gz" + tarball.write_bytes(data) + return f"{LOCAL_ARTIFACT_PREFIX}{tarball}" diff --git a/app/services/harbor/rewards.py b/app/services/harbor/rewards.py new file mode 100644 index 00000000..9d91f734 --- /dev/null +++ b/app/services/harbor/rewards.py @@ -0,0 +1,90 @@ +"""Normalize platform metric scores into Harbor rewards in [0, 1].""" + +from __future__ import annotations + +import re +from typing import Any, Iterable, Optional + +AGGREGATE_REWARD_KEY = "reward" + + +def metric_reward_key(name: str, taken: set[str]) -> str: + """Stable, unique snake_case key for a metric name inside one rewards dict.""" + base = re.sub(r"[^a-z0-9]+", "_", (name or "").lower()).strip("_") or "metric" + if base == AGGREGATE_REWARD_KEY: + base = f"metric_{base}" + key = base + suffix = 2 + while key in taken: + key = f"{base}_{suffix}" + suffix += 1 + taken.add(key) + return key + + +def _metric_type(entry: dict[str, Any], metric: Any) -> str: + raw = entry.get("type") or getattr(metric, "metric_type", None) or "" + if hasattr(raw, "value"): + raw = raw.value + return str(raw).lower() + + +def normalize_metric_score(entry: Optional[dict[str, Any]], metric: Any) -> Optional[float]: + """Map one ``metric_scores`` entry to [0, 1]; ``None`` when it shouldn't count. + + Rating values are already 0–1 (see ``score_utils.normalize_score``). Booleans map + to 1/0. Numbers need a ``custom_config`` min/max range. Categorization child + labels, text, enum and errored/skipped scores don't contribute. + """ + if not isinstance(entry, dict) or getattr(metric, "parent_metric_id", None): + return None + if entry.get("error") or entry.get("skipped"): + return None + value = entry.get("value") + if value is None: + return None + + metric_type = _metric_type(entry, metric) + if metric_type == "rating": + try: + return max(0.0, min(1.0, float(value))) + except (TypeError, ValueError): + return None + if metric_type == "boolean": + if isinstance(value, str): + return 1.0 if value.strip().lower() in ("true", "yes", "1") else 0.0 + return 1.0 if bool(value) else 0.0 + if metric_type == "number": + cfg = getattr(metric, "custom_config", None) or {} + lo, hi = cfg.get("min"), cfg.get("max") + try: + lo, hi, v = float(lo), float(hi), float(value) + except (TypeError, ValueError): + return None + if hi <= lo: + return None + return max(0.0, min(1.0, (v - lo) / (hi - lo))) + return None + + +def compute_rewards( + metric_scores: dict[str, Any], + metrics: Iterable[Any], +) -> tuple[Optional[float], dict[str, float], dict[str, str]]: + """Return ``(aggregate, rewards_by_key, metric_id_by_key)``. + + The aggregate is the unweighted mean of every scorable metric, or ``None`` + when nothing was scorable. + """ + rewards: dict[str, float] = {} + metric_ids: dict[str, str] = {} + taken: set[str] = {AGGREGATE_REWARD_KEY} + for metric in metrics: + score = normalize_metric_score(metric_scores.get(str(metric.id)), metric) + if score is None: + continue + key = metric_reward_key(metric.name, taken) + rewards[key] = round(score, 4) + metric_ids[key] = str(metric.id) + aggregate = round(sum(rewards.values()) / len(rewards), 4) if rewards else None + return aggregate, rewards, metric_ids diff --git a/app/services/harbor/success_criteria.py b/app/services/harbor/success_criteria.py new file mode 100644 index 00000000..4991d988 --- /dev/null +++ b/app/services/harbor/success_criteria.py @@ -0,0 +1,414 @@ +"""Per-metric success rules for Harbor simulation runs.""" + +from __future__ import annotations + +from typing import Any, Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Metric +from app.models.schemas import MetricSuccessCriterion +from app.services.classification_metric_scores import classification_facets_from_entry +from app.services.metric_classification_validation import is_classification_metric +from app.services.metric_failure_policy import normalize_label +from app.services.metrics.option_labels import ( + classification_choice_labels_from_config, + classification_score_labels_from_config, + enum_labels_equal, + enum_option_labels_from_config, +) + + +def _is_categorization_parent(metric: Metric) -> bool: + return bool(metric.selection_mode and not metric.parent_metric_id) + + +def infer_criterion_kind(metric: Metric) -> str: + if is_classification_metric(custom_data_type=metric.custom_data_type): + return "classification" + if _is_categorization_parent(metric): + mode = (metric.selection_mode or "").lower() + return "category_multi" if mode == "multi_label" else "category_single" + if (metric.custom_data_type or "").lower() == "enum": + return "enum" + mtype = (metric.metric_type or "").lower() + if mtype in ("boolean", "rating", "number", "text"): + return mtype + return "text" + + +def _coerce_bool(value: Any) -> Optional[bool]: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("true", "yes", "1"): + return True + if lowered in ("false", "no", "0"): + return False + if isinstance(value, (int, float)): + return bool(value) + return None + + +def _bool_matches_success(value: Any, success_when: str) -> bool: + b = _coerce_bool(value) + if b is None: + return False + want_true = success_when == "yes" + return b is want_true + + +def _numeric_compare(value: Any, op: str, threshold: float) -> bool: + try: + v = float(value) + except (TypeError, ValueError): + return False + if op == "lt": + return v < threshold + if op == "lte": + return v <= threshold + if op == "gt": + return v > threshold + if op == "gte": + return v >= threshold + return False + + +def _entry_failed(entry: Optional[dict[str, Any]]) -> bool: + if not isinstance(entry, dict): + return True + if entry.get("error") or entry.get("skipped"): + return True + if entry.get("value") is None and entry.get("type") != "category": + return True + return False + + +def criterion_passes( + criterion: dict[str, Any] | MetricSuccessCriterion, + *, + metric_scores: dict[str, Any], + metric: Metric, + children_by_parent: dict[str, list[Metric]], +) -> bool: + if isinstance(criterion, MetricSuccessCriterion): + crit = criterion.model_dump(mode="json") + else: + crit = dict(criterion) + kind = crit.get("kind") + mid = str(crit.get("metric_id") or metric.id) + entry = metric_scores.get(mid) + + if kind == "text": + return True + + if kind == "boolean": + if _entry_failed(entry): + return False + return _bool_matches_success(entry.get("value"), crit.get("success_when") or "yes") + + if kind in ("rating", "number"): + if _entry_failed(entry): + return False + threshold = float(crit.get("threshold", 0)) + return _numeric_compare(entry.get("value"), crit.get("op") or "gte", threshold) + + if kind == "enum": + if _entry_failed(entry): + return False + raw = entry.get("value") + if raw is None: + return False + allowed = crit.get("success_values") or [] + return any(enum_labels_equal(str(raw), v) for v in allowed) + + if kind == "category_single": + if _entry_failed(entry): + return False + chosen = entry.get("chosen_child_name") or entry.get("value") + if chosen is None: + return False + allowed = {normalize_label(v) for v in (crit.get("success_values") or [])} + return normalize_label(str(chosen)) in allowed + + if kind == "category_multi": + rules = crit.get("label_rules") or [] + if not rules: + return False + for rule in rules: + child_id = str(rule.get("child_id")) + child_entry = metric_scores.get(child_id) + if _entry_failed(child_entry): + return False + expect = rule.get("expect") or "yes" + if not _bool_matches_success(child_entry.get("value"), expect): + return False + return True + + if kind == "classification": + if _entry_failed(entry): + return False + questions = crit.get("questions") or {} + if not questions: + return False + facets = classification_facets_from_entry(entry) + used_any = False + noul = questions.get("noul") + if noul: + used_any = True + yes_no = facets.get("yes_no") + if yes_no is None: + return False + if not _bool_matches_success(yes_no, noul.get("success_when") or "yes"): + return False + choice = questions.get("choice") + if choice: + used_any = True + val = facets.get("choice") + if not val: + return False + success_vals = choice.get("success_values") or [] + if not any(enum_labels_equal(val, v) for v in success_vals): + return False + score_q = questions.get("score") + if score_q: + used_any = True + level = facets.get("level") + if not level: + return False + success_vals = score_q.get("success_values") or [] + if not any(enum_labels_equal(level, v) for v in success_vals): + return False + return used_any + + return False + + +def is_gating_criterion(criterion: dict[str, Any] | MetricSuccessCriterion) -> bool: + kind = criterion.get("kind") if isinstance(criterion, dict) else criterion.kind + return kind != "text" + + +def apply_success_criteria( + metric_scores: dict[str, Any], + criteria: Iterable[dict[str, Any] | MetricSuccessCriterion], + metrics_by_id: dict[str, Metric], + children_by_parent: dict[str, list[Metric]], +) -> tuple[bool, Optional[float], dict[str, Any]]: + """Return ``(trial_passed, reward_fraction, updated_metric_scores)``.""" + updated = dict(metric_scores) + gating: list[tuple[dict[str, Any], bool]] = [] + + for raw in criteria: + crit = raw if isinstance(raw, dict) else raw.model_dump(mode="json") + mid = str(crit.get("metric_id")) + metric = metrics_by_id.get(mid) + if metric is None: + continue + passed = criterion_passes( + crit, + metric_scores=updated, + metric=metric, + children_by_parent=children_by_parent, + ) + if not is_gating_criterion(crit): + continue + gating.append((crit, passed)) + entry = updated.get(mid) + if isinstance(entry, dict): + entry = dict(entry) + entry["passed"] = passed + updated[mid] = entry + kind = crit.get("kind") + if kind == "category_multi": + parent_entry = updated.get(mid) + if isinstance(parent_entry, dict): + parent_copy = dict(parent_entry) + parent_copy["passed"] = passed + updated[mid] = parent_copy + for rule in crit.get("label_rules") or []: + child_id = str(rule.get("child_id")) + child_entry = updated.get(child_id) + child_ok = not _entry_failed(child_entry) and _bool_matches_success( + child_entry.get("value") if isinstance(child_entry, dict) else None, + rule.get("expect") or "yes", + ) + if isinstance(child_entry, dict): + child_copy = dict(child_entry) + child_copy["passed"] = child_ok + updated[child_id] = child_copy + + if not gating: + return False, None, updated + n_pass = sum(1 for _, ok in gating if ok) + reward = round(n_pass / len(gating), 4) + trial_passed = n_pass == len(gating) + return trial_passed, reward, updated + + +def _load_metric_context( + db: Session, + organization_id: UUID, + metric_ids: list[str], +) -> tuple[dict[str, Metric], dict[str, list[Metric]]]: + ids = [UUID(mid) for mid in metric_ids] + metrics = ( + db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.id.in_(ids)) + .all() + ) + by_id = {str(m.id): m for m in metrics} + parent_ids = [m.id for m in metrics if _is_categorization_parent(m)] + children_by_parent: dict[str, list[Metric]] = {} + if parent_ids: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(parent_ids), + ) + .all() + ) + for child in children: + children_by_parent.setdefault(str(child.parent_metric_id), []).append(child) + return by_id, children_by_parent + + +def validate_metric_success_criteria( + db: Session, + *, + organization_id: UUID, + metric_ids: list[UUID], + criteria: list[MetricSuccessCriterion], +) -> None: + """Raise ValueError when rules do not match metrics or cannot gate a run.""" + from app.services.evaluators.evaluator_helpers import normalize_metric_ids_for_storage + + normalized_ids = normalize_metric_ids_for_storage( + db, organization_id, [str(m) for m in metric_ids] + ) + by_id, children_by_parent = _load_metric_context(db, organization_id, normalized_ids) + if len(by_id) != len(set(normalized_ids)): + raise ValueError("One or more metric_ids were not found") + + gating_kinds: list[str] = [] + for rule in criteria: + mid = str(rule.metric_id) + if mid not in by_id: + raise ValueError(f"No metric found for criterion metric_id={mid}") + metric = by_id[mid] + expected_kind = infer_criterion_kind(metric) + if rule.kind != expected_kind: + raise ValueError( + f"Metric '{metric.name}' expects success criteria kind '{expected_kind}', " + f"got '{rule.kind}'" + ) + _validate_rule_shape(rule, metric, children_by_parent.get(mid, [])) + if rule.kind != "text": + gating_kinds.append(rule.kind) + + if not gating_kinds: + raise ValueError("At least one non-text metric with success criteria is required") + + +def _validate_rule_shape( + rule: MetricSuccessCriterion, + metric: Metric, + children: list[Metric], +) -> None: + kind = rule.kind + if kind == "boolean": + if rule.success_when not in ("yes", "no"): + raise ValueError(f"Boolean metric '{metric.name}' requires success_when yes or no") + return + if kind == "rating": + if rule.op not in ("lt", "lte", "gt", "gte") or rule.threshold is None: + raise ValueError(f"Rating metric '{metric.name}' requires op and threshold") + if not 0 <= rule.threshold <= 1: + raise ValueError(f"Rating threshold for '{metric.name}' must be between 0 and 1") + return + if kind == "number": + if rule.op not in ("lt", "lte", "gt", "gte") or rule.threshold is None: + raise ValueError(f"Number metric '{metric.name}' requires op and threshold") + return + if kind == "enum": + options = _enum_options(metric) + if not rule.success_values: + raise ValueError(f"Enum metric '{metric.name}' requires at least one success value") + unknown = [ + v + for v in rule.success_values + if not any(enum_labels_equal(v, o) for o in options) + ] + if unknown: + raise ValueError(f"Enum metric '{metric.name}' has unknown success values: {unknown}") + return + if kind == "category_single": + if not rule.success_values: + raise ValueError(f"Category '{metric.name}' requires at least one success label") + child_names = {normalize_label(c.name) for c in children} + for val in rule.success_values: + if normalize_label(val) not in child_names: + raise ValueError(f"Category '{metric.name}' has unknown label '{val}'") + return + if kind == "category_multi": + if not rule.label_rules: + raise ValueError(f"Multi-label category '{metric.name}' requires label_rules") + child_ids = {str(c.id) for c in children} + for lr in rule.label_rules: + if str(lr.child_id) not in child_ids: + raise ValueError(f"Multi-label '{metric.name}' has unknown child_id") + if lr.expect not in ("yes", "no"): + raise ValueError(f"Multi-label '{metric.name}' label_rules need expect yes or no") + return + if kind == "classification": + cfg = metric.custom_config or {} + questions = rule.questions + if questions is None: + raise ValueError(f"Classification metric '{metric.name}' requires questions") + used = 0 + if questions.noul and _classification_block_enabled(cfg, "noul"): + used += 1 + if questions.noul.success_when not in ("yes", "no"): + raise ValueError(f"Classification noul for '{metric.name}' needs success_when") + if questions.choice and _classification_block_enabled(cfg, "choice"): + used += 1 + opts = classification_choice_labels_from_config(cfg) + if not questions.choice.success_values: + raise ValueError(f"Classification choice for '{metric.name}' needs success_values") + for val in questions.choice.success_values: + if not any(enum_labels_equal(val, o) for o in opts): + raise ValueError(f"Classification choice unknown value '{val}'") + if questions.score and _classification_block_enabled(cfg, "score"): + used += 1 + levels = classification_score_labels_from_config(cfg) + if not questions.score.success_values: + raise ValueError(f"Classification score for '{metric.name}' needs success_values") + for val in questions.score.success_values: + if not any(enum_labels_equal(val, level) for level in levels): + raise ValueError(f"Classification score unknown level '{val}'") + if used == 0: + raise ValueError( + f"Classification metric '{metric.name}' must use at least one enabled question" + ) + return + if kind == "text": + return + raise ValueError(f"Unknown success criteria kind '{kind}'") + + +def _classification_block_enabled(cfg: dict[str, Any], key: str) -> bool: + block = cfg.get(key) + return isinstance(block, dict) and bool(block.get("enabled")) + + +def _enum_options(metric: Metric) -> list[str]: + return enum_option_labels_from_config(metric.custom_config or {}) + + +def criteria_to_storage(criteria: list[MetricSuccessCriterion]) -> list[dict[str, Any]]: + return [c.model_dump(mode="json") for c in criteria] diff --git a/app/services/harbor/task_exporter.py b/app/services/harbor/task_exporter.py new file mode 100644 index 00000000..2f7521cd --- /dev/null +++ b/app/services/harbor/task_exporter.py @@ -0,0 +1,214 @@ +"""Export simulation run scenarios × personas as Harbor task directories.""" + +from __future__ import annotations + +import json +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import Agent, Metric, Persona, Scenario, SimulationRun +from app.services.metrics.evaluation_scope import ( + applicable_metrics, + load_agent_link_map, + load_scenario_link_map, + scope_metric_id, +) +from app.services.testing.test_agent_simulation_prompt import ( + build_persona_description_for_bridge, + build_test_agent_system_prompt, + resolve_persona_max_turns, +) + +# Per-trial budgets; the conversation loop is bounded by persona max_turns. +AGENT_TIMEOUT_SEC = 900.0 +VERIFIER_TIMEOUT_SEC = 600.0 + +_TEST_SH = """#!/bin/bash +# Portable grader for running this task outside EfficientAI. EfficientAI runs +# the host-side EfficientAIMetricVerifier instead and never executes this file. +set -euo pipefail +uvx harbor-rewardkit /tests +""" + + +@dataclass(frozen=True) +class ExportedTask: + name: str + path: Path + scenario_id: UUID + persona_id: UUID + + +def run_work_dir(run: SimulationRun) -> Path: + return Path(settings.HARBOR_WORK_DIR).expanduser().resolve() / str(run.id) + + +def _slug(text: str, limit: int) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-") + return (slug[:limit].rstrip("-")) or "x" + + +def _toml_str(value: object) -> str: + # JSON string escapes are a subset of TOML basic-string escapes. + return json.dumps(str(value)) + + +def task_name_for(scenario: Scenario, persona: Persona) -> str: + # Harbor keeps only the first 32 chars of a task name in trial names. + return ( + f"{_slug(scenario.name, 11)}-{str(scenario.id)[:6]}" + f"__{_slug(persona.name, 7)}-{str(persona.id)[:4]}" + ) + + +def _task_toml(run: SimulationRun, scenario: Scenario, persona: Persona) -> str: + return "\n".join( + [ + 'schema_version = "1.4"', + "", + "[metadata]", + f"simulation_run_id = {_toml_str(run.id)}", + f"agent_id = {_toml_str(run.agent_id)}", + f"scenario_id = {_toml_str(scenario.id)}", + f"persona_id = {_toml_str(persona.id)}", + f"scenario_name = {_toml_str(scenario.name)}", + f"persona_name = {_toml_str(persona.name)}", + "", + "[agent]", + f"timeout_sec = {AGENT_TIMEOUT_SEC}", + "", + "[verifier]", + f"timeout_sec = {VERIFIER_TIMEOUT_SEC}", + "", + ] + ) + + +def _rubric_criterion(metric: Metric) -> list[str] | None: + metric_type = getattr(metric.metric_type, "value", metric.metric_type) + description = (metric.description or metric.name or "").strip() + lines = [ + "[[criterion]]", + f"id = {_toml_str(metric.id)}", + f"name = {_toml_str(metric.name)}", + f"description = {_toml_str(description)}", + ] + if metric_type == "boolean": + lines.append('type = "binary"') + elif metric_type == "rating": + lines += ['type = "numeric"', "min = 0", "max = 10"] + elif metric_type == "number" and isinstance(metric.custom_config, dict): + lo, hi = metric.custom_config.get("min"), metric.custom_config.get("max") + if lo is None or hi is None: + return None + lines += ['type = "numeric"', f"min = {lo}", f"max = {hi}"] + else: + return None + return lines + + +def _rubric_toml(metrics: list[Metric]) -> str: + """RewardKit mirror of the platform metrics so exported tasks stay portable.""" + lines = [ + "[judge]", + 'atif-trajectory = "/logs/agent/trajectory.json"', + "", + ] + for metric in metrics: + criterion = _rubric_criterion(metric) + if criterion: + lines += criterion + [""] + lines += ["[scoring]", 'aggregation = "weighted-mean"', ""] + return "\n".join(lines) + + +def export_tasks( + db: Session, + *, + run: SimulationRun, + agent: Agent, + metrics: list[Metric], +) -> list[ExportedTask]: + """Write one Harbor task per scenario × persona under ``/tasks``. + + Raises ``ValueError`` for missing scenarios/personas: a task that fails to load + aborts the whole Harbor job, so everything is validated up front. + """ + scenario_ids = [UUID(str(sid)) for sid in run.scenario_ids or []] + persona_ids = [UUID(str(pid)) for pid in run.persona_ids or []] + if not scenario_ids or not persona_ids: + raise ValueError("Simulation run needs at least one scenario and one persona") + + scenarios = ( + db.query(Scenario) + .filter( + Scenario.id.in_(scenario_ids), + Scenario.organization_id == run.organization_id, + Scenario.workspace_id == run.workspace_id, + ) + .all() + ) + personas = ( + db.query(Persona) + .filter( + Persona.id.in_(persona_ids), + Persona.organization_id == run.organization_id, + Persona.workspace_id == run.workspace_id, + ) + .all() + ) + missing = {str(i) for i in scenario_ids} - {str(s.id) for s in scenarios} + missing |= {str(i) for i in persona_ids} - {str(p.id) for p in personas} + if missing: + raise ValueError(f"Scenarios/personas not found: {', '.join(sorted(missing))}") + + tasks_root = run_work_dir(run) / "tasks" + shutil.rmtree(tasks_root, ignore_errors=True) + scope_ids = {scope_metric_id(m) for m in metrics} + agent_link_map = load_agent_link_map(db, scope_ids) + scenario_link_map = load_scenario_link_map(db, scope_ids) + run_agent_id = UUID(str(run.agent_id)) if run.agent_id else None + + exported: list[ExportedTask] = [] + for scenario in sorted(scenarios, key=lambda s: str(s.name)): + scenario_metrics = applicable_metrics( + metrics, + agent_id=run_agent_id, + scenario_id=scenario.id, + agent_link_map=agent_link_map, + scenario_link_map=scenario_link_map, + ) + rubric = _rubric_toml(scenario_metrics) + for persona in sorted(personas, key=lambda p: str(p.name)): + name = task_name_for(scenario, persona) + if (tasks_root / name).exists(): # id-prefix collision; keep names unique + name = f"{name[:26]}-{len(exported):05d}" + task_dir = tasks_root / name + (task_dir / "environment").mkdir(parents=True, exist_ok=True) + (task_dir / "tests").mkdir(parents=True, exist_ok=True) + (task_dir / "instruction.md").write_text( + build_test_agent_system_prompt( + agent, + persona, + scenario, + persona_description=build_persona_description_for_bridge(persona), + max_turns=resolve_persona_max_turns(persona), + ) + ) + (task_dir / "task.toml").write_text(_task_toml(run, scenario, persona)) + test_sh = task_dir / "tests" / "test.sh" + test_sh.write_text(_TEST_SH) + test_sh.chmod(0o755) + (task_dir / "tests" / "rubric.toml").write_text(rubric) + exported.append( + ExportedTask( + name=name, path=task_dir, scenario_id=scenario.id, persona_id=persona.id + ) + ) + return exported diff --git a/app/services/harbor/tool_checks.py b/app/services/harbor/tool_checks.py new file mode 100644 index 00000000..044708c0 --- /dev/null +++ b/app/services/harbor/tool_checks.py @@ -0,0 +1,174 @@ +"""Deterministic tool-use checks for a trial (no LLM): success rate and scenario expectations. + +A scenario's ``expected_tool_calls`` is a list of expectations:: + + {"tool": "search_knowledge_base", "args_contains": {"query": "refund"}, "min_calls": 1} + {"tool": "slack__post_message", "forbidden": true} + {"tool": "postgres__query", "must_succeed": true, "mock_response": {...}} + +``args_contains`` matches case-insensitive substrings of the call's arguments. +``mock_response`` is used by the toolset, not by these checks. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + +from app.services.tools.base import TOOL_STATUS_ERROR + +KB_TOOL = "search_knowledge_base" + + +def _args_match(arguments: dict[str, Any], expected: dict[str, Any]) -> bool: + for key, value in (expected or {}).items(): + actual = arguments.get(key) + if actual is None or str(value).lower() not in str(actual).lower(): + return False + return True + + +def _check_expectation(expectation: dict[str, Any], calls: list[dict[str, Any]]) -> dict[str, Any]: + tool = str(expectation.get("tool") or "") + forbidden = bool(expectation.get("forbidden")) + matching = [ + c + for c in calls + if c.get("tool") == tool + and _args_match(c.get("arguments") or {}, expectation.get("args_contains") or {}) + ] + if forbidden: + satisfied = not matching + reason = ( + None if satisfied else f"{tool} was called {len(matching)} time(s) but is forbidden" + ) + return { + **expectation, + "satisfied": satisfied, + "matched_calls": len(matching), + "reason": reason, + } + + if expectation.get("must_succeed"): + matching = [c for c in matching if c.get("status") != TOOL_STATUS_ERROR] + min_calls = int(expectation.get("min_calls") or 1) + max_calls = expectation.get("max_calls") + reason = None + if len(matching) < min_calls: + what = "successful matching" if expectation.get("must_succeed") else "matching" + reason = f"expected at least {min_calls} {what} call(s) to {tool}, got {len(matching)}" + elif max_calls is not None and len(matching) > int(max_calls): + reason = f"expected at most {max_calls} call(s) to {tool}, got {len(matching)}" + return { + **expectation, + "satisfied": reason is None, + "matched_calls": len(matching), + "reason": reason, + } + + +@dataclass +class ToolCheckReport: + total_calls: int = 0 + errored_calls: int = 0 + kb_calls: int = 0 + kb_calls_with_hits: int = 0 + expectations: list[dict[str, Any]] = field(default_factory=list) + + @property + def success_rate(self) -> Optional[float]: + if not self.total_calls: + return None + return round((self.total_calls - self.errored_calls) / self.total_calls, 4) + + @property + def kb_hit_rate(self) -> Optional[float]: + return round(self.kb_calls_with_hits / self.kb_calls, 4) if self.kb_calls else None + + @property + def has_expectations(self) -> bool: + return bool(self.expectations) + + @property + def all_expectations_met(self) -> bool: + return all(e["satisfied"] for e in self.expectations) + + @property + def expectation_score(self) -> Optional[float]: + if not self.expectations: + return None + met = sum(1 for e in self.expectations if e["satisfied"]) + return round(met / len(self.expectations), 4) + + @property + def forbidden_violations(self) -> int: + return sum(1 for e in self.expectations if e.get("forbidden") and not e["satisfied"]) + + def rewards(self) -> dict[str, float]: + """``tool_*`` reward components for Harbor (only the ones that apply).""" + values = { + "tool_success_rate": self.success_rate, + "tool_expectations_met": self.expectation_score, + "kb_retrieval_hit_rate": self.kb_hit_rate, + } + return {key: value for key, value in values.items() if value is not None} + + def as_dict(self) -> dict[str, Any]: + return { + "total_calls": self.total_calls, + "errored_calls": self.errored_calls, + "success_rate": self.success_rate, + "kb_calls": self.kb_calls, + "kb_hit_rate": self.kb_hit_rate, + "expectations": self.expectations, + "expectation_score": self.expectation_score, + "all_expectations_met": self.all_expectations_met if self.has_expectations else None, + "forbidden_violations": self.forbidden_violations, + } + + +def check_tool_calls( + tool_calls: Optional[list[dict[str, Any]]], + expected_tool_calls: Optional[list[dict[str, Any]]], +) -> ToolCheckReport: + calls = [ + c + for c in tool_calls or [] + if isinstance(c, dict) and c.get("side", "production") == "production" + ] + kb_calls = [ + c for c in calls if c.get("tool") == KB_TOOL and c.get("status") != TOOL_STATUS_ERROR + ] + return ToolCheckReport( + total_calls=len(calls), + errored_calls=sum(1 for c in calls if c.get("status") == TOOL_STATUS_ERROR), + kb_calls=len(kb_calls), + kb_calls_with_hits=sum(1 for c in kb_calls if (c.get("hit_count") or 0) > 0), + expectations=[ + _check_expectation(e, calls) + for e in expected_tool_calls or [] + if isinstance(e, dict) and e.get("tool") + ], + ) + + +def format_tool_calls_for_judge( + tool_calls: Optional[list[dict[str, Any]]], limit: int = 600 +) -> str: + """``## Tool Calls`` section appended to the judge's transcript (empty when none).""" + calls = [c for c in tool_calls or [] if isinstance(c, dict)] + if not calls: + return "" + lines = [ + "## Tool Calls (made by the production agent, Speaker 2)", + "Each call shows its turn, arguments, status and the result the agent saw.", + ] + for call in calls: + result = (call.get("result_preview") or "").strip().replace("\n", " ") + if len(result) > limit: + result = result[:limit] + "…" + lines.append( + f"- turn {call.get('turn_index')}: {call.get('tool')}({call.get('arguments') or {}}) " + f"→ {call.get('status')}: {result}" + ) + return "\n".join(lines) diff --git a/app/services/harbor/trial_store.py b/app/services/harbor/trial_store.py new file mode 100644 index 00000000..0f5baeb6 --- /dev/null +++ b/app/services/harbor/trial_store.py @@ -0,0 +1,478 @@ +"""DB side of a Harbor trial: one ``EvaluatorResult`` per trial (synchronous).""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.models.database import ( + Agent, + EvaluatorResult, + EvaluatorResultStatus, + Persona, + Scenario, + SimulationRun, +) +from app.models.enums import ModelProvider +from app.services.agents.chat_llm_config import ResolvedSimulationLlm +from app.services.agents.chat_production_leg import uses_live_production_leg +from app.services.evaluators.evaluator_run_service import generate_unique_result_id +from app.services.harbor.rewards import AGGREGATE_REWARD_KEY, compute_rewards, metric_reward_key +from app.services.harbor.tool_checks import check_tool_calls +from app.services.live_entity_storage import sync_evaluator_result +from app.services.testing.llm_to_llm_evaluator_simulation import ( + TextConversation, + run_text_conversation, + write_conversation_to_result, +) +from app.services.testing.simulation_llms import describe_simulation_llms +from app.services.testing.simulation_prompts import build_agent_system_prompt +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + usage_context_for_test_agent_simulation, +) + +ATIF_SCHEMA_VERSION = "ATIF-v1.7" +TRAJECTORY_AGENT_NAME = "efficientai-conversation" +TRAJECTORY_AGENT_VERSION = "1.0.0" + + +@dataclass(frozen=True) +class TrialRefs: + simulation_run_id: UUID + scenario_id: UUID + persona_id: UUID + trial_name: str + + +def llm_override(provider: Optional[str], model: Optional[str]) -> Optional[ResolvedSimulationLlm]: + if not (provider or "").strip() or not (model or "").strip(): + return None + return ResolvedSimulationLlm( + provider=ModelProvider(provider.strip().lower()), + model=model.strip(), + llm_config=None, + credential_id=None, + source="simulation_run_override", + ) + + +def _commit(db: Session, result: EvaluatorResult) -> None: + sync_evaluator_result(db, result) + db.commit() + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _atif_tool_fields(step_id: int, tool_calls: list[dict[str, Any]]) -> dict[str, Any]: + """ATIF ``tool_calls`` + ``observation`` for one agent step from our call records.""" + atif_calls: list[dict[str, Any]] = [] + results: list[dict[str, Any]] = [] + for index, call in enumerate(tool_calls, start=1): + call_id = str(call.get("call_id") or f"call_{step_id}_{index}") + atif_calls.append( + { + "tool_call_id": call_id, + "function_name": str(call.get("tool") or "unknown"), + "arguments": call.get("arguments") or {}, + "extra": { + "provider": call.get("provider"), + "status": call.get("status"), + "latency_ms": call.get("latency_ms"), + }, + } + ) + results.append( + { + "source_call_id": call_id, + "content": call.get("result_preview") or "", + "extra": {"status": call.get("status")}, + } + ) + return {"tool_calls": atif_calls, "observation": {"results": results}} + + +class TrajectoryWriter: + """Writes an ATIF trajectory (user = test agent, agent = production agent) after every turn.""" + + def __init__(self, path: Path, *, model_name: Optional[str], extra: dict[str, Any]): + self.path = path + self.model_name = model_name + self.extra = extra + self.steps: list[dict[str, Any]] = [] + + def add( + self, + source: str, + message: str, + *, + latency_seconds: Optional[float] = None, + tool_calls: Optional[list[dict[str, Any]]] = None, + ) -> None: + step_id = len(self.steps) + 1 + step: dict[str, Any] = { + "step_id": step_id, + "timestamp": _now_iso(), + "source": source, + "message": message, + } + if source == "agent" and self.model_name: + step["model_name"] = self.model_name + if source == "agent" and tool_calls: + step.update(_atif_tool_fields(step_id, tool_calls)) + if latency_seconds is not None: + step["extra"] = {"latency_ms": round(latency_seconds * 1000)} + self.steps.append(step) + self.flush() + + def flush(self) -> None: + payload = { + "schema_version": ATIF_SCHEMA_VERSION, + "agent": { + "name": TRAJECTORY_AGENT_NAME, + "version": TRAJECTORY_AGENT_VERSION, + "model_name": self.model_name, + "extra": self.extra, + }, + "steps": self.steps, + "final_metrics": {"total_steps": len(self.steps)}, + } + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _load_trial_entities(db: Session, refs: TrialRefs): + run = db.query(SimulationRun).filter(SimulationRun.id == refs.simulation_run_id).first() + if not run: + raise ValueError(f"Simulation run {refs.simulation_run_id} not found") + agent = db.query(Agent).filter(Agent.id == run.agent_id).first() + persona = db.query(Persona).filter(Persona.id == refs.persona_id).first() + scenario = db.query(Scenario).filter(Scenario.id == refs.scenario_id).first() + if not agent or not persona or not scenario: + raise ValueError("Agent, persona or scenario for this trial no longer exists") + return run, agent, persona, scenario + + +def upsert_trial_result( + db: Session, + *, + run: SimulationRun, + persona: Persona, + scenario: Scenario, + trial_name: str, +) -> EvaluatorResult: + """Create the trial's result row, or reset it when Harbor retries the same trial.""" + result = ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.simulation_run_id == run.id, + EvaluatorResult.harbor_trial_name == trial_name, + ) + .first() + ) + if result is None: + result = EvaluatorResult( + result_id=generate_unique_result_id(db), + organization_id=run.organization_id, + workspace_id=run.workspace_id, + agent_id=run.agent_id, + persona_id=persona.id, + scenario_id=scenario.id, + simulation_run_id=run.id, + harbor_trial_name=trial_name, + name=scenario.name, + created_by=run.created_by, + ) + db.add(result) + result.status = EvaluatorResultStatus.CALL_IN_PROGRESS.value + result.provider_platform = "internal" + result.error_message = None + result.transcription = None + result.speaker_segments = None + result.metric_scores = None + result.call_data = None + result.reward = None + result.passed = None + _commit(db, result) + return result + + +def run_trial_conversation( + db: Session, + *, + refs: TrialRefs, + trajectory_path: Path, + production_override: Optional[ResolvedSimulationLlm], + test_override: Optional[ResolvedSimulationLlm], +) -> dict[str, Any]: + """Run one test-agent ↔ production-agent conversation for a Harbor trial.""" + run, agent, persona, scenario = _load_trial_entities(db, refs) + result = upsert_trial_result( + db, run=run, persona=persona, scenario=scenario, trial_name=refs.trial_name + ) + ids = { + "agent_id": str(agent.id), + "persona_id": str(persona.id), + "scenario_id": str(scenario.id), + "evaluator_result_id": str(result.id), + "simulation_run_id": str(run.id), + } + caller_ctx = usage_context_for_test_agent_simulation( + organization_id=run.organization_id, + workspace_id=run.workspace_id, + agent_id=agent.id, + persona_id=persona.id, + scenario_id=scenario.id, + evaluator_result_id=result.id, + provider_platform="internal", + ) + agent_ctx = LLMUsageContext( + organization_id=run.organization_id, + workspace_id=run.workspace_id, + product_section=LLMUsageProductSection.EVALUATORS, + resource_id=run.id, + resource_type="simulation_run", + extra={ + **ids, + "synthetic_testing": "pre_prod", + "simulation_leg": "production_agent", + "provider_platform": "internal", + }, + ) + production_model = ( + f"{production_override.provider.value}/{production_override.model}" + if production_override + else (f"{agent.main_llm_provider}/{agent.main_llm_model}" if agent.main_llm_model else None) + ) + writer = TrajectoryWriter(trajectory_path, model_name=production_model, extra=ids) + if not uses_live_production_leg(agent): + writer.add("system", build_agent_system_prompt(agent)) + + conversation = TextConversation() + trial_call_data = { + "simulation_run_id": str(run.id), + "harbor_trial_name": refs.trial_name, + "llms": describe_simulation_llms( + db, agent=agent, production_override=production_override, test_override=test_override + ), + } + + def _on_turn(entry: dict[str, Any], latency_seconds: float) -> None: + source = "agent" if entry["speaker"] == "Speaker 2" else "user" + writer.add( + source, + entry["text"], + latency_seconds=latency_seconds, + tool_calls=entry.get("tool_calls"), + ) + write_conversation_to_result( + result, conversation, agent=agent, extra_call_data=trial_call_data + ) + _commit(db, result) + + try: + run_text_conversation( + agent=agent, + persona=persona, + scenario=scenario, + organization_id=run.organization_id, + db=db, + conversation=conversation, + caller_ctx=caller_ctx, + agent_ctx=agent_ctx, + evaluator_result=result, + test_llm_override=test_override, + production_llm_override=production_override, + on_turn=_on_turn, + ) + except Exception as exc: + err_text = str(exc) + db.rollback() + result = ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.simulation_run_id == run.id, + EvaluatorResult.harbor_trial_name == refs.trial_name, + ) + .one() + ) + if conversation.transcript: + write_conversation_to_result( + result, conversation, agent=agent, extra_call_data=trial_call_data + ) + else: + merged = dict(result.call_data or {}) + merged.update(trial_call_data) + result.call_data = merged + call_data = dict(result.call_data or {}) + call_data["trial_error"] = err_text + result.call_data = call_data + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = err_text + _commit(db, result) + raise + + write_conversation_to_result(result, conversation, agent=agent, extra_call_data=trial_call_data) + result.call_data["partial"] = False + result.status = EvaluatorResultStatus.EVALUATING.value + _commit(db, result) + return {**ids, "exchanges": conversation.exchanges, "turns": len(conversation.transcript)} + + +def score_trial( + db: Session, + *, + simulation_run_id: UUID, + trial_name: str, + pass_threshold: float, +) -> tuple[dict[str, float], dict[str, Any]]: + """Score a trial's transcript with platform metrics; returns Harbor rewards + details.""" + from app.workers.tasks.process_evaluator_result import evaluate_transcript_metrics + + result = ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.simulation_run_id == simulation_run_id, + EvaluatorResult.harbor_trial_name == trial_name, + ) + .first() + ) + if result is None or not (result.transcription or "").strip(): + raise ValueError(f"No transcript recorded for trial {trial_name}") + run = db.query(SimulationRun).filter(SimulationRun.id == simulation_run_id).first() + agent = db.query(Agent).filter(Agent.id == result.agent_id).first() + persona = db.query(Persona).filter(Persona.id == result.persona_id).first() + scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() + + metric_scores, metrics = evaluate_transcript_metrics( + db, + result=result, + agent=agent, + persona=persona, + scenario=scenario, + metric_ids=run.metric_ids if run else None, + ) + criteria = (run.metric_success_criteria if run else None) or [] + if criteria: + from app.models.database import Metric + from app.services.metrics.evaluation_scope import ( + load_agent_link_map, + load_scenario_link_map, + metric_applies_to_agent, + metric_applies_to_scenario, + ) + + parent_ids = {UUID(str(c["metric_id"])) for c in criteria if c.get("metric_id")} + parents = ( + db.query(Metric) + .filter( + Metric.id.in_(parent_ids), + Metric.organization_id == result.organization_id, + ) + .all() + ) + agent_link_map = load_agent_link_map(db, parent_ids) + scenario_link_map = load_scenario_link_map(db, parent_ids) + parents_by_id = {p.id: p for p in parents} + filtered: list[dict] = [] + for c in criteria: + raw_id = c.get("metric_id") + if not raw_id: + continue + parent = parents_by_id.get(UUID(str(raw_id))) + if parent is None: + continue + if not metric_applies_to_agent( + parent, + result.agent_id, + link_map=agent_link_map, + ): + continue + if not metric_applies_to_scenario( + parent, + result.scenario_id, + link_map=scenario_link_map, + ): + continue + filtered.append(c) + criteria = filtered + if criteria: + from app.services.harbor.success_criteria import ( + _load_metric_context, + apply_success_criteria, + ) + + metric_ids_for_ctx = [str(c.get("metric_id")) for c in criteria if c.get("metric_id")] + metrics_by_id, children_by_parent = _load_metric_context( + db, agent.organization_id, metric_ids_for_ctx + ) + trial_passed, aggregate, metric_scores = apply_success_criteria( + metric_scores, + criteria, + metrics_by_id, + children_by_parent, + ) + _, rewards, metric_ids_by_key = compute_rewards(metric_scores, metrics) + result.passed = trial_passed + result.error_message = ( + None if aggregate is not None else "No metrics apply to this trial" + ) + else: + aggregate, rewards, metric_ids_by_key = compute_rewards(metric_scores, metrics) + result.passed = aggregate is not None and aggregate >= pass_threshold + result.error_message = ( + None if aggregate is not None else "No metrics apply to this trial" + ) + + tool_report = check_tool_calls( + (result.call_data or {}).get("tool_calls"), + getattr(scenario, "expected_tool_calls", None), + ) + taken = {AGGREGATE_REWARD_KEY, *rewards} + for name, value in tool_report.rewards().items(): + rewards[metric_reward_key(name, taken)] = value + if tool_report.has_expectations: + # Expected tool calls gate the pass; with no scorable metrics they are the score. + if aggregate is None: + aggregate = tool_report.expectation_score + result.error_message = None + result.passed = tool_report.all_expectations_met + else: + result.passed = bool(result.passed) and tool_report.all_expectations_met + call_data = dict(result.call_data or {}) + call_data["tool_evaluation"] = tool_report.as_dict() + result.call_data = call_data + + result.metric_scores = metric_scores + flag_modified(result, "metric_scores") + flag_modified(result, "call_data") + result.reward = aggregate + result.status = EvaluatorResultStatus.COMPLETED.value + _commit(db, result) + logger.info( + "[Harbor] Trial {} scored reward={} passed={} ({} metrics)", + trial_name, + aggregate, + result.passed, + len(rewards), + ) + details = { + "evaluator_result_id": str(result.id), + "pass_threshold": pass_threshold, + "passed": result.passed, + "metric_ids": metric_ids_by_key, + "metric_scores": metric_scores, + "tool_evaluation": tool_report.as_dict(), + } + return {AGGREGATE_REWARD_KEY: aggregate if aggregate is not None else 0.0, **rewards}, details diff --git a/app/services/import_eval_llm_output_budgets.py b/app/services/import_eval_llm_output_budgets.py new file mode 100644 index 00000000..e5ce8841 --- /dev/null +++ b/app/services/import_eval_llm_output_budgets.py @@ -0,0 +1,9 @@ +"""Shared LLM output token budgets for import-eval analysis jobs.""" + +# Two-step pattern: first call, then one retry at double if truncated or JSON invalid. +IMPORT_EVAL_LLM_OUTPUT_FIRST = 32_768 +IMPORT_EVAL_LLM_OUTPUT_RETRY = 65_536 + +# Diariser for Gemini 3.x and GLM: smaller pair (see llm_diarisation). +DIARISER_COMPACT_OUTPUT_FIRST = 8_192 +DIARISER_COMPACT_OUTPUT_RETRY = 16_384 diff --git a/app/services/judge_alignment/gepa_bridge.py b/app/services/judge_alignment/gepa_bridge.py index 6a12506b..107a9136 100644 --- a/app/services/judge_alignment/gepa_bridge.py +++ b/app/services/judge_alignment/gepa_bridge.py @@ -244,6 +244,8 @@ def _execute_judge_gepa_with_context( evaluator: Evaluator, db: Session, ) -> Dict[str, Any]: + from app.services.optimization.gepa_service import _ensure_gepa + dev_ids: List[str] = cfg.get("dev_sample_ids", []) if not dev_ids: raise RuntimeError("Optimisation run has no dev_sample_ids in config") diff --git a/app/services/knowledge_bases/agent_links.py b/app/services/knowledge_bases/agent_links.py new file mode 100644 index 00000000..c1e1ed67 --- /dev/null +++ b/app/services/knowledge_bases/agent_links.py @@ -0,0 +1,66 @@ +"""Sync agent ↔ knowledge base attachments (separately for the production and test sides).""" + +from __future__ import annotations + +from typing import Iterable, List +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Agent, AgentKnowledgeBase, KnowledgeBase + +KB_ROLE_PRODUCTION = "production" +KB_ROLE_TEST = "test" +KB_ROLES = (KB_ROLE_PRODUCTION, KB_ROLE_TEST) + + +def get_agent_knowledge_base_ids( + db: Session, *, agent_id: UUID, role: str = KB_ROLE_PRODUCTION +) -> List[UUID]: + rows = ( + db.query(AgentKnowledgeBase.knowledge_base_id) + .filter(AgentKnowledgeBase.agent_id == agent_id, AgentKnowledgeBase.role == role) + .all() + ) + return [row[0] for row in rows] + + +def replace_agent_knowledge_bases( + db: Session, + *, + agent: Agent, + organization_id: UUID, + workspace_id: UUID, + knowledge_base_ids: Iterable[UUID], + role: str = KB_ROLE_PRODUCTION, +) -> None: + if role not in KB_ROLES: + raise ValueError(f"Unknown knowledge base role: {role}") + desired = list(dict.fromkeys(knowledge_base_ids)) + if desired: + valid = ( + db.query(KnowledgeBase.id) + .filter( + KnowledgeBase.id.in_(desired), + KnowledgeBase.organization_id == organization_id, + KnowledgeBase.workspace_id == workspace_id, + ) + .all() + ) + valid_ids = {row[0] for row in valid} + missing = [str(kb_id) for kb_id in desired if kb_id not in valid_ids] + if missing: + raise ValueError(f"Knowledge base(s) not found in workspace: {', '.join(missing)}") + db.query(AgentKnowledgeBase).filter( + AgentKnowledgeBase.agent_id == agent.id, AgentKnowledgeBase.role == role + ).delete() + for kb_id in desired: + db.add( + AgentKnowledgeBase( + agent_id=agent.id, + knowledge_base_id=kb_id, + role=role, + organization_id=organization_id, + workspace_id=workspace_id, + ) + ) diff --git a/app/services/knowledge_bases/chat_tools.py b/app/services/knowledge_bases/chat_tools.py new file mode 100644 index 00000000..90641403 --- /dev/null +++ b/app/services/knowledge_bases/chat_tools.py @@ -0,0 +1,74 @@ +"""Tool-aware chat completion for internal production simulation.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.enums import ModelProvider +from app.services.knowledge_bases.agent_links import KB_ROLE_PRODUCTION, KB_ROLE_TEST +from app.services.knowledge_bases.prompt import append_knowledge_base_instructions +from app.services.tools.loop import run_tool_loop +from app.services.tools.providers.knowledge_base import KnowledgeBaseToolProvider +from app.services.tools.toolset import Toolset + +MAX_TOOL_ROUNDS = 3 + + +def with_knowledge_base_instructions( + messages: List[Dict[str, Any]], *, simulated_caller: bool = False +) -> List[Dict[str, Any]]: + working = list(messages) + if working and working[0].get("role") == "system": + working[0] = { + "role": "system", + "content": append_knowledge_base_instructions( + working[0].get("content") or "", simulated_caller=simulated_caller + ), + } + return working + + +def generate_production_reply_with_knowledge_tools( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: UUID, + messages: List[Dict[str, str]], + llm_provider: ModelProvider, + llm_model: str, + llm_config: Optional[Dict[str, Any]] = None, + credential_id: Optional[UUID] = None, + task_defaults: Optional[Dict[str, Any]] = None, + simulated_caller: bool = False, +) -> tuple[str, List[Dict[str, Any]]]: + """Run LiteLLM with search_knowledge_base tool until assistant text or round cap. + + ``simulated_caller`` searches the test agent's knowledge bases instead of the + production agent's. + """ + provider = KnowledgeBaseToolProvider( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + llm_model=llm_model, + role=KB_ROLE_TEST if simulated_caller else KB_ROLE_PRODUCTION, + ) + text, _records, _tail = run_tool_loop( + db, + toolset=Toolset([provider]), + messages=with_knowledge_base_instructions(messages, simulated_caller=simulated_caller), + organization_id=organization_id, + llm_provider=llm_provider, + llm_model=llm_model, + llm_config=llm_config, + credential_id=credential_id, + task_defaults=task_defaults, + max_rounds=MAX_TOOL_ROUNDS, + side="test" if simulated_caller else "production", + ) + return text, provider.legacy_log diff --git a/app/services/knowledge_bases/chunking.py b/app/services/knowledge_bases/chunking.py new file mode 100644 index 00000000..3abac636 --- /dev/null +++ b/app/services/knowledge_bases/chunking.py @@ -0,0 +1,44 @@ +"""Text chunking for knowledge base ingestion.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterator, List, Optional + + +@dataclass(frozen=True) +class TextChunk: + text: str + chunk_index: int + page_number: Optional[int] = None + + +def chunk_text_pages( + pages: List[tuple[Optional[int], str]], + *, + max_chars: int = 3200, + overlap_chars: int = 400, +) -> List[TextChunk]: + """Split page text into overlapping chunks (~800 tokens at 4 chars/token).""" + chunks: List[TextChunk] = [] + idx = 0 + for page_number, page_text in pages: + normalized = (page_text or "").strip() + if not normalized: + continue + start = 0 + length = len(normalized) + while start < length: + end = min(start + max_chars, length) + piece = normalized[start:end].strip() + if piece: + chunks.append(TextChunk(text=piece, chunk_index=idx, page_number=page_number)) + idx += 1 + if end >= length: + break + start = max(0, end - overlap_chars) + return chunks + + +def chunk_plain_text(text: str, **kwargs) -> List[TextChunk]: + return chunk_text_pages([(None, text)], **kwargs) diff --git a/app/services/knowledge_bases/document_view.py b/app/services/knowledge_bases/document_view.py new file mode 100644 index 00000000..06adccdf --- /dev/null +++ b/app/services/knowledge_bases/document_view.py @@ -0,0 +1,69 @@ +"""Build API payloads for in-app knowledge document viewing.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from app.services.knowledge_bases.extract import extract_text_from_bytes +from app.services.storage.blob_storage_service import blob_storage_service + + +def _extension(filename: Optional[str]) -> str: + if not filename or "." not in filename: + return "txt" + return filename.rsplit(".", 1)[-1].lower() + + +def build_document_view_payload( + *, + original_filename: Optional[str], + content_type: Optional[str], + blob_key: str, +) -> Dict[str, Any]: + ext = _extension(original_filename) + file_bytes = blob_storage_service.download_file_by_key(blob_key) + ct = (content_type or "").lower() + + if ext in ("txt", "md") or ct.startswith("text/"): + text = file_bytes.decode("utf-8", errors="replace") + return { + "view_mode": "text", + "original_filename": original_filename, + "content_type": content_type or "text/plain", + "text_content": text, + "pages": None, + } + + if ext == "pdf" or ct == "application/pdf": + pages_raw = extract_text_from_bytes(file_bytes, extension="pdf") + pages: List[Dict[str, Any]] = [] + for page_num, page_text in pages_raw: + pages.append({"page": page_num, "text": page_text}) + return { + "view_mode": "pdf", + "original_filename": original_filename, + "content_type": content_type or "application/pdf", + "text_content": None, + "pages": pages, + "page_count": len(pages), + } + + if ext == "docx": + pages_raw = extract_text_from_bytes(file_bytes, extension="docx") + combined = "\n\n".join(t for _, t in pages_raw if t) + return { + "view_mode": "docx", + "original_filename": original_filename, + "content_type": content_type + or "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "text_content": combined, + "pages": None, + } + + return { + "view_mode": "binary", + "original_filename": original_filename, + "content_type": content_type or "application/octet-stream", + "text_content": None, + "pages": None, + } diff --git a/app/services/knowledge_bases/embedding_config.py b/app/services/knowledge_bases/embedding_config.py new file mode 100644 index 00000000..029180a6 --- /dev/null +++ b/app/services/knowledge_bases/embedding_config.py @@ -0,0 +1,89 @@ +"""Resolve and validate knowledge-base embedding settings.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import KnowledgeBase +from app.models.enums import ModelProvider +from app.services.credentials.resolver import resolve_ai_provider +from app.services.knowledge_bases.embeddings_catalog import ( + DEFAULT_EMBEDDING_MODEL, + DEFAULT_EMBEDDING_PROVIDER, + MAX_EMBEDDING_MODEL_LENGTH, + normalize_embedding_model, + normalize_embedding_provider, +) + + +@dataclass(frozen=True) +class EmbeddingProfile: + provider: str + model: str + credential_id: Optional[UUID] + + def cache_key(self) -> tuple[str, str, Optional[str]]: + cred = str(self.credential_id) if self.credential_id else None + return (self.provider, self.model, cred) + + +def profile_from_kb(kb: KnowledgeBase | None) -> EmbeddingProfile: + if kb is None: + return EmbeddingProfile( + provider=DEFAULT_EMBEDDING_PROVIDER, + model=DEFAULT_EMBEDDING_MODEL, + credential_id=None, + ) + return EmbeddingProfile( + provider=normalize_embedding_provider(getattr(kb, "embedding_provider", None)), + model=normalize_embedding_model(getattr(kb, "embedding_model", None)), + credential_id=getattr(kb, "embedding_credential_id", None), + ) + + +def validate_embedding_credential( + db: Session, + *, + organization_id: UUID, + provider: ModelProvider, + credential_id: Optional[UUID], +) -> Optional[UUID]: + if credential_id is None: + return None + row = resolve_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) + if not row: + raise ValueError( + "embedding_credential_id does not match an active credential for this provider" + ) + return credential_id + + +def parse_embedding_fields( + db: Session, + *, + organization_id: UUID, + embedding_provider: str | None, + embedding_model: str | None, + embedding_credential_id: UUID | None, +) -> EmbeddingProfile: + provider = normalize_embedding_provider(embedding_provider) + model = normalize_embedding_model(embedding_model) + provider_enum = ModelProvider(provider) + cred = validate_embedding_credential( + db, + organization_id=organization_id, + provider=provider_enum, + credential_id=embedding_credential_id, + ) + return EmbeddingProfile(provider=provider, model=model, credential_id=cred) + + +def embedding_profile_changed(kb: KnowledgeBase, profile: EmbeddingProfile) -> bool: + current = profile_from_kb(kb) + return current.cache_key() != profile.cache_key() diff --git a/app/services/knowledge_bases/embeddings.py b/app/services/knowledge_bases/embeddings.py new file mode 100644 index 00000000..f108fc5d --- /dev/null +++ b/app/services/knowledge_bases/embeddings.py @@ -0,0 +1,116 @@ +"""Embedding helpers for knowledge base chunks (multi-provider via LiteLLM).""" + +from __future__ import annotations + +from typing import List, Optional +from uuid import UUID + +import litellm +from sqlalchemy.orm import Session + +from app.models.enums import ModelProvider +from app.services.ai.llm_gateway import ( + apply_llm_gateway, + resolve_effective_routing, + resolve_litellm_api_key, + resolve_litellm_model, +) +from app.services.ai.llm_service import canonical_litellm_model_id, llm_service +from app.services.knowledge_bases.embeddings_catalog import ( + DEFAULT_EMBEDDING_MODEL, + DEFAULT_EMBEDDING_PROVIDER, + EMBEDDING_DIMENSIONS, + normalize_embedding_model, + normalize_embedding_provider, +) + + +def embed_texts( + db: Session, + *, + organization_id: UUID, + texts: List[str], + provider: str = DEFAULT_EMBEDDING_PROVIDER, + model: str = DEFAULT_EMBEDDING_MODEL, + credential_id: Optional[UUID] = None, + dimensions: int = EMBEDDING_DIMENSIONS, +) -> List[List[float]]: + if not texts: + return [] + provider_value = normalize_embedding_provider(provider) + model = normalize_embedding_model(model) + llm_provider = ModelProvider(provider_value) + + ai_provider, credential_ctx = llm_service._resolve_credential_context( # noqa: SLF001 + llm_provider, + db, + organization_id, + credential_id=credential_id, + ) + if ai_provider: + api_key = resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) + else: + api_key = llm_service._resolve_api_key( # noqa: SLF001 + llm_provider, db, organization_id, credential_id=credential_id + ) + + _, effective_routing = resolve_effective_routing( + organization_id, db, credential_ctx + ) + workload_model_str = llm_service._litellm_model_name(llm_provider, model) # noqa: SLF001 + model_str = canonical_litellm_model_id( + resolve_litellm_model( + workload_model_str=workload_model_str, + gateway_active=effective_routing != "direct", + credential=credential_ctx, + ) + ) + + kwargs: dict = {"model": model_str, "input": texts} + if api_key: + kwargs["api_key"] = api_key + if ( + llm_provider == ModelProvider.OPENAI + and model.startswith("text-embedding-3-") + ): + kwargs["dimensions"] = dimensions + + kwargs = apply_llm_gateway( + kwargs, + organization_id=organization_id, + db=db, + model=model_str, + credential=credential_ctx, + ) + + try: + response = litellm.embedding(**kwargs) + except Exception as exc: + raise RuntimeError( + f"Embedding request failed for {provider_value}/{model}: {exc}" + ) from exc + + data = getattr(response, "data", None) or [] + ordered = sorted(data, key=lambda row: getattr(row, "index", 0)) + vectors: List[List[float]] = [] + for row in ordered: + emb = getattr(row, "embedding", None) + if emb is None and isinstance(row, dict): + emb = row.get("embedding") + if not emb: + raise RuntimeError("Embedding API returned empty vector") + vec = list(emb) + if len(vec) != dimensions: + raise RuntimeError( + f"Embedding model returned {len(vec)} dimensions; " + f"knowledge bases require {dimensions}. " + "Choose a model that outputs 1536 vectors (e.g. OpenAI text-embedding-3-* " + "with dimensions=1536) or re-index after changing pgvector size." + ) + vectors.append(vec) + return vectors diff --git a/app/services/knowledge_bases/embeddings_catalog.py b/app/services/knowledge_bases/embeddings_catalog.py new file mode 100644 index 00000000..6f6043b7 --- /dev/null +++ b/app/services/knowledge_bases/embeddings_catalog.py @@ -0,0 +1,89 @@ +"""Embedding defaults and provider hints for knowledge bases (pgvector is 1536-dim).""" + +from __future__ import annotations + +from typing import Dict, List, Tuple + +from app.models.enums import ModelProvider + +DEFAULT_EMBEDDING_PROVIDER = "openai" +DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small" +EMBEDDING_DIMENSIONS = 1536 +MAX_EMBEDDING_MODEL_LENGTH = 512 + +# Voice / TTS platforms — not offered in the KB embedding picker. +_EMBEDDING_EXCLUDED = { + ModelProvider.DEEPGRAM, + ModelProvider.CARTESIA, + ModelProvider.ELEVENLABS, + ModelProvider.MURF, + ModelProvider.VOICEMAKER, + ModelProvider.SMALLEST, + ModelProvider.CUSTOM, +} + +EMBEDDING_CAPABLE_PROVIDERS: List[ModelProvider] = [ + p for p in ModelProvider if p not in _EMBEDDING_EXCLUDED +] + +PROVIDER_LABELS: Dict[str, str] = { + ModelProvider.OPENAI.value: "OpenAI", + ModelProvider.OPENROUTER.value: "OpenRouter", + ModelProvider.ANTHROPIC.value: "Anthropic", + ModelProvider.GOOGLE.value: "Google", + ModelProvider.XAI.value: "xAI", + ModelProvider.FIREWORKS.value: "Fireworks AI", + ModelProvider.COHERE.value: "Cohere", + ModelProvider.MISTRAL.value: "Mistral", + ModelProvider.META.value: "Meta", + ModelProvider.TOGETHER.value: "Together", + ModelProvider.TYPESAFE.value: "TypeSafe", + ModelProvider.PERPLEXITY.value: "Perplexity", + ModelProvider.AZURE.value: "Azure", + ModelProvider.AWS.value: "AWS", + ModelProvider.SARVAM.value: "Sarvam", +} + +# Curated models known to output EMBEDDING_DIMENSIONS (1536) vectors for pgvector. +SUGGESTED_EMBEDDING_MODELS: Dict[str, List[Tuple[str, str]]] = { + "openai": [ + ("text-embedding-3-small", "text-embedding-3-small"), + ("text-embedding-3-large", "text-embedding-3-large (1536 dims)"), + ("text-embedding-ada-002", "text-embedding-ada-002"), + ], + "openrouter": [ + ("openai/text-embedding-3-small", "openai/text-embedding-3-small"), + ("openai/text-embedding-3-large", "openai/text-embedding-3-large"), + ], + "azure": [ + ("text-embedding-3-small", "Deployment name (text-embedding-3-small)"), + ], +} + +# Backward-compatible flat list for legacy clients. +EMBEDDING_MODEL_CHOICES: List[Tuple[str, str]] = SUGGESTED_EMBEDDING_MODELS["openai"] + + +def normalize_embedding_provider(provider: str | None) -> str: + raw = (provider or DEFAULT_EMBEDDING_PROVIDER).strip().lower() + try: + ModelProvider(raw) + except ValueError as exc: + allowed = ", ".join(sorted(p.value for p in EMBEDDING_CAPABLE_PROVIDERS)) + raise ValueError( + f"Unsupported embedding provider '{raw}'. Choose one of: {allowed}" + ) from exc + if ModelProvider(raw) not in EMBEDDING_CAPABLE_PROVIDERS: + raise ValueError(f"Provider '{raw}' is not available for knowledge base embeddings") + return raw + + +def normalize_embedding_model(model: str | None) -> str: + raw = (model or DEFAULT_EMBEDDING_MODEL).strip() + if not raw: + return DEFAULT_EMBEDDING_MODEL + if len(raw) > MAX_EMBEDDING_MODEL_LENGTH: + raise ValueError( + f"embedding_model must be at most {MAX_EMBEDDING_MODEL_LENGTH} characters" + ) + return raw diff --git a/app/services/knowledge_bases/extract.py b/app/services/knowledge_bases/extract.py new file mode 100644 index 00000000..eccfdbe3 --- /dev/null +++ b/app/services/knowledge_bases/extract.py @@ -0,0 +1,28 @@ +"""Extract plain text from knowledge base source files.""" + +from __future__ import annotations + +import io +from typing import List, Optional, Tuple + + +def extract_text_from_bytes(file_bytes: bytes, *, extension: str) -> List[Tuple[Optional[int], str]]: + ext = extension.lower().lstrip(".") + if ext in ("txt", "md"): + text = file_bytes.decode("utf-8", errors="replace") + return [(None, text)] + if ext == "pdf": + from pypdf import PdfReader + + reader = PdfReader(io.BytesIO(file_bytes)) + pages: List[Tuple[Optional[int], str]] = [] + for i, page in enumerate(reader.pages): + pages.append((i + 1, page.extract_text() or "")) + return pages + if ext == "docx": + import docx + + document = docx.Document(io.BytesIO(file_bytes)) + paragraphs = "\n".join(p.text for p in document.paragraphs if p.text) + return [(None, paragraphs)] + raise ValueError(f"Unsupported extension: {extension}") diff --git a/app/services/knowledge_bases/ingest.py b/app/services/knowledge_bases/ingest.py new file mode 100644 index 00000000..71b6c3db --- /dev/null +++ b/app/services/knowledge_bases/ingest.py @@ -0,0 +1,147 @@ +"""Ingest a knowledge document: extract, chunk, embed, persist.""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.models.database import KnowledgeBase, KnowledgeDocument +from app.services.knowledge_bases.chunking import chunk_text_pages +from app.services.knowledge_bases.embeddings import embed_texts +from app.services.knowledge_bases.embedding_config import profile_from_kb +from app.services.knowledge_bases.extract import extract_text_from_bytes +from app.services.storage.blob_storage_service import blob_storage_service + +_INGEST_LOCK_NS = 0x4B42_444F # "KBDO" — serialize ingest per document across workers + + +def _signed_int32(value: int) -> int: + """pg_advisory_lock(int, int) requires signed 32-bit keys.""" + value &= 0xFFFFFFFF + if value > 0x7FFFFFFF: + return value - 0x100000000 + return value + + +def _ingest_advisory_keys(document_id: UUID) -> tuple[int, int]: + b = document_id.bytes + doc_key = int.from_bytes(b[0:4], "big") ^ int.from_bytes(b[4:8], "big") + return (_signed_int32(_INGEST_LOCK_NS), _signed_int32(doc_key)) + + +def ingest_knowledge_document(db: Session, *, document_id: UUID) -> None: + lock_k1, lock_k2 = _ingest_advisory_keys(document_id) + bind = db.get_bind() + with bind.connect() as lock_conn: + lock_conn.execute( + text("SELECT pg_advisory_lock(:k1, :k2)"), + {"k1": lock_k1, "k2": lock_k2}, + ) + lock_conn.commit() + work = Session(bind=lock_conn, expire_on_commit=False) + try: + _ingest_knowledge_document_locked(work, document_id=document_id) + work.commit() + finally: + work.close() + lock_conn.execute( + text("SELECT pg_advisory_unlock(:k1, :k2)"), + {"k1": lock_k1, "k2": lock_k2}, + ) + lock_conn.commit() + + +def _ingest_knowledge_document_locked(db: Session, *, document_id: UUID) -> None: + doc = ( + db.query(KnowledgeDocument) + .filter(KnowledgeDocument.id == document_id) + .with_for_update() + .first() + ) + if not doc: + return + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == doc.knowledge_base_id).first() + embed_profile = profile_from_kb(kb) + doc.status = "processing" + doc.error_text = None + db.commit() + + try: + file_bytes = blob_storage_service.download_file_by_key(doc.blob_key) + extension = (doc.original_filename or "file.txt").rsplit(".", 1)[-1].lower() + pages = extract_text_from_bytes(file_bytes, extension=extension) + chunks = chunk_text_pages(pages) + if not chunks: + raise ValueError("No text could be extracted from this document") + + db.execute( + text("DELETE FROM knowledge_chunks WHERE document_id = :document_id"), + {"document_id": str(doc.id)}, + ) + db.commit() + + batch_size = 32 + for start in range(0, len(chunks), batch_size): + batch = chunks[start : start + batch_size] + vectors = embed_texts( + db, + organization_id=doc.organization_id, + texts=[c.text for c in batch], + provider=embed_profile.provider, + model=embed_profile.model, + credential_id=embed_profile.credential_id, + ) + for chunk, vector in zip(batch, vectors): + db.execute( + text( + """ + INSERT INTO knowledge_chunks + (id, document_id, chunk_index, text, page_number, embedding, embedding_model) + VALUES + (:id, :document_id, :chunk_index, :text, :page_number, + CAST(:embedding AS vector), :embedding_model) + """ + ), + { + "id": str(uuid4()), + "document_id": str(doc.id), + "chunk_index": chunk.chunk_index, + "text": chunk.text, + "page_number": chunk.page_number, + "embedding": _vector_literal(vector), + "embedding_model": embed_profile.model, + }, + ) + db.refresh(kb) + current_profile = profile_from_kb(kb) + if ( + current_profile.provider != embed_profile.provider + or current_profile.model != embed_profile.model + or current_profile.credential_id != embed_profile.credential_id + ): + doc.status = "pending" + doc.error_text = None + db.commit() + from app.workers.tasks.ingest_knowledge_document import ( + ingest_knowledge_document_task, + ) + + ingest_knowledge_document_task.delay(str(doc.id)) + return + doc.status = "ready" + doc.error_text = None + db.commit() + except Exception as exc: + db.rollback() + doc = db.query(KnowledgeDocument).filter(KnowledgeDocument.id == document_id).first() + if doc: + doc.status = "failed" + doc.error_text = str(exc)[:2000] + db.commit() + return + + +def _vector_literal(values: list[float]) -> str: + return "[" + ",".join(f"{v:.8f}" for v in values) + "]" diff --git a/app/services/knowledge_bases/paths.py b/app/services/knowledge_bases/paths.py new file mode 100644 index 00000000..0abd211d --- /dev/null +++ b/app/services/knowledge_bases/paths.py @@ -0,0 +1,44 @@ +"""Blob key helpers for knowledge base uploads.""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + + +ALLOWED_KB_EXTENSIONS = frozenset({"pdf", "txt", "md", "docx"}) +MAX_KB_UPLOAD_BYTES = 25 * 1024 * 1024 + + +def knowledge_document_blob_key( + organization_id: Any, + workspace_id: Any, + knowledge_base_id: Any, + document_id: Any, + extension: str, +) -> str: + from app.services.storage.s3_service import s3_service + + ext = extension.lower().lstrip(".") + return ( + f"{s3_service.prefix}organizations/{organization_id}/workspaces/{workspace_id}/" + f"knowledge-bases/{knowledge_base_id}/documents/{document_id}.{ext}" + ) + + +def validate_kb_upload_filename(filename: str) -> tuple[str, str]: + name = (filename or "").strip() + if not name or "." not in name: + raise ValueError("Filename must include an extension") + ext = name.rsplit(".", 1)[-1].lower() + if ext not in ALLOWED_KB_EXTENSIONS: + raise ValueError( + f"Unsupported file type. Allowed: {', '.join(sorted(ALLOWED_KB_EXTENSIONS))}" + ) + content_types = { + "pdf": "application/pdf", + "txt": "text/plain", + "md": "text/markdown", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + } + return ext, content_types.get(ext, "application/octet-stream") diff --git a/app/services/knowledge_bases/prompt.py b/app/services/knowledge_bases/prompt.py new file mode 100644 index 00000000..61afa4f8 --- /dev/null +++ b/app/services/knowledge_bases/prompt.py @@ -0,0 +1,36 @@ +"""System prompt additions for knowledge-base-enabled agents.""" + +from __future__ import annotations + +KB_SYSTEM_APPENDIX = ( + "\n\nYou have access to organization knowledge bases. " + "On voice calls, relevant document excerpts are retrieved automatically from what the " + "other party says; you may also call search_knowledge_base when you need more detail. " + "Prefer retrieved or tool results over guessing, and cite the document name when answering." + "\n\nWhen tools are available, look up what you need in as few calls as possible, " + "then answer the customer in plain text. Do not repeat list or describe calls once " + "you already have the rows you need." +) + +# Playground / simulated caller: production speaks as "user"; caller must use injected passages. +CALLER_KB_VOICE_APPENDIX = ( + "\n\nWhen a system message contains 'Retrieved passages' and the production agent's latest " + "message asks a factual question those passages answer, answer from the passages first " + "(accurately and briefly). You may then continue your persona and scenario." +) + + +def append_knowledge_base_instructions( + system_prompt: str, + *, + simulated_caller: bool = False, +) -> str: + base = (system_prompt or "").strip() + appendix = KB_SYSTEM_APPENDIX + if simulated_caller: + appendix = appendix + CALLER_KB_VOICE_APPENDIX + if not base: + return appendix.strip() + if "search_knowledge_base" in base and "Retrieved passages" in base: + return base + return base + appendix diff --git a/app/services/knowledge_bases/retrieval.py b/app/services/knowledge_bases/retrieval.py new file mode 100644 index 00000000..9a332198 --- /dev/null +++ b/app/services/knowledge_bases/retrieval.py @@ -0,0 +1,202 @@ +"""Vector retrieval for agent-attached knowledge bases.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.models.database import AgentKnowledgeBase +from app.services.knowledge_bases.agent_links import KB_ROLE_PRODUCTION +from app.services.knowledge_bases.embedding_config import EmbeddingProfile +from app.services.knowledge_bases.embeddings import embed_texts +from app.services.knowledge_bases.embeddings_catalog import ( + DEFAULT_EMBEDDING_MODEL, + DEFAULT_EMBEDDING_PROVIDER, +) + +TOP_K = 5 +# Cosine-style scores from pgvector; below this, chunks are usually unrelated noise. +DEFAULT_MIN_SCORE = 0.32 + + +def agent_has_ready_knowledge( + db: Session, *, agent_id: UUID, role: str = KB_ROLE_PRODUCTION +) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM agent_knowledge_bases akb + JOIN knowledge_documents kd ON kd.knowledge_base_id = akb.knowledge_base_id + WHERE akb.agent_id = :agent_id AND akb.role = :role AND kd.status = 'ready' + LIMIT 1 + """ + ), + {"agent_id": str(agent_id), "role": role}, + ).first() + return row is not None + + +def list_agent_knowledge_base_ids( + db: Session, *, agent_id: UUID, role: str = KB_ROLE_PRODUCTION +) -> List[UUID]: + rows = ( + db.query(AgentKnowledgeBase.knowledge_base_id) + .filter(AgentKnowledgeBase.agent_id == agent_id, AgentKnowledgeBase.role == role) + .all() + ) + return [row[0] for row in rows] + + +def _agent_embedding_profiles( + db: Session, *, agent_id: UUID, role: str = KB_ROLE_PRODUCTION +) -> List[EmbeddingProfile]: + rows = db.execute( + text( + """ + SELECT DISTINCT + COALESCE(kb.embedding_provider, :default_provider), + COALESCE(kb.embedding_model, :default_model), + kb.embedding_credential_id + FROM agent_knowledge_bases akb + JOIN knowledge_bases kb ON kb.id = akb.knowledge_base_id + WHERE akb.agent_id = :agent_id AND akb.role = :role + """ + ), + { + "agent_id": str(agent_id), + "role": role, + "default_provider": DEFAULT_EMBEDDING_PROVIDER, + "default_model": DEFAULT_EMBEDDING_MODEL, + }, + ).fetchall() + if not rows: + return [ + EmbeddingProfile( + provider=DEFAULT_EMBEDDING_PROVIDER, + model=DEFAULT_EMBEDDING_MODEL, + credential_id=None, + ) + ] + profiles: List[EmbeddingProfile] = [] + seen: set[tuple[str, str, str | None]] = set() + for provider, model, credential_id in rows: + cred_str = str(credential_id) if credential_id else None + key = (str(provider), str(model), cred_str) + if key in seen: + continue + seen.add(key) + profiles.append( + EmbeddingProfile( + provider=str(provider), + model=str(model), + credential_id=credential_id, + ) + ) + return profiles + + +def search_knowledge_base( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: UUID, + query: str, + top_k: int = TOP_K, + min_score: Optional[float] = None, + role: str = KB_ROLE_PRODUCTION, +) -> List[Dict[str, Any]]: + query = (query or "").strip() + if not query: + return [] + threshold = DEFAULT_MIN_SCORE if min_score is None else min_score + merged: List[Dict[str, Any]] = [] + for profile in _agent_embedding_profiles(db, agent_id=agent_id, role=role): + vectors = embed_texts( + db, + organization_id=organization_id, + texts=[query], + provider=profile.provider, + model=profile.model, + credential_id=profile.credential_id, + ) + if not vectors: + continue + embedding_literal = _vector_literal(vectors[0]) + rows = db.execute( + text( + """ + SELECT + kc.text, + kd.original_filename, + kc.page_number, + kb.name AS knowledge_base_name, + 1 - (kc.embedding <=> CAST(:embedding AS vector)) AS score + FROM agent_knowledge_bases akb + JOIN knowledge_bases kb ON kb.id = akb.knowledge_base_id + JOIN knowledge_documents kd ON kd.knowledge_base_id = kb.id + JOIN knowledge_chunks kc ON kc.document_id = kd.id + WHERE akb.agent_id = :agent_id + AND akb.role = :role + AND akb.organization_id = :organization_id + AND akb.workspace_id = :workspace_id + AND kd.status = 'ready' + AND kc.embedding IS NOT NULL + AND COALESCE(kb.embedding_provider, :default_provider) = :embedding_provider + AND COALESCE(kb.embedding_model, :default_model) = :embedding_model + AND kb.embedding_credential_id IS NOT DISTINCT FROM CAST(:credential_id AS uuid) + ORDER BY kc.embedding <=> CAST(:embedding AS vector) + LIMIT :limit + """ + ), + { + "agent_id": str(agent_id), + "role": role, + "organization_id": str(organization_id), + "workspace_id": str(workspace_id), + "embedding": embedding_literal, + "embedding_provider": profile.provider, + "embedding_model": profile.model, + "credential_id": str(profile.credential_id) + if profile.credential_id + else None, + "default_provider": DEFAULT_EMBEDDING_PROVIDER, + "default_model": DEFAULT_EMBEDDING_MODEL, + "limit": top_k, + }, + ).mappings().all() + for row in rows: + score = float(row["score"]) if row["score"] is not None else None + if score is not None and score < threshold: + continue + merged.append( + { + "text": row["text"], + "document_name": row["original_filename"] or "document", + "page": row["page_number"], + "knowledge_base_name": row["knowledge_base_name"], + "score": score, + } + ) + merged.sort(key=lambda hit: hit.get("score") or 0.0, reverse=True) + return merged[:top_k] + + +def format_search_results_for_tool(results: List[Dict[str, Any]]) -> str: + if not results: + return "No matching passages found in the attached knowledge bases." + parts: List[str] = [] + for i, hit in enumerate(results, start=1): + doc = hit.get("document_name") or "document" + page = hit.get("page") + page_suffix = f", page {page}" if page is not None else "" + parts.append(f"[{i}] {doc}{page_suffix}:\n{hit.get('text', '').strip()}") + return "\n\n".join(parts) + + +def _vector_literal(values: List[float]) -> str: + return "[" + ",".join(f"{v:.8f}" for v in values) + "]" diff --git a/app/services/knowledge_bases/tool_call_logging.py b/app/services/knowledge_bases/tool_call_logging.py new file mode 100644 index 00000000..fa815dc6 --- /dev/null +++ b/app/services/knowledge_bases/tool_call_logging.py @@ -0,0 +1,91 @@ +"""Structured logging for search_knowledge_base invocations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable, Dict, List, Optional + +from loguru import logger + +NotifyFn = Callable[[Dict[str, Any]], Awaitable[None]] + + +def record_knowledge_base_tool_call( + *, + channel: str, + query: str, + hit_count: int, + agent_id: str, + llm_model: Optional[str] = None, + tool_calls_log: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Log to terminal and optionally append to an in-memory session log.""" + entry: Dict[str, Any] = { + "tool": "search_knowledge_base", + "query": query, + "hit_count": hit_count, + "agent_id": agent_id, + "channel": channel, + "at": datetime.now(timezone.utc).isoformat(), + } + if llm_model: + entry["llm_model"] = llm_model + + logger.info( + "[KnowledgeBase] tool_call channel={} model={} agent_id={} query={!r} hit_count={}", + channel, + llm_model or "—", + agent_id, + query, + hit_count, + ) + + if tool_calls_log is not None: + tool_calls_log.append(entry) + return entry + + +class KnowledgeBaseVoiceNotifier: + """Optional live RTVI push when a voice-bundle KB search runs.""" + + def __init__( + self, + *, + agent_id: str, + llm_model: Optional[str] = None, + tool_calls_log: Optional[List[Dict[str, Any]]] = None, + ) -> None: + self.agent_id = agent_id + self.llm_model = llm_model + self.tool_calls_log = tool_calls_log + self._rtvi: Any = None + self._notify_fn: Optional[NotifyFn] = None + + def bind_rtvi(self, rtvi: Any) -> None: + self._rtvi = rtvi + + def bind_notify(self, notify_fn: NotifyFn) -> None: + self._notify_fn = notify_fn + + async def on_tool_call( + self, *, query: str, hit_count: int, channel: str = "voice_bundle" + ) -> None: + entry = record_knowledge_base_tool_call( + channel=channel, + query=query, + hit_count=hit_count, + agent_id=self.agent_id, + llm_model=self.llm_model, + tool_calls_log=self.tool_calls_log, + ) + payload = {"type": "knowledge_base_tool_call", **entry} + if self._notify_fn is not None: + try: + await self._notify_fn(payload) + except Exception as exc: + logger.warning("[KnowledgeBase] live notify failed: {}", exc) + elif self._rtvi is not None: + try: + await self._rtvi.send_server_message(payload) + except Exception as exc: + logger.warning("[KnowledgeBase] RTVI notify failed: {}", exc) diff --git a/app/services/knowledge_bases/tool_schema.py b/app/services/knowledge_bases/tool_schema.py new file mode 100644 index 00000000..fcef2cb3 --- /dev/null +++ b/app/services/knowledge_bases/tool_schema.py @@ -0,0 +1,31 @@ +"""OpenAI-style tool definition for knowledge search.""" + +from __future__ import annotations + +from typing import Any, Dict, List + + +def search_knowledge_base_tool_definition() -> Dict[str, Any]: + return { + "type": "function", + "function": { + "name": "search_knowledge_base", + "description": ( + "Search attached knowledge bases for relevant passages from uploaded documents." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Natural language search query.", + } + }, + "required": ["query"], + }, + }, + } + + +def knowledge_base_tools() -> List[Dict[str, Any]]: + return [search_knowledge_base_tool_definition()] diff --git a/app/services/knowledge_bases/voice_tools.py b/app/services/knowledge_bases/voice_tools.py new file mode 100644 index 00000000..0747c630 --- /dev/null +++ b/app/services/knowledge_bases/voice_tools.py @@ -0,0 +1,80 @@ +"""Register knowledge-base search on voice-bundle LLM pipelines.""" + +from __future__ import annotations + +import asyncio +from typing import Any +from uuid import UUID + +from efficientai.adapters.schemas.function_schema import FunctionSchema +from efficientai.adapters.schemas.tools_schema import ToolsSchema + +from app.database import SessionLocal +from app.services.knowledge_bases.agent_links import KB_ROLE_PRODUCTION +from app.services.knowledge_bases.retrieval import format_search_results_for_tool, search_knowledge_base +from app.services.knowledge_bases.tool_call_logging import KnowledgeBaseVoiceNotifier +from app.services.knowledge_bases.tool_schema import search_knowledge_base_tool_definition + + +def register_voice_bundle_knowledge_tools( + llm: Any, + context: Any, + *, + organization_id: str, + workspace_id: str, + agent_id: str, + kb_notifier: KnowledgeBaseVoiceNotifier | None = None, + role: str = KB_ROLE_PRODUCTION, +) -> None: + """Attach search_knowledge_base to the bundle LLM context.""" + org_uuid = UUID(organization_id) + ws_uuid = UUID(workspace_id) + agent_uuid = UUID(agent_id) + + tool_def = search_knowledge_base_tool_definition()["function"] + + schema = FunctionSchema( + name=tool_def["name"], + description=tool_def["description"], + properties=tool_def["parameters"]["properties"], + required=tool_def["parameters"].get("required") or [], + ) + + def _search_sync(search_query: str) -> tuple[list, str]: + db = SessionLocal() + try: + hits = search_knowledge_base( + db, + organization_id=org_uuid, + workspace_id=ws_uuid, + agent_id=agent_uuid, + query=search_query, + role=role, + ) + return hits, format_search_results_for_tool(hits) + finally: + db.close() + + async def _handler(params: Any) -> None: + query = str((params.arguments or {}).get("query") or "").strip() + hits: list = [] + try: + hits, text = await asyncio.to_thread(_search_sync, query) + except Exception as exc: + from loguru import logger + + logger.warning("[KnowledgeBase] voice tool search failed: {}", exc) + text = f"Knowledge search failed: {exc}" + if kb_notifier is not None: + await kb_notifier.on_tool_call(query=query, hit_count=len(hits)) + await params.result_callback(text) + + from loguru import logger + + llm.register_function(schema.name, _handler) + context.set_tools(ToolsSchema(standard_tools=[schema])) + logger.info( + "[KnowledgeBase] registered voice tool {} for agent_id={}", + schema.name, + agent_id, + ) diff --git a/app/services/metric_failure_policy.py b/app/services/metric_failure_policy.py index 281013bc..b78aac6d 100644 --- a/app/services/metric_failure_policy.py +++ b/app/services/metric_failure_policy.py @@ -97,6 +97,18 @@ def suggest_failure_policy( numeric_mean: Optional[float] = None, ) -> MetricFailurePolicy: metric_id = str(metric.id) + from app.services.metrics.computed_metrics import ( + COMPUTED_TRANSCRIPT_SCORING_METHODS, + metric_scoring_method_key, + ) + + if metric_scoring_method_key(metric) in COMPUTED_TRANSCRIPT_SCORING_METHODS: + return MetricFailurePolicy( + metric_id=metric_id, + failure_values=[], + numeric_rule={"op": "gt", "threshold": 0.25}, + ) + selection_mode = (getattr(metric, "selection_mode", None) or "").lower() is_parent = bool(selection_mode and not getattr(metric, "parent_metric_id", None)) diff --git a/app/services/metric_studio/source_resolver.py b/app/services/metric_studio/source_resolver.py index c659e8cf..0958a8e0 100644 --- a/app/services/metric_studio/source_resolver.py +++ b/app/services/metric_studio/source_resolver.py @@ -41,6 +41,10 @@ class ResolvedCallSample: call_data: Optional[dict] agent_id: Optional[UUID] metadata: Dict[str, Any] = field(default_factory=dict) + diarised_segments: Optional[list] = None + diarised_speaker_swap: bool = False + diarised_native_transcript: str = "" + speaker_aliases: Optional[Dict[str, Any]] = None def _resolve_call_import_row( @@ -107,6 +111,20 @@ def _resolve_call_import_row( audio_s3_key=(row.recording_s3_key or "").strip() or None, call_data=None, agent_id=None, + diarised_segments=( + row.diarised_segments + if isinstance(row.diarised_segments, list) + else None + ), + diarised_speaker_swap=bool(getattr(row, "diarised_speaker_swap", False)), + diarised_native_transcript=( + getattr(row, "diarised_native_transcript", None) or "" + ).strip(), + speaker_aliases=( + call_import.speaker_aliases + if isinstance(getattr(call_import, "speaker_aliases", None), dict) + else None + ), metadata={ "call_import_id": str(row.call_import_id), "call_import_name": getattr(call_import, "name", None), diff --git a/app/services/metrics/computed_metrics.py b/app/services/metrics/computed_metrics.py new file mode 100644 index 00000000..147c20d0 --- /dev/null +++ b/app/services/metrics/computed_metrics.py @@ -0,0 +1,116 @@ +"""Registry of deterministic call-import metric scorers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple + +from app.models.database import Metric +from app.models.enums import MetricScoringMethod, MetricType +from app.services.evaluation.transcript_error_rates import ( + compute_transcript_error_rates, + select_comparison_texts, +) + + +@dataclass(frozen=True) +class ComputedMetricContext: + production_transcript: str + diarised_transcript: str + diarised_segments: Optional[List[Dict[str, Any]]] + diarised_speaker_swap: bool + speaker_aliases: Optional[Mapping[str, Tuple[str, ...]]] = None + diarised_native_transcript: str = "" + + +def _score_transcript_agreement(ctx: ComputedMetricContext) -> Dict[str, Any]: + """Production user turns are the reference. Agent turns are ignored.""" + selected = select_comparison_texts( + production_transcript=ctx.production_transcript, + diarised_transcript=ctx.diarised_transcript, + diarised_segments=ctx.diarised_segments, + diarised_native_transcript=ctx.diarised_native_transcript, + diarised_speaker_swap=ctx.diarised_speaker_swap, + speaker_aliases=ctx.speaker_aliases, + ) + rates = compute_transcript_error_rates( + selected["reference"], + selected["hypothesis"], + mode=selected["mode"], + script_mismatch=selected["script_mismatch"], + ) + if rates.get("cer") is None or rates.get("wer") is None: + return {"skipped": "production_user_empty"} + return rates + + +def _chart_value(rate: float) -> float: + return round(min(1.0, float(rate)), 4) + + +def _score_user_wer(metric: Metric, ctx: ComputedMetricContext) -> Dict[str, Any]: + rates = _score_transcript_agreement(ctx) + if "skipped" in rates: + return rates + return { + "value": _chart_value(rates["wer"]), + "type": MetricType.NUMBER.value, + "metric_name": metric.name, + "details": rates, + } + + +def _score_user_cer(metric: Metric, ctx: ComputedMetricContext) -> Dict[str, Any]: + rates = _score_transcript_agreement(ctx) + if "skipped" in rates: + return rates + return { + "value": _chart_value(rates["cer"]), + "type": MetricType.NUMBER.value, + "metric_name": metric.name, + "details": rates, + } + + +COMPUTED_TRANSCRIPT_SCORING_METHODS = frozenset( + { + MetricScoringMethod.USER_WER.value, + MetricScoringMethod.USER_CER.value, + } +) + + +def metric_scoring_method_key(metric: Metric) -> str: + raw = getattr(metric, "scoring_method", None) or MetricScoringMethod.LLM.value + return str(raw).strip().lower() + + +def metrics_require_transcript_agreement(metrics: Sequence[Metric]) -> bool: + return any( + metric_scoring_method_key(metric) in COMPUTED_TRANSCRIPT_SCORING_METHODS + for metric in metrics + ) + + +COMPUTED_SCORERS: Dict[str, Callable[[Metric, ComputedMetricContext], Dict[str, Any]]] = { + MetricScoringMethod.USER_WER.value: _score_user_wer, + MetricScoringMethod.USER_CER.value: _score_user_cer, +} + + +def is_computed_metric(metric: Metric) -> bool: + return metric_scoring_method_key(metric) in COMPUTED_SCORERS + + +def run_computed_scorer( + metric: Metric, + ctx: ComputedMetricContext, +) -> Dict[str, Any]: + key = metric_scoring_method_key(metric) + scorer = COMPUTED_SCORERS.get(key) + if not scorer: + return {"skipped": "unknown_computed_scorer"} + result = scorer(metric, ctx) + if "skipped" in result: + return result + return result diff --git a/app/services/metrics/evaluation_scope.py b/app/services/metrics/evaluation_scope.py new file mode 100644 index 00000000..4fff58e7 --- /dev/null +++ b/app/services/metrics/evaluation_scope.py @@ -0,0 +1,163 @@ +"""Agent and scenario scope for metrics (golden = no links on that dimension).""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Agent, Metric, MetricAgentAssignment, MetricScenarioAssignment, Scenario +from app.services.metrics.scenario_scope import ( + is_golden_scope, + load_scenario_link_map, + metric_applies_to_scenario, + replace_metric_scenarios, + scenario_ids_for_response, + scope_metric_id, + validate_scenario_links, +) + +__all__ = [ + "agent_ids_for_response", + "applicable_metrics", + "is_fully_golden", + "is_golden_scope", + "load_agent_link_map", + "load_scenario_link_map", + "metric_applies_to_agent", + "metric_applies_to_scenario", + "replace_metric_agents", + "replace_metric_scenarios", + "scenario_ids_for_response", + "scope_metric_id", + "validate_agent_links", + "validate_scenario_links", +] + + +def load_agent_link_map( + db: Session, metric_ids: Iterable[UUID] +) -> dict[UUID, frozenset[UUID]]: + ids = list({UUID(str(i)) for i in metric_ids}) + if not ids: + return {} + rows = ( + db.query(MetricAgentAssignment.metric_id, MetricAgentAssignment.agent_id) + .filter(MetricAgentAssignment.metric_id.in_(ids)) + .all() + ) + out: dict[UUID, set[UUID]] = {} + for mid, aid in rows: + out.setdefault(mid, set()).add(aid) + return {mid: frozenset(aids) for mid, aids in out.items()} + + +def agent_ids_for_response(metric: Metric) -> list[UUID]: + if metric.parent_metric_id is not None: + parent = metric.parent + if parent is None: + return [] + metric = parent + return [a.id for a in sorted(metric.scoped_agents or [], key=lambda a: a.name or str(a.id))] + + +def metric_applies_to_agent( + metric: Metric, + agent_id: Optional[UUID], + *, + link_map: dict[UUID, frozenset[UUID]], +) -> bool: + if agent_id is None: + return True + sid = scope_metric_id(metric) + linked = link_map.get(sid) + if linked is None or len(linked) == 0: + return True + return agent_id in linked + + +def is_fully_golden( + scope_id: UUID, + *, + agent_link_map: dict[UUID, frozenset[UUID]], + scenario_link_map: dict[UUID, frozenset[UUID]], +) -> bool: + return is_golden_scope(agent_link_map, scope_id) and is_golden_scope( + scenario_link_map, scope_id + ) + + +def applicable_metrics( + metrics: list[Metric], + *, + agent_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, + db: Optional[Session] = None, + agent_link_map: Optional[dict[UUID, frozenset[UUID]]] = None, + scenario_link_map: Optional[dict[UUID, frozenset[UUID]]] = None, +) -> list[Metric]: + if agent_id is None and scenario_id is None: + return metrics + scope_ids = {scope_metric_id(m) for m in metrics} + if agent_link_map is None or scenario_link_map is None: + if db is None: + raise ValueError("db or both link maps are required when filtering by scope") + if agent_link_map is None: + agent_link_map = load_agent_link_map(db, scope_ids) + if scenario_link_map is None: + scenario_link_map = load_scenario_link_map(db, scope_ids) + return [ + m + for m in metrics + if metric_applies_to_agent(m, agent_id, link_map=agent_link_map) + and metric_applies_to_scenario(m, scenario_id, link_map=scenario_link_map) + ] + + +def validate_agent_links( + db: Session, + *, + organization_id: UUID, + metric: Metric, + agent_ids: list[UUID], +) -> None: + if metric.parent_metric_id is not None: + raise ValueError("Child metrics cannot have agent links") + if not agent_ids: + return + unique = list(dict.fromkeys(agent_ids)) + agents = ( + db.query(Agent) + .filter(Agent.id.in_(unique), Agent.organization_id == organization_id) + .all() + ) + found = {a.id for a in agents} + missing = [str(a) for a in unique if a not in found] + if missing: + raise ValueError(f"Agents not found: {', '.join(missing)}") + if metric.workspace_id is not None: + bad = [str(a.id) for a in agents if a.workspace_id != metric.workspace_id] + if bad: + raise ValueError( + "Workspace-scoped metrics can only link to agents in the same workspace" + ) + + +def replace_metric_agents( + db: Session, + *, + organization_id: UUID, + metric: Metric, + agent_ids: Optional[list[UUID]], +) -> None: + if agent_ids is None: + return + validate_agent_links( + db, organization_id=organization_id, metric=metric, agent_ids=agent_ids + ) + db.query(MetricAgentAssignment).filter( + MetricAgentAssignment.metric_id == metric.id + ).delete(synchronize_session=False) + for aid in dict.fromkeys(agent_ids): + db.add(MetricAgentAssignment(metric_id=metric.id, agent_id=aid)) diff --git a/app/services/metrics/option_labels.py b/app/services/metrics/option_labels.py new file mode 100644 index 00000000..8f94ac98 --- /dev/null +++ b/app/services/metrics/option_labels.py @@ -0,0 +1,218 @@ +"""Normalize metric enum / classification option shapes from DB or LLM.""" + +import re +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, List, Optional + + +def _scalar_option_to_string(val: Any) -> Optional[str]: + if val is None: + return None + if isinstance(val, str): + stripped = val.strip() + return stripped or None + if isinstance(val, bool): + return str(val).strip() or None + if isinstance(val, int): + return str(val) + if isinstance(val, float): + if val == int(val): + return str(int(val)) + text = str(val).strip() + return text or None + return None + + +def coerce_metric_option_label(opt: Any) -> Optional[str]: + if isinstance(opt, dict): + for key in ("value", "label", "name", "description"): + if key not in opt: + continue + label = _scalar_option_to_string(opt.get(key)) + if label is not None: + return label + return None + return _scalar_option_to_string(opt) + + +def enum_labels_equal(a: str, b: str) -> bool: + """Strict label equality for pass/fail rules (case-insensitive, no numeric aliasing).""" + return a.strip().lower() == b.strip().lower() + + +def _is_integer_label(label: str) -> bool: + return bool(re.fullmatch(r"-?\d+", label.strip())) + + +def _numeric_option_same(candidate: str, option: str) -> bool: + """Whether ``candidate`` can refer to ``option`` when strings differ only numerically.""" + a, b = candidate.strip(), option.strip() + if a.lower() == b.lower(): + return True + if _is_integer_label(a) and _is_integer_label(b): + return int(a, 10) == int(b, 10) and a == b + try: + return Decimal(a) == Decimal(b) + except (InvalidOperation, ValueError): + return False + + +def canonical_enum_option(text: str, options: List[str]) -> Optional[str]: + """Resolve LLM/user text to a single configured option, or None if ambiguous.""" + needle = text.strip() + if not needle or not options: + return None + needle_lower = needle.lower() + for opt in options: + if opt.strip().lower() == needle_lower: + return opt + numeric_hits = [opt for opt in options if _numeric_option_same(needle, opt)] + if len(numeric_hits) == 1: + return numeric_hits[0] + return None + + +def enum_numeric_matches_ambiguous(text: str, options: List[str]) -> bool: + """True when more than one option could match numerically (skip loose fallbacks).""" + needle = text.strip() + if not needle: + return False + hits = [opt for opt in options if _numeric_option_same(needle, opt)] + return len(hits) > 1 + + +def _dedupe_labels(labels: List[str]) -> List[str]: + seen: set[str] = set() + out: List[str] = [] + for label in labels: + key = label.strip().lower() + if not key or key in seen: + continue + seen.add(key) + out.append(label) + return out + + +def enum_option_labels_from_config(custom_config: Any) -> List[str]: + if not isinstance(custom_config, dict): + return [] + raw = custom_config.get("options") + if not isinstance(raw, list): + return [] + labels: List[str] = [] + for opt in raw: + label = coerce_metric_option_label(opt) + if label is not None: + labels.append(label) + return _dedupe_labels(labels) + + +def classification_choice_labels_from_config(custom_config: Any) -> List[str]: + normalized = _normalize_classification_choice_criteria( + _classification_choice_criteria_raw(custom_config) + ) + return list(normalized.keys()) if normalized else [] + + +def _classification_choice_criteria_raw(custom_config: Any) -> Any: + if not isinstance(custom_config, dict): + return None + choice = custom_config.get("choice") + if not isinstance(choice, dict): + return None + return choice.get("criteria") + + +def _normalize_classification_choice_criteria(criteria: Any) -> Optional[Dict[str, str]]: + if isinstance(criteria, dict): + out: Dict[str, str] = {} + for raw_key, raw_desc in criteria.items(): + key = coerce_metric_option_label(raw_key) or _scalar_option_to_string(raw_key) + if not key or key in out: + continue + desc = raw_desc if isinstance(raw_desc, str) else str(raw_desc or "") + out[key] = desc.strip() or key + return out + if isinstance(criteria, list): + out: Dict[str, str] = {} + for item in criteria: + if not isinstance(item, dict): + label = coerce_metric_option_label(item) + if label and label not in out: + out[label] = label + continue + key = coerce_metric_option_label(item) + if not key or key in out: + continue + desc = item.get("description") or item.get("desc") or key + out[key] = str(desc).strip() or key + return out if out else None + return None + + +def classification_score_labels_from_config(custom_config: Any) -> List[str]: + if not isinstance(custom_config, dict): + return [] + score = custom_config.get("score") + if not isinstance(score, dict): + return [] + criteria = score.get("criteria") + if not isinstance(criteria, list): + return [] + labels: List[str] = [] + for level in criteria: + label = coerce_metric_option_label(level) + if label is not None: + labels.append(label) + return _dedupe_labels(labels) + + +def _normalize_classification_config(custom_config: dict[str, Any]) -> tuple[dict[str, Any], bool]: + updated = dict(custom_config) + changed = False + + choice = updated.get("choice") + if isinstance(choice, dict): + raw_criteria = choice.get("criteria") + normalized_choice = _normalize_classification_choice_criteria(raw_criteria) + if normalized_choice is not None and normalized_choice != raw_criteria: + updated["choice"] = {**choice, "criteria": normalized_choice} + changed = True + + score = updated.get("score") + if isinstance(score, dict) and isinstance(score.get("criteria"), list): + normalized_levels = classification_score_labels_from_config(updated) + if normalized_levels != score.get("criteria"): + updated["score"] = {**score, "criteria": normalized_levels} + changed = True + + return updated, changed + + +def custom_config_for_api(custom_data_type: Optional[str], custom_config: Any) -> Any: + if not isinstance(custom_config, dict): + return custom_config + updated = dict(custom_config) + changed = False + dtype = (custom_data_type or "").strip().lower() + + if dtype == "enum": + options = updated.get("options") + if isinstance(options, list): + normalized = enum_option_labels_from_config(updated) + if normalized != options: + updated["options"] = normalized + changed = True + + if dtype == "classification": + updated, cls_changed = _normalize_classification_config(updated) + changed = changed or cls_changed + + return updated if changed else custom_config + + +def coerce_custom_config_for_storage( + custom_data_type: Optional[str], custom_config: Any +) -> Any: + """Persist canonical option labels (same rules as API responses).""" + return custom_config_for_api(custom_data_type, custom_config) diff --git a/app/services/metrics/scenario_scope.py b/app/services/metrics/scenario_scope.py new file mode 100644 index 00000000..53989cbe --- /dev/null +++ b/app/services/metrics/scenario_scope.py @@ -0,0 +1,114 @@ +"""Scenario scope for metrics: golden (all scenarios) vs linked to specific scenarios.""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Metric, MetricScenarioAssignment, Scenario + + +def load_scenario_link_map( + db: Session, metric_ids: Iterable[UUID] +) -> dict[UUID, frozenset[UUID]]: + """Return metric_id -> linked scenario ids. Missing keys mean golden (no links).""" + ids = list({UUID(str(i)) for i in metric_ids}) + if not ids: + return {} + rows = ( + db.query(MetricScenarioAssignment.metric_id, MetricScenarioAssignment.scenario_id) + .filter(MetricScenarioAssignment.metric_id.in_(ids)) + .all() + ) + out: dict[UUID, set[UUID]] = {} + for mid, sid in rows: + out.setdefault(mid, set()).add(sid) + return {mid: frozenset(sids) for mid, sids in out.items()} + + +def scope_metric_id(metric: Metric) -> UUID: + return metric.parent_metric_id or metric.id + + +def scenario_ids_for_response(metric: Metric) -> list[UUID]: + """Scenario links for API responses; children inherit the parent's links.""" + if metric.parent_metric_id is not None: + parent = metric.parent + if parent is None: + return [] + metric = parent + return [s.id for s in sorted(metric.scoped_scenarios or [], key=lambda s: s.name or str(s.id))] + + +def is_golden_scope(link_map: dict[UUID, frozenset[UUID]], scope_id: UUID) -> bool: + linked = link_map.get(scope_id) + return linked is None or len(linked) == 0 + + +def metric_applies_to_scenario( + metric: Metric, + scenario_id: Optional[UUID], + *, + link_map: dict[UUID, frozenset[UUID]], +) -> bool: + if scenario_id is None: + return True + sid = scope_metric_id(metric) + linked = link_map.get(sid) + if linked is None or len(linked) == 0: + return True + return scenario_id in linked + + +def validate_scenario_links( + db: Session, + *, + organization_id: UUID, + metric: Metric, + scenario_ids: list[UUID], +) -> None: + if metric.parent_metric_id is not None: + raise ValueError("Child metrics cannot have scenario links") + if not scenario_ids: + return + unique = list(dict.fromkeys(scenario_ids)) + scenarios = ( + db.query(Scenario) + .filter( + Scenario.id.in_(unique), + Scenario.organization_id == organization_id, + ) + .all() + ) + found = {s.id for s in scenarios} + missing = [str(s) for s in unique if s not in found] + if missing: + raise ValueError(f"Scenarios not found: {', '.join(missing)}") + if metric.workspace_id is not None: + bad = [str(s.id) for s in scenarios if s.workspace_id != metric.workspace_id] + if bad: + raise ValueError( + "Workspace-scoped metrics can only link to scenarios in the same workspace" + ) + + +def replace_metric_scenarios( + db: Session, + *, + organization_id: UUID, + metric: Metric, + scenario_ids: Optional[list[UUID]], +) -> None: + """Replace links. ``None`` leaves existing links unchanged; ``[]`` clears (golden).""" + if scenario_ids is None: + return + validate_scenario_links( + db, organization_id=organization_id, metric=metric, scenario_ids=scenario_ids + ) + db.query(MetricScenarioAssignment).filter( + MetricScenarioAssignment.metric_id == metric.id + ).delete(synchronize_session=False) + for sid in dict.fromkeys(scenario_ids): + db.add(MetricScenarioAssignment(metric_id=metric.id, scenario_id=sid)) diff --git a/app/services/personas/persona_prompt_generation.py b/app/services/personas/persona_prompt_generation.py index f13e4d32..cc3d2beb 100644 --- a/app/services/personas/persona_prompt_generation.py +++ b/app/services/personas/persona_prompt_generation.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import Session from app.models.database import Agent -from app.models.enums import ModelProvider +from app.models.enums import CallMediumEnum, ModelProvider from app.services.ai.llm_service import llm_service SCENARIO_REFERENCE_HEADING = "## Test scenarios (reference)" @@ -39,6 +39,28 @@ def strip_scenario_reference_appendix(markdown: str) -> str: "- Return ONLY the persona prompt text." ) +GENERATE_CHAT_PERSONA_PROMPT_SYSTEM = ( + "You are an expert at writing persona prompts for synthetic chat test customers.\n\n" + "A persona prompt describes how the simulated customer writes and behaves when texting " + "a chat agent (web chat, WhatsApp, SMS). It is appended to the customer LLM system prompt " + "during evaluation runs.\n\n" + "Guidelines:\n" + "- Write 2-5 concise sentences in plain prose (not markdown headings).\n" + "- Describe personality, tone, patience, and texting style (message length, typos, " + "emoji, how much detail they volunteer up front).\n" + "- The customer is chatting with an AI agent — stay in character as a realistic customer.\n" + "- Do NOT describe voice, accent, or phone-call behavior.\n" + "- Do NOT mention testing, QA, automation, LLM, or system prompts.\n" + "- Do NOT repeat the agent's full system prompt verbatim.\n" + "- Infer a complementary customer profile from the agent under test.\n" + "- Return ONLY the persona prompt text." +) + + +def _is_chat_agent(agent: Agent) -> bool: + medium = getattr(agent.call_medium, "value", agent.call_medium) or "" + return str(medium).lower() == CallMediumEnum.CHAT.value + @dataclass class PersonaPromptGenerationResult: @@ -92,9 +114,8 @@ def build_persona_prompt_user_message( parts.append(f"Persona gender: {persona_gender}") if additional_context and additional_context.strip(): parts.append(f"Additional context:\n{additional_context.strip()}") - parts.append( - "\nGenerate a persona prompt for the synthetic caller who will interact with this agent." - ) + who = "customer who will chat with" if _is_chat_agent(agent) else "caller who will interact with" + parts.append(f"\nGenerate a persona prompt for the synthetic {who} this agent.") return "\n".join(parts) @@ -138,7 +159,14 @@ def generate_persona_prompt_from_agent( ) messages = [ - {"role": "system", "content": GENERATE_PERSONA_PROMPT_SYSTEM}, + { + "role": "system", + "content": ( + GENERATE_CHAT_PERSONA_PROMPT_SYSTEM + if _is_chat_agent(agent) + else GENERATE_PERSONA_PROMPT_SYSTEM + ), + }, { "role": "user", "content": build_persona_prompt_user_message( diff --git a/app/services/reporting/call_import_evaluation_pdf_api.py b/app/services/reporting/call_import_evaluation_pdf_api.py new file mode 100644 index 00000000..690145bd --- /dev/null +++ b/app/services/reporting/call_import_evaluation_pdf_api.py @@ -0,0 +1,962 @@ +"""Call Import evaluation PDF report API orchestration.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import re +from datetime import date, datetime, timezone +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID, uuid4 + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import desc +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.core.auth import Principal +from app.models.database import ( + CallImport, + CallImportEvaluation, + CallImportEvaluationPdfReport, + CallImportEvaluationReportSnapshot, + CallImportEvaluationRow, + CallImportRow, + Metric, + Workspace, +) +from app.services.metric_failure_policy import effective_policies +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.api.v1.routes.call_import_evaluations._shared import ( + _metrics_for_ids, + _serialize_selected_metric_ids, +) +from app.api.v1.routes.call_import_evaluations.aggregate import ( + _aggregate_to_dict, + _audit_summary_text_from_tldr, + _baseline_candidate_evaluations, + _benchmark_context_for_evaluation, + _child_names_by_parent, + _compute_metric_aggregates, + _evaluation_rows_for_period, + _metric_insights_from_tldr, + _metric_is_user_insight, + _period_deltas_from_evaluation, + _period_deltas_with_explanations, + _report_period_from_rows, + _resolve_baseline_evaluation, +) +from app.api.v1.routes.call_import_evaluations.clusters import ( + _metric_clusters_payload, + _selected_metric_clusters_for_pdf, +) +from app.api.v1.routes.call_import_evaluations.insights import ( + _selected_generated_user_insights, + _tldr_summary_payload, + _user_insights_payload, +) +from app.api.v1.routes.call_import_evaluations.prompts import ( + _prompt_improvements_payload, + _selected_prompt_improvements_for_pdf, +) + +class CallImportEvaluationPdfReportRequest(BaseModel): + vendor_name: str = Field(..., min_length=1, max_length=120) + report_type: Literal["external", "internal"] = "external" + include_weekly_delta: bool = False + include_period_delta: bool = False + baseline_evaluation_id: Optional[str] = None + period_label: Optional[str] = Field(default=None, max_length=64) + use_case: Optional[str] = Field(default=None, max_length=120) + internal_brand_image_id: Optional[str] = None + external_brand_image_id: Optional[str] = None + report_config: Dict[str, Any] = Field(default_factory=dict) + platform_base_url: Optional[str] = Field( + default=None, + max_length=512, + description="Frontend origin for deep links to example calls in internal PDFs.", + ) + + @field_validator("vendor_name") + @classmethod + def _clean_vendor_name(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("Vendor name is required.") + return cleaned + + +class CallImportEvaluationPdfReportResponse(BaseModel): + id: str + filename: str + preview_url: Optional[str] = None + download_url: Optional[str] = None + created_at: datetime + created_by: Optional[str] = None + report_type: str + vendor_name: str + config_summary: Optional[str] = None + storage_available: bool = True + cache_hit: bool = False + + +class CallImportEvaluationPdfReportListItem(BaseModel): + id: str + filename: Optional[str] = None + vendor_name: str + report_type: str + created_by: Optional[str] = None + created_at: datetime + config_summary: Optional[str] = None + cache_fingerprint: Optional[str] = None + + +class CallImportEvaluationPdfReportListResponse(BaseModel): + items: List[CallImportEvaluationPdfReportListItem] + + +class CallImportEvaluationBaselineCandidate(BaseModel): + evaluation_id: str + name: str + dataset: str + period_label: Optional[str] = None + period_start: Optional[date] = None + period_end: Optional[date] = None + period_display: str + completed_rows: int + created_at: datetime + is_default: bool = False + + +class CallImportEvaluationBaselineCandidatesResponse(BaseModel): + items: List[CallImportEvaluationBaselineCandidate] + default_evaluation_id: Optional[str] = None + +def _report_filename_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") + return slug or "client" + + +def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: + created_by = principal.email + if not created_by and principal.user_id: + created_by = str(principal.user_id) + return created_by, principal.user_id + + +def _pdf_report_response_from_row( + row: CallImportEvaluationPdfReport, + *, + cache_hit: bool = False, +) -> CallImportEvaluationPdfReportResponse: + filename = row.filename or "report.pdf" + preview_url, download_url = presigned_urls_for_pdf_report( + row.s3_key or "", + filename, + ) + return CallImportEvaluationPdfReportResponse( + id=str(row.id), + filename=filename, + preview_url=preview_url, + download_url=download_url, + created_at=row.created_at or datetime.now(timezone.utc), + created_by=row.created_by, + report_type=row.report_type, + vendor_name=row.vendor_name, + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + storage_available=bool(row.s3_key), + cache_hit=cache_hit, + ) + + +def _pdf_report_list_item_from_row( + row: CallImportEvaluationPdfReport, +) -> CallImportEvaluationPdfReportListItem: + return CallImportEvaluationPdfReportListItem( + id=str(row.id), + filename=row.filename, + vendor_name=row.vendor_name, + report_type=row.report_type, + created_by=row.created_by, + created_at=row.created_at or datetime.now(timezone.utc), + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + cache_fingerprint=row.cache_fingerprint, + ) + + +def _report_branding_for_import_workspace( + db: Session, + organization_id: UUID, + workspace_id: UUID, + *, + internal_brand_image_id: Optional[str] = None, + external_brand_image_id: Optional[str] = None, +) -> tuple[dict[str, str] | list[str], Optional[str]]: + workspace = ( + db.query(Workspace) + .filter( + Workspace.id == workspace_id, + Workspace.organization_id == organization_id, + ) + .first() + ) + raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} + images = raw.get("images") if isinstance(raw.get("images"), list) else [] + loaded_images: list[dict[str, str]] = [] + for item in images: + if not isinstance(item, dict) or not item.get("s3_key"): + continue + content_type = str(item.get("content_type") or "image/png") + try: + from app.services.storage.s3_service import s3_service + + image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Unable to load report branding image for workspace {}: {}", + workspace_id, + exc, + ) + continue + encoded = base64.b64encode(image_bytes).decode("ascii") + role = str(item.get("role") or "generic") + if role not in {"internal", "external", "generic"}: + role = "generic" + loaded_images.append( + { + "id": str(item.get("id") or ""), + "role": role, + "data_uri": f"data:{content_type};base64,{encoded}", + } + ) + + def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: + if selected_id: + for loaded in loaded_images: + if loaded["id"] == selected_id: + return loaded["data_uri"] + for loaded in loaded_images: + if loaded["role"] == role: + return loaded["data_uri"] + return None + + logo_data_uris: dict[str, str] = {} + internal_uri = _pick("internal", internal_brand_image_id) + external_uri = _pick("external", external_brand_image_id) + if internal_uri: + logo_data_uris["internal"] = internal_uri + if external_uri: + logo_data_uris["external"] = external_uri + if ( + not logo_data_uris + and not internal_brand_image_id + and not external_brand_image_id + ): + # Backward compatibility for workspaces that only had a generic logo + # library before the two-slot report header existed. + generic_uris = [ + loaded["data_uri"] + for loaded in loaded_images + if loaded.get("data_uri") + ] + if generic_uris: + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return generic_uris[:4], heading + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return logo_data_uris, heading + + +def _display_metrics_for_pdf_report( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, +) -> list[Metric]: + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + parent_id = UUID(parent_str) + except (TypeError, ValueError): + continue + if parent_id not in lookup_ids: + lookup_ids.append(parent_id) + + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + child_ids_in_groups: set[str] = set() + for child_strs in groups_raw.values(): + if not isinstance(child_strs, list): + continue + child_ids_in_groups.update(str(child_id) for child_id in child_strs) + + metrics_by_id = {str(metric.id): metric for metric in metrics} + display: list[Metric] = [] + seen: set[str] = set() + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(str(parent_str)) + if parent and str(parent.id) not in seen: + display.append(parent) + seen.add(str(parent.id)) + + for metric in metrics: + metric_id = str(metric.id) + if metric_id in seen or metric_id in child_ids_in_groups: + continue + if metric.selection_mode and not metric.parent_metric_id: + continue + display.append(metric) + seen.add(metric_id) + + return display + + +def _sample_evidence_for_metrics( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], + metric_ids: set[str], +) -> dict[str, list[dict[str, str]]]: + samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} + for eval_row, source_row in rows: + scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + for metric_id in metric_ids: + if len(samples.get(metric_id, [])) >= 4: + continue + score = scores.get(metric_id) + if not isinstance(score, dict): + continue + rationale = score.get("rationale") + transcript = source_row.diarised_transcript or source_row.transcript or "" + quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] + if quote: + samples.setdefault(metric_id, []).append( + { + "conversation_id": source_row.conversation_id, + "quote": str(quote).strip()[:500], + } + ) + return samples + + +def _fallback_report_narrative( + insight_aggregates: list[dict[str, Any]], + evidence_samples: dict[str, list[dict[str, str]]], +) -> dict[str, Any]: + observations: dict[str, str] = {} + evidence: dict[str, dict[str, str]] = {} + design_notes: list[str] = [] + for aggregate in insight_aggregates: + metric_id = str(aggregate.get("metric_id") or "") + name = str(aggregate.get("metric_name") or "Insight") + counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] + if counts: + top = counts[0] + total = int(aggregate.get("count") or 0) or sum( + int(item.get("count") or 0) for item in counts if isinstance(item, dict) + ) + pct = (int(top.get("count") or 0) / total) * 100 if total else 0 + observations[metric_id] = ( + f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." + ) + design_notes.append( + f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." + ) + sample = (evidence_samples.get(metric_id) or [{}])[0] + if sample: + evidence[metric_id] = sample + return { + "observations": observations, + "evidence": evidence, + "design_notes": design_notes[:7], + "audit_summary": None, + } + + +def _generate_report_narrative( + db: Session, + organization_id: UUID, + *, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, dict[str, str]], + evidence_samples: dict[str, list[dict[str, str]]], + report_config: dict[str, Any], +) -> dict[str, Any]: + if not insight_aggregates: + return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} + try: + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + from app.services.ai.llm_service import llm_service + + llm_provider = (report_config.get("llm_provider") or "").strip() or None + llm_model = (report_config.get("llm_model") or "").strip() or None + llm_credential_id = report_config.get("llm_credential_id") + if not llm_provider and not llm_credential_id: + return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, + db, + llm_provider, + llm_model, + credential_id=llm_credential_id, + ) + prompt = ( + "You are writing a vendor-safe external call quality audit report. " + "Return strict JSON with keys observations (object keyed by metric_id), " + "evidence (object keyed by metric_id with conversation_id and quote), " + "design_notes (array of concise numbered-note strings), and audit_summary (string). " + "Use only the supplied aggregates and evidence samples.\n\n" + + json.dumps( + { + "metric_aggregates": metric_aggregates[:30], + "insight_aggregates": insight_aggregates, + "period_deltas": period_delta_by_metric, + "evidence_samples": evidence_samples, + "report_config": report_config, + }, + default=str, + ) + ) + llm_result = llm_service.generate_response( + messages=[ + {"role": "system", "content": "Return JSON only. No markdown."}, + {"role": "user", "content": prompt}, + ], + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.2, + max_tokens=1200, + ) + parsed = json.loads(str(llm_result.content or "{}")) + if isinstance(parsed, dict): + fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) + return { + "observations": parsed.get("observations") or fallback["observations"], + "evidence": parsed.get("evidence") or fallback["evidence"], + "design_notes": parsed.get("design_notes") or fallback["design_notes"], + "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], + } + except Exception as exc: # noqa: BLE001 + logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) + return _fallback_report_narrative(insight_aggregates, evidence_samples) + +def baseline_candidates_response( + db: Session, + *, + call_import: CallImport, + evaluation: CallImportEvaluation, + organization_id: UUID, +) -> CallImportEvaluationBaselineCandidatesResponse: + from app.services.call_imports.evaluation_report_period import ( + ensure_evaluation_report_period, + report_period_tuple, + ) + + ensure_evaluation_report_period(db, evaluation) + period_start, _, _, _ = report_period_tuple(evaluation) + candidates = _baseline_candidate_evaluations( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + ) + db.commit() + default_evaluation_id = next( + (item["evaluation_id"] for item in candidates if item.get("is_default")), + None, + ) + return CallImportEvaluationBaselineCandidatesResponse( + items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], + default_evaluation_id=default_evaluation_id, + ) + + +async def generate_call_import_evaluation_pdf_report( + db: Session, + *, + call_import: CallImport, + evaluation: CallImportEvaluation, + organization_id: UUID, + eval_id: UUID, + call_import_id: UUID, + payload: CallImportEvaluationPdfReportRequest, + principal: Principal, +): + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + is_internal = payload.report_type == "internal" + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + report_config = payload.report_config if isinstance(payload.report_config, dict) else {} + metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) + configured_quality_ids = { + str(item) + for item in report_config.get("quality_metric_ids", []) + if item + } + configured_insight_ids = { + str(item.get("metric_id") or item) + for item in report_config.get("insights", []) + if item + } + if configured_quality_ids or configured_insight_ids: + allowed_ids = configured_quality_ids | configured_insight_ids + metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] + + eval_rows = [eval_row for eval_row, _source_row in rows] + aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) + selected_report_metric_ids = {str(metric.id) for metric in metrics} + aggregate_dicts = [ + _aggregate_to_dict(aggregate) + for aggregate in aggregate_models + if aggregate.metric_id in selected_report_metric_ids + ] + insight_metric_ids = { + str(metric.id) + for metric in metrics + if _metric_is_user_insight(metric) + } + metric_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids + ] + insight_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids + ] + period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) + period_label = (payload.period_label or derived_period_label or "").strip() or None + include_period_delta = ( + payload.include_period_delta or payload.include_weekly_delta + ) + previous_snapshot = None + period_delta_by_metric: dict[str, dict[str, str]] = {} + baseline_evaluation: Optional[CallImportEvaluation] = None + if include_period_delta and period_start: + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + payload.baseline_evaluation_id, + ) + if baseline_evaluation: + period_delta_by_metric = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates, + evaluation, + [eval_row for eval_row, _ in rows], + ) + period_delta_by_metric = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + period_delta_by_metric, + ) + benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) + evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) + cached_tldr_summary = _tldr_summary_payload(evaluation) + cached_user_insights = _user_insights_payload(evaluation) + cached_metric_clusters = _metric_clusters_payload(evaluation) + cached_prompt_improvements = _prompt_improvements_payload(evaluation) + generated_insights_for_pdf = _selected_generated_user_insights( + cached_user_insights, + report_config, + ) + metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( + cached_metric_clusters, + report_config, + ) + prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( + cached_prompt_improvements, + report_config, + ) + branding_images, custom_heading = _report_branding_for_import_workspace( + db, + organization_id, + call_import.workspace_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + ) + eval_row_list = [eval_row for eval_row, _ in rows] + pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) + pdf_parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + pdf_child_map = _child_names_by_parent( + db, evaluation.organization_id, pdf_parent_ids + ) + failure_policies_for_pdf, _fp_source = effective_policies( + evaluation, + metrics, + pdf_aggregates, + child_names_by_parent=pdf_child_map, + ) + + from app.services.storage.s3_service import s3_service + + config_fingerprint = compute_pdf_report_config_fingerprint( + report_type=payload.report_type, + include_period_delta=bool(payload.include_period_delta), + include_weekly_delta=bool(payload.include_weekly_delta), + baseline_evaluation_id=payload.baseline_evaluation_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + use_case=payload.use_case, + report_config=report_config, + report_heading=custom_heading, + vendor_name=payload.vendor_name, + platform_base_url=payload.platform_base_url, + period_label=period_label, + ) + content_fingerprint = compute_pdf_report_content_fingerprint( + evaluation_status=evaluation.status, + completed_rows=int(evaluation.completed_rows or 0), + total_rows=int(evaluation.total_rows or 0), + failed_rows=int(evaluation.failed_rows or 0), + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + period_delta_by_metric=period_delta_by_metric, + benchmark_context=benchmark_context, + metric_metadata=[ + { + "id": str(metric.id), + "name": metric.name, + "description": metric.description, + } + for metric in metrics + ], + failure_policies=failure_policies_for_pdf, + tldr_summary=cached_tldr_summary, + user_insights_for_pdf=generated_insights_for_pdf, + metric_clusters_for_pdf=metric_clusters_for_pdf, + prompt_improvements_for_pdf=prompt_improvements_for_pdf, + ) + cache_fingerprint = compute_pdf_report_cache_fingerprint( + config_fingerprint=config_fingerprint, + content_fingerprint=content_fingerprint, + ) + if s3_service.is_enabled(): + cached_pdf_report = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if cached_pdf_report is not None: + logger.info( + "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", + cached_pdf_report.id, + eval_id, + ) + return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) + + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + ) + from app.services.usage.context import llm_usage_context + + with llm_usage_context( + call_import_evaluation_usage_context( + organization_id=organization_id, + workspace_id=getattr(call_import, "workspace_id", None) + or evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) + ): + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + + generated_at = datetime.now(timezone.utc) + try: + pdf_started = datetime.now(timezone.utc) + pdf_bytes = await asyncio.to_thread( + call_import_evaluation_pdf_report_service.render_pdf, + vendor_name=payload.vendor_name, + call_import=call_import, + evaluation=evaluation, + metrics=metrics, + rows=rows, + failure_policies=failure_policies_for_pdf, + generated_at=generated_at, + internal=is_internal, + logo_data_uris=branding_images, + custom_heading=custom_heading, + include_weekly_delta=include_period_delta, + period_delta_by_metric=period_delta_by_metric, + use_case=payload.use_case, + period_display=period_display, + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + report_config=report_config, + narrative=narrative, + audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), + metric_insights=_metric_insights_from_tldr(cached_tldr_summary), + benchmark_context=benchmark_context, + generated_user_insights=generated_insights_for_pdf, + user_insights_overview=( + cached_user_insights.overview if cached_user_insights else None + ), + metric_clusters=metric_clusters_for_pdf, + metric_clusters_overview=( + cached_metric_clusters.overview if cached_metric_clusters else None + ), + prompt_improvements=prompt_improvements_for_pdf, + platform_base_url=payload.platform_base_url, + ) + logger.info( + "PDF report render finished in {:.1f}s for evaluation {}", + (datetime.now(timezone.utc) - pdf_started).total_seconds(), + eval_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to generate PDF report for call import {} evaluation {}", + call_import_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to generate PDF report: {exc}", + ) from exc + + snapshot = CallImportEvaluationReportSnapshot( + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + period_label=period_label, + period_start=period_start, + period_end=period_end, + report_config=report_config, + selected_metric_ids=[str(metric.id) for metric in metrics], + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + narrative=narrative, + total_calls=evaluation.total_rows, + selected_metric_count=len(metrics), + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + ) + db.add(snapshot) + db.flush() + + filename = ( + f"{_report_filename_slug(payload.vendor_name)}-" + f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" + ) + + if not s3_service.is_enabled(): + db.commit() + return StreamingResponse( + iter([pdf_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + report_id = uuid4() + s3_key = build_pdf_report_s3_key( + organization_id=organization_id, + call_import_id=call_import.id, + evaluation_id=evaluation.id, + report_id=report_id, + ) + try: + s3_service.upload_file_by_key( + file_content=pdf_bytes, + key=s3_key, + content_type="application/pdf", + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to upload PDF report for evaluation {} to object storage", + eval_id, + ) + db.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to store PDF report: {exc}", + ) from exc + + created_by, created_by_user_id = _pdf_report_actor(principal) + pdf_report = CallImportEvaluationPdfReport( + id=report_id, + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + snapshot_id=snapshot.id, + vendor_name=payload.vendor_name, + report_type=payload.report_type, + filename=filename, + s3_key=s3_key, + report_config=report_config, + cache_fingerprint=cache_fingerprint, + created_by=created_by, + created_by_user_id=created_by_user_id, + ) + db.add(pdf_report) + try: + db.commit() + except IntegrityError: + db.rollback() + try: + s3_service.delete_file_by_key(s3_key) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to delete orphan PDF after cache race for evaluation {}", + eval_id, + ) + raced_winner = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if raced_winner is not None: + logger.info( + "PDF report cache race resolved for evaluation {} (winner {})", + eval_id, + raced_winner.id, + ) + return _pdf_report_response_from_row(raced_winner, cache_hit=True) + raise HTTPException( + status_code=500, + detail="Failed to store PDF report due to a concurrent duplicate request.", + ) from None + db.refresh(pdf_report) + from app.services.billing.flexprice_service import ( + record_call_import_pdf_report_generated, + ) + + record_call_import_pdf_report_generated( + organization_id, + pdf_report.id, + workspace_id=pdf_report.workspace_id, + evaluation_id=evaluation.id, + call_import_id=call_import_id, + report_type=pdf_report.report_type, + ) + return _pdf_report_response_from_row(pdf_report) + +def pdf_report_download_response( + db: Session, + *, + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + organization_id: UUID, +) -> StreamingResponse: + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + raise HTTPException( + status_code=503, + detail="Object storage is not enabled or not configured.", + ) + try: + file_bytes = s3_service.download_file_by_key(row.s3_key) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to download PDF report {} for evaluation {}", + report_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to download PDF report: {exc}", + ) from exc + filename = row.filename or "report.pdf" + safe_name = filename.replace('"', "'") + return StreamingResponse( + iter([file_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{safe_name}"'}, + ) + + +pdf_report_response_from_row = _pdf_report_response_from_row +pdf_report_list_item_from_row = _pdf_report_list_item_from_row diff --git a/app/services/reporting/call_import_evaluation_pdf_report.py b/app/services/reporting/call_import_evaluation_pdf_report.py index ef651c25..5ab8217f 100644 --- a/app/services/reporting/call_import_evaluation_pdf_report.py +++ b/app/services/reporting/call_import_evaluation_pdf_report.py @@ -96,6 +96,7 @@ class MetricReportSummary: weekly_delta_detail: str | None = None weekly_delta_why: str | None = None group_name: str = "Quality Metrics" + lower_is_better: bool = False class CallImportEvaluationPdfReportService: @@ -205,6 +206,7 @@ def _summarize_metrics( description=metric.description or "", is_business_metric=self._is_user_insight_metric(metric), group_name=self._metric_group_name(metric), + lower_is_better=self._metric_lower_is_better(metric), ) for metric in metrics ] @@ -242,7 +244,7 @@ def _summarize_metrics( is_failure = ( is_metric_failure(eval_row, metric, policy) if metric is not None and policy is not None - else self._is_flagged(value) + else self._is_flagged(value, metric) ) if is_failure: summary.flagged_count += 1 @@ -388,10 +390,22 @@ def _score_value(self, score: dict[str, Any], metric: Metric | None) -> Any: return score.get("chosen_child_name") or score.get("value") return score.get("value") - def _is_flagged(self, value: Any) -> bool: + def _metric_lower_is_better(self, metric: Metric | None) -> bool: + if metric is None: + return False + from app.services.metrics.computed_metrics import ( + COMPUTED_TRANSCRIPT_SCORING_METHODS, + metric_scoring_method_key, + ) + + return metric_scoring_method_key(metric) in COMPUTED_TRANSCRIPT_SCORING_METHODS + + def _is_flagged(self, value: Any, metric: Metric | None = None) -> bool: if isinstance(value, bool): return value number = self._coerce_number(value) + if number is not None and self._metric_lower_is_better(metric): + return number > 0.25 if number is not None: return number < 0.5 if isinstance(value, str): @@ -445,6 +459,8 @@ def _measurement_label(self, summary: MetricReportSummary) -> str: return "Categorical metric measured as response distribution" def _metric_direction_label(self, summary: MetricReportSummary) -> str: + if summary.lower_is_better: + return "Lower is better" if summary.numeric_values: return "Higher score is better" return "Lower rate is better" @@ -2181,262 +2197,318 @@ def _render_html(self, payload: dict[str, Any]) -> str: + + + {repeat_header_markup} +
+ {brand_header_markup} +
Quality Metric Audit · Weekly
+
+

{html.escape(payload["title"])}

+
{html.escape(payload["subtitle"])}
+
+
Client{html.escape(payload["vendor_name"])}
+
Use Case{html.escape(payload.get("use_case") or "Not specified")}
+
Window{html.escape(payload.get("period_display") or "Not specified")}
+
Calls{payload["evaluation"].total_rows}
+
Audit Set{len(metrics)} metrics
+
+
+ {f''' +
+

01 Audit Summary

+
{audit_summary_markup}
+ {top_metric_strip_markup} + {top_metric_delta_markup} +
+ ''' if show_audit_summary else ''} + {quality_panel_markup} + {business_section} + {failure_diagnostics_section} + {prompt_improvements_section} + {design_notes_section} + {f''' +
+

This report is generated from completed Call Import evaluation results. Metrics are derived from the saved evaluation outputs and the transcript source configured for the run.{html.escape(weekly_methodology)}

+
+ ''' if show_methodology else ''} +
+
+
Powered by EfficientAI
+
Voice AI Evaluation Platform
+ +
+
+
Evaluation ID: {html.escape(str(getattr(payload["evaluation"], "id", "")))}
+
Generated: {html.escape(payload["generated_at_iso"])}
+
Call import reports by EfficientAI Cloud
+
+
+ + + """ + + def _report_css(self) -> str: + """Shared stylesheet for EfficientAI PDF reports.""" + return """ + @page { size: A4; margin: 18mm 14mm 17mm 14mm; - @top-left {{ content: element(repeatHeader); }} - @bottom-center {{ content: "Page " counter(page) " of " counter(pages); color: #667085; font-size: 9px; }} - }} - @page:first {{ - @top-left {{ content: none; }} - }} - body {{ font-family: Helvetica, Arial, sans-serif; color: #111827; font-size: 11px; line-height: 1.4; }} - header {{ border-bottom: 0; padding-bottom: 8px; margin-bottom: 12px; }} - .brand-header {{ margin-bottom: 10px; }} - .brand-logo-row {{ display: flex; align-items: center; justify-content: space-between; gap: 32px; min-height: 58px; margin-bottom: 8px; }} - .brand-logo-slot {{ flex: 1; min-width: 0; display: flex; flex-direction: column; gap: 5px; }} - .brand-logo-slot-internal {{ align-items: flex-start; }} - .brand-logo-slot-external {{ align-items: flex-end; text-align: right; }} - .brand-placeholder {{ display: block; min-height: 1px; }} - .brand-logo {{ max-width: 230px; max-height: 58px; object-fit: contain; }} - .repeat-page-header {{ position: running(repeatHeader); height: 10mm; }} - .repeat-page-logo {{ max-width: 92px; max-height: 28px; object-fit: contain; }} - .brand-text {{ margin-top: 2px; font-size: 15px; font-weight: 800; line-height: 1.1; color: #111827; letter-spacing: .16em; text-transform: uppercase; }} - .title-rule {{ height: 2px; background: #b85f4b; margin: 10px 0 10px; }} - .eyebrow {{ text-transform: uppercase; letter-spacing: .34em; font-size: 10px; color: #b85f4b; font-weight: 800; margin-bottom: 10px; }} - .eyebrow .dotsep {{ color: #7a756d; padding: 0 16px; letter-spacing: 0; }} - .eyebrow .muted {{ color: #7a756d; }} - h1 {{ font-size: 28px; font-weight: 800; margin: 4px 0; letter-spacing: .1px; }} - .subtitle {{ font-size: 13px; color: #666; margin-bottom: 10px; }} - h2 {{ font-size: 16px; margin: 8px 0 6px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; break-after: avoid; page-break-after: avoid; }} - h3 {{ font-size: 13px; margin: 0; break-after: avoid; page-break-after: avoid; }} - .report-body section {{ margin: 0 0 10px; padding: 0; }} - .report-body section > .method:first-of-type {{ margin-top: 0; margin-bottom: 6px; }} - .metric-group {{ margin-bottom: 8px; break-inside: auto; page-break-inside: auto; }} - .meta {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; }} - .summary {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }} - .metric-group-title {{ font-size: 13px; margin: 8px 0 6px; color: #374151; text-transform: uppercase; letter-spacing: .08em; break-after: avoid; page-break-after: avoid; }} - .box {{ border: 1px solid #d9dee8; padding: 7px 8px; border-radius: 6px; background: #f7f9fc; }} - .box strong {{ display: block; font-size: 13px; color: #111827; }} - .box span {{ color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .04em; }} - .metric {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 8px; background: #fff; }} - .metric-compact-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-bottom: 6px; break-inside: auto; page-break-inside: auto; }} - .metric-compact {{ padding: 9px 10px 8px; margin-bottom: 0; min-height: 0; break-inside: avoid; page-break-inside: avoid; }} - .metric-compact-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 6px; }} - .metric-compact-head .metric-title {{ font-size: 9px; letter-spacing: .08em; color: #667085; font-weight: 800; }} - .metric-compact-status {{ width: 7px; height: 7px; border-radius: 50%; background: #16a34a; margin-top: 2px; flex-shrink: 0; }} - .metric-compact-body {{ display: flex; gap: 8px; align-items: stretch; }} - .metric-compact-hero {{ display: flex; align-items: flex-end; justify-content: space-between; gap: 8px; }} - .metric-compact-main {{ flex: 1; min-width: 0; }} - .metric-compact .metric-percent {{ font-size: 24px; margin-bottom: 4px; }} - .metric-compact-delta {{ margin-top: 2px; }} - .metric-compact-delta-row {{ display: grid; grid-template-columns: 68px 1fr auto; gap: 6px; align-items: center; font-size: 9px; color: #475569; }} - .metric-compact-delta-row span {{ text-transform: lowercase; color: #667085; }} - .metric-compact-delta-row strong {{ font-size: 10px; text-align: right; }} - .metric-compact-delta-row small {{ color: #667085; white-space: nowrap; }} - .metric-compact-delta.delta-good strong {{ color: #047857; }} - .metric-compact-delta.delta-bad strong {{ color: #b42318; }} - .metric-compact-delta.delta-flat strong {{ color: #475569; }} - .metric-compact-delta-detail {{ display: block; margin-top: 3px; color: #667085; font-size: 9px; line-height: 1.3; }} - .metric-compact-sparkline {{ flex-shrink: 0; width: 118px; }} - .metric-sparkline {{ display: block; width: 100%; max-width: 118px; }} - .metric-sparkline-area {{ fill: rgba(22, 163, 74, 0.12); stroke: none; }} - .metric-sparkline-line {{ fill: none; stroke: #16a34a; stroke-width: 1.6; stroke-linecap: round; stroke-linejoin: round; }} - .metric-sparkline-dot {{ fill: #16a34a; }} - .metric-sparkline-label {{ fill: #94a3b8; font-size: 7px; font-family: Helvetica, Arial, sans-serif; }} - .metric-compact-meaning {{ margin: 8px 0 0; font-size: 10px; line-height: 1.35; color: #374151; }} - .metric-compact-meta {{ margin-top: 6px; color: #94a3b8; font-size: 8px; letter-spacing: .02em; }} - .metric-head {{ display: flex; justify-content: space-between; gap: 12px; border-bottom: 1px solid #eef1f5; padding-bottom: 6px; margin-bottom: 7px; }} - .metric-title {{ font-size: 12px; text-transform: uppercase; letter-spacing: .04em; font-weight: 800; color: #1f2937; }} - .dot {{ display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: #b85f4b; margin-right: 8px; }} - .metric-type {{ color: #7a756d; font-size: 10px; text-transform: uppercase; margin-top: 3px; }} - .direction-badge {{ border: 1px solid #d7cfc2; background: #fff8f5; padding: 5px 8px; color: #7a756d; align-self: start; text-align: right; min-width: 104px; }} - .direction-badge span {{ display: block; font-size: 9px; text-transform: uppercase; font-weight: 800; letter-spacing: .04em; }} - .direction-badge strong {{ display: block; margin-top: 2px; color: #b85f4b; font-size: 11px; text-transform: uppercase; }} - .metric-hero {{ display: flex; align-items: flex-end; justify-content: space-between; gap: 12px; }} - .metric-percent {{ font-size: 27px; line-height: 1; font-weight: 800; color: #111; }} - .metric-count {{ color: #7a756d; font-size: 10px; letter-spacing: .04em; font-weight: 800; }} - .metric-count strong {{ color: #111827; font-size: 13px; letter-spacing: 0; margin-left: 8px; }} - .metric-bar {{ height: 8px; background: #e7ddd1; border: 1px solid #d7cfc2; margin: 8px 0; }} - .metric-bar-fill {{ height: 100%; background: #c7725e; }} - .metric-grid {{ display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 8px; }} - .metric-grid div {{ background: #f8fafc; border-radius: 6px; padding: 7px 8px; min-height: 40px; }} - .metric-grid strong, .measurement-grid strong {{ display: block; color: #111827; font-size: 13px; line-height: 1.25; overflow-wrap: anywhere; word-break: normal; }} - .metric-grid small, .measurement-grid small {{ display: block; margin-top: 4px; color: #667085; font-size: 9px; line-height: 1.2; text-transform: uppercase; letter-spacing: .04em; }} - .delta-card {{ margin-top: 8px; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; }} - .delta-head {{ display: flex; align-items: center; justify-content: space-between; gap: 10px; margin-bottom: 5px; }} - .delta-head span {{ color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .05em; font-weight: 800; }} - .delta-head strong {{ font-size: 12px; }} - .delta-good .delta-head strong {{ color: #047857; }} - .delta-bad .delta-head strong {{ color: #b42318; }} - .delta-flat .delta-head strong {{ color: #475569; }} - .delta-bars {{ display: grid; gap: 4px; }} - .delta-row {{ display: grid; grid-template-columns: 58px 1fr 42px; gap: 7px; align-items: center; }} - .delta-row span {{ color: #475569; font-size: 9px; font-weight: 700; }} - .delta-row b {{ text-align: right; font-size: 9px; }} - .delta-track {{ height: 7px; background: #e7ddd1; border: 1px solid #d7cfc2; }} - .delta-fill {{ height: 100%; }} - .delta-fill-prev {{ background: #94a3b8; }} - .delta-fill-current {{ background: #c7725e; }} - .delta-benchmark {{ margin-top: 4px; color: #475569; font-size: 9px; line-height: 1.3; }} - .measurement-grid {{ display: grid; grid-template-columns: 2fr 1fr; gap: 8px; margin-top: 8px; }} - .measurement-grid div {{ background: #fff8f5; border: 1px solid #f3d3c5; border-radius: 6px; padding: 7px 8px; min-height: 40px; }} - .distribution {{ margin: 8px 0 0; color: #374151; }} - .meaning {{ margin: 8px 0 0; color: #444; font-size: 13px; line-height: 1.45; }} - .distribution-bars {{ margin-top: 10px; }} - .subhead {{ font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: #7a756d; font-weight: 800; margin-bottom: 5px; }} - .dist-row {{ display: grid; grid-template-columns: 110px 1fr 74px; gap: 8px; align-items: center; margin: 4px 0; }} - .dist-label {{ font-size: 10px; font-weight: 700; color: #1f2937; overflow-wrap: anywhere; }} - .dist-track {{ height: 9px; background: #e7ddd1; border: 1px solid #d7cfc2; }} - .dist-fill {{ height: 100%; background: #c7725e; }} - .dist-value {{ text-align: right; font-size: 10px; font-weight: 800; color: #111827; }} - .empty-bars {{ color: #667085; font-size: 10px; font-style: italic; }} - .insight-block {{ break-inside: auto; page-break-inside: auto; margin: 8px 0 10px; }} - .insight-heading {{ margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} - .insight-heading h3 {{ margin: 0; }} - .insight-body {{ display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 10px; align-items: start; break-inside: auto; page-break-inside: auto; }} - .insight-table-col {{ min-width: 0; break-inside: auto; page-break-inside: auto; }} - .insight-sidecol {{ display: grid; gap: 8px; min-width: 0; }} - .insight-overview-box {{ break-inside: auto; page-break-inside: auto; margin: 0 0 10px; padding: 10px 12px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; }} - .insight-overview-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #7a756d; font-weight: 800; margin-bottom: 6px; }} - .insight-overview-box p {{ margin: 0; color: #374151; line-height: 1.5; }} - .insight-callout {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; }} - .insight-observation-box {{ background: #fdf3f0; border-color: #f0d4cb; }} - .insight-evidence-box {{ background: #faf6ef; border-color: #eadfce; }} - .insight-callout-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #b85f4b; font-weight: 800; margin-bottom: 6px; }} - .insight-callout p {{ margin: 0 0 6px; color: #374151; line-height: 1.45; }} - .insight-callout p:last-child {{ margin-bottom: 0; }} - .insight-evidence-id {{ color: #7a756d; font-size: 9px; margin-top: 4px; }} - .insight-table td:nth-child(2), .insight-table td:nth-child(4) {{ white-space: nowrap; font-weight: 800; }} - .insight-track {{ height: 9px; background: #e8edff; border: 1px solid #cfd8ff; min-width: 120px; }} - .insight-fill {{ height: 100%; background: #4f46e5; }} - .failure-diagnostics {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} - .failure-diagnostics-intro {{ break-inside: auto; margin-bottom: 6px; max-width: 100%; }} - .failure-diagnostics-intro h2 {{ margin-bottom: 4px; }} - .failure-diagnostics .method {{ margin: 0 0 6px; }} - .failure-diagnostics .insight-block {{ margin: 6px 0 8px; break-inside: auto; }} - .failure-diagnostics-metrics {{ display: grid; gap: 8px; max-width: 100%; }} - .fd-summary-box {{ + @top-left { content: element(repeatHeader); } + @bottom-center { content: "Page " counter(page) " of " counter(pages); color: #667085; font-size: 9px; } + } + @page:first { + @top-left { content: none; } + } + body { font-family: Helvetica, Arial, sans-serif; color: #111827; font-size: 11px; line-height: 1.4; } + header { border-bottom: 0; padding-bottom: 8px; margin-bottom: 12px; } + .brand-header { margin-bottom: 10px; } + .brand-logo-row { display: flex; align-items: center; justify-content: space-between; gap: 32px; min-height: 58px; margin-bottom: 8px; } + .brand-logo-slot { flex: 1; min-width: 0; display: flex; flex-direction: column; gap: 5px; } + .brand-logo-slot-internal { align-items: flex-start; } + .brand-logo-slot-external { align-items: flex-end; text-align: right; } + .brand-placeholder { display: block; min-height: 1px; } + .brand-logo { max-width: 230px; max-height: 58px; object-fit: contain; } + .repeat-page-header { position: running(repeatHeader); height: 10mm; } + .repeat-page-logo { max-width: 92px; max-height: 28px; object-fit: contain; } + .brand-text { margin-top: 2px; font-size: 15px; font-weight: 800; line-height: 1.1; color: #111827; letter-spacing: .16em; text-transform: uppercase; } + .title-rule { height: 2px; background: #b85f4b; margin: 10px 0 10px; } + .eyebrow { text-transform: uppercase; letter-spacing: .34em; font-size: 10px; color: #b85f4b; font-weight: 800; margin-bottom: 10px; } + .eyebrow .dotsep { color: #7a756d; padding: 0 16px; letter-spacing: 0; } + .eyebrow .muted { color: #7a756d; } + h1 { font-size: 28px; font-weight: 800; margin: 4px 0; letter-spacing: .1px; } + .subtitle { font-size: 13px; color: #666; margin-bottom: 10px; } + h2 { font-size: 16px; margin: 8px 0 6px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; break-after: avoid; page-break-after: avoid; } + h3 { font-size: 13px; margin: 0; break-after: avoid; page-break-after: avoid; } + .report-body section { margin: 0 0 10px; padding: 0; } + .report-body section > .method:first-of-type { margin-top: 0; margin-bottom: 6px; } + .metric-group { margin-bottom: 8px; break-inside: auto; page-break-inside: auto; } + .meta { display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; } + .summary { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; } + .metric-group-title { font-size: 13px; margin: 8px 0 6px; color: #374151; text-transform: uppercase; letter-spacing: .08em; break-after: avoid; page-break-after: avoid; } + .box { border: 1px solid #d9dee8; padding: 7px 8px; border-radius: 6px; background: #f7f9fc; } + .box strong { display: block; font-size: 13px; color: #111827; } + .box span { color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .04em; } + .metric { break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 8px; background: #fff; } + .metric-compact-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-bottom: 6px; break-inside: auto; page-break-inside: auto; } + .metric-compact { padding: 9px 10px 8px; margin-bottom: 0; min-height: 0; break-inside: avoid; page-break-inside: avoid; } + .metric-compact-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 6px; } + .metric-compact-head .metric-title { font-size: 9px; letter-spacing: .08em; color: #667085; font-weight: 800; } + .metric-compact-status { width: 7px; height: 7px; border-radius: 50%; background: #16a34a; margin-top: 2px; flex-shrink: 0; } + .metric-compact-body { display: flex; gap: 8px; align-items: stretch; } + .metric-compact-hero { display: flex; align-items: flex-end; justify-content: space-between; gap: 8px; } + .metric-compact-main { flex: 1; min-width: 0; } + .metric-compact .metric-percent { font-size: 24px; margin-bottom: 4px; } + .metric-compact-delta { margin-top: 2px; } + .metric-compact-delta-row { display: grid; grid-template-columns: 68px 1fr auto; gap: 6px; align-items: center; font-size: 9px; color: #475569; } + .metric-compact-delta-row span { text-transform: lowercase; color: #667085; } + .metric-compact-delta-row strong { font-size: 10px; text-align: right; } + .metric-compact-delta-row small { color: #667085; white-space: nowrap; } + .metric-compact-delta.delta-good strong { color: #047857; } + .metric-compact-delta.delta-bad strong { color: #b42318; } + .metric-compact-delta.delta-flat strong { color: #475569; } + .metric-compact-delta-detail { display: block; margin-top: 3px; color: #667085; font-size: 9px; line-height: 1.3; } + .metric-compact-sparkline { flex-shrink: 0; width: 118px; } + .metric-sparkline { display: block; width: 100%; max-width: 118px; } + .metric-sparkline-area { fill: rgba(22, 163, 74, 0.12); stroke: none; } + .metric-sparkline-line { fill: none; stroke: #16a34a; stroke-width: 1.6; stroke-linecap: round; stroke-linejoin: round; } + .metric-sparkline-dot { fill: #16a34a; } + .metric-sparkline-label { fill: #94a3b8; font-size: 7px; font-family: Helvetica, Arial, sans-serif; } + .metric-compact-meaning { margin: 8px 0 0; font-size: 10px; line-height: 1.35; color: #374151; } + .metric-compact-meta { margin-top: 6px; color: #94a3b8; font-size: 8px; letter-spacing: .02em; } + .metric-head { display: flex; justify-content: space-between; gap: 12px; border-bottom: 1px solid #eef1f5; padding-bottom: 6px; margin-bottom: 7px; } + .metric-title { font-size: 12px; text-transform: uppercase; letter-spacing: .04em; font-weight: 800; color: #1f2937; } + .dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: #b85f4b; margin-right: 8px; } + .metric-type { color: #7a756d; font-size: 10px; text-transform: uppercase; margin-top: 3px; } + .direction-badge { border: 1px solid #d7cfc2; background: #fff8f5; padding: 5px 8px; color: #7a756d; align-self: start; text-align: right; min-width: 104px; } + .direction-badge span { display: block; font-size: 9px; text-transform: uppercase; font-weight: 800; letter-spacing: .04em; } + .direction-badge strong { display: block; margin-top: 2px; color: #b85f4b; font-size: 11px; text-transform: uppercase; } + .metric-hero { display: flex; align-items: flex-end; justify-content: space-between; gap: 12px; } + .metric-percent { font-size: 27px; line-height: 1; font-weight: 800; color: #111; } + .metric-count { color: #7a756d; font-size: 10px; letter-spacing: .04em; font-weight: 800; } + .metric-count strong { color: #111827; font-size: 13px; letter-spacing: 0; margin-left: 8px; } + .metric-bar { height: 8px; background: #e7ddd1; border: 1px solid #d7cfc2; margin: 8px 0; } + .metric-bar-fill { height: 100%; background: #c7725e; } + .metric-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 8px; } + .metric-grid div { background: #f8fafc; border-radius: 6px; padding: 7px 8px; min-height: 40px; } + .metric-grid strong, .measurement-grid strong { display: block; color: #111827; font-size: 13px; line-height: 1.25; overflow-wrap: anywhere; word-break: normal; } + .metric-grid small, .measurement-grid small { display: block; margin-top: 4px; color: #667085; font-size: 9px; line-height: 1.2; text-transform: uppercase; letter-spacing: .04em; } + .delta-card { margin-top: 8px; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; } + .delta-head { display: flex; align-items: center; justify-content: space-between; gap: 10px; margin-bottom: 5px; } + .delta-head span { color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .05em; font-weight: 800; } + .delta-head strong { font-size: 12px; } + .delta-good .delta-head strong { color: #047857; } + .delta-bad .delta-head strong { color: #b42318; } + .delta-flat .delta-head strong { color: #475569; } + .delta-bars { display: grid; gap: 4px; } + .delta-row { display: grid; grid-template-columns: 58px 1fr 42px; gap: 7px; align-items: center; } + .delta-row span { color: #475569; font-size: 9px; font-weight: 700; } + .delta-row b { text-align: right; font-size: 9px; } + .delta-track { height: 7px; background: #e7ddd1; border: 1px solid #d7cfc2; } + .delta-fill { height: 100%; } + .delta-fill-prev { background: #94a3b8; } + .delta-fill-current { background: #c7725e; } + .delta-benchmark { margin-top: 4px; color: #475569; font-size: 9px; line-height: 1.3; } + .measurement-grid { display: grid; grid-template-columns: 2fr 1fr; gap: 8px; margin-top: 8px; } + .measurement-grid div { background: #fff8f5; border: 1px solid #f3d3c5; border-radius: 6px; padding: 7px 8px; min-height: 40px; } + .distribution { margin: 8px 0 0; color: #374151; } + .meaning { margin: 8px 0 0; color: #444; font-size: 13px; line-height: 1.45; } + .distribution-bars { margin-top: 10px; } + .subhead { font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: #7a756d; font-weight: 800; margin-bottom: 5px; } + .dist-row { display: grid; grid-template-columns: 110px 1fr 74px; gap: 8px; align-items: center; margin: 4px 0; } + .dist-label { font-size: 10px; font-weight: 700; color: #1f2937; overflow-wrap: anywhere; } + .dist-track { height: 9px; background: #e7ddd1; border: 1px solid #d7cfc2; } + .dist-fill { height: 100%; background: #c7725e; } + .dist-value { text-align: right; font-size: 10px; font-weight: 800; color: #111827; } + .empty-bars { color: #667085; font-size: 10px; font-style: italic; } + .insight-block { break-inside: auto; page-break-inside: auto; margin: 8px 0 10px; } + .insight-heading { margin-bottom: 6px; break-after: avoid; page-break-after: avoid; } + .insight-heading h3 { margin: 0; } + .insight-body { display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 10px; align-items: start; break-inside: auto; page-break-inside: auto; } + .insight-table-col { min-width: 0; break-inside: auto; page-break-inside: auto; } + .insight-sidecol { display: grid; gap: 8px; min-width: 0; } + .insight-overview-box { break-inside: auto; page-break-inside: auto; margin: 0 0 10px; padding: 10px 12px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; } + .insight-overview-label { font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #7a756d; font-weight: 800; margin-bottom: 6px; } + .insight-overview-box p { margin: 0; color: #374151; line-height: 1.5; } + .insight-callout { break-inside: avoid; page-break-inside: avoid; border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; } + .insight-observation-box { background: #fdf3f0; border-color: #f0d4cb; } + .insight-evidence-box { background: #faf6ef; border-color: #eadfce; } + .insight-callout-label { font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #b85f4b; font-weight: 800; margin-bottom: 6px; } + .insight-callout p { margin: 0 0 6px; color: #374151; line-height: 1.45; } + .insight-callout p:last-child { margin-bottom: 0; } + .insight-evidence-id { color: #7a756d; font-size: 9px; margin-top: 4px; } + .insight-table td:nth-child(2), .insight-table td:nth-child(4) { white-space: nowrap; font-weight: 800; } + .insight-track { height: 9px; background: #e8edff; border: 1px solid #cfd8ff; min-width: 120px; } + .insight-fill { height: 100%; background: #4f46e5; } + .failure-diagnostics { margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; } + .failure-diagnostics-intro { break-inside: auto; margin-bottom: 6px; max-width: 100%; } + .failure-diagnostics-intro h2 { margin-bottom: 4px; } + .failure-diagnostics .method { margin: 0 0 6px; } + .failure-diagnostics .insight-block { margin: 6px 0 8px; break-inside: auto; } + .failure-diagnostics-metrics { display: grid; gap: 8px; max-width: 100%; } + .fd-summary-box { break-inside: auto; margin: 0 0 8px; padding: 8px 10px; max-width: 100%; overflow: hidden; box-sizing: border-box; - }} - .fd-summary-box .insight-table {{ + } + .fd-summary-box .insight-table { margin-top: 4px; width: 100%; max-width: 100%; table-layout: fixed; - }} - .fd-summary-table th, .fd-level1-table th {{ font-size: 9px; }} - .fd-summary-table td, .fd-level1-table td {{ + } + .fd-summary-table th, .fd-level1-table th { font-size: 9px; } + .fd-summary-table td, .fd-level1-table td { font-size: 10px; padding: 5px 6px; overflow-wrap: anywhere; white-space: normal; - }} - .fd-summary-table .fd-col-metric {{ width: 24%; }} - .fd-summary-table .fd-col-num {{ width: 11%; }} - .fd-summary-table .fd-col-preview {{ width: 54%; }} - .failure-diagnostics .fd-summary-table td.fd-col-num {{ + } + .fd-summary-table .fd-col-metric { width: 24%; } + .fd-summary-table .fd-col-num { width: 11%; } + .fd-summary-table .fd-col-preview { width: 54%; } + .failure-diagnostics .fd-summary-table td.fd-col-num { white-space: nowrap; font-weight: 800; text-align: right; - }} - .failure-diagnostics .fd-summary-table td.fd-col-preview {{ + } + .failure-diagnostics .fd-summary-table td.fd-col-preview { white-space: normal; font-weight: 400; - }} + } .failure-diagnostics .fd-level1-table td:nth-child(2), - .failure-diagnostics .fd-level1-table td:nth-child(3) {{ + .failure-diagnostics .fd-level1-table td:nth-child(3) { white-space: normal; font-weight: 400; - }} - .fd-preview-list {{ + } + .fd-preview-list { margin: 0; padding-left: 14px; list-style: disc; color: #374151; line-height: 1.35; - }} - .fd-preview-list li {{ margin: 0 0 3px; }} - .fd-preview-list li:last-child {{ margin-bottom: 0; }} - .fd-preview-more {{ color: #6b7280; font-style: italic; list-style: none; margin-left: -14px; }} - .fd-level1-table {{ + } + .fd-preview-list li { margin: 0 0 3px; } + .fd-preview-list li:last-child { margin-bottom: 0; } + .fd-preview-more { color: #6b7280; font-style: italic; list-style: none; margin-left: -14px; } + .fd-level1-table { width: 100%; max-width: 100%; table-layout: fixed; margin: 0 0 8px; - }} - .fd-level1-table .fd-col-index {{ width: 6%; }} - .fd-level1-table .fd-col-cluster {{ width: 34%; }} - .fd-level1-table .fd-col-gap {{ width: 14%; }} - .fd-level1-table .fd-col-num {{ width: 9%; }} - .fd-level1-table .fd-col-share {{ width: 9%; }} - .fd-level1-table .fd-col-bar {{ width: 28%; }} + } + .fd-level1-table .fd-col-index { width: 6%; } + .fd-level1-table .fd-col-cluster { width: 34%; } + .fd-level1-table .fd-col-gap { width: 14%; } + .fd-level1-table .fd-col-num { width: 9%; } + .fd-level1-table .fd-col-share { width: 9%; } + .fd-level1-table .fd-col-bar { width: 28%; } .failure-diagnostics .fd-level1-table td.fd-col-num, - .failure-diagnostics .fd-level1-table td:nth-child(5) {{ + .failure-diagnostics .fd-level1-table td:nth-child(5) { white-space: nowrap; font-weight: 800; - }} - .failure-diagnostics .insight-track {{ + } + .failure-diagnostics .insight-track { min-width: 0; max-width: 100%; width: 100%; - }} - .fd-summary-discovered-row td {{ background: #f8fafc; }} - .fd-num {{ color: #6b7280; font-weight: 800; width: 24px; }} - .fd-breakdown-caption {{ margin: 0 0 4px; font-size: 9px; color: #6b7280; text-transform: uppercase; letter-spacing: .04em; font-weight: 700; }} - .fd-discovered-block {{ margin-top: 4px; padding-top: 6px; border-top: 1px solid #e5e7eb; max-width: 100%; }} - .fd-rca-block {{ margin: 10px 0 12px; max-width: 100%; overflow: hidden; box-sizing: border-box; }} - .fd-rca-block h3 {{ margin: 0 0 4px; font-size: 12px; }} - .fd-rca-base {{ margin: 0 0 6px; font-size: 9px; }} - .fd-rca-patterns-wrap {{ width: 100%; max-width: 100%; margin: 0 auto; overflow: hidden; box-sizing: border-box; }} - .fd-rca-table {{ table-layout: fixed; width: 100%; max-width: 100%; box-sizing: border-box; }} - .fd-rca-table td, .fd-rca-table th {{ font-size: 10px; padding: 5px 4px; overflow-wrap: anywhere; word-break: break-word; box-sizing: border-box; }} - .fd-rca-patterns-table {{ margin-left: auto; margin-right: auto; }} - .fd-rca-patterns-table .fd-rca-col-finding {{ width: 41%; text-align: left; }} - .fd-rca-patterns-table th.fd-rca-col-finding {{ text-align: left; }} - .fd-rca-patterns-table .fd-rca-col-pct {{ width: 12%; white-space: nowrap; text-align: center; font-weight: 800; vertical-align: top; }} - .fd-rca-patterns-table th.fd-rca-col-pct {{ text-align: center; }} - .fd-rca-patterns-table .fd-rca-col-bar {{ width: 29%; vertical-align: middle; padding-left: 6px; padding-right: 6px; }} - .fd-rca-patterns-table th.fd-rca-col-bar {{ text-align: center; }} - .fd-rca-patterns-table .fd-rca-col-count {{ width: 18%; white-space: nowrap; text-align: center; font-weight: 800; vertical-align: top; }} - .fd-rca-patterns-table th.fd-rca-col-count {{ text-align: center; }} - .fd-rca-track {{ height: 9px; background: #e7ddd1; border: 1px solid #d7cfc2; min-width: 0; max-width: 100%; width: 100%; box-sizing: border-box; }} - .fd-rca-fill {{ height: 100%; background: #c7725e; max-width: 100%; }} - .fd-rca-hotspots-table .fd-rca-col-pct {{ white-space: nowrap; text-align: center; font-weight: 800; width: 14%; }} - .fd-rca-hotspots-table .fd-rca-col-bar {{ width: 28%; }} - .fd-rca-hotspots-table .fd-rca-col-count {{ white-space: nowrap; text-align: center; font-weight: 800; width: 14%; vertical-align: top; }} - .fd-finding-sub {{ margin-top: 3px; font-size: 9px; color: #6b7280; font-weight: 400; line-height: 1.35; }} - .fd-evidence-line {{ font-size: 8px; color: #6b7280; font-weight: 600; line-height: 1.3; }} - .fd-evidence-primary {{ font-size: 10px; font-weight: 800; color: #111827; line-height: 1.3; }} - .fd-cluster-appendix {{ margin-top: 14px; padding-top: 10px; border-top: 1px solid #e5e7eb; break-inside: avoid; }} - .fd-cluster-appendix h3 {{ margin: 0 0 4px; font-size: 11px; }} - .fd-rca-interpretation {{ margin-top: 6px; }} - .fd-rca-summary-grid {{ display: grid; grid-template-columns: 1.2fr 1fr; gap: 10px; align-items: start; }} - .fd-rca-twocol {{ display: grid; gap: 8px; }} - .fd-rca-mini-table {{ table-layout: fixed; width: 100%; }} - .fd-why-flagged {{ margin: 0 0 6px; font-size: 10px; color: #374151; line-height: 1.4; }} - .insight-evidence-id a {{ color: #b85f4b; font-weight: 700; text-decoration: none; }} - .metric-cluster-metric-group {{ + } + .fd-summary-discovered-row td { background: #f8fafc; } + .fd-num { color: #6b7280; font-weight: 800; width: 24px; } + .fd-breakdown-caption { margin: 0 0 4px; font-size: 9px; color: #6b7280; text-transform: uppercase; letter-spacing: .04em; font-weight: 700; } + .fd-discovered-block { margin-top: 4px; padding-top: 6px; border-top: 1px solid #e5e7eb; max-width: 100%; } + .fd-rca-block { margin: 10px 0 12px; max-width: 100%; overflow: hidden; box-sizing: border-box; } + .fd-rca-block h3 { margin: 0 0 4px; font-size: 12px; } + .fd-rca-base { margin: 0 0 6px; font-size: 9px; } + .fd-rca-patterns-wrap { width: 100%; max-width: 100%; margin: 0 auto; overflow: hidden; box-sizing: border-box; } + .fd-rca-table { table-layout: fixed; width: 100%; max-width: 100%; box-sizing: border-box; } + .fd-rca-table td, .fd-rca-table th { font-size: 10px; padding: 5px 4px; overflow-wrap: anywhere; word-break: break-word; box-sizing: border-box; } + .fd-rca-patterns-table { margin-left: auto; margin-right: auto; } + .fd-rca-patterns-table .fd-rca-col-finding { width: 41%; text-align: left; } + .fd-rca-patterns-table th.fd-rca-col-finding { text-align: left; } + .fd-rca-patterns-table .fd-rca-col-pct { width: 12%; white-space: nowrap; text-align: center; font-weight: 800; vertical-align: top; } + .fd-rca-patterns-table th.fd-rca-col-pct { text-align: center; } + .fd-rca-patterns-table .fd-rca-col-bar { width: 29%; vertical-align: middle; padding-left: 6px; padding-right: 6px; } + .fd-rca-patterns-table th.fd-rca-col-bar { text-align: center; } + .fd-rca-patterns-table .fd-rca-col-count { width: 18%; white-space: nowrap; text-align: center; font-weight: 800; vertical-align: top; } + .fd-rca-patterns-table th.fd-rca-col-count { text-align: center; } + .fd-rca-track { height: 9px; background: #e7ddd1; border: 1px solid #d7cfc2; min-width: 0; max-width: 100%; width: 100%; box-sizing: border-box; } + .fd-rca-fill { height: 100%; background: #c7725e; max-width: 100%; } + .fd-rca-hotspots-table .fd-rca-col-pct { white-space: nowrap; text-align: center; font-weight: 800; width: 14%; } + .fd-rca-hotspots-table .fd-rca-col-bar { width: 28%; } + .fd-rca-hotspots-table .fd-rca-col-count { white-space: nowrap; text-align: center; font-weight: 800; width: 14%; vertical-align: top; } + .fd-finding-sub { margin-top: 3px; font-size: 9px; color: #6b7280; font-weight: 400; line-height: 1.35; } + .fd-evidence-line { font-size: 8px; color: #6b7280; font-weight: 600; line-height: 1.3; } + .fd-evidence-primary { font-size: 10px; font-weight: 800; color: #111827; line-height: 1.3; } + .fd-cluster-appendix { margin-top: 14px; padding-top: 10px; border-top: 1px solid #e5e7eb; break-inside: avoid; } + .fd-cluster-appendix h3 { margin: 0 0 4px; font-size: 11px; } + .fd-rca-interpretation { margin-top: 6px; } + .fd-rca-summary-grid { display: grid; grid-template-columns: 1.2fr 1fr; gap: 10px; align-items: start; } + .fd-rca-twocol { display: grid; gap: 8px; } + .fd-rca-mini-table { table-layout: fixed; width: 100%; } + .fd-why-flagged { margin: 0 0 6px; font-size: 10px; color: #374151; line-height: 1.4; } + .insight-evidence-id a { color: #b85f4b; font-weight: 700; text-decoration: none; } + .metric-cluster-metric-group { margin: 0; padding: 8px 0; border-top: 1px solid #e5e7eb; break-inside: auto; - }} - .metric-cluster-metric-group:first-child {{ border-top: none; padding-top: 4px; }} - .metric-cluster-metric-head {{ + } + .metric-cluster-metric-group:first-child { border-top: none; padding-top: 4px; } + .metric-cluster-metric-head { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; margin-bottom: 4px; - }} - .metric-cluster-metric-head h3 {{ margin: 0; font-size: 13px; }} - .metric-cluster-metric-head span {{ + } + .metric-cluster-metric-head h3 { margin: 0; font-size: 13px; } + .metric-cluster-metric-head span { color: #6b7280; font-size: 9px; font-weight: 700; @@ -2446,23 +2518,23 @@ def _render_html(self, payload: dict[str, Any]) -> str: text-align: right; flex-shrink: 1; min-width: 0; - }} - .metric-cluster-detail-section-title {{ margin-top: 6px; }} - .metric-cluster-details {{ display: grid; gap: 6px; }} - .metric-cluster-detail {{ + } + .metric-cluster-detail-section-title { margin-top: 6px; } + .metric-cluster-details { display: grid; gap: 6px; } + .metric-cluster-detail { border: 1px solid #e5e7eb; border-radius: 6px; background: #fff; padding: 6px 8px; break-inside: auto; page-break-inside: auto; - }} - .metric-cluster-detail-compact .insight-callout {{ margin-top: 4px; }} - .fd-cluster-observation {{ margin: 0 0 4px; font-size: 10px; color: #374151; line-height: 1.35; }} - .fd-omitted-metrics {{ margin: 6px 0 8px; font-size: 9px; }} - .metric-cluster-detail-head {{ display: flex; justify-content: space-between; align-items: baseline; gap: 8px; margin-bottom: 4px; }} - .metric-cluster-detail-head h4 {{ margin: 0; font-size: 11px; font-weight: 800; color: #111827; line-height: 1.3; }} - .metric-cluster-detail-head span {{ + } + .metric-cluster-detail-compact .insight-callout { margin-top: 4px; } + .fd-cluster-observation { margin: 0 0 4px; font-size: 10px; color: #374151; line-height: 1.35; } + .fd-omitted-metrics { margin: 6px 0 8px; font-size: 9px; } + .metric-cluster-detail-head { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; margin-bottom: 4px; } + .metric-cluster-detail-head h4 { margin: 0; font-size: 11px; font-weight: 800; color: #111827; line-height: 1.3; } + .metric-cluster-detail-head span { color: #6b7280; font-size: 9px; font-weight: 700; @@ -2472,65 +2544,65 @@ def _render_html(self, payload: dict[str, Any]) -> str: text-align: right; flex-shrink: 1; min-width: 0; - }} - .metric-cluster-detail-body {{ grid-template-columns: 1.1fr 0.9fr; gap: 10px; }} - .metric-cluster-detail-body .subhead {{ margin-bottom: 3px; }} - .metric-cluster-sub-bars {{ margin: 0; }} - .metric-cluster-sub-row {{ margin: 2px 0; grid-template-columns: 96px 1fr 62px; gap: 6px; }} - .metric-cluster-sub-fill {{ background: #6366f1; }} - .insight-callout-empty p {{ margin: 0; }} - .insight-observation, .insight-evidence {{ margin: 0; color: #374151; }} - .insight-evidence span {{ color: #7a756d; font-size: 9px; margin-left: 6px; }} - .design-notes {{ margin: 8px 0 0; padding-left: 18px; }} - .design-notes li {{ margin-bottom: 8px; line-height: 1.5; }} - .gap-badge {{ display: inline-block; font-size: 9px; font-weight: 800; letter-spacing: .04em; color: #b42318; text-transform: uppercase; }} - table {{ width: 100%; border-collapse: collapse; margin-top: 6px; break-inside: auto; page-break-inside: auto; }} - thead {{ display: table-header-group; }} - tr {{ break-inside: avoid; page-break-inside: avoid; }} - th, td {{ text-align: left; border-bottom: 1px solid #e5e7eb; padding: 7px; vertical-align: top; }} - th {{ background: #111827; color: #fff; font-size: 10px; text-transform: uppercase; }} - .method {{ color: #475467; line-height: 1.5; }} - .audit-summary p {{ margin: 0 0 7px; }} - .audit-summary ul {{ margin: 7px 0 10px; padding-left: 16px; }} - .audit-summary li {{ margin-bottom: 5px; }} - .audit-summary-section {{ margin-bottom: 0; break-after: auto; page-break-after: auto; }} - .audit-summary-section + section {{ margin-top: 0; }} - .audit-summary-section + section h2 {{ margin-top: 8px; }} - .audit-stat-strip {{ display: flex; gap: 0; margin: 10px 0 0; border-top: 1px solid #111827; break-after: auto; page-break-after: auto; }} - .audit-stat-strip:last-child {{ margin-bottom: 10px; }} - .audit-stat-card {{ flex: 1; min-width: 0; padding: 10px 12px 12px 0; }} - .audit-stat-card + .audit-stat-card {{ border-left: 1px solid #d1d5db; padding-left: 12px; }} - .audit-stat-rule {{ display: none; }} - .audit-stat-label {{ font-size: 8px; font-weight: 800; letter-spacing: .08em; text-transform: uppercase; color: #d16532; margin-bottom: 6px; line-height: 1.3; }} - .audit-stat-value {{ font-family: Georgia, 'Times New Roman', serif; font-size: 28px; font-weight: 700; color: #111827; line-height: 1; }} - .audit-delta-panel {{ break-inside: auto; page-break-inside: auto; margin: 6px 0 8px; padding-top: 6px; border-top: 1px solid #e5e7eb; }} - .audit-delta-title {{ font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} - .audit-delta-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; break-inside: auto; page-break-inside: auto; }} - .audit-delta-card {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 0; }} - .audit-delta-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 4px; }} - .audit-delta-head-main {{ min-width: 0; }} - .audit-delta-head span {{ display: block; color: #111827; font-size: 9px; font-weight: 900; letter-spacing: .06em; line-height: 1.25; text-transform: uppercase; overflow-wrap: anywhere; }} - .audit-delta-direction {{ display: block; margin-top: 2px; color: #94a3b8; font-size: 7px; font-weight: 600; letter-spacing: .03em; text-transform: none; }} - .audit-delta-head strong {{ flex-shrink: 0; font-size: 10px; line-height: 1.2; text-align: right; white-space: nowrap; }} - .audit-delta-card.delta-good .audit-delta-head strong {{ color: #047857; }} - .audit-delta-card.delta-bad .audit-delta-head strong {{ color: #b42318; }} - .audit-delta-card.delta-flat .audit-delta-head strong {{ color: #475569; }} - .audit-delta-chart {{ min-height: 50px; }} - .audit-delta-chart .metric-sparkline {{ width: 100%; max-width: 100%; }} - .audit-delta-empty {{ height: 50px; display: flex; align-items: center; justify-content: center; border: 1px dashed #cbd5e1; background: #fff; color: #667085; font-size: 9px; }} - .audit-delta-reason {{ margin: 6px 0 0; padding-top: 5px; border-top: 1px solid #e5e7eb; color: #374151; font-size: 10px; font-weight: 600; line-height: 1.45; }} - .report-footer {{ margin-top: 14px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; break-inside: avoid; page-break-inside: avoid; }} - .footer-left, .footer-right {{ display: flex; flex-direction: column; gap: 3px; }} - .footer-right {{ text-align: right; }} - .brand-title {{ color: #111827; font-weight: 800; font-size: 11px; letter-spacing: .2px; }} - .brand-link {{ color: #d16532; font-weight: 700; text-decoration: none; }} - .internal-report .insight-block {{ margin: 6px 0 8px; }} + } + .metric-cluster-detail-body { grid-template-columns: 1.1fr 0.9fr; gap: 10px; } + .metric-cluster-detail-body .subhead { margin-bottom: 3px; } + .metric-cluster-sub-bars { margin: 0; } + .metric-cluster-sub-row { margin: 2px 0; grid-template-columns: 96px 1fr 62px; gap: 6px; } + .metric-cluster-sub-fill { background: #6366f1; } + .insight-callout-empty p { margin: 0; } + .insight-observation, .insight-evidence { margin: 0; color: #374151; } + .insight-evidence span { color: #7a756d; font-size: 9px; margin-left: 6px; } + .design-notes { margin: 8px 0 0; padding-left: 18px; } + .design-notes li { margin-bottom: 8px; line-height: 1.5; } + .gap-badge { display: inline-block; font-size: 9px; font-weight: 800; letter-spacing: .04em; color: #b42318; text-transform: uppercase; } + table { width: 100%; border-collapse: collapse; margin-top: 6px; break-inside: auto; page-break-inside: auto; } + thead { display: table-header-group; } + tr { break-inside: avoid; page-break-inside: avoid; } + th, td { text-align: left; border-bottom: 1px solid #e5e7eb; padding: 7px; vertical-align: top; } + th { background: #111827; color: #fff; font-size: 10px; text-transform: uppercase; } + .method { color: #475467; line-height: 1.5; } + .audit-summary p { margin: 0 0 7px; } + .audit-summary ul { margin: 7px 0 10px; padding-left: 16px; } + .audit-summary li { margin-bottom: 5px; } + .audit-summary-section { margin-bottom: 0; break-after: auto; page-break-after: auto; } + .audit-summary-section + section { margin-top: 0; } + .audit-summary-section + section h2 { margin-top: 8px; } + .audit-stat-strip { display: flex; gap: 0; margin: 10px 0 0; border-top: 1px solid #111827; break-after: auto; page-break-after: auto; } + .audit-stat-strip:last-child { margin-bottom: 10px; } + .audit-stat-card { flex: 1; min-width: 0; padding: 10px 12px 12px 0; } + .audit-stat-card + .audit-stat-card { border-left: 1px solid #d1d5db; padding-left: 12px; } + .audit-stat-rule { display: none; } + .audit-stat-label { font-size: 8px; font-weight: 800; letter-spacing: .08em; text-transform: uppercase; color: #d16532; margin-bottom: 6px; line-height: 1.3; } + .audit-stat-value { font-family: Georgia, 'Times New Roman', serif; font-size: 28px; font-weight: 700; color: #111827; line-height: 1; } + .audit-delta-panel { break-inside: auto; page-break-inside: auto; margin: 6px 0 8px; padding-top: 6px; border-top: 1px solid #e5e7eb; } + .audit-delta-title { font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 6px; break-after: avoid; page-break-after: avoid; } + .audit-delta-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; break-inside: auto; page-break-inside: auto; } + .audit-delta-card { break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 0; } + .audit-delta-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 4px; } + .audit-delta-head-main { min-width: 0; } + .audit-delta-head span { display: block; color: #111827; font-size: 9px; font-weight: 900; letter-spacing: .06em; line-height: 1.25; text-transform: uppercase; overflow-wrap: anywhere; } + .audit-delta-direction { display: block; margin-top: 2px; color: #94a3b8; font-size: 7px; font-weight: 600; letter-spacing: .03em; text-transform: none; } + .audit-delta-head strong { flex-shrink: 0; font-size: 10px; line-height: 1.2; text-align: right; white-space: nowrap; } + .audit-delta-card.delta-good .audit-delta-head strong { color: #047857; } + .audit-delta-card.delta-bad .audit-delta-head strong { color: #b42318; } + .audit-delta-card.delta-flat .audit-delta-head strong { color: #475569; } + .audit-delta-chart { min-height: 50px; } + .audit-delta-chart .metric-sparkline { width: 100%; max-width: 100%; } + .audit-delta-empty { height: 50px; display: flex; align-items: center; justify-content: center; border: 1px dashed #cbd5e1; background: #fff; color: #667085; font-size: 9px; } + .audit-delta-reason { margin: 6px 0 0; padding-top: 5px; border-top: 1px solid #e5e7eb; color: #374151; font-size: 10px; font-weight: 600; line-height: 1.45; } + .report-footer { margin-top: 14px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; break-inside: avoid; page-break-inside: avoid; } + .footer-left, .footer-right { display: flex; flex-direction: column; gap: 3px; } + .footer-right { text-align: right; } + .brand-title { color: #111827; font-weight: 800; font-size: 11px; letter-spacing: .2px; } + .brand-link { color: #d16532; font-weight: 700; text-decoration: none; } + .internal-report .insight-block { margin: 6px 0 8px; } .internal-report .failure-diagnostics-intro h2, - .internal-report .prompt-improvements > h2 {{ + .internal-report .prompt-improvements > h2 { margin-top: 8px; - }} - .prompt-improvements {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} - .prompt-improvement-card {{ + } + .prompt-improvements { margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; } + .prompt-improvement-card { margin: 0 0 10px; padding: 8px 10px; border: 1px solid #e5e7eb; @@ -2540,19 +2612,19 @@ def _render_html(self, payload: dict[str, Any]) -> str: box-sizing: border-box; break-inside: auto; page-break-inside: auto; - }} - .prompt-improvements .metric-cluster-metric-head h3 {{ + } + .prompt-improvements .metric-cluster-metric-head h3 { flex: 1; min-width: 0; overflow-wrap: anywhere; word-break: break-word; - }} - .prompt-improvements .method {{ + } + .prompt-improvements .method { overflow-wrap: anywhere; word-break: break-word; max-width: 100%; - }} - .prompt-suggestion-block {{ + } + .prompt-suggestion-block { margin: 6px 0; padding: 8px 10px; border: 1px solid #e5e7eb; @@ -2568,14 +2640,14 @@ def _render_html(self, payload: dict[str, Any]) -> str: max-width: 100%; width: 100%; box-sizing: border-box; - }} - .prompt-improvement-meta {{ + } + .prompt-improvement-meta { display: inline-flex; flex-wrap: wrap; align-items: center; gap: 6px; - }} - .prompt-change-badge {{ + } + .prompt-change-badge { display: inline-block; font-size: 8px; font-weight: 800; @@ -2584,113 +2656,63 @@ def _render_html(self, payload: dict[str, Any]) -> str: border-radius: 4px; padding: 2px 5px; border: 1px solid transparent; - }} - .prompt-change-badge-edit {{ + } + .prompt-change-badge-edit { color: #9f1239; background: #fff1f2; border-color: #fecdd3; - }} - .prompt-change-badge-add {{ + } + .prompt-change-badge-add { color: #065f46; background: #ecfdf5; border-color: #a7f3d0; - }} - .prompt-change-grid {{ + } + .prompt-change-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin: 8px 0; - }} - .prompt-change-box {{ + } + .prompt-change-box { border-radius: 6px; padding: 8px 10px; min-width: 0; - }} - .prompt-change-before {{ + } + .prompt-change-before { border: 1px solid #fecdd3; background: #fff1f2; - }} - .prompt-change-after {{ + } + .prompt-change-after { border: 1px solid #a7f3d0; background: #ecfdf5; - }} - .prompt-change-add {{ + } + .prompt-change-add { border: 1px solid #e5e7eb; background: #fff; margin: 8px 0; - }} - .prompt-change-label {{ + } + .prompt-change-label { font-size: 8px; font-weight: 800; text-transform: uppercase; letter-spacing: .08em; margin-bottom: 6px; - }} - .prompt-change-before .prompt-change-label {{ color: #be123c; }} - .prompt-change-after .prompt-change-label {{ color: #047857; }} - .prompt-change-add .prompt-change-label {{ color: #6b7280; }} - .prompt-change-text {{ + } + .prompt-change-before .prompt-change-label { color: #be123c; } + .prompt-change-after .prompt-change-label { color: #047857; } + .prompt-change-add .prompt-change-label { color: #6b7280; } + .prompt-change-text { font-size: 9px; line-height: 1.45; color: #111827; white-space: pre-wrap; overflow-wrap: anywhere; word-break: break-word; - }} - .prompt-improvement-rationale {{ + } + .prompt-improvement-rationale { margin-top: 6px; - }} - - - - {repeat_header_markup} -
- {brand_header_markup} -
Quality Metric Audit · Weekly
-
-

{html.escape(payload["title"])}

-
{html.escape(payload["subtitle"])}
-
-
Client{html.escape(payload["vendor_name"])}
-
Use Case{html.escape(payload.get("use_case") or "Not specified")}
-
Window{html.escape(payload.get("period_display") or "Not specified")}
-
Calls{payload["evaluation"].total_rows}
-
Audit Set{len(metrics)} metrics
-
-
- {f''' -
-

01 Audit Summary

-
{audit_summary_markup}
- {top_metric_strip_markup} - {top_metric_delta_markup} -
- ''' if show_audit_summary else ''} - {quality_panel_markup} - {business_section} - {failure_diagnostics_section} - {prompt_improvements_section} - {design_notes_section} - {f''' -
-

This report is generated from completed Call Import evaluation results. Metrics are derived from the saved evaluation outputs and the transcript source configured for the run.{html.escape(weekly_methodology)}

-
- ''' if show_methodology else ''} -
-
-
Powered by EfficientAI
-
Voice AI Evaluation Platform
- -
-
-
Evaluation ID: {html.escape(str(getattr(payload["evaluation"], "id", "")))}
-
Generated: {html.escape(payload["generated_at_iso"])}
-
Call import reports by EfficientAI Cloud
-
-
- - - """ + } +""" def _render_weasyprint( self, diff --git a/app/services/reporting/simulation_run_pdf_report.py b/app/services/reporting/simulation_run_pdf_report.py new file mode 100644 index 00000000..6ba96e71 --- /dev/null +++ b/app/services/reporting/simulation_run_pdf_report.py @@ -0,0 +1,368 @@ +"""PDF report for a Harbor simulation run: KPIs, failure diagnostics and prompt fixes. + +Reuses the call-import report's stylesheet, failure-diagnostics and prompt-improvement +sections so both reports read the same. +""" + +from __future__ import annotations + +import html +from datetime import datetime, timezone +from typing import Any, Optional +from urllib.parse import quote + +from app.models.database import Agent, SimulationRun +from app.services.reporting.call_import_evaluation_pdf_report import ( + CallImportEvaluationPdfReportService, +) + +_COMBO_ROWS_MAX = 25 + + +def _pct(value: Any) -> str: + if value is None: + return "—" + try: + return f"{float(value) * 100:.0f}%" + except (TypeError, ValueError): + return "—" + + +def _num(value: Any) -> str: + if value is None: + return "—" + return html.escape(str(value)) + + +class SimulationRunPdfReportService(CallImportEvaluationPdfReportService): + def _evaluation_call_deep_link( + self, + *, + platform_base_url: str | None, + call_import_id: str, + evaluation_id: str, + conversation_id: str | None, + evaluation_row_id: str | None = None, + ) -> str | None: + # ``evaluation_id`` carries the run's 6-digit ID; evidence rows are trial IDs. + del call_import_id, conversation_id + base = (platform_base_url or "").strip().rstrip("/") + if not base or not evaluation_row_id: + return None + return ( + f"{base}/simulations/{evaluation_id}?tab=analysis" + f"&trial={quote(str(evaluation_row_id), safe='')}" + ) + + def render_pdf( + self, + *, + run: SimulationRun, + agent: Agent, + summary: dict[str, Any], + success_rules: dict[str, str], + metric_clusters: Optional[dict[str, Any]], + prompt_improvements: Optional[dict[str, Any]], + sections: Optional[dict[str, bool]] = None, + platform_base_url: Optional[str] = None, + generated_at: Optional[datetime] = None, + ) -> bytes: + payload = { + "run": run, + "agent": agent, + "summary": summary or {}, + "success_rules": success_rules or {}, + "metric_clusters": metric_clusters, + "prompt_improvements": prompt_improvements, + "sections": sections or {}, + "platform_base_url": platform_base_url, + "generated_at_iso": (generated_at or datetime.now(timezone.utc)).strftime( + "%Y-%m-%d %H:%M UTC" + ), + } + pdf_bytes = self._render_weasyprint( + self._render_simulation_html(payload), + label=f"simulation run {run.run_id}", + ) + if pdf_bytes is not None: + return pdf_bytes + return self._render_basic_pdf(self._simulation_plain_text_lines(payload)) + + def _run_title(self, run: SimulationRun, agent: Agent) -> str: + return run.name or f"{agent.name} · run {run.run_id}" + + def _llm_label(self, run: SimulationRun, side: str) -> str: + llms = run.resolved_llms if isinstance(run.resolved_llms, dict) else {} + entry = llms.get(side) + if not isinstance(entry, dict): + return "Not resolved" + label = str(entry.get("label") or "") + source = str(entry.get("source_label") or "") + return f"{label} ({source})" if source else label + + def _kpi_cards(self, summary: dict[str, Any]) -> list[tuple[str, str]]: + uses_criteria = bool(summary.get("uses_success_criteria")) + cards = [ + ("Pass rate", _pct(summary.get("pass_rate"))), + ("Criteria met" if uses_criteria else "Mean score", _pct(summary.get("mean_reward"))), + ("Consistency (pass^k)", _pct(summary.get("pass_all_rate"))), + ("Pass@k", _pct(summary.get("pass_any_rate"))), + ("Errored trials", _num(summary.get("n_failed") or 0)), + ("Avg turns", _num(summary.get("avg_turns"))), + ] + tools = summary.get("tools") + if isinstance(tools, dict) and tools.get("total_calls"): + cards.append(("Tool calls succeeded", _pct(tools.get("success_rate")))) + return cards + + def _run_summary_section_html(self, payload: dict[str, Any]) -> str: + summary = payload["summary"] + cards = "".join( + f""" +
+
+
{html.escape(label.upper())}
+
{value}
+
+ """ + for label, value in self._kpi_cards(summary) + ) + run: SimulationRun = payload["run"] + n_completed = int(summary.get("n_completed") or 0) + n_passed = int(summary.get("n_passed") or 0) + verdict = ( + f"{n_passed} of {n_completed} completed trials met every success rule" + if summary.get("uses_success_criteria") + else f"{n_passed} of {n_completed} completed trials scored at or above " + f"the {run.pass_threshold:.2f} pass threshold" + ) + return f""" +
+

01 Run Summary

+

{html.escape(verdict)}. + Production agent: {html.escape(self._llm_label(run, "production"))}. + Simulated caller: {html.escape(self._llm_label(run, "test"))}.

+
{cards}
+
+ """ + + def _metric_results_section_html(self, payload: dict[str, Any]) -> str: + summary = payload["summary"] + rules: dict[str, str] = payload["success_rules"] + metrics = [m for m in summary.get("metrics") or [] if isinstance(m, dict)] + if not metrics: + return "" + uses_criteria = bool(summary.get("uses_success_criteria")) + rows = "".join( + f""" + + {html.escape(str(m.get("name") or "Metric"))} + {html.escape(rules.get(str(m.get("metric_id")), "—"))} + {_pct(m.get("pass_rate") if uses_criteria else m.get("mean"))} + {_num(m.get("n"))} + + """ + for m in metrics + ) + value_header = "Pass rate" if uses_criteria else "Mean score" + intro = ( + "Each trial is judged against the success rule set for every metric when the run " + "started. Metrics are sorted from weakest to strongest." + if uses_criteria + else "No success rules were set for this run; metrics show the mean normalised score." + ) + return f""" +
+

02 Success Rules & Metric Results

+

{intro}

+ + + {rows} +
MetricSuccess rule{value_header}Trials
+
+ """ + + def _scenario_results_section_html(self, payload: dict[str, Any]) -> str: + combos = [c for c in payload["summary"].get("combos") or [] if isinstance(c, dict)] + if not combos: + return "" + rows = "".join( + f""" + + {html.escape(str(c.get("scenario_name") or "Scenario"))} + {html.escape(str(c.get("persona_name") or "Persona"))} + {_num(c.get("passed"))} / {_num(c.get("attempts"))} + {_pct(c.get("mean_reward"))} + + """ + for c in combos[:_COMBO_ROWS_MAX] + ) + more = ( + f'

Showing the {_COMBO_ROWS_MAX} weakest of {len(combos)} combinations.

' + if len(combos) > _COMBO_ROWS_MAX + else "" + ) + return f""" +
+

03 Scenario Results

+

Scenario × persona combinations, weakest first.

+ + + {rows} +
ScenarioPersonaPassedMean score
+ {more} +
+ """ + + def _render_simulation_html(self, payload: dict[str, Any]) -> str: + run: SimulationRun = payload["run"] + agent: Agent = payload["agent"] + summary = payload["summary"] + sections = payload["sections"] + + run_summary = ( + self._run_summary_section_html(payload) + + self._metric_results_section_html(payload) + + self._scenario_results_section_html(payload) + if sections.get("run_summary", True) + else "" + ) + + failure_diagnostics = "" + clusters = payload.get("metric_clusters") + if ( + sections.get("failure_diagnostics", True) + and isinstance(clusters, dict) + and clusters.get("status") == "completed" + ): + failure_diagnostics = self._failure_diagnostics_section_html( + clusters, + clusters.get("overview"), + platform_base_url=payload.get("platform_base_url"), + evaluation_id=run.run_id, + ) + + prompt_section = "" + improvements = payload.get("prompt_improvements") + if ( + sections.get("prompt_improvements", True) + and isinstance(improvements, dict) + and improvements.get("status") == "completed" + ): + prompt_section = self._prompt_improvements_section_html(improvements).replace( + ", aligned with the Visualizations → Prompt / Agent Improvements view.", + ", based on the failure clusters above.", + ) + + methodology = ( + """ +
+

This report is generated from a simulation run: a test agent played + each scenario × persona against the production agent, and an LLM judge scored every + trial. A metric counts as failed in a trial when it missed the success rule set for the + run. Failed trials are clustered by an LLM into failure patterns with engineering gap + labels, and prompt recommendations target the largest clusters.

+
+ """ + if sections.get("methodology", True) + else "" + ) + + meta_boxes = [ + ("Agent", html.escape(agent.name or "Agent")), + ("Run", html.escape(run.run_id)), + ("Trials", _num(summary.get("n_trials") or run.expected_trials)), + ("Scenarios", _num(len(run.scenario_ids or []))), + ("Personas", _num(len(run.persona_ids or []))), + ] + meta = "".join( + f'
{label}{value}
' + for label, value in meta_boxes + ) + return f""" + + + + + + + +
+
Simulation Run Report · {html.escape(run.status)}
+
+

{html.escape(self._run_title(run, agent))}

+
Simulation results, failure diagnostics and prompt recommendations
+
{meta}
+
+ {run_summary} + {failure_diagnostics} + {prompt_section} + {methodology} +
+
+
Powered by EfficientAI
+
Voice AI Evaluation Platform
+ +
+
+
Simulation run: {html.escape(run.run_id)}
+
Generated: {html.escape(payload["generated_at_iso"])}
+
+
+ + + """ + + def _simulation_plain_text_lines(self, payload: dict[str, Any]) -> list[str]: + run: SimulationRun = payload["run"] + agent: Agent = payload["agent"] + summary = payload["summary"] + sections = payload.get("sections") or {} + lines = [ + "SIMULATION RUN REPORT", + self._run_title(run, agent), + f"Agent: {agent.name}", + f"Run: {run.run_id}", + f"Generated: {payload['generated_at_iso']}", + ] + if sections.get("run_summary", True): + lines.extend(["", "01 Run Summary"]) + lines.extend( + f"{label}: {html.unescape(value)}" for label, value in self._kpi_cards(summary) + ) + lines.extend(["", "02 Success Rules & Metric Results"]) + for m in summary.get("metrics") or []: + if isinstance(m, dict): + rule = payload["success_rules"].get(str(m.get("metric_id")), "-") + lines.append(f"{m.get('name')}: {_pct(m.get('mean'))} (rule: {rule})") + clusters = payload.get("metric_clusters") + if ( + sections.get("failure_diagnostics", True) + and isinstance(clusters, dict) + and clusters.get("status") == "completed" + ): + lines.extend(["", "04 Failure Diagnostics"]) + for group in clusters.get("groups") or []: + if not isinstance(group, dict): + continue + lines.append(f"{group.get('metric_name')}: {group.get('flagged_count')} flagged") + for cluster in group.get("clusters") or []: + if isinstance(cluster, dict): + lines.append(f" - {cluster.get('label')} ({cluster.get('count')})") + improvements = payload.get("prompt_improvements") + if ( + sections.get("prompt_improvements", True) + and isinstance(improvements, dict) + and improvements.get("status") == "completed" + ): + lines.extend(["", "05 Prompt Improvement Recommendations"]) + for item in improvements.get("suggestions") or []: + if isinstance(item, dict): + lines.append(f"- {item.get('target_section') or item.get('metric_name')}") + lines.append(f" {item.get('suggested_text') or ''}") + return lines + + +simulation_run_pdf_report_service = SimulationRunPdfReportService() diff --git a/app/services/storage/azure_blob_service.py b/app/services/storage/azure_blob_service.py index fa443889..74d8dd62 100644 --- a/app/services/storage/azure_blob_service.py +++ b/app/services/storage/azure_blob_service.py @@ -621,6 +621,63 @@ def generate_presigned_url_by_key( except Exception as e: raise StorageError(f"Failed to generate SAS URL: {str(e)}") + def generate_presigned_put_url_by_key( + self, + key: str, + *, + content_type: str, + expiration: int = 3600, + ) -> str: + """SAS PUT URL for direct browser upload to a blob key.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = ( + self._initialization_error + or "Azure Blob Storage is not enabled or not configured" + ) + raise StorageError(error_msg) + + _, _, generate_blob_sas, BlobSasPermissions, _ = _get_azure_libs() + account_key = self._get_account_key_for_sas() + account_name = self._resolved_account_name or self._resolve_credentials()[0] + if not account_key or not account_name: + raise StorageError( + "Azure Blob Storage SAS URLs require an account key. " + "Set azure.connection_string or azure.account_key." + ) + try: + sas_token = generate_blob_sas( + account_name=account_name, + container_name=self.bucket_name, + blob_name=key, + account_key=account_key, + permission=BlobSasPermissions(write=True, create=True), + expiry=datetime.now(UTC) + timedelta(seconds=expiration), + ) + blob_client = self.container_client.get_blob_client(key) + return f"{blob_client.url}?{sas_token}" + except Exception as e: + raise StorageError(f"Failed to generate SAS PUT URL: {str(e)}") + + def head_object_by_key(self, key: str) -> Optional[dict]: + """Return blob metadata or None when missing.""" + self._ensure_initialized() + if not self.is_enabled(): + return None + _, ResourceNotFoundError, _, _, _ = _get_azure_libs() + try: + props = self.container_client.get_blob_client(key).get_blob_properties() + return { + "ContentLength": int(props.size or 0), + "ContentType": props.content_settings.content_type + if props.content_settings + else None, + } + except ResourceNotFoundError: + return None + except Exception as e: + raise StorageError(f"Failed to head object {key}: {e}") from e + # Singleton instance azure_blob_service = AzureBlobService() diff --git a/app/services/storage/blob_storage_service.py b/app/services/storage/blob_storage_service.py index b8779272..dc3995d3 100644 --- a/app/services/storage/blob_storage_service.py +++ b/app/services/storage/blob_storage_service.py @@ -175,5 +175,21 @@ def generate_presigned_url_by_key( response_content_disposition=response_content_disposition, ) + def generate_presigned_put_url_by_key( + self, + key: str, + *, + content_type: str, + expiration: int = 3600, + ) -> str: + return self._backend().generate_presigned_put_url_by_key( + key, + content_type=content_type, + expiration=expiration, + ) + + def head_object_by_key(self, key: str) -> Optional[dict]: + return self._backend().head_object_by_key(key) + blob_storage_service = BlobStorageService() diff --git a/app/services/storage/gcs_service.py b/app/services/storage/gcs_service.py index 1846bf67..45e03d0d 100644 --- a/app/services/storage/gcs_service.py +++ b/app/services/storage/gcs_service.py @@ -618,6 +618,71 @@ def generate_presigned_url_by_key( except Exception as e: raise StorageError(f"Unexpected error generating signed URL: {str(e)}") + def generate_presigned_put_url_by_key( + self, + key: str, + *, + content_type: str, + expiration: int = 3600, + ) -> str: + """Signed PUT URL for direct browser upload to an object key.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, _, GoogleCloudError = _get_gcs_exception_types() + credentials = self._get_signing_credentials() + iam_params = None if credentials is not None else self._get_iam_signing_params() + if credentials is None and iam_params is None: + raise StorageError(_GCS_SIGNING_UNAVAILABLE_MSG) + + signed_url_kwargs: dict = { + "version": "v4", + "expiration": timedelta(seconds=expiration), + "method": "PUT", + "content_type": content_type, + } + try: + blob = self.bucket.blob(key) + if credentials is not None: + return blob.generate_signed_url( + credentials=credentials, + **signed_url_kwargs, + ) + sa_email, access_token = iam_params + return blob.generate_signed_url( + service_account_email=sa_email, + access_token=access_token, + **signed_url_kwargs, + ) + except GoogleCloudError as e: + raise StorageError(f"Failed to generate signed PUT URL: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error generating signed PUT URL: {str(e)}") + + def head_object_by_key(self, key: str) -> Optional[dict]: + """Return blob metadata or None when missing.""" + self._ensure_initialized() + if not self.is_enabled(): + return None + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + try: + blob = self.bucket.blob(key) + if not blob.exists(): + return None + blob.reload() + return { + "ContentLength": int(blob.size or 0), + "ContentType": blob.content_type, + } + except NotFound: + return None + except GoogleCloudError as e: + raise StorageError(f"Failed to head object {key}: {e}") from e + except Exception as e: + raise StorageError(f"Unexpected error reading object metadata: {e}") from e + # Singleton instance gcs_service = GcsService() diff --git a/app/services/storage/s3_service.py b/app/services/storage/s3_service.py index 2cd92823..854cb8e1 100644 --- a/app/services/storage/s3_service.py +++ b/app/services/storage/s3_service.py @@ -796,6 +796,48 @@ def generate_presigned_url_by_key( except Exception as e: raise StorageError(f"Unexpected error generating presigned URL: {str(e)}") + def generate_presigned_put_url_by_key( + self, + key: str, + *, + content_type: str, + expiration: int = 3600, + ) -> str: + """Presigned PUT for direct browser upload to an object key.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "S3 is not enabled or not configured" + raise StorageError(error_msg) + + try: + return self.s3_client.generate_presigned_url( + "put_object", + Params={ + "Bucket": self.bucket_name, + "Key": key, + "ContentType": content_type, + }, + ExpiresIn=expiration, + ) + except ClientError as e: + raise StorageError(f"Failed to generate presigned PUT URL: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error generating presigned PUT URL: {str(e)}") + + def head_object_by_key(self, key: str) -> Optional[dict]: + """Return S3 HeadObject payload or None when missing.""" + self._ensure_initialized() + if not self.is_enabled(): + return None + try: + return self.s3_client.head_object(Bucket=self.bucket_name, Key=key) + except ClientError as e: + if e.response.get("Error", {}).get("Code") in {"404", "NoSuchKey", "NotFound"}: + return None + raise StorageError(f"Failed to head object {key}: {e}") from e + except Exception as e: + raise StorageError(f"Unexpected error reading object metadata: {e}") from e + def __getattr__(name: str): """Lazy alias to avoid circular import with blob_storage_service.""" diff --git a/app/services/telephony/recording_download.py b/app/services/telephony/recording_download.py index ccc1c737..5e50de11 100644 --- a/app/services/telephony/recording_download.py +++ b/app/services/telephony/recording_download.py @@ -525,6 +525,150 @@ def download_recording_url( return body, content_type +def stream_recording_url_to_path( + recording_url: str, + dest_path: str, + *, + auth: Optional[Union[Tuple[str, str], httpx.Auth]] = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + max_bytes: int = DEFAULT_MAX_RECORDING_BYTES, + user_supplied: bool = False, + credential_fingerprint: Optional[str] = None, +) -> str: + """Stream a recording to ``dest_path``; return content_type.""" + if not recording_url: + raise ExotelInvalidContentError("recording_url is empty") + if user_supplied and auth is not None: + raise ExotelInvalidContentError( + "User-supplied recording URLs must not be fetched with credentials" + ) + + if auth is not None: + allowed_suffixes = list(_CREDENTIALED_ALLOWED_HOST_SUFFIXES) + else: + allowed_suffixes = _allowed_host_suffixes() + + assert_recording_url_safe( + recording_url, + user_supplied=user_supplied, + allowed_suffixes=allowed_suffixes, + ) + + request_hooks: Optional[dict[str, List[Callable[..., None]]]] = None + if not user_supplied: + request_hooks = _provider_recording_redirect_hooks( + allowed_suffixes=allowed_suffixes if auth is not None else None, + ) + + follow_redirects = not user_supplied + + try: + with httpx.Client( + timeout=timeout_seconds, + follow_redirects=follow_redirects, + event_hooks=request_hooks, + ) as client: + with client.stream("GET", recording_url, auth=auth) as resp: + if resp.status_code in (401, 403): + if auth is not None: + retry_after = _raise_credentialed_throttle( + fingerprint=credential_fingerprint, + message=f"Recording URL rejected credentials (HTTP {resp.status_code})", + ) + raise CredentialedRecordingThrottledError( + f"Recording URL rejected credentials (HTTP {resp.status_code})", + retry_after_seconds=retry_after, + ) + raise ExotelAuthError( + f"Recording URL rejected credentials (HTTP {resp.status_code})" + ) + if resp.status_code == 404: + raise ExotelNotFoundError(f"Recording not found at {recording_url}") + if resp.status_code == 429: + retry_after_header = _parse_retry_after_header( + resp.headers.get("Retry-After") + ) + if auth is not None: + retry_after = _raise_credentialed_throttle( + fingerprint=credential_fingerprint, + message="Rate limited fetching recording (HTTP 429)", + retry_after_seconds=retry_after_header, + ) + raise CredentialedRecordingThrottledError( + "Rate limited fetching recording (HTTP 429)", + retry_after_seconds=retry_after, + ) + raise ExotelTransientError( + f"Rate limited fetching recording (HTTP 429): {resp.text[:200]}" + ) + if 500 <= resp.status_code < 600: + raise ExotelTransientError( + f"Server error fetching recording (HTTP {resp.status_code})" + ) + if resp.status_code == 400: + if auth is not None: + raise ExotelTransientError( + f"Unexpected HTTP 400 fetching recording: {resp.text[:200]}" + ) + raise ExotelInvalidContentError( + f"Unexpected HTTP 400 fetching recording: {resp.text[:200]}" + ) + if resp.status_code >= 400: + raise ExotelInvalidContentError( + f"Unexpected HTTP {resp.status_code} fetching recording: {resp.text[:200]}" + ) + + content_type = ( + (resp.headers.get("content-type") or "") + .split(";", 1)[0] + .strip() + .lower() + ) + if not content_type.startswith("audio/"): + raise ExotelInvalidContentError( + "Recording URL returned non-audio content type: " + f"{content_type or 'unknown'}" + ) + + total = 0 + with open(dest_path, "wb") as handle: + for chunk in resp.iter_bytes(): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + raise ExotelRecordingTooLargeError( + f"Recording size exceeds cap of {max_bytes} bytes" + ) + handle.write(chunk) + if total == 0: + raise ExotelInvalidContentError("Recording response was empty") + return content_type + except ExotelInvalidContentError: + raise + except httpx.TimeoutException as exc: + raise ExotelTransientError(f"Timeout fetching recording: {exc}") from exc + except httpx.HTTPError as exc: + raise ExotelTransientError(f"HTTP error fetching recording: {exc}") from exc + + +def download_public_recording_to_path( + recording_url: str, + dest_path: str, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + max_bytes: int = DEFAULT_MAX_RECORDING_BYTES, +) -> str: + return stream_recording_url_to_path( + recording_url, + dest_path, + auth=None, + timeout_seconds=timeout_seconds, + max_bytes=max_bytes, + user_supplied=True, + ) + + def _raise_credentialed_throttle( *, fingerprint: Optional[str], diff --git a/app/services/testing/agent_test_setup_generation.py b/app/services/testing/agent_test_setup_generation.py index e9a6a0f6..8e9b5fb7 100644 --- a/app/services/testing/agent_test_setup_generation.py +++ b/app/services/testing/agent_test_setup_generation.py @@ -75,6 +75,101 @@ AgentTestPromptSection = TestAgentPromptSection +GENERATE_PRODUCTION_PROMPT_SYSTEM = ( + "You are an expert at writing system prompts for customer-facing AI agents.\n\n" + "The user will describe the agent they want to build. Write a complete production system prompt " + "for a text-chat agent that will be evaluated in automated simulations.\n\n" + "The prompt should:\n" + "- Define role, goals, tone, and boundaries clearly\n" + "- Include how to greet and how to handle common user intents\n" + "- Stay realistic for SMS/web chat (concise replies, no stage directions)\n" + "- Avoid mentioning tests, QA, evaluators, or that this is synthetic\n\n" + "Return ONLY the system prompt text. No markdown fences, no title, no explanation." +) + + +@dataclass +class ProductionPromptGenerationResult: + production_prompt: str + provider: str + model: str + + +def build_production_prompt_user_message( + *, + agent_name: str, + brief: str, + language: Optional[str] = None, + call_type: Optional[str] = None, + additional_context: Optional[str] = None, +) -> str: + parts = [f"Agent name: {agent_name}", "", f"Agent brief:\n{brief.strip()}"] + if language: + parts.append(f"\nLanguage: {language}") + if call_type: + parts.append(f"Interaction type: {call_type}") + if additional_context and additional_context.strip(): + parts.append(f"\nAdditional context:\n{additional_context.strip()}") + parts.append("\nWrite the production system prompt.") + return "\n".join(parts) + + +def generate_production_prompt_from_brief( + brief: str, + *, + agent_name: str, + language: Optional[str], + call_type: Optional[str], + additional_context: Optional[str], + llm_provider: ModelProvider, + llm_model: str, + organization_id: UUID, + db: Session, + llm_config: Optional[Dict[str, Any]] = None, + credential_id: Optional[UUID] = None, +) -> ProductionPromptGenerationResult: + """Generate a production system prompt from a short agent brief.""" + if not brief.strip(): + raise ValueError("Agent brief is required") + if not agent_name.strip(): + raise ValueError("Agent name is required") + + messages = [ + {"role": "system", "content": GENERATE_PRODUCTION_PROMPT_SYSTEM}, + { + "role": "user", + "content": build_production_prompt_user_message( + agent_name=agent_name, + brief=brief, + language=language, + call_type=call_type, + additional_context=additional_context, + ), + }, + ] + + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + task_defaults={"temperature": 0.5, "max_tokens": 4000}, + credential_id=credential_id, + ) + + production_prompt = _strip_llm_text_wrapper(result["text"]) + if len(production_prompt.split()) < 10: + raise ValueError("Generated production prompt was too short") + + return ProductionPromptGenerationResult( + production_prompt=production_prompt, + provider=llm_provider.value, + model=llm_model, + ) + + @dataclass class TestPromptGenerationResult: sections: List[TestAgentPromptSection] diff --git a/app/services/testing/llm_to_llm_evaluator_simulation.py b/app/services/testing/llm_to_llm_evaluator_simulation.py index eaca050b..ad2af9b5 100644 --- a/app/services/testing/llm_to_llm_evaluator_simulation.py +++ b/app/services/testing/llm_to_llm_evaluator_simulation.py @@ -1,445 +1,602 @@ -"""Text-based LLM-to-LLM simulation for evaluator runs without an external voice provider.""" - -from __future__ import annotations - -import re -import time -from typing import Any, Optional -from uuid import UUID - -from loguru import logger -from sqlalchemy.orm import Session - -from app.models.database import Agent, Evaluator, EvaluatorResult, Persona, Scenario -from app.services.agents.chat_connection import normalized_chat_connection_type -from app.services.agents.chat_llm_config import resolve_simulation_llm -from app.services.agents.chat_connection_config_store import chat_connection_config_for_runtime -from app.services.agents.chat_production_leg import ( - generate_production_chat_reply, - uses_live_production_leg, -) -from app.services.agents.provider_platform_chat import ( - ProviderChatState, - close_provider_chat_session, -) -from app.services.testing.evaluator_simulation_errors import ( - ProductionChatLegError, - TestAgentLlmLegError, -) -from app.models.enums import ModelProvider -from app.services.ai.llm_service import llm_service -from app.services.testing.test_agent_simulation_prompt import ( - build_persona_description_for_bridge, - build_test_agent_system_prompt, - is_chat_agent, - resolve_persona_max_turns, -) -from app.services.testing.test_agent_template import ( - TestAgentFirstMessage, - ensure_opening_includes_persona_name, - resolve_caller_opening_text, - resolve_first_message_from_agent, - should_caller_speak_first, -) -from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, - usage_context_for_test_agent_simulation, -) - -_GOODBYE_RE = re.compile( - r"\b(goodbye|bye|talk\s+to\s+you\s+later|have\s+a\s+(?:good|great)\s+(?:day|one))\b", - re.IGNORECASE, -) -_THANKS_CLOSING_RE = re.compile( - r"\b(?:thank\s+you|thanks)\b\s*[.!]?\s*$", - re.IGNORECASE, -) - -_SIMULATION_LLM_RETRIES = 3 -_SIMULATION_TASK_DEFAULTS = {"temperature": 0.7, "max_tokens": 2048} - - -def _production_turn_needs_user_seed(transcript: list[dict[str, str]]) -> bool: - if not transcript: - return True - last = transcript[-1] - speaker = (last.get("speaker") or "").strip() - text = (last.get("text") or "").strip() - return speaker != "Speaker 1" or not text - - -def _build_eval_user_opener( - *, - agent: Agent, - persona: Persona, - first_message_config: TestAgentFirstMessage, - scenario_first_message: Optional[str], -) -> str: - persona_name = (persona.name or "Test Caller").strip() - opening: Optional[str] = None - if should_caller_speak_first(first_message_config): - opening = resolve_caller_opening_text( - first_message=first_message_config, - persona_name=persona_name, - scenario_first_message=scenario_first_message, - ) - phone_default = f"Hello, this is {persona_name} calling." - if is_chat_agent(agent) and opening == phone_default: - opening = f"Hi, I'm {persona_name}. I need some help." - else: - if scenario_first_message and str(scenario_first_message).strip(): - opening = ensure_opening_includes_persona_name( - str(scenario_first_message).strip(), - persona_name, - ) - else: - opening = f"Hi, I'm {persona_name}. I need some help." - if not (opening or "").strip(): - opening = f"Hi, I'm {persona_name}. I need some help." - return opening.strip() - - -def _should_end_conversation(text: str, *, turn_index: int, min_turns: int = 2) -> bool: - if turn_index < min_turns: - return False - stripped = (text or "").strip() - if not stripped: - return False - if _GOODBYE_RE.search(stripped): - return True - return bool(_THANKS_CLOSING_RE.search(stripped)) - - -_CUSTOMER_CHAT_TURN_NUDGE = ( - "The company's agent sent the last message above. " - "Write the customer's next chat reply only (plain text, stay in character as the customer)." -) - - -def _caller_messages( - system_prompt: str, - transcript: list[dict[str, str]], - *, - chat: bool = False, -) -> list[dict[str, str]]: - messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] - for entry in transcript: - speaker = entry.get("speaker") - text = (entry.get("text") or "").strip() - if not text: - continue - if chat: - if speaker == "Speaker 1": - messages.append({"role": "user", "content": text}) - else: - messages.append({"role": "assistant", "content": text}) - elif speaker == "Speaker 1": - messages.append({"role": "assistant", "content": text}) - else: - messages.append({"role": "user", "content": text}) - if len(messages) == 1: - messages.append( - { - "role": "user", - "content": ( - "Start the conversation as the customer (one short message)." - if chat - else "The call has just connected. Start the conversation." - ), - } - ) - elif chat: - messages.append({"role": "user", "content": _CUSTOMER_CHAT_TURN_NUDGE}) - return messages - - -def _agent_messages(system_prompt: str, transcript: list[dict[str, str]]) -> list[dict[str, str]]: - messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] - for entry in transcript: - speaker = entry.get("speaker") - text = (entry.get("text") or "").strip() - if not text: - continue - if speaker == "Speaker 2": - messages.append({"role": "assistant", "content": text}) - else: - messages.append({"role": "user", "content": text}) - return messages - - -def _generate_turn( - *, - messages: list[dict[str, str]], - llm_provider: ModelProvider, - llm_model: str, - organization_id: UUID, - db: Session, - llm_config: Optional[dict], - credential_id: Optional[UUID], - leg_label: str = "simulation", -) -> str: - last_result: Optional[dict[str, Any]] = None - for attempt in range(_SIMULATION_LLM_RETRIES): - override_llm_config: Optional[dict[str, Any]] = None - if attempt > 0: - override_llm_config = { - "temperature": min(1.0, 0.7 + 0.15 * attempt), - } - time.sleep(0.35 * attempt) - try: - result = llm_service.generate_response( - messages=messages, - llm_provider=llm_provider, - llm_model=llm_model, - organization_id=organization_id, - db=db, - llm_config=llm_config, - task_defaults=_SIMULATION_TASK_DEFAULTS, - override_llm_config=override_llm_config, - credential_id=credential_id, - ) - except RuntimeError as exc: - raise RuntimeError( - f"{leg_label} failed ({llm_provider.value}/{llm_model}): {exc}" - ) from exc - last_result = result - text = (result.get("text") or "").strip() - if text: - return text - logger.warning( - "[LLM simulation] Empty {} response (attempt {}/{}, provider={}, model={}, finish_reason={})", - leg_label, - attempt + 1, - _SIMULATION_LLM_RETRIES, - llm_provider.value, - llm_model, - result.get("finish_reason"), - ) - - result = last_result or {} - refusal = (result.get("refusal") or "").strip() - if refusal: - raise ValueError( - f"LLM refused the simulation request ({leg_label}; " - f"provider={llm_provider.value}, model={llm_model}): {refusal}" - ) - finish_reason = result.get("finish_reason") - hint = "" - if finish_reason == "length" or result.get("truncated"): - hint = ( - " The model hit the output token limit before producing visible text " - "(common with reasoning models like gpt-oss-120b or gpt-5-mini); " - "raise max_tokens or set reasoning_effort=minimal in test agent LLM config." - ) - elif finish_reason: - hint = f" finish_reason={finish_reason}." - raise ValueError( - f"LLM returned an empty simulation response ({leg_label}; " - f"provider={llm_provider.value}, model={llm_model}).{hint} " - f"Tried {_SIMULATION_LLM_RETRIES} times. " - "Check provider rate limits and worker logs for [LLMService] empty assistant text." - ) - - -def run_llm_to_llm_evaluator_simulation( - *, - evaluator: Evaluator, - result: EvaluatorResult, - agent: Agent, - persona: Persona, - scenario: Scenario, - organization_id: UUID, - db: Session, -) -> dict[str, Any]: - """Run a text simulation and populate the evaluator result transcript.""" - conn_type = normalized_chat_connection_type(agent) - - test_llm = resolve_simulation_llm( - db, agent=agent, organization_id=organization_id, leg="test" - ) - provider_chat_state = ProviderChatState() - production_leg_meta: dict[str, Any] = {} - - max_turns = resolve_persona_max_turns(persona) - persona_description = build_persona_description_for_bridge(persona) - caller_system = build_test_agent_system_prompt( - agent, - persona, - scenario, - persona_description=persona_description, - max_turns=max_turns, - ) - caller_ctx = usage_context_for_test_agent_simulation( - organization_id=organization_id, - workspace_id=evaluator.workspace_id, - agent_id=agent.id, - evaluator_id=evaluator.id, - persona_id=persona.id, - scenario_id=scenario.id, - evaluator_result_id=result.id, - provider_platform="internal", - ) - agent_ctx = LLMUsageContext( - organization_id=organization_id, - workspace_id=evaluator.workspace_id, - product_section=LLMUsageProductSection.EVALUATORS, - resource_id=evaluator.id, - resource_type="evaluator", - extra={ - "agent_id": str(agent.id), - "evaluator_id": str(evaluator.id), - "persona_id": str(persona.id), - "scenario_id": str(scenario.id), - "evaluator_result_id": str(result.id), - "synthetic_testing": "pre_prod", - "simulation_leg": "production_agent", - "provider_platform": "internal", - }, - ) - - transcript: list[dict[str, str]] = [] - chat_mode = is_chat_agent(agent) - first_message_config = resolve_first_message_from_agent(agent) - scenario_first_message = None - if isinstance(scenario.required_info, dict): - scenario_first_message = scenario.required_info.get("first_message") - exchanges = 0 - - def _write_transcript_to_result() -> None: - result.transcription = "\n".join( - f"{entry['speaker']}: {entry['text']}" - for entry in transcript - if entry.get("text") - ) - result.speaker_segments = [ - { - "speaker": entry["speaker"], - "text": entry["text"], - "start": float(idx), - "end": float(idx) + 1.0, - } - for idx, entry in enumerate(transcript) - ] - result.provider_platform = "internal" - result.call_data = { - "source": "llm_to_llm_simulation", - "simulation": "llm_to_llm", - "modality": "chat" if chat_mode else "voice", - "chat_connection_type": conn_type, - **production_leg_meta, - "test_llm_source": test_llm.source, - "exchanges": exchanges, - "messages": transcript, - "partial": exchanges < max_turns, - } - result.duration_seconds = float(max(1, len(transcript))) - - try: - while exchanges < max_turns: - if ( - uses_live_production_leg(agent) - and _production_turn_needs_user_seed(transcript) - ): - transcript.append( - { - "speaker": "Speaker 1", - "text": _build_eval_user_opener( - agent=agent, - persona=persona, - first_message_config=first_message_config, - scenario_first_message=scenario_first_message, - ), - } - ) - elif ( - not uses_live_production_leg(agent) - and not transcript - and should_caller_speak_first(first_message_config) - ): - opener = _build_eval_user_opener( - agent=agent, - persona=persona, - first_message_config=first_message_config, - scenario_first_message=scenario_first_message, - ) - transcript.append({"speaker": "Speaker 1", "text": opener}) - - try: - with llm_usage_context(agent_ctx): - agent_text, leg_meta = generate_production_chat_reply( - db, - agent=agent, - organization_id=organization_id, - transcript=transcript, - provider_state=provider_chat_state, - evaluator_result=result, - ) - except Exception as exc: - leg = ( - provider_chat_state.extra.get("production_leg") - or provider_chat_state.platform - or conn_type - ) - raise ProductionChatLegError( - str(leg), - f"Production chat leg failed ({leg}): {exc}", - ) from exc - production_leg_meta.update(leg_meta) - transcript.append({"speaker": "Speaker 2", "text": agent_text}) - exchanges += 1 - if _should_end_conversation(agent_text, turn_index=exchanges): - break - - try: - with llm_usage_context(caller_ctx): - caller_text = _generate_turn( - messages=_caller_messages(caller_system, transcript, chat=chat_mode), - llm_provider=test_llm.provider, - llm_model=test_llm.model, - organization_id=organization_id, - db=db, - llm_config=test_llm.llm_config, - credential_id=test_llm.credential_id, - leg_label="Simulated customer (test agent LLM)", - ) - except Exception as exc: - raise TestAgentLlmLegError(f"Test agent LLM failed: {exc}") from exc - transcript.append({"speaker": "Speaker 1", "text": caller_text}) - exchanges += 1 - if _should_end_conversation(caller_text, turn_index=exchanges): - break - except Exception: - if transcript: - _write_transcript_to_result() - raise - finally: - try: - close_provider_chat_session( - db, - agent=agent, - organization_id=organization_id, - state=provider_chat_state, - ) - except Exception: - logger.warning( - "[LLM simulation] Provider chat session teardown failed for evaluator {}", - evaluator.evaluator_id, - ) - - _write_transcript_to_result() - if isinstance(result.call_data, dict): - result.call_data["partial"] = False - - logger.info( - "[LLM simulation] Completed evaluator {} result {} with {} transcript lines", - evaluator.evaluator_id, - result.result_id, - len(transcript), - ) - return { - "transcript_lines": len(transcript), - "exchanges": exchanges, - "provider_platform": "internal", - } +"""Text-based LLM-to-LLM simulation for evaluator runs without an external voice provider.""" + +from __future__ import annotations + +import re +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import Agent, Evaluator, EvaluatorResult, Persona, Scenario +from app.services.agents.chat_connection import normalized_chat_connection_type +from app.services.agents.chat_llm_config import ResolvedSimulationLlm, resolve_simulation_llm +from app.services.agents.chat_production_leg import ( + generate_production_chat_reply, + uses_live_production_leg, +) +from app.services.agents.provider_platform_chat import ( + ProviderChatState, + close_provider_chat_session, +) +from app.services.testing.evaluator_simulation_errors import ( + ProductionChatLegError, + TestAgentLlmLegError, +) +from app.models.enums import ModelProvider +from app.services.ai.llm_service import llm_service +from app.services.testing.test_agent_simulation_prompt import ( + build_persona_description_for_bridge, + build_test_agent_system_prompt, + is_chat_agent, + resolve_persona_max_turns, +) +from app.services.testing.test_agent_template import ( + TestAgentFirstMessage, + ensure_opening_includes_persona_name, + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, +) +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + usage_context_for_test_agent_simulation, +) + +_GOODBYE_RE = re.compile( + r"\b(goodbye|bye|talk\s+to\s+you\s+later|have\s+a\s+(?:good|great)\s+(?:day|one))\b", + re.IGNORECASE, +) +_THANKS_CLOSING_RE = re.compile( + r"\b(?:thank\s+you|thanks)\b\s*[.!]?\s*$", + re.IGNORECASE, +) + +_SIMULATION_LLM_RETRIES = 3 +_SIMULATION_TASK_DEFAULTS = {"temperature": 0.7, "max_tokens": 2048} + + +def _production_turn_needs_user_seed(transcript: list[dict[str, str]]) -> bool: + if not transcript: + return True + last = transcript[-1] + speaker = (last.get("speaker") or "").strip() + text = (last.get("text") or "").strip() + return speaker != "Speaker 1" or not text + + +def _build_eval_user_opener( + *, + agent: Agent, + persona: Persona, + first_message_config: TestAgentFirstMessage, + scenario_first_message: Optional[str], +) -> str: + persona_name = (persona.name or "Test Caller").strip() + opening: Optional[str] = None + if should_caller_speak_first(first_message_config): + opening = resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona_name, + scenario_first_message=scenario_first_message, + ) + phone_default = f"Hello, this is {persona_name} calling." + if is_chat_agent(agent) and opening == phone_default: + opening = f"Hi, I'm {persona_name}. I need some help." + else: + if scenario_first_message and str(scenario_first_message).strip(): + opening = ensure_opening_includes_persona_name( + str(scenario_first_message).strip(), + persona_name, + ) + else: + opening = f"Hi, I'm {persona_name}. I need some help." + if not (opening or "").strip(): + opening = f"Hi, I'm {persona_name}. I need some help." + return opening.strip() + + +def _should_end_conversation(text: str, *, turn_index: int, min_turns: int = 2) -> bool: + if turn_index < min_turns: + return False + stripped = (text or "").strip() + if not stripped: + return False + if _GOODBYE_RE.search(stripped): + return True + return bool(_THANKS_CLOSING_RE.search(stripped)) + + +_CUSTOMER_CHAT_TURN_NUDGE = ( + "The company's agent sent the last message above. " + "Write the customer's next chat reply only (plain text, stay in character as the customer)." +) + + +def _caller_messages( + system_prompt: str, + transcript: list[dict[str, str]], + *, + chat: bool = False, +) -> list[dict[str, str]]: + messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] + for entry in transcript: + speaker = entry.get("speaker") + text = (entry.get("text") or "").strip() + if not text: + continue + if chat: + if speaker == "Speaker 1": + messages.append({"role": "user", "content": text}) + else: + messages.append({"role": "assistant", "content": text}) + elif speaker == "Speaker 1": + messages.append({"role": "assistant", "content": text}) + else: + messages.append({"role": "user", "content": text}) + if len(messages) == 1: + messages.append( + { + "role": "user", + "content": ( + "Start the conversation as the customer (one short message)." + if chat + else "The call has just connected. Start the conversation." + ), + } + ) + elif chat: + messages.append({"role": "user", "content": _CUSTOMER_CHAT_TURN_NUDGE}) + return messages + + +def _agent_messages(system_prompt: str, transcript: list[dict[str, str]]) -> list[dict[str, str]]: + messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] + for entry in transcript: + speaker = entry.get("speaker") + text = (entry.get("text") or "").strip() + if not text: + continue + if speaker == "Speaker 2": + messages.append({"role": "assistant", "content": text}) + else: + messages.append({"role": "user", "content": text}) + return messages + + +def _generate_test_customer_turn( + *, + messages: list[dict[str, str]], + agent: Agent, + organization_id: UUID, + db: Session, + llm_provider: ModelProvider, + llm_model: str, + llm_config: Optional[dict], + credential_id: Optional[UUID], +) -> tuple[str, list[dict[str, Any]]]: + """Simulated customer turn; uses the test agent's knowledge bases when they have ready docs.""" + leg_label = "Simulated customer (test agent LLM)" + kb_tool_calls: list[dict[str, Any]] = [] + workspace_id = getattr(agent, "workspace_id", None) + if workspace_id: + from app.services.knowledge_bases.chat_tools import ( + generate_production_reply_with_knowledge_tools, + ) + from app.services.knowledge_bases.agent_links import KB_ROLE_TEST + from app.services.knowledge_bases.retrieval import agent_has_ready_knowledge + + if agent_has_ready_knowledge(db, agent_id=agent.id, role=KB_ROLE_TEST): + try: + text, kb_tool_calls = generate_production_reply_with_knowledge_tools( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent.id, + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + llm_config=llm_config, + credential_id=credential_id, + task_defaults=_SIMULATION_TASK_DEFAULTS, + simulated_caller=True, + ) + except RuntimeError as exc: + raise RuntimeError( + f"{leg_label} failed ({llm_provider.value}/{llm_model}): {exc}" + ) from exc + if (text or "").strip(): + return text.strip(), kb_tool_calls + text = _generate_turn( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + credential_id=credential_id, + leg_label=leg_label, + ) + return text, kb_tool_calls + + +def _generate_turn( + *, + messages: list[dict[str, str]], + llm_provider: ModelProvider, + llm_model: str, + organization_id: UUID, + db: Session, + llm_config: Optional[dict], + credential_id: Optional[UUID], + leg_label: str = "simulation", +) -> str: + last_result: Optional[dict[str, Any]] = None + for attempt in range(_SIMULATION_LLM_RETRIES): + override_llm_config: Optional[dict[str, Any]] = None + if attempt > 0: + override_llm_config = { + "temperature": min(1.0, 0.7 + 0.15 * attempt), + } + time.sleep(0.35 * attempt) + try: + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + task_defaults=_SIMULATION_TASK_DEFAULTS, + override_llm_config=override_llm_config, + credential_id=credential_id, + ) + except RuntimeError as exc: + raise RuntimeError( + f"{leg_label} failed ({llm_provider.value}/{llm_model}): {exc}" + ) from exc + last_result = result + text = (result.get("text") or "").strip() + if text: + return text + logger.warning( + "[LLM simulation] Empty {} response (attempt {}/{}, provider={}, model={}, finish_reason={})", + leg_label, + attempt + 1, + _SIMULATION_LLM_RETRIES, + llm_provider.value, + llm_model, + result.get("finish_reason"), + ) + + result = last_result or {} + refusal = (result.get("refusal") or "").strip() + if refusal: + raise ValueError( + f"LLM refused the simulation request ({leg_label}; " + f"provider={llm_provider.value}, model={llm_model}): {refusal}" + ) + finish_reason = result.get("finish_reason") + hint = "" + if finish_reason == "length" or result.get("truncated"): + hint = ( + " The model hit the output token limit before producing visible text " + "(common with reasoning models like gpt-oss-120b or gpt-5-mini); " + "raise max_tokens or set reasoning_effort=minimal in test agent LLM config." + ) + elif finish_reason: + hint = f" finish_reason={finish_reason}." + raise ValueError( + f"LLM returned an empty simulation response ({leg_label}; " + f"provider={llm_provider.value}, model={llm_model}).{hint} " + f"Tried {_SIMULATION_LLM_RETRIES} times. " + "Check provider rate limits and worker logs for [LLMService] empty assistant text." + ) + + +@dataclass +class TextConversation: + """Mutable conversation state; stays readable when ``run_text_conversation`` raises.""" + + transcript: list[dict[str, str]] = field(default_factory=list) + exchanges: int = 0 + max_turns: int = 0 + production_leg_meta: dict[str, Any] = field(default_factory=dict) + # Production-agent tool calls across all turns, each stamped with ``turn_index``. + tool_calls: list[dict[str, Any]] = field(default_factory=list) + test_kb_tool_calls: list[dict[str, Any]] = field(default_factory=list) + test_llm_source: str = "" + + +# Called after every appended turn with the transcript entry and LLM/endpoint latency in seconds. +# Production entries carry a ``tool_calls`` list when the agent called tools that turn. +OnTurn = Callable[[dict[str, Any], float], None] + + +def run_text_conversation( + *, + agent: Agent, + persona: Persona, + scenario: Scenario, + organization_id: UUID, + db: Session, + conversation: TextConversation, + caller_ctx: LLMUsageContext, + agent_ctx: LLMUsageContext, + evaluator_result: Optional[EvaluatorResult] = None, + test_llm_override: Optional[ResolvedSimulationLlm] = None, + production_llm_override: Optional[ResolvedSimulationLlm] = None, + on_turn: Optional[OnTurn] = None, +) -> TextConversation: + """Alternate test-agent and production-agent turns until goodbye or ``max_turns``.""" + test_llm = test_llm_override or resolve_simulation_llm( + db, agent=agent, organization_id=organization_id, leg="test" + ) + conversation.test_llm_source = test_llm.source + provider_chat_state = ProviderChatState() + conversation.max_turns = resolve_persona_max_turns(persona) + caller_system = build_test_agent_system_prompt( + agent, + persona, + scenario, + persona_description=build_persona_description_for_bridge(persona), + max_turns=conversation.max_turns, + ) + transcript = conversation.transcript + first_message_config = resolve_first_message_from_agent(agent) + scenario_first_message = None + if isinstance(scenario.required_info, dict): + scenario_first_message = scenario.required_info.get("first_message") + conn_type = normalized_chat_connection_type(agent) + production_toolset = None + if not uses_live_production_leg(agent): + from app.services.tools.toolset import build_agent_toolset, scenario_mock_responses + + production_toolset = build_agent_toolset( + db, + agent=agent, + organization_id=organization_id, + llm_model=(production_llm_override.model if production_llm_override else None) + or agent.main_llm_model, + mock_responses=scenario_mock_responses(scenario), + ) + + def _append( + speaker: str, + text: str, + started: float, + tool_calls: Optional[list[dict[str, Any]]] = None, + ) -> None: + entry: dict[str, Any] = {"speaker": speaker, "text": text} + if tool_calls: + entry["tool_calls"] = tool_calls + transcript.append(entry) + if on_turn is not None: + on_turn(entry, time.monotonic() - started) + + try: + while conversation.exchanges < conversation.max_turns: + if uses_live_production_leg(agent) and _production_turn_needs_user_seed(transcript): + _append( + "Speaker 1", + _build_eval_user_opener( + agent=agent, + persona=persona, + first_message_config=first_message_config, + scenario_first_message=scenario_first_message, + ), + time.monotonic(), + ) + elif ( + not uses_live_production_leg(agent) + and not transcript + and should_caller_speak_first(first_message_config) + ): + opener = _build_eval_user_opener( + agent=agent, + persona=persona, + first_message_config=first_message_config, + scenario_first_message=scenario_first_message, + ) + _append("Speaker 1", opener, time.monotonic()) + + started = time.monotonic() + try: + with llm_usage_context(agent_ctx): + agent_text, leg_meta = generate_production_chat_reply( + db, + agent=agent, + organization_id=organization_id, + transcript=transcript, + provider_state=provider_chat_state, + evaluator_result=evaluator_result, + llm_override=production_llm_override, + toolset=production_toolset, + ) + except Exception as exc: + leg = ( + provider_chat_state.extra.get("production_leg") + or provider_chat_state.platform + or conn_type + ) + raise ProductionChatLegError( + str(leg), + f"Production chat leg failed ({leg}): {exc}", + ) from exc + turn_tool_calls = [ + {**call, "turn_index": len(transcript)} + for call in leg_meta.pop("tool_calls", None) or [] + ] + conversation.tool_calls.extend(turn_tool_calls) + conversation.production_leg_meta.update(leg_meta) + _append("Speaker 2", agent_text, started, turn_tool_calls) + conversation.exchanges += 1 + if _should_end_conversation(agent_text, turn_index=conversation.exchanges): + break + + started = time.monotonic() + try: + with llm_usage_context(caller_ctx): + caller_text, turn_kb_calls = _generate_test_customer_turn( + messages=_caller_messages( + caller_system, transcript, chat=is_chat_agent(agent) + ), + agent=agent, + organization_id=organization_id, + db=db, + llm_provider=test_llm.provider, + llm_model=test_llm.model, + llm_config=test_llm.llm_config, + credential_id=test_llm.credential_id, + ) + if turn_kb_calls: + conversation.test_kb_tool_calls.extend(turn_kb_calls) + except Exception as exc: + raise TestAgentLlmLegError(f"Test agent LLM failed: {exc}") from exc + _append("Speaker 1", caller_text, started) + conversation.exchanges += 1 + if _should_end_conversation(caller_text, turn_index=conversation.exchanges): + break + finally: + if production_toolset is not None: + production_toolset.close() + try: + close_provider_chat_session( + db, + agent=agent, + organization_id=organization_id, + state=provider_chat_state, + ) + except Exception: + logger.warning( + "[LLM simulation] Provider chat session teardown failed for agent {}", + agent.id, + ) + return conversation + + +def write_conversation_to_result( + result: EvaluatorResult, + conversation: TextConversation, + *, + agent: Agent, + extra_call_data: Optional[dict[str, Any]] = None, +) -> None: + """Store transcript, speaker segments and call_data on an evaluator result.""" + transcript = conversation.transcript + result.transcription = "\n".join( + f"{entry['speaker']}: {entry['text']}" + for entry in transcript + if entry.get("text") + ) + result.speaker_segments = [ + { + "speaker": entry["speaker"], + "text": entry["text"], + "start": float(idx), + "end": float(idx) + 1.0, + } + for idx, entry in enumerate(transcript) + ] + result.provider_platform = "internal" + call_data: dict[str, Any] = { + "source": "llm_to_llm_simulation", + "simulation": "llm_to_llm", + "modality": "chat" if is_chat_agent(agent) else "voice", + "chat_connection_type": normalized_chat_connection_type(agent), + **conversation.production_leg_meta, + "test_llm_source": conversation.test_llm_source, + "exchanges": conversation.exchanges, + "messages": transcript, + "partial": conversation.exchanges < conversation.max_turns, + } + if conversation.tool_calls: + call_data["tool_calls"] = conversation.tool_calls + call_data["knowledge_base_tool_calls"] = [ + call for call in conversation.tool_calls if call.get("provider") == "knowledge_base" + ] + if conversation.test_kb_tool_calls: + call_data["test_agent_knowledge_base_tool_calls"] = conversation.test_kb_tool_calls + if extra_call_data: + call_data.update(extra_call_data) + result.call_data = call_data + result.duration_seconds = float(max(1, len(transcript))) + + +def run_llm_to_llm_evaluator_simulation( + *, + evaluator: Evaluator, + result: EvaluatorResult, + agent: Agent, + persona: Persona, + scenario: Scenario, + organization_id: UUID, + db: Session, +) -> dict[str, Any]: + """Run a text simulation and populate the evaluator result transcript.""" + caller_ctx = usage_context_for_test_agent_simulation( + organization_id=organization_id, + workspace_id=evaluator.workspace_id, + agent_id=agent.id, + evaluator_id=evaluator.id, + persona_id=persona.id, + scenario_id=scenario.id, + evaluator_result_id=result.id, + provider_platform="internal", + ) + agent_ctx = LLMUsageContext( + organization_id=organization_id, + workspace_id=evaluator.workspace_id, + product_section=LLMUsageProductSection.EVALUATORS, + resource_id=evaluator.id, + resource_type="evaluator", + extra={ + "agent_id": str(agent.id), + "evaluator_id": str(evaluator.id), + "persona_id": str(persona.id), + "scenario_id": str(scenario.id), + "evaluator_result_id": str(result.id), + "synthetic_testing": "pre_prod", + "simulation_leg": "production_agent", + "provider_platform": "internal", + }, + ) + + conversation = TextConversation() + try: + run_text_conversation( + agent=agent, + persona=persona, + scenario=scenario, + organization_id=organization_id, + db=db, + conversation=conversation, + caller_ctx=caller_ctx, + agent_ctx=agent_ctx, + evaluator_result=result, + ) + except Exception: + if conversation.transcript: + write_conversation_to_result(result, conversation, agent=agent) + raise + + write_conversation_to_result(result, conversation, agent=agent) + if isinstance(result.call_data, dict): + result.call_data["partial"] = False + + logger.info( + "[LLM simulation] Completed evaluator {} result {} with {} transcript lines", + evaluator.evaluator_id, + result.result_id, + len(conversation.transcript), + ) + return { + "transcript_lines": len(conversation.transcript), + "exchanges": conversation.exchanges, + "provider_platform": "internal", + } diff --git a/app/services/testing/scenario_metric_generation.py b/app/services/testing/scenario_metric_generation.py new file mode 100644 index 00000000..72563469 --- /dev/null +++ b/app/services/testing/scenario_metric_generation.py @@ -0,0 +1,335 @@ +"""Generate evaluation metric drafts from scenario definitions and agent prompts.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import Metric +from app.models.enums import ModelProvider +from app.services.ai.llm_service import llm_service +from app.services.metrics.surfaces import ALLOWED_METRIC_SURFACES +from app.services.testing.scenario_generation_modality import scenario_modality_from_call_medium + + +@dataclass +class ScenarioMetricInput: + name: str + description: str + goal: Optional[str] = None + # When set to boolean|rating|number|text, the generated metric must use that type. + metric_type: Optional[str] = None + + +@dataclass +class ScenarioMetricDraft: + name: str + description: str + metric_type: str + custom_data_type: Optional[str] + custom_config: Dict[str, Any] + supported_surfaces: List[str] + enabled_surfaces: List[str] + tags: List[str] + scenario_name: str + + +@dataclass +class ScenarioMetricGenerationResult: + metrics: List[ScenarioMetricDraft] + provider: str + model: str + + +def evaluation_surface_for_call_medium(call_medium: Optional[str]) -> str: + return "chat_agent" if scenario_modality_from_call_medium(call_medium) == "chat" else "agent" + + +def _extract_json_array(text: str) -> List[Any]: + cleaned = (text or "").strip() + if cleaned.startswith("```"): + cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned) + cleaned = re.sub(r"\s*```$", "", cleaned) + start = cleaned.find("[") + end = cleaned.rfind("]") + if start == -1 or end == -1 or end <= start: + raise ValueError("LLM response did not contain a JSON array") + return json.loads(cleaned[start : end + 1]) + + +def _build_messages( + *, + agent_name: str, + production_prompt: str, + surface: str, + scenarios: Sequence[ScenarioMetricInput], +) -> List[Dict[str, str]]: + scenario_blocks = [] + for index, scenario in enumerate(scenarios, start=1): + parts = [ + f"Scenario {index}: {scenario.name}", + f"Description:\n{scenario.description.strip()}", + ] + if scenario.goal: + parts.append(f"Goal: {scenario.goal.strip()}") + requested_type = (scenario.metric_type or "").strip().lower() + if requested_type in {"boolean", "rating", "number", "text"}: + parts.append( + f'Requested metric type: {requested_type} ' + f'(you MUST set "metric_type" to "{requested_type}")' + ) + else: + parts.append( + "Requested metric type: auto (choose the best structured type for this scenario)" + ) + scenario_blocks.append("\n".join(parts)) + + schema_block = f""" +You MUST respond with ONLY a JSON array (no markdown, no commentary) with exactly {len(scenarios)} objects, in the same order as the scenarios below. Each object: +{{ + "name": str (concise, Title Case, <= 60 chars), + "description": str (1-3 sentences: LLM-judge rubric for whether the agent handled THIS scenario per the production prompt), + "metric_type": "rating" | "boolean" | "number" | "text", + "custom_data_type": "boolean" | "enum" | "number_range" | null, + "custom_config": {{}}, + "supported_surfaces": ["{surface}"], + "enabled_surfaces": ["{surface}"] +}} + +Rules: + - Prefer structured types (boolean, rating/enum, number) over free text. + - metric_type must align with custom_data_type (boolean/boolean, rating/enum, number/number_range). + - Each metric must be specific to its scenario's success criteria and edge cases. + - Do not reuse identical names across scenarios. +""" + + system_message = ( + "You are an expert evaluation designer. For each test scenario, define one " + "LLM-as-judge metric that scores whether an agent conversation satisfied that " + "scenario given the agent's production system prompt. Always respond with valid JSON only." + ) + + user_message = ( + f"Agent Name: {agent_name}\n" + f"Evaluation surface: {surface}\n\n" + f"Production system prompt:\n{production_prompt.strip()}\n\n" + f"Scenarios:\n\n" + + "\n\n---\n\n".join(scenario_blocks) + + schema_block + ) + + return [ + {"role": "system", "content": system_message}, + {"role": "user", "content": user_message}, + ] + + +def normalize_generated_metric_fields( + parsed: Dict[str, Any], + *, + surface: str, + forced_metric_type: Optional[str] = None, +) -> Dict[str, Any]: + allowed_surfaces = set(ALLOWED_METRIC_SURFACES) + supported = [s for s in (parsed.get("supported_surfaces") or []) if s in allowed_surfaces] + if surface not in supported: + supported = list({*supported, surface}) + enabled_surfaces = [s for s in (parsed.get("enabled_surfaces") or supported) if s in supported] + if not enabled_surfaces: + enabled_surfaces = list(supported) + + forced = (forced_metric_type or "").strip().lower() + if forced in {"rating", "boolean", "number", "text"}: + metric_type = forced + else: + metric_type = (parsed.get("metric_type") or "rating").lower() + if metric_type not in {"rating", "boolean", "number", "text"}: + metric_type = "rating" + + custom_data_type = parsed.get("custom_data_type") + if custom_data_type not in {"boolean", "enum", "number_range", None}: + custom_data_type = None + + if metric_type == "text": + custom_data_type = None + custom_config: Dict[str, Any] = {} + else: + if custom_data_type is None: + custom_data_type = ( + "boolean" + if metric_type == "boolean" + else "number_range" + if metric_type == "number" + else "enum" + ) + custom_config = parsed.get("custom_config") or {} + if custom_data_type == "enum" and not isinstance(custom_config.get("options"), list): + custom_config = {"options": ["Excellent", "Good", "Neutral", "Poor"]} + if custom_data_type == "number_range": + custom_config = { + "min": float(custom_config.get("min", 0)), + "max": float(custom_config.get("max", 10)), + "step": float(custom_config.get("step", 1)), + } + if custom_data_type == "boolean": + custom_config = {} + + name = (parsed.get("name") or "Scenario Metric").strip()[:60] + description = (parsed.get("description") or "").strip()[:1000] + + from app.services.metrics.option_labels import coerce_custom_config_for_storage + + custom_config = coerce_custom_config_for_storage(custom_data_type, custom_config) + + return { + "name": name, + "description": description, + "metric_type": metric_type, + "custom_data_type": custom_data_type, + "custom_config": custom_config, + "supported_surfaces": supported, + "enabled_surfaces": enabled_surfaces, + } + + +def _dedupe_metric_name( + db: Session, + organization_id: UUID, + workspace_id: UUID, + name: str, +) -> str: + workspace_filter = Metric.workspace_id == workspace_id + existing = ( + db.query(Metric) + .filter( + Metric.name == name, + Metric.organization_id == organization_id, + workspace_filter, + Metric.parent_metric_id.is_(None), + ) + .first() + ) + if not existing: + return name + suffix = 2 + while True: + suffix_text = f" ({suffix})" + candidate = f"{name[: 60 - len(suffix_text)]}{suffix_text}" + collision = ( + db.query(Metric) + .filter( + Metric.name == candidate, + Metric.organization_id == organization_id, + workspace_filter, + Metric.parent_metric_id.is_(None), + ) + .first() + ) + if not collision: + return candidate + suffix += 1 + + +def _build_tags(agent_name: str, extra: Optional[List[str]] = None) -> List[str]: + reserved = {agent_name.strip(), "auto-generated"} + tags: List[str] = [] + for value in [agent_name.strip(), "auto-generated"]: + if value and value not in tags: + tags.append(value) + for item in extra or []: + text = str(item).strip() + if text and text not in reserved and text not in tags: + tags.append(text) + return tags[:12] + + +def generate_metrics_from_scenarios( + *, + agent_name: str, + production_prompt: str, + call_medium: Optional[str], + scenarios: Sequence[ScenarioMetricInput], + organization_id: UUID, + workspace_id: UUID, + db: Session, + llm_provider: ModelProvider, + llm_model: str, + llm_config: Optional[Dict[str, Any]] = None, + credential_id: Optional[UUID] = None, +) -> ScenarioMetricGenerationResult: + if not production_prompt.strip(): + raise ValueError("Production prompt is required") + if not scenarios: + raise ValueError("At least one scenario is required") + if len(scenarios) > 10: + raise ValueError("At most 10 scenarios per request") + + surface = evaluation_surface_for_call_medium(call_medium) + messages = _build_messages( + agent_name=agent_name, + production_prompt=production_prompt, + surface=surface, + scenarios=scenarios, + ) + + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + task_defaults={"temperature": 0.4, "max_tokens": 8000}, + credential_id=credential_id, + ) + + try: + raw = _extract_json_array(result["text"]) + except (ValueError, json.JSONDecodeError) as exc: + logger.warning(f"[ScenarioMetrics] Failed to parse metric JSON array: {exc}") + raise ValueError("Could not parse generated metrics from LLM response") from exc + + if not isinstance(raw, list) or len(raw) < len(scenarios): + raise ValueError("LLM response did not contain enough metric definitions") + + drafts: List[ScenarioMetricDraft] = [] + for index, scenario in enumerate(scenarios): + item = raw[index] + if not isinstance(item, dict): + raise ValueError(f"Metric at index {index} is not an object") + forced_type = (scenario.metric_type or "").strip().lower() + if forced_type == "auto": + forced_type = "" + normalized = normalize_generated_metric_fields( + item, + surface=surface, + forced_metric_type=forced_type or None, + ) + name = _dedupe_metric_name( + db, organization_id, workspace_id, normalized["name"] + ) + drafts.append( + ScenarioMetricDraft( + name=name, + description=normalized["description"], + metric_type=normalized["metric_type"], + custom_data_type=normalized["custom_data_type"], + custom_config=normalized["custom_config"], + supported_surfaces=normalized["supported_surfaces"], + enabled_surfaces=normalized["enabled_surfaces"], + tags=_build_tags(agent_name, item.get("suggested_tags")), + scenario_name=scenario.name, + ) + ) + + return ScenarioMetricGenerationResult( + metrics=drafts, + provider=llm_provider.value, + model=llm_model, + ) diff --git a/app/services/testing/simulation_llms.py b/app/services/testing/simulation_llms.py new file mode 100644 index 00000000..bf68f8e7 --- /dev/null +++ b/app/services/testing/simulation_llms.py @@ -0,0 +1,66 @@ +"""Which LLM (or live endpoint) answers on each side of a simulation, for display.""" + +from __future__ import annotations + +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from app.models.database import Agent +from app.services.agents.chat_connection import normalized_chat_connection_type +from app.services.agents.chat_llm_config import ResolvedSimulationLlm, resolve_simulation_llm +from app.services.agents.chat_production_leg import uses_live_production_leg + +SOURCE_LABELS = { + "main_llm": "agent production LLM", + "test_llm": "agent test LLM", + "voice_bundle": "voice bundle LLM", + "simulation_run_override": "run override", +} + + +def describe_llm(llm: ResolvedSimulationLlm) -> dict[str, Any]: + return { + "kind": "llm", + "provider": llm.provider.value, + "model": llm.model, + "source": llm.source, + "label": f"{llm.provider.value}/{llm.model}", + "source_label": SOURCE_LABELS.get(llm.source, llm.source), + } + + +def describe_simulation_llms( + db: Session, + *, + agent: Agent, + production_override: Optional[ResolvedSimulationLlm] = None, + test_override: Optional[ResolvedSimulationLlm] = None, +) -> dict[str, Optional[dict[str, Any]]]: + """``{"production": ..., "test": ...}``; ``None`` for a side that can't be resolved.""" + + def _resolve(leg: str, override: Optional[ResolvedSimulationLlm]): + if override is not None: + return describe_llm(override) + try: + return describe_llm( + resolve_simulation_llm( + db, agent=agent, organization_id=agent.organization_id, leg=leg + ) + ) + except ValueError: + return None + + if uses_live_production_leg(agent): + connection = normalized_chat_connection_type(agent) + production = { + "kind": "live", + "provider": None, + "model": None, + "source": connection, + "label": f"Live endpoint ({connection})", + "source_label": "agent connection", + } + else: + production = _resolve("main", production_override) + return {"production": production, "test": _resolve("test", test_override)} diff --git a/app/services/testing/simulation_run_naming.py b/app/services/testing/simulation_run_naming.py new file mode 100644 index 00000000..90cd0821 --- /dev/null +++ b/app/services/testing/simulation_run_naming.py @@ -0,0 +1,36 @@ +"""Display names for simulation runs (re-run clones).""" + +from __future__ import annotations + +from typing import Optional + + +def base_simulation_run_name(name: Optional[str], run_id: str) -> str: + """Strip stacked ``(re-run)`` / ``(re-run failed)`` suffixes from legacy names.""" + raw = (name or f"Run {run_id}").strip() + while raw: + lowered = raw.lower() + if lowered.endswith(" · failed trials"): + raw = raw[: -len(" · failed trials")].rstrip() + continue + if lowered.endswith(" (re-run failed)"): + raw = raw[: -len(" (re-run failed)")].rstrip() + continue + if lowered.endswith(" (re-run)"): + raw = raw[: -len(" (re-run)")].rstrip() + continue + break + return raw or f"Run {run_id}" + + +def simulation_run_name_for_rerun( + name: Optional[str], + run_id: str, + *, + only_failed: bool = False, +) -> str: + """Name for a cloned run — same base title; list uses ``run_id`` + started time.""" + base = base_simulation_run_name(name, run_id) + if only_failed: + return f"{base} · failed trials" + return base diff --git a/app/services/tools/__init__.py b/app/services/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/services/tools/base.py b/app/services/tools/base.py new file mode 100644 index 00000000..30da3993 --- /dev/null +++ b/app/services/tools/base.py @@ -0,0 +1,109 @@ +"""Provider-agnostic tool types shared by every tool source (knowledge bases, MCP, HTTP, ...).""" + +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass, field +from typing import Any, Optional, Protocol + +RESULT_PREVIEW_CHARS = 2000 + +TOOL_STATUS_OK = "ok" +TOOL_STATUS_ERROR = "error" +TOOL_STATUS_MOCKED = "mocked" + +SIDE_EFFECT_READ = "read" +SIDE_EFFECT_WRITE = "write" + +# How a provider's tools behave inside simulations. +SIM_MODE_LIVE = "live" +SIM_MODE_MOCK_WRITES = "mock_writes" +SIM_MODE_MOCK_ALL = "mock_all" +SIM_MODES = (SIM_MODE_LIVE, SIM_MODE_MOCK_WRITES, SIM_MODE_MOCK_ALL) + +_WRITE_VERBS = re.compile( + r"(^|_)(add|append|archive|assign|create|delete|drop|insert|invite|move|post|publish|" + r"put|remove|reply|send|set|share|update|upload|upsert|write|patch|react|comment)(_|$)" +) + + +def classify_side_effect( + name: str, + *, + read_only_hint: Optional[bool] = None, + destructive_hint: Optional[bool] = None, +) -> str: + """Read or write: trust explicit MCP annotations, else guess from the tool name's verbs.""" + if read_only_hint is True: + return SIDE_EFFECT_READ + if destructive_hint is True or read_only_hint is False: + return SIDE_EFFECT_WRITE + words = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|[-.\s]+", "_", name).lower() + return SIDE_EFFECT_WRITE if _WRITE_VERBS.search(words) else SIDE_EFFECT_READ + + +@dataclass(frozen=True) +class ToolSpec: + """One callable tool as exposed to the LLM (OpenAI function-tool shape).""" + + name: str + description: str + parameters: dict[str, Any] + provider: str + side_effect: str = SIDE_EFFECT_READ + + def openai_definition(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + }, + } + + +@dataclass +class ToolResult: + content: str + is_error: bool = False + # Provider-specific facts worth keeping on the call record (e.g. KB hit_count). + extra: dict[str, Any] = field(default_factory=dict) + + +class ToolProvider(Protocol): + """A tool source. Optional attributes: ``simulation_mode`` (default mock_writes) + and ``close()`` for providers holding sessions or connections.""" + + name: str + + def list_tools(self) -> list[ToolSpec]: ... + + def call(self, tool_name: str, arguments: dict[str, Any]) -> ToolResult: ... + + +@dataclass +class ToolCallRecord: + """One executed (or mocked) tool call, stored in ``call_data.tool_calls``.""" + + call_id: Optional[str] + tool: str + provider: str + arguments: dict[str, Any] + status: str + result_preview: str + latency_ms: int + side: str = "production" + turn_index: Optional[int] = None + error: Optional[str] = None + extra: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + data = asdict(self) + extra = data.pop("extra") or {} + return {**data, **extra} + + +def preview(text: str, limit: int = RESULT_PREVIEW_CHARS) -> str: + text = text or "" + return text if len(text) <= limit else text[:limit] + "…" diff --git a/app/services/tools/connections.py b/app/services/tools/connections.py new file mode 100644 index 00000000..5e75aa85 --- /dev/null +++ b/app/services/tools/connections.py @@ -0,0 +1,298 @@ +"""Tool connections: validation, encrypted secrets, tool discovery, providers and agent links.""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timezone +from typing import Any, Iterable, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Agent, AgentToolConnection, ToolConnection +from app.services.tools.base import SIM_MODES, ToolProvider + +CONNECTION_TYPE_MCP = "mcp" +CONNECTION_TYPE_HTTP = "http" +CONNECTION_TYPE_POSTGRES = "postgres" +CONNECTION_TYPES = (CONNECTION_TYPE_MCP, CONNECTION_TYPE_HTTP, CONNECTION_TYPE_POSTGRES) + +# Secret keys per type: dicts of header / env values, or the Postgres DSN. +SECRET_FIELDS = { + CONNECTION_TYPE_MCP: ("headers", "env"), + CONNECTION_TYPE_HTTP: ("headers",), + CONNECTION_TYPE_POSTGRES: ("dsn",), +} + +_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +def slugify(name: str) -> str: + return _SLUG_RE.sub("_", (name or "").lower()).strip("_")[:40] or "tools" + + +def unique_slug( + db: Session, *, workspace_id: UUID, name: str, exclude_id: Optional[UUID] = None +) -> str: + base = slugify(name) + slug, suffix = base, 2 + while True: + query = db.query(ToolConnection.id).filter( + ToolConnection.workspace_id == workspace_id, ToolConnection.slug == slug + ) + if exclude_id is not None: + query = query.filter(ToolConnection.id != exclude_id) + if query.first() is None: + return slug + slug, suffix = f"{base}_{suffix}", suffix + 1 + + +def encrypt_secrets(secrets: dict[str, Any]) -> Optional[str]: + from app.core.encryption import encrypt_api_key + + cleaned = {k: v for k, v in (secrets or {}).items() if v not in (None, "", {})} + return encrypt_api_key(json.dumps(cleaned)) if cleaned else None + + +def decrypt_secrets(connection: ToolConnection) -> dict[str, Any]: + from app.core.encryption import decrypt_api_key + + if not connection.secrets: + return {} + return json.loads(decrypt_api_key(connection.secrets)) + + +def secret_keys(connection: ToolConnection) -> list[str]: + """Names of stored secrets (``headers.Authorization``, ``dsn``) — never the values.""" + try: + secrets = decrypt_secrets(connection) + except Exception: + return [] + keys: list[str] = [] + for field, value in secrets.items(): + if isinstance(value, dict): + keys.extend(f"{field}.{name}" for name in value) + else: + keys.append(field) + return sorted(keys) + + +def merge_secrets( + existing: dict[str, Any], updates: Optional[dict[str, Any]], connection_type: str +) -> dict[str, Any]: + """Apply a secrets patch: dict fields merge per key (``""`` deletes), scalars replace.""" + if not updates: + return existing + allowed = SECRET_FIELDS[connection_type] + merged = dict(existing) + for field, value in updates.items(): + if field not in allowed: + raise ValueError(f"Unknown secret field for {connection_type}: {field}") + if isinstance(value, dict): + current = dict(merged.get(field) or {}) + for key, item in value.items(): + if item in (None, ""): + current.pop(key, None) + else: + current[str(key)] = str(item) + merged[field] = current + elif value in (None, ""): + merged.pop(field, None) + else: + merged[field] = str(value) + return merged + + +def validate_connection(connection_type: str, config: dict[str, Any], simulation_mode: str) -> None: + from app.services.tools.providers.http import validate_http_tools + from app.services.tools.providers.mcp import TRANSPORTS + + if connection_type not in CONNECTION_TYPES: + raise ValueError(f"Unknown connection type: {connection_type}") + if simulation_mode not in SIM_MODES: + raise ValueError(f"Unknown simulation mode: {simulation_mode}") + if connection_type == CONNECTION_TYPE_MCP: + transport = config.get("transport") or "streamable_http" + if transport not in TRANSPORTS: + raise ValueError(f"Unknown MCP transport: {transport}") + if transport == "stdio" and not (config.get("command") or "").strip(): + raise ValueError("stdio MCP servers need a command") + if transport != "stdio" and not (config.get("url") or "").strip(): + raise ValueError("MCP server URL is required") + elif connection_type == CONNECTION_TYPE_HTTP: + validate_http_tools(config) + + +def build_provider( + connection: ToolConnection, + *, + enabled_tools: Optional[list[str]] = None, +) -> Optional[ToolProvider]: + """Provider for a connection's cached tools, or ``None`` if it has none yet.""" + from app.config import settings + + timeout = settings.TOOL_CALL_TIMEOUT_SECONDS + secrets = decrypt_secrets(connection) + common = dict( + name=connection.name, + prefix=connection.slug, + config=dict(connection.config or {}), + secrets=secrets, + enabled_tools=enabled_tools, + simulation_mode=connection.simulation_mode, + timeout=timeout, + ) + if connection.connection_type == CONNECTION_TYPE_HTTP: + from app.services.tools.providers.http import HttpToolProvider + + return HttpToolProvider(**common) + if connection.connection_type == CONNECTION_TYPE_POSTGRES: + from app.services.tools.providers.postgres import PostgresToolProvider + + return PostgresToolProvider(**common) + if not connection.cached_tools: + return None + from app.services.tools.providers.mcp import McpToolProvider + + return McpToolProvider(cached_tools=list(connection.cached_tools), **common) + + +def discover_tools(connection: ToolConnection) -> list[dict[str, Any]]: + """Fetch the connection's tool list from its source, proving the credentials work.""" + from app.config import settings + + secrets = decrypt_secrets(connection) + config = dict(connection.config or {}) + timeout = settings.TOOL_CALL_TIMEOUT_SECONDS + if connection.connection_type == CONNECTION_TYPE_MCP: + from app.services.tools.providers.mcp import discover_mcp_tools + + return discover_mcp_tools(config, secrets, timeout=timeout) + if connection.connection_type == CONNECTION_TYPE_HTTP: + from app.services.tools.providers.http import tool_entries_from_http_config + + return tool_entries_from_http_config(config) + from app.services.tools.providers.postgres import ( + PostgresToolProvider, + tool_entries_for_postgres, + ) + + if not secrets.get("dsn"): + raise ValueError("Add a connection string (dsn)") + provider = PostgresToolProvider( + name=connection.name, + prefix=connection.slug, + config=config, + secrets=secrets, + timeout=timeout, + ) + try: + result = provider.call(f"{connection.slug}__list_tables", {}) + finally: + provider.close() + if result.is_error: + raise ValueError(result.content) + return tool_entries_for_postgres() + + +def refresh_connection_tools(db: Session, connection: ToolConnection) -> bool: + """Run discovery and store the result; returns whether it succeeded.""" + try: + tools = discover_tools(connection) + except Exception as exc: + connection.last_error = str(exc)[:2000] or exc.__class__.__name__ + connection.last_tested_at = datetime.now(timezone.utc) + return False + connection.cached_tools = tools + connection.last_error = None + connection.last_tested_at = datetime.now(timezone.utc) + return True + + +def agent_tool_links(db: Session, *, agent_id: UUID) -> list[AgentToolConnection]: + return db.query(AgentToolConnection).filter(AgentToolConnection.agent_id == agent_id).all() + + +def replace_agent_tool_connections( + db: Session, + *, + agent: Agent, + links: Iterable[dict[str, Any]], +) -> None: + """``links``: ``[{"tool_connection_id": UUID, "enabled_tools": [..] | None}]``.""" + desired = {UUID(str(link["tool_connection_id"])): link.get("enabled_tools") for link in links} + if desired: + found = { + row[0] + for row in db.query(ToolConnection.id) + .filter( + ToolConnection.id.in_(desired), + ToolConnection.organization_id == agent.organization_id, + ToolConnection.workspace_id == agent.workspace_id, + ) + .all() + } + missing = [str(cid) for cid in desired if cid not in found] + if missing: + raise ValueError(f"Tool connection(s) not found in workspace: {', '.join(missing)}") + db.query(AgentToolConnection).filter(AgentToolConnection.agent_id == agent.id).delete() + for connection_id, enabled_tools in desired.items(): + db.add( + AgentToolConnection( + agent_id=agent.id, + tool_connection_id=connection_id, + organization_id=agent.organization_id, + workspace_id=agent.workspace_id, + enabled_tools=list(enabled_tools) if enabled_tools is not None else None, + ) + ) + + +def agent_connection_providers(db: Session, *, agent: Any) -> list[ToolProvider]: + """Providers for every tool connection attached to the agent.""" + from loguru import logger + + agent_id = getattr(agent, "id", None) + if agent_id is None: + return [] + rows = ( + db.query(AgentToolConnection, ToolConnection) + .join(ToolConnection, ToolConnection.id == AgentToolConnection.tool_connection_id) + .filter(AgentToolConnection.agent_id == agent_id) + .all() + ) + providers: list[ToolProvider] = [] + for link, connection in rows: + try: + provider = build_provider(connection, enabled_tools=link.enabled_tools) + except Exception as exc: + logger.warning("[Tools] Skipping connection {}: {}", connection.name, exc) + continue + if provider is not None: + providers.append(provider) + return providers + + +def connection_tool_views(connection: ToolConnection) -> list[dict[str, Any]]: + """Cached tools with their exposed name and read/write classification.""" + from app.services.tools.base import classify_side_effect + + overrides = (connection.config or {}).get("tool_side_effects") or {} + views = [] + for entry in connection.cached_tools or []: + views.append( + { + "name": entry["name"], + "exposed_name": f"{connection.slug}__{entry['name']}", + "description": entry.get("description"), + "input_schema": entry.get("input_schema"), + "side_effect": overrides.get(entry["name"]) + or classify_side_effect( + entry["name"], + read_only_hint=entry.get("read_only_hint"), + destructive_hint=entry.get("destructive_hint"), + ), + } + ) + return views diff --git a/app/services/tools/loop.py b/app/services/tools/loop.py new file mode 100644 index 00000000..b42034f7 --- /dev/null +++ b/app/services/tools/loop.py @@ -0,0 +1,123 @@ +"""LLM ↔ tool loop: call the model with tools until it answers in text or hits the round cap.""" + +from __future__ import annotations + +import json +from typing import Any, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.enums import ModelProvider +from app.services.ai.llm_service import llm_service +from app.services.tools.base import TOOL_STATUS_ERROR, ToolCallRecord +from app.services.tools.toolset import Toolset + +DEFAULT_MAX_TOOL_ROUNDS = 8 + + +def _parse_arguments(raw: Any) -> tuple[dict[str, Any], Optional[str]]: + if isinstance(raw, dict): + return dict(raw), None + try: + parsed = json.loads(raw or "{}") + except (TypeError, json.JSONDecodeError) as exc: + return {}, f"Invalid JSON arguments: {exc}" + if not isinstance(parsed, dict): + return {}, "Tool arguments must be a JSON object" + return parsed, None + + +def run_tool_loop( + db: Session, + *, + toolset: Toolset, + messages: list[dict[str, Any]], + organization_id: UUID, + llm_provider: ModelProvider, + llm_model: str, + llm_config: Optional[dict[str, Any]] = None, + credential_id: Optional[UUID] = None, + task_defaults: Optional[dict[str, Any]] = None, + max_rounds: int = DEFAULT_MAX_TOOL_ROUNDS, + side: str = "production", +) -> tuple[str, list[ToolCallRecord], list[dict[str, Any]]]: + """Return ``(reply_text, tool_call_records, messages_appended)``; tool failures go back to the model.""" + records: list[ToolCallRecord] = [] + working = list(messages) + initial_len = len(working) + empty_retries = 0 + for _ in range(max_rounds): + result = llm_service.generate_response_with_tools( + messages=working, + tools=toolset.definitions(), + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + credential_id=credential_id, + task_defaults=task_defaults, + ) + tool_calls = result.get("tool_calls") + if not tool_calls: + text = (result.get("text") or "").strip() + # length with no text means the output budget was used up (often by + # reasoning) before a reply or a tool call was finished. Try again + # instead of treating that as the customer's answer. + if text: + return text, records, working[initial_len:] + if result.get("finish_reason") == "length" or empty_retries < 1: + empty_retries += 1 + continue + return "", records, working[initial_len:] + working.append( + {"role": "assistant", "content": result.get("text") or None, "tool_calls": tool_calls} + ) + for call in tool_calls: + fn = call.get("function") or {} + name = fn.get("name") or "" + arguments, parse_error = _parse_arguments(fn.get("arguments")) + if parse_error: + content = parse_error + records.append( + ToolCallRecord( + call_id=call.get("id"), + tool=name, + provider="unknown", + arguments={"raw": fn.get("arguments")}, + status=TOOL_STATUS_ERROR, + result_preview=parse_error, + latency_ms=0, + side=side, + error=parse_error, + ) + ) + else: + tool_result, record = toolset.execute( + name, arguments, call_id=call.get("id"), side=side + ) + content = tool_result.content + records.append(record) + working.append({"role": "tool", "tool_call_id": call.get("id"), "content": content}) + + working.append( + { + "role": "user", + "content": ( + "Reply to the customer now in plain text, using the tool results above. " + "Do not call another tool." + ), + } + ) + final = llm_service.generate_response( + messages=working, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + credential_id=credential_id, + task_defaults=task_defaults, + ) + return (final.get("text") or "").strip(), records, working[initial_len:] diff --git a/app/services/tools/providers/__init__.py b/app/services/tools/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/services/tools/providers/http.py b/app/services/tools/providers/http.py new file mode 100644 index 00000000..3e271941 --- /dev/null +++ b/app/services/tools/providers/http.py @@ -0,0 +1,168 @@ +"""User-defined HTTP endpoints as tools (a customer's own order / CRM / booking APIs). + +Connection config:: + + {"base_url": "https://api.example.com", + "headers": {"Accept": "application/json"}, + "tools": [{"name": "get_order", "description": "Look up an order", + "method": "GET", "path": "/orders/{order_id}", + "parameters": {"type": "object", + "properties": {"order_id": {"type": "string"}}, + "required": ["order_id"]}, + "side_effect": "read"}]} + +Secrets: ``{"headers": {"Authorization": "Bearer ..."}}``. ``{placeholders}`` in the +path are filled from the arguments; the rest go in the query string for GET/DELETE +and in a JSON body otherwise. +""" + +from __future__ import annotations + +import re +from typing import Any, Optional +from urllib.parse import quote + +import httpx + +from app.services.tools.base import ( + SIDE_EFFECT_READ, + SIDE_EFFECT_WRITE, + SIM_MODE_MOCK_WRITES, + ToolResult, + ToolSpec, +) + +MAX_RESULT_CHARS = 12000 +HTTP_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE") +_TOOL_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") +_PLACEHOLDER_RE = re.compile(r"{([a-zA-Z0-9_]+)}") + + +def validate_http_tools(config: dict[str, Any]) -> list[dict[str, Any]]: + """Normalized tool definitions; raises ``ValueError`` on bad config.""" + tools = config.get("tools") or [] + if not isinstance(tools, list) or not tools: + raise ValueError("Define at least one HTTP tool") + seen: set[str] = set() + normalized = [] + for tool in tools: + name = str(tool.get("name") or "").strip() + if not _TOOL_NAME_RE.match(name): + raise ValueError(f"Invalid tool name {name!r}: use letters, digits, _ or -") + if name in seen: + raise ValueError(f"Duplicate tool name {name!r}") + seen.add(name) + method = str(tool.get("method") or "GET").upper() + if method not in HTTP_METHODS: + raise ValueError(f"{name}: unsupported method {method}") + if not (tool.get("path") or tool.get("url") or config.get("base_url")): + raise ValueError(f"{name}: needs a path or URL") + parameters = tool.get("parameters") or {"type": "object", "properties": {}} + if not isinstance(parameters, dict) or parameters.get("type") != "object": + raise ValueError(f"{name}: parameters must be a JSON schema object") + side_effect = tool.get("side_effect") or ( + SIDE_EFFECT_READ if method == "GET" else SIDE_EFFECT_WRITE + ) + normalized.append( + { + "name": name, + "description": str(tool.get("description") or name), + "method": method, + "path": tool.get("path") or "", + "url": tool.get("url") or "", + "parameters": parameters, + "side_effect": side_effect, + } + ) + return normalized + + +def tool_entries_from_http_config(config: dict[str, Any]) -> list[dict[str, Any]]: + """``cached_tools`` entries, in the same shape MCP discovery produces.""" + return [ + { + "name": tool["name"], + "description": tool["description"], + "input_schema": tool["parameters"], + "read_only_hint": tool["side_effect"] == SIDE_EFFECT_READ, + "destructive_hint": None, + } + for tool in validate_http_tools(config) + ] + + +class HttpToolProvider: + def __init__( + self, + *, + name: str, + prefix: str, + config: dict[str, Any], + secrets: dict[str, Any], + enabled_tools: Optional[list[str]] = None, + simulation_mode: str = SIM_MODE_MOCK_WRITES, + timeout: float = 30.0, + ): + self.name = name + self.simulation_mode = simulation_mode + self.timeout = timeout + self.base_url = (config.get("base_url") or "").rstrip("/") + self.headers = {**(config.get("headers") or {}), **(secrets.get("headers") or {})} + enabled = set(enabled_tools) if enabled_tools is not None else None + self._tools = { + f"{prefix}__{tool['name']}": tool + for tool in validate_http_tools(config) + if enabled is None or tool["name"] in enabled + } + + def list_tools(self) -> list[ToolSpec]: + return [ + ToolSpec( + name=exposed, + description=tool["description"], + parameters=tool["parameters"], + provider=self.name, + side_effect=tool["side_effect"], + ) + for exposed, tool in self._tools.items() + ] + + def _build_request(self, tool: dict[str, Any], arguments: dict[str, Any]): + remaining = dict(arguments) + + def _fill(match: re.Match) -> str: + key = match.group(1) + if key not in remaining: + raise ValueError(f"Missing path argument: {key}") + return quote(str(remaining.pop(key)), safe="") + + target = tool["url"] or f"{self.base_url}{tool['path']}" + url = _PLACEHOLDER_RE.sub(_fill, target) + if tool["method"] in ("GET", "DELETE"): + return url, {k: v for k, v in remaining.items() if v is not None}, None + return url, None, remaining + + def call(self, tool_name: str, arguments: dict[str, Any]) -> ToolResult: + from app.config import settings + from app.services.telephony.recording_download import assert_outbound_http_url_safe + + tool = self._tools.get(tool_name) + if tool is None: + return ToolResult(f"Unknown tool: {tool_name}", is_error=True) + try: + url, params, body = self._build_request(tool, arguments) + assert_outbound_http_url_safe(url, allow_loopback=bool(settings.DEBUG)) + except Exception as exc: + return ToolResult(f"Request not sent: {exc}", is_error=True) + with httpx.Client(timeout=self.timeout, follow_redirects=False) as client: + response = client.request( + tool["method"], url, params=params, json=body, headers=self.headers + ) + text = response.text + if len(text) > MAX_RESULT_CHARS: + text = text[:MAX_RESULT_CHARS] + "…" + if response.status_code >= 400: + return ToolResult(f"HTTP {response.status_code}: {text}", is_error=True) + return ToolResult( + text or f"HTTP {response.status_code}", extra={"http_status": response.status_code} + ) diff --git a/app/services/tools/providers/knowledge_base.py b/app/services/tools/providers/knowledge_base.py new file mode 100644 index 00000000..9dac2080 --- /dev/null +++ b/app/services/tools/providers/knowledge_base.py @@ -0,0 +1,82 @@ +"""``search_knowledge_base`` as a tool provider over the agent's attached knowledge bases.""" + +from __future__ import annotations + +from typing import Any, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.services.knowledge_bases.agent_links import KB_ROLE_PRODUCTION +from app.services.knowledge_bases.retrieval import ( + format_search_results_for_tool, + search_knowledge_base, +) +from app.services.knowledge_bases.tool_call_logging import record_knowledge_base_tool_call +from app.services.knowledge_bases.tool_schema import search_knowledge_base_tool_definition +from app.services.tools.base import ToolResult, ToolSpec + +KNOWLEDGE_BASE_PROVIDER = "knowledge_base" +KNOWLEDGE_BASE_TOOL = "search_knowledge_base" + + +class KnowledgeBaseToolProvider: + name = KNOWLEDGE_BASE_PROVIDER + + def __init__( + self, + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: UUID, + channel: str = "chat_tools", + llm_model: Optional[str] = None, + role: str = KB_ROLE_PRODUCTION, + ): + self.db = db + self.organization_id = organization_id + self.workspace_id = workspace_id + self.agent_id = agent_id + self.channel = channel + self.llm_model = llm_model + self.role = role + # Legacy-shaped log ({tool, query, hit_count, ...}) kept for existing callers. + self.legacy_log: list[dict[str, Any]] = [] + + def list_tools(self) -> list[ToolSpec]: + fn = search_knowledge_base_tool_definition()["function"] + return [ + ToolSpec( + name=fn["name"], + description=fn["description"], + parameters=fn["parameters"], + provider=self.name, + ) + ] + + def call(self, tool_name: str, arguments: dict[str, Any]) -> ToolResult: + query = str(arguments.get("query") or "").strip() + if not query: + return ToolResult("search_knowledge_base needs a non-empty query.", is_error=True) + hits = search_knowledge_base( + self.db, + organization_id=self.organization_id, + workspace_id=self.workspace_id, + agent_id=self.agent_id, + query=query, + role=self.role, + ) + record_knowledge_base_tool_call( + channel=self.channel, + query=query, + hit_count=len(hits), + agent_id=str(self.agent_id), + llm_model=self.llm_model, + tool_calls_log=self.legacy_log, + ) + documents = sorted({hit.get("document_name") or "document" for hit in hits}) + return ToolResult( + format_search_results_for_tool(hits), + extra={"hit_count": len(hits), "documents": documents}, + ) diff --git a/app/services/tools/providers/mcp.py b/app/services/tools/providers/mcp.py new file mode 100644 index 00000000..8e8b2ee3 --- /dev/null +++ b/app/services/tools/providers/mcp.py @@ -0,0 +1,306 @@ +"""MCP servers (Slack, Notion, Google Drive, ...) as a tool provider. + +The tool list comes from the connection's cached ``tools/list`` result, so building a +toolset never touches the network. The server session opens on the first call and +stays open for the conversation; ``close()`` ends it. + +The MCP client is async and its transports use anyio cancel scopes, which must be +entered and exited by the same task. ``McpSession`` therefore runs one long-lived +task on a private event loop thread that owns the session and serves requests from +a queue; sync callers wait on ``concurrent.futures.Future`` results. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import threading +from contextlib import AsyncExitStack +from datetime import timedelta +from typing import Any, Awaitable, Callable, Optional + +from loguru import logger + +from app.services.tools.base import ( + SIM_MODE_MOCK_WRITES, + ToolResult, + ToolSpec, + classify_side_effect, +) + +TRANSPORT_STREAMABLE_HTTP = "streamable_http" +TRANSPORT_SSE = "sse" +TRANSPORT_STDIO = "stdio" +TRANSPORTS = (TRANSPORT_STREAMABLE_HTTP, TRANSPORT_SSE, TRANSPORT_STDIO) + +MAX_RESULT_CHARS = 12000 + + +class McpConnectionError(RuntimeError): + pass + + +def assert_transport_allowed(config: dict[str, Any]) -> None: + from app.config import settings + from app.services.telephony.recording_download import assert_outbound_http_url_safe + + transport = config.get("transport") or TRANSPORT_STREAMABLE_HTTP + if transport not in TRANSPORTS: + raise McpConnectionError(f"Unknown MCP transport: {transport}") + if transport == TRANSPORT_STDIO: + if not settings.TOOL_CONNECTIONS_ALLOW_STDIO: + raise McpConnectionError( + "stdio MCP servers are disabled on this deployment " + "(set TOOL_CONNECTIONS_ALLOW_STDIO=true on self-hosted installs)" + ) + if not (config.get("command") or "").strip(): + raise McpConnectionError("stdio MCP servers need a command") + return + url = (config.get("url") or "").strip() + if not url: + raise McpConnectionError("MCP server URL is required") + try: + assert_outbound_http_url_safe(url, allow_loopback=bool(settings.DEBUG)) + except Exception as exc: + raise McpConnectionError(f"MCP server URL is not allowed: {exc}") from exc + + +async def _open_client_session( + stack: AsyncExitStack, + config: dict[str, Any], + secrets: dict[str, Any], + timeout: float, +): + from mcp import ClientSession + + transport = config.get("transport") or TRANSPORT_STREAMABLE_HTTP + headers = {**(config.get("headers") or {}), **(secrets.get("headers") or {})} + if transport == TRANSPORT_STDIO: + from mcp.client.stdio import StdioServerParameters, stdio_client + + params = StdioServerParameters( + command=config["command"], + args=list(config.get("args") or []), + env={**(config.get("env") or {}), **(secrets.get("env") or {})} or None, + ) + read, write = await stack.enter_async_context(stdio_client(params)) + elif transport == TRANSPORT_SSE: + from mcp.client.sse import sse_client + + read, write = await stack.enter_async_context( + sse_client(config["url"], headers=headers, timeout=timeout) + ) + else: + import httpx + from mcp.client.streamable_http import streamable_http_client + + # Own client: the SDK default follows redirects, which would bypass the SSRF check. + http_client = await stack.enter_async_context( + httpx.AsyncClient(headers=headers, timeout=timeout, follow_redirects=False) + ) + read, write, _ = await stack.enter_async_context( + streamable_http_client(config["url"], http_client=http_client) + ) + session = await stack.enter_async_context(ClientSession(read, write)) + await session.initialize() + return session + + +class McpSession: + """A live MCP client session driven from synchronous code.""" + + def __init__(self, config: dict[str, Any], secrets: dict[str, Any], *, timeout: float): + self.timeout = timeout + self._config = config + self._secrets = secrets + self._ready: concurrent.futures.Future = concurrent.futures.Future() + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._queue: Optional[asyncio.Queue] = None + self._thread = threading.Thread(target=self._run, name="mcp-session", daemon=True) + self._thread.start() + try: + self._ready.result(timeout=timeout) + except concurrent.futures.TimeoutError as exc: + self.close() + raise McpConnectionError(f"MCP server did not respond within {timeout:.0f}s") from exc + except Exception as exc: + self.close() + raise McpConnectionError(f"Could not connect to MCP server: {exc}") from exc + + def _run(self) -> None: + asyncio.run(self._main()) + + async def _main(self) -> None: + self._loop = asyncio.get_running_loop() + self._queue = asyncio.Queue() + try: + async with AsyncExitStack() as stack: + session = await _open_client_session( + stack, self._config, self._secrets, self.timeout + ) + self._ready.set_result(None) + while True: + item = await self._queue.get() + if item is None: + break + fn, future = item + try: + future.set_result(await fn(session)) + except Exception as exc: + future.set_exception(exc) + except BaseException as exc: # noqa: BLE001 - surface any startup failure to the caller + if not self._ready.done(): + self._ready.set_exception(exc if isinstance(exc, Exception) else RuntimeError(exc)) + else: + logger.warning("[MCP] Session ended with error: {}", exc) + + def run(self, fn: Callable[[Any], Awaitable[Any]]) -> Any: + if self._loop is None or self._queue is None or not self._thread.is_alive(): + raise McpConnectionError("MCP session is closed") + future: concurrent.futures.Future = concurrent.futures.Future() + self._loop.call_soon_threadsafe(self._queue.put_nowait, (fn, future)) + try: + return future.result(timeout=self.timeout) + except concurrent.futures.TimeoutError as exc: + raise McpConnectionError(f"MCP call timed out after {self.timeout:.0f}s") from exc + + def close(self) -> None: + if self._loop is not None and self._queue is not None and self._thread.is_alive(): + self._loop.call_soon_threadsafe(self._queue.put_nowait, None) + self._thread.join(timeout=5) + + +def tool_entry_from_mcp(tool: Any) -> dict[str, Any]: + """Serializable form of an ``mcp.types.Tool`` for ``ToolConnection.cached_tools``.""" + annotations = getattr(tool, "annotations", None) + return { + "name": tool.name, + "description": tool.description or getattr(tool, "title", None) or tool.name, + "input_schema": tool.inputSchema or {"type": "object", "properties": {}}, + "read_only_hint": getattr(annotations, "readOnlyHint", None), + "destructive_hint": getattr(annotations, "destructiveHint", None), + } + + +def discover_mcp_tools( + config: dict[str, Any], secrets: dict[str, Any], *, timeout: float +) -> list[dict[str, Any]]: + """Connect, run ``tools/list`` (all pages) and disconnect.""" + assert_transport_allowed(config) + + async def _list(session) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + cursor = None + while True: + page = await session.list_tools(cursor=cursor) + entries.extend(tool_entry_from_mcp(tool) for tool in page.tools) + cursor = page.nextCursor + if not cursor: + return entries + + session = McpSession(config, secrets, timeout=timeout) + try: + return session.run(_list) + finally: + session.close() + + +def _result_text(result: Any) -> str: + parts: list[str] = [] + for item in getattr(result, "content", None) or []: + kind = getattr(item, "type", None) + if kind == "text": + parts.append(item.text) + elif kind == "resource": + resource = getattr(item, "resource", None) + parts.append( + getattr(resource, "text", None) or f"[resource {getattr(resource, 'uri', '')}]" + ) + else: + parts.append(f"[{kind} content]") + if not parts and getattr(result, "structuredContent", None) is not None: + import json + + parts.append(json.dumps(result.structuredContent)) + text = "\n".join(parts).strip() or "(empty result)" + return text if len(text) <= MAX_RESULT_CHARS else text[:MAX_RESULT_CHARS] + "…" + + +class McpToolProvider: + """Tools of one MCP connection, exposed as ``__``.""" + + def __init__( + self, + *, + name: str, + prefix: str, + config: dict[str, Any], + secrets: dict[str, Any], + cached_tools: list[dict[str, Any]], + enabled_tools: Optional[list[str]] = None, + simulation_mode: str = SIM_MODE_MOCK_WRITES, + timeout: float = 30.0, + ): + self.name = name + self.prefix = prefix + self.config = config + self.secrets = secrets + self.simulation_mode = simulation_mode + self.timeout = timeout + overrides = config.get("tool_side_effects") or {} + enabled = set(enabled_tools) if enabled_tools is not None else None + self._tools: dict[str, dict[str, Any]] = {} + self._specs: list[ToolSpec] = [] + for entry in cached_tools: + if enabled is not None and entry["name"] not in enabled: + continue + exposed = f"{prefix}__{entry['name']}" + self._tools[exposed] = entry + self._specs.append( + ToolSpec( + name=exposed, + description=entry.get("description") or entry["name"], + parameters=entry.get("input_schema") or {"type": "object", "properties": {}}, + provider=name, + side_effect=overrides.get(entry["name"]) + or classify_side_effect( + entry["name"], + read_only_hint=entry.get("read_only_hint"), + destructive_hint=entry.get("destructive_hint"), + ), + ) + ) + self._session: Optional[McpSession] = None + self._lock = threading.Lock() + + def list_tools(self) -> list[ToolSpec]: + return list(self._specs) + + def _ensure_session(self) -> McpSession: + with self._lock: + if self._session is None: + assert_transport_allowed(self.config) + self._session = McpSession(self.config, self.secrets, timeout=self.timeout) + return self._session + + def call(self, tool_name: str, arguments: dict[str, Any]) -> ToolResult: + entry = self._tools.get(tool_name) + if entry is None: + return ToolResult(f"Unknown tool: {tool_name}", is_error=True) + session = self._ensure_session() + + async def _call(client): + return await client.call_tool( + entry["name"], + arguments, + read_timeout_seconds=timedelta(seconds=self.timeout), + ) + + result = session.run(_call) + return ToolResult(_result_text(result), is_error=bool(getattr(result, "isError", False))) + + def close(self) -> None: + with self._lock: + session, self._session = self._session, None + if session is not None: + session.close() diff --git a/app/services/tools/providers/postgres.py b/app/services/tools/providers/postgres.py new file mode 100644 index 00000000..c2958906 --- /dev/null +++ b/app/services/tools/providers/postgres.py @@ -0,0 +1,411 @@ +"""Read-only Postgres access for agents that look up data (orders, accounts, ...). + +Every statement runs inside ``SET TRANSACTION READ ONLY`` with a statement timeout, +so a write attempted by the model fails in the database, not just in our checks. +Secrets: ``{"dsn": "postgresql://user:pass@host:5432/db"}``. Config: optional +``schemas`` allow-list (default ``["public"]``) and ``max_rows`` (default 50). +""" + +from __future__ import annotations + +import json +import re +import threading +from typing import Any, Optional + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, make_url +from sqlalchemy.pool import NullPool + +from app.services.tools.base import SIDE_EFFECT_READ, SIM_MODE_LIVE, ToolResult, ToolSpec + +DEFAULT_MAX_ROWS = 50 +STATEMENT_TIMEOUT_MS = 10000 +MAX_RESULT_CHARS = 12000 + +POSTGRES_TOOLS = [ + { + "name": "list_tables", + "description": ( + "List the tables you can query. Call this at most once per conversation, " + "then query. Do not call it again if you already know the table names." + ), + "input_schema": {"type": "object", "properties": {}}, + }, + { + "name": "describe_table", + "description": "Columns and types of one table.", + "input_schema": { + "type": "object", + "properties": { + "table": {"type": "string", "description": "Table name, optionally schema.table"} + }, + "required": ["table"], + }, + }, + { + "name": "query", + "description": ( + "Run one read-only SQL SELECT and get the rows back as JSON. " + "Always add a WHERE clause or LIMIT. After the rows come back, " + "answer the customer instead of calling more tools." + ), + "input_schema": { + "type": "object", + "properties": {"sql": {"type": "string", "description": "A single SELECT statement"}}, + "required": ["sql"], + }, + }, +] + + +def tool_entries_for_postgres() -> list[dict[str, Any]]: + return [{**tool, "read_only_hint": True, "destructive_hint": None} for tool in POSTGRES_TOOLS] + + +def normalize_dsn(dsn: str) -> str: + """psycopg2 SQLAlchemy URL from a ``postgres://`` / ``postgresql://`` DSN.""" + raw = dsn.strip() + if raw.startswith("postgres://"): # Heroku / Supabase / Neon style + raw = "postgresql://" + raw[len("postgres://") :] + url = make_url(raw) + if url.get_backend_name() != "postgresql": + raise ValueError("Only PostgreSQL connection strings are supported") + return url.set(drivername="postgresql+psycopg2").render_as_string(hide_password=False) + + +def assert_dsn_host_allowed(dsn: str) -> None: + from app.config import settings + from app.services.telephony.recording_download import assert_outbound_http_url_safe + + raw = dsn.strip() + url = make_url(raw) + if not url.host: + raise ValueError("Connection string needs a host") + assert_outbound_http_url_safe(f"http://{url.host}", allow_loopback=bool(settings.DEBUG)) + + query = url.query if isinstance(url.query, dict) else {} + if not query and url.query: + for part in str(url.query).split("&"): + if "=" in part: + key, _value = part.split("=", 1) + query[key.lower()] = _value + for override in ("hostaddr", "host"): + if override in {k.lower() for k in query}: + raise ValueError( + f"Connection string must not override the host via {override}" + ) + + +def _single_statement(sql: str) -> str: + stripped = (sql or "").strip().rstrip(";").strip() + if not stripped: + raise ValueError("SQL is empty") + if ";" in stripped: + raise ValueError("Send one statement at a time") + return stripped + + +_FORBIDDEN_SQL_SCHEMAS = frozenset({"pg_catalog", "information_schema"}) +_FROM_OR_JOIN = re.compile(r"\b(?:from|join)\b", re.IGNORECASE) +_TABLE_PREFIX_WORDS = frozenset({"only", "lateral"}) +_FROM_STOP_WORDS = frozenset({ + "where", + "group", + "order", + "limit", + "union", + "except", + "intersect", + "window", + "having", + "offset", + "fetch", + "for", + "join", + "inner", + "left", + "right", + "full", + "cross", + "natural", + "on", + "using", +}) +_IDENT = re.compile(r'"([^"]+)"|([A-Za-z_][\w$]*)') +_DOLLAR_QUOTE = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)?\$") + + +def _mask_dollar_quote(sql: str, index: int, out: list[str]) -> int | None: + """Blank one ``$$...$$`` or ``$tag$...$tag$`` literal. None if this ``$`` is not one.""" + opener = _DOLLAR_QUOTE.match(sql, index) + if opener is None: + return None + delimiter = opener.group(0) + close = sql.find(delimiter, opener.end()) + end = len(sql) if close < 0 else close + len(delimiter) + out.extend(" " * (end - index)) + return end + + +def _continues_identifier(sql: str, index: int) -> bool: + """True when ``sql[index]`` extends an unquoted identifier (``$`` included).""" + if index <= 0: + return False + previous = sql[index - 1] + return previous.isalnum() or previous in "_$" + + +def _copy_quoted_identifier(sql: str, index: int, out: list[str]) -> int: + """Copy one double-quoted identifier, including a ``$`` inside it.""" + out.append('"') + index += 1 + length = len(sql) + while index < length: + out.append(sql[index]) + index += 1 + if out[-1] != '"': + continue + if index < length and sql[index] == '"': + out.append('"') + index += 1 + continue + break + return index + + +def _mask_sql_strings_and_comments(sql: str) -> str: + """Blank string literals and comments so schema checks ignore them.""" + out: list[str] = [] + i = 0 + length = len(sql) + while i < length: + if sql.startswith("--", i): + while i < length and sql[i] != "\n": + out.append(" ") + i += 1 + continue + if sql.startswith("/*", i): + end = sql.find("*/", i + 2) + end = length if end < 0 else end + 2 + out.extend(" " * (end - i)) + i = end + continue + if sql[i] == '"': + i = _copy_quoted_identifier(sql, i, out) + continue + if sql[i] == "$" and not _continues_identifier(sql, i): + end = _mask_dollar_quote(sql, i, out) + if end is not None: + i = end + continue + if sql[i] == "'": + out.append(" ") + i += 1 + while i < length: + if sql[i] == "'" and i + 1 < length and sql[i + 1] == "'": + out.extend(" ") + i += 2 + continue + ch = sql[i] + out.append(" ") + i += 1 + if ch == "'": + break + continue + out.append(sql[i]) + i += 1 + return "".join(out) + + +def _skip_ws(sql: str, index: int) -> int: + while index < len(sql) and sql[index].isspace(): + index += 1 + return index + + +def _read_ident(sql: str, index: int) -> tuple[str, int] | None: + match = _IDENT.match(sql, index) + if match is None: + return None + return (match.group(1) or match.group(2)), match.end() + + +def _skip_table_alias(sql: str, index: int) -> int: + index = _skip_ws(sql, index) + ident = _read_ident(sql, index) + if ident is None: + return index + name, end = ident + lowered = name.lower() + if lowered in _FROM_STOP_WORDS: + return index + if lowered == "as": + index = _skip_ws(sql, end) + alias = _read_ident(sql, index) + return alias[1] if alias is not None else index + return end + + +def _schemas_named_in_query(sql: str) -> list[str]: + """Schema qualifiers on table refs and ``schema.func(`` calls.""" + masked = _mask_sql_strings_and_comments(sql) + found: list[str] = [] + for match in re.finditer( + r'(?:"([^"]+)"|([A-Za-z_][\w$]*))\s*\.\s*(?:"[^"]+"|[A-Za-z_][\w$]*)\s*\(', + masked, + ): + found.append((match.group(1) or match.group(2)).lower()) + + for match in _FROM_OR_JOIN.finditer(masked): + index = match.end() + while index < len(masked): + index = _skip_ws(masked, index) + if index >= len(masked) or masked[index] == "(": + break + ident = _read_ident(masked, index) + if ident is None: + break + name, index = ident + if name.lower() in _TABLE_PREFIX_WORDS: + continue + index = _skip_ws(masked, index) + if index < len(masked) and masked[index] == ".": + found.append(name.lower()) + index += 1 + index = _skip_ws(masked, index) + second = _read_ident(masked, index) + if second is not None: + index = second[1] + index = _skip_table_alias(masked, index) + index = _skip_ws(masked, index) + if index < len(masked) and masked[index] == ",": + index += 1 + continue + break + return found + + +def _assert_query_uses_allowed_schemas(sql: str, allowed_schemas: list[str]) -> None: + masked = _mask_sql_strings_and_comments(sql) + lowered = masked.lower() + for blocked in _FORBIDDEN_SQL_SCHEMAS: + if re.search(rf"\b{blocked}\b", lowered): + raise ValueError(f"Schema {blocked} is not allowed") + allowed = {schema.lower() for schema in allowed_schemas} + for schema in _schemas_named_in_query(sql): + if schema not in allowed: + raise ValueError(f"Schema {schema} is not allowed") + + +class PostgresToolProvider: + def __init__( + self, + *, + name: str, + prefix: str, + config: dict[str, Any], + secrets: dict[str, Any], + enabled_tools: Optional[list[str]] = None, + simulation_mode: str = SIM_MODE_LIVE, + timeout: float = 30.0, + ): + self.name = name + self.simulation_mode = simulation_mode + self.prefix = prefix + self.dsn = secrets.get("dsn") or "" + self.schemas = list(config.get("schemas") or ["public"]) + self.max_rows = int(config.get("max_rows") or DEFAULT_MAX_ROWS) + self.timeout = timeout + enabled = set(enabled_tools) if enabled_tools is not None else None + self._tools = { + f"{prefix}__{tool['name']}": tool + for tool in POSTGRES_TOOLS + if enabled is None or tool["name"] in enabled + } + self._engine: Optional[Engine] = None + self._lock = threading.Lock() + + def list_tools(self) -> list[ToolSpec]: + return [ + ToolSpec( + name=exposed, + description=tool["description"], + parameters=tool["input_schema"], + provider=self.name, + side_effect=SIDE_EFFECT_READ, + ) + for exposed, tool in self._tools.items() + ] + + def _get_engine(self) -> Engine: + with self._lock: + if self._engine is None: + assert_dsn_host_allowed(self.dsn) + self._engine = create_engine( + normalize_dsn(self.dsn), + poolclass=NullPool, + connect_args={"connect_timeout": int(min(self.timeout, 30))}, + ) + return self._engine + + def _read_only(self, sql: str, params: Optional[dict[str, Any]] = None) -> list[dict[str, Any]]: + with self._get_engine().connect() as conn: + with conn.begin(): + conn.execute(text("SET TRANSACTION READ ONLY")) + conn.execute(text(f"SET LOCAL statement_timeout = {STATEMENT_TIMEOUT_MS}")) + schemas_sql = ", ".join( + f'"{schema.replace(chr(34), "")}"' for schema in self.schemas + ) + conn.execute(text(f"SET LOCAL search_path TO {schemas_sql}")) + result = conn.execute(text(sql), params or {}) + rows = result.mappings().fetchmany(self.max_rows + 1) + return [dict(row) for row in rows] + + def _format(self, rows: list[dict[str, Any]]) -> str: + truncated = len(rows) > self.max_rows + body = json.dumps(rows[: self.max_rows], default=str) + if truncated: + body += f"\n(only the first {self.max_rows} rows are shown)" + return body if len(body) <= MAX_RESULT_CHARS else body[:MAX_RESULT_CHARS] + "…" + + def call(self, tool_name: str, arguments: dict[str, Any]) -> ToolResult: + tool = self._tools.get(tool_name) + if tool is None: + return ToolResult(f"Unknown tool: {tool_name}", is_error=True) + try: + if tool["name"] == "list_tables": + rows = self._read_only( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_schema = ANY(:schemas) ORDER BY 1, 2", + {"schemas": self.schemas}, + ) + elif tool["name"] == "describe_table": + raw = str(arguments.get("table") or "").strip() + schema, _, table = raw.rpartition(".") + if schema and schema not in self.schemas: + return ToolResult(f"Schema {schema} is not allowed", is_error=True) + rows = self._read_only( + "SELECT column_name, data_type, is_nullable FROM information_schema.columns " + "WHERE table_name = :table AND table_schema = ANY(:schemas) " + "ORDER BY ordinal_position", + {"table": table, "schemas": [schema] if schema else self.schemas}, + ) + if not rows: + return ToolResult(f"Table {raw} not found", is_error=True) + else: + sql = _single_statement(str(arguments.get("sql") or "")) + if not sql.lstrip().lower().startswith("select"): + return ToolResult("Only SELECT queries are allowed", is_error=True) + _assert_query_uses_allowed_schemas(sql, self.schemas) + rows = self._read_only(sql) + except Exception as exc: + message = str(getattr(exc, "orig", None) or exc).strip().splitlines()[0] + return ToolResult(f"Query failed: {message}", is_error=True) + return ToolResult(self._format(rows), extra={"row_count": min(len(rows), self.max_rows)}) + + def close(self) -> None: + with self._lock: + engine, self._engine = self._engine, None + if engine is not None: + engine.dispose() diff --git a/app/services/tools/toolset.py b/app/services/tools/toolset.py new file mode 100644 index 00000000..c901287b --- /dev/null +++ b/app/services/tools/toolset.py @@ -0,0 +1,188 @@ +"""Collect an agent's tools from every provider and execute calls by name.""" + +from __future__ import annotations + +import json +import time +from typing import Any, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.services.tools.base import ( + SIDE_EFFECT_WRITE, + SIM_MODE_LIVE, + SIM_MODE_MOCK_ALL, + SIM_MODE_MOCK_WRITES, + TOOL_STATUS_ERROR, + TOOL_STATUS_MOCKED, + TOOL_STATUS_OK, + ToolCallRecord, + ToolProvider, + ToolResult, + ToolSpec, + preview, +) + +DEFAULT_WRITE_MOCK = {"ok": True, "simulated": True} + + +class Toolset: + """Name → provider map that executes calls and records them. + + In a simulation each provider's ``simulation_mode`` decides what runs for real: + ``live`` runs everything, ``mock_writes`` (default) fakes write tools and + ``mock_all`` fakes every tool. ``mock_responses`` (tool name → response) comes + from a scenario's ``expected_tool_calls[].mock_response`` and always wins. + """ + + def __init__( + self, + providers: list[ToolProvider], + *, + simulation: bool = True, + mock_responses: Optional[dict[str, Any]] = None, + ): + self.providers = providers + self.simulation = simulation + self.mock_responses = dict(mock_responses or {}) + self._specs: dict[str, tuple[ToolSpec, ToolProvider]] = {} + for provider in providers: + try: + specs = provider.list_tools() + except Exception as exc: + logger.warning("[Tools] {} has no usable tools: {}", provider.name, exc) + continue + for spec in specs: + if spec.name in self._specs: + logger.warning("[Tools] Duplicate tool name {} ignored", spec.name) + continue + self._specs[spec.name] = (spec, provider) + + def __bool__(self) -> bool: + return bool(self._specs) + + @property + def tool_names(self) -> list[str]: + return list(self._specs) + + def definitions(self) -> list[dict[str, Any]]: + return [spec.openai_definition() for spec, _ in self._specs.values()] + + def provider(self, name: str) -> Optional[ToolProvider]: + return next((p for p in self.providers if p.name == name), None) + + def spec(self, name: str) -> Optional[ToolSpec]: + entry = self._specs.get(name) + return entry[0] if entry else None + + def _should_mock(self, spec: ToolSpec, provider: ToolProvider) -> bool: + if spec.name in self.mock_responses: + return True + if not self.simulation: + return False + mode = getattr(provider, "simulation_mode", SIM_MODE_MOCK_WRITES) + if mode == SIM_MODE_LIVE: + return False + return mode == SIM_MODE_MOCK_ALL or spec.side_effect == SIDE_EFFECT_WRITE + + def should_mock(self, name: str) -> bool: + entry = self._specs.get(name) + return entry is not None and self._should_mock(*entry) + + def close(self) -> None: + for provider in self.providers: + close = getattr(provider, "close", None) + if close is None: + continue + try: + close() + except Exception as exc: + logger.warning("[Tools] Closing {} failed: {}", provider.name, exc) + + def execute( + self, + name: str, + arguments: dict[str, Any], + *, + call_id: Optional[str] = None, + side: str = "production", + ) -> tuple[ToolResult, ToolCallRecord]: + started = time.monotonic() + entry = self._specs.get(name) + provider_name = entry[0].provider if entry else "unknown" + status = TOOL_STATUS_OK + if entry is None: + result = ToolResult(f"Unknown tool: {name}", is_error=True) + elif self._should_mock(*entry): + mock = self.mock_responses.get(name, DEFAULT_WRITE_MOCK) + content = mock if isinstance(mock, str) else json.dumps(mock) + result = ToolResult(content) + status = TOOL_STATUS_MOCKED + else: + try: + result = entry[1].call(name, arguments) + except Exception as exc: + logger.warning("[Tools] {} failed: {}", name, exc) + result = ToolResult(f"Tool error: {exc}", is_error=True) + if result.is_error: + status = TOOL_STATUS_ERROR + record = ToolCallRecord( + call_id=call_id, + tool=name, + provider=provider_name, + arguments=arguments, + status=status, + result_preview=preview(result.content), + latency_ms=round((time.monotonic() - started) * 1000), + side=side, + error=result.content if result.is_error else None, + extra=result.extra, + ) + return result, record + + +def build_agent_toolset( + db: Session, + *, + agent: Any, + organization_id: UUID, + llm_model: Optional[str] = None, + channel: str = "chat_tools", + mock_responses: Optional[dict[str, Any]] = None, +) -> Toolset: + """Every tool the production agent can call: its knowledge bases plus attached connections.""" + from app.services.knowledge_bases.retrieval import agent_has_ready_knowledge + from app.services.tools.connections import agent_connection_providers + from app.services.tools.providers.knowledge_base import KnowledgeBaseToolProvider + + providers: list[ToolProvider] = [] + workspace_id = getattr(agent, "workspace_id", None) + if workspace_id and agent_has_ready_knowledge(db, agent_id=agent.id): + providers.append( + KnowledgeBaseToolProvider( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent.id, + channel=channel, + llm_model=llm_model, + ) + ) + if workspace_id: + providers.extend(agent_connection_providers(db, agent=agent)) + return Toolset(providers, mock_responses=mock_responses) + + +def scenario_mock_responses(scenario: Any) -> dict[str, Any]: + """``tool → mock_response`` from a scenario's ``expected_tool_calls``.""" + mocks: dict[str, Any] = {} + for expectation in getattr(scenario, "expected_tool_calls", None) or []: + if ( + isinstance(expectation, dict) + and expectation.get("tool") + and expectation.get("mock_response") is not None + ): + mocks[str(expectation["tool"])] = expectation["mock_response"] + return mocks diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index 38c9455f..932818bf 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -1404,7 +1404,8 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID) -> int: _release_flush_lock(organization_id) flushed += _flush_pending_buffer(db, organization_id, pricing_resolver=pricing_resolver) - invalidate_org_usage_read_cache(organization_id) + if flushed: + invalidate_org_usage_read_cache(organization_id) return flushed diff --git a/app/services/usage/pricing_overrides.py b/app/services/usage/pricing_overrides.py index 77b84c2b..967f0dbf 100644 --- a/app/services/usage/pricing_overrides.py +++ b/app/services/usage/pricing_overrides.py @@ -389,6 +389,9 @@ def upsert_override( db.commit() invalidate_org_pricing_cache(organization_id) + from app.services.usage.read_cache import invalidate_org_usage_read_cache + + invalidate_org_usage_read_cache(organization_id) recompute_job_id = None recompute_enqueued = False @@ -493,6 +496,9 @@ def delete_override( db.commit() invalidate_org_pricing_cache(organization_id) + from app.services.usage.read_cache import invalidate_org_usage_read_cache + + invalidate_org_usage_read_cache(organization_id) recompute_job_id = None recompute_enqueued = False diff --git a/app/services/usage/read_cache.py b/app/services/usage/read_cache.py index 5db010f0..796f492e 100644 --- a/app/services/usage/read_cache.py +++ b/app/services/usage/read_cache.py @@ -5,7 +5,8 @@ import hashlib import json import os -from typing import Any, Optional +import time +from typing import Any, Callable, Optional from uuid import UUID import redis @@ -15,6 +16,10 @@ from app.services.usage.access import UsageAccessResult _redis: redis.Redis | None = None +_VERSION_TTL_SECONDS = 7 * 24 * 60 * 60 +_STALE_TTL_SECONDS = 120 +_LOCK_TTL_SECONDS = 10 +_FOLLOWER_WAIT_SECONDS = 1.5 def _client() -> redis.Redis: @@ -24,12 +29,31 @@ def _client() -> redis.Redis: return _redis -def cache_ttl_seconds() -> int: - raw = os.environ.get("USAGE_READ_CACHE_TTL_SECONDS", "90") +def _ttl_from_env(name: str, default: int) -> int: + raw = os.environ.get(name, str(default)) try: return max(0, int(raw)) except (TypeError, ValueError): - return 90 + return default + + +def cache_ttl_seconds() -> int: + return _ttl_from_env("USAGE_READ_CACHE_TTL_SECONDS", 90) + + +def historical_cache_ttl_seconds() -> int: + return _ttl_from_env("USAGE_HISTORICAL_READ_CACHE_TTL_SECONDS", 900) + + +def empty_cache_ttl_seconds() -> int: + return _ttl_from_env("USAGE_EMPTY_READ_CACHE_TTL_SECONDS", 60) + + +def ttl_for_payload(*, historical: bool, empty: bool) -> int: + ttl = historical_cache_ttl_seconds() if historical else cache_ttl_seconds() + if empty: + return min(ttl, empty_cache_ttl_seconds()) + return ttl def cache_key_for(access: UsageAccessResult, **params: Any) -> str: @@ -59,8 +83,39 @@ def cache_key_for(access: UsageAccessResult, **params: Any) -> str: return digest[:32] -def _redis_key(organization_id: UUID, endpoint: str, cache_key: str) -> str: - return f"usage:read:{organization_id}:{endpoint}:{cache_key}" +def _version_key(organization_id: UUID) -> str: + return f"usage:read_ver:{organization_id}" + + +def _read_version(organization_id: UUID) -> str: + try: + raw = _client().get(_version_key(organization_id)) + return str(raw or "0") + except redis.RedisError: + return "0" + + +def read_cache_version(organization_id: UUID) -> str: + """Epoch captured before a usage query. Writes must use this same value.""" + return _read_version(organization_id) + + +def _redis_key( + organization_id: UUID, + endpoint: str, + cache_key: str, + version: Optional[str] = None, +) -> str: + current = version if version is not None else _read_version(organization_id) + return f"usage:read:{organization_id}:v{current}:{endpoint}:{cache_key}" + + +def _stale_key(organization_id: UUID, endpoint: str, cache_key: str) -> str: + return f"usage:read_stale:{organization_id}:{endpoint}:{cache_key}" + + +def _lock_key(organization_id: UUID, endpoint: str, cache_key: str) -> str: + return f"usage:read_lock:{organization_id}:{endpoint}:{cache_key}" def _is_empty_usage_payload(endpoint: str, payload: dict[str, Any]) -> bool: @@ -95,17 +150,18 @@ def get_cached_response( organization_id: UUID, endpoint: str, cache_key: str, + version: Optional[str] = None, ) -> Optional[dict[str, Any]]: ttl = cache_ttl_seconds() if ttl <= 0: return None try: - raw = _client().get(_redis_key(organization_id, endpoint, cache_key)) + raw = _client().get( + _redis_key(organization_id, endpoint, cache_key, version) + ) if not raw: return None payload = json.loads(raw) - if _is_empty_usage_payload(endpoint, payload): - return None return payload except (redis.RedisError, json.JSONDecodeError) as exc: logger.debug("usage read cache miss/error org={} endpoint={}: {}", organization_id, endpoint, exc) @@ -117,35 +173,150 @@ def set_cached_response( endpoint: str, cache_key: str, payload: dict[str, Any], + *, + historical: bool = False, + version: Optional[str] = None, ) -> None: - ttl = cache_ttl_seconds() + """Cache a usage read. Empty ranges use a short TTL; closed history uses a longer one. + + ``version`` is the epoch read before the database query. A flush bumps that + epoch while the query is running; the stale result is then discarded instead + of being stored under the new epoch. + """ + ttl = ttl_for_payload( + historical=historical, + empty=_is_empty_usage_payload(endpoint, payload), + ) if ttl <= 0: return - if _is_empty_usage_payload(endpoint, payload): - return + expected = version if version is not None else _read_version(organization_id) try: - _client().set( - _redis_key(organization_id, endpoint, cache_key), - json.dumps(payload, default=str), + client = _client() + if _read_version(organization_id) != expected: + return + encoded = json.dumps(payload, default=str) + client.set( + _redis_key(organization_id, endpoint, cache_key, expected), + encoded, ex=ttl, ) + if _read_version(organization_id) != expected: + return + client.set( + _stale_key(organization_id, endpoint, cache_key), + encoded, + ex=ttl + _STALE_TTL_SECONDS, + ) except redis.RedisError as exc: logger.debug("usage read cache set failed org={}: {}", organization_id, exc) def invalidate_org_usage_read_cache(organization_id: UUID) -> int: - pattern = f"usage:read:{organization_id}:*" - deleted = 0 + """Bump the org read epoch. Old keys expire on their own TTL.""" try: client = _client() - batch: list[str] = [] - for key in client.scan_iter(match=pattern, count=200): - batch.append(key) - if len(batch) >= 200: - deleted += client.delete(*batch) - batch.clear() - if batch: - deleted += client.delete(*batch) + pipe = client.pipeline() + pipe.incr(_version_key(organization_id)) + pipe.expire(_version_key(organization_id), _VERSION_TTL_SECONDS) + pipe.execute() + return 1 except redis.RedisError as exc: logger.warning("usage read cache invalidate failed org={}: {}", organization_id, exc) - return deleted + return 0 + + +def _load_json(raw: Optional[str]) -> Optional[dict[str, Any]]: + if not raw: + return None + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def acquire_read_lock(organization_id: UUID, endpoint: str, cache_key: str) -> bool: + try: + return bool( + _client().set( + _lock_key(organization_id, endpoint, cache_key), + "1", + nx=True, + ex=_LOCK_TTL_SECONDS, + ) + ) + except redis.RedisError: + return True + + +def release_read_lock(organization_id: UUID, endpoint: str, cache_key: str) -> None: + try: + _client().delete(_lock_key(organization_id, endpoint, cache_key)) + except redis.RedisError: + pass + + +def get_stale_response( + organization_id: UUID, + endpoint: str, + cache_key: str, +) -> Optional[dict[str, Any]]: + try: + return _load_json(_client().get(_stale_key(organization_id, endpoint, cache_key))) + except redis.RedisError: + return None + + +def wait_for_cached_response( + organization_id: UUID, + endpoint: str, + cache_key: str, + *, + timeout_seconds: float = _FOLLOWER_WAIT_SECONDS, +) -> Optional[dict[str, Any]]: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + time.sleep(0.05) + cached = get_cached_response(organization_id, endpoint, cache_key) + if cached is not None: + return cached + return None + + +def get_or_compute_cached_response( + organization_id: UUID, + endpoint: str, + cache_key: str, + compute_fn: Callable[[], dict[str, Any]], + *, + historical: bool = False, +) -> dict[str, Any]: + """One database read per cache miss. Followers reuse the leader result or the last payload.""" + if cache_ttl_seconds() <= 0: + return compute_fn() + version = read_cache_version(organization_id) + cached = get_cached_response(organization_id, endpoint, cache_key, version) + if cached is not None: + return cached + leader = acquire_read_lock(organization_id, endpoint, cache_key) + if not leader: + stale = get_stale_response(organization_id, endpoint, cache_key) + if stale is not None: + return stale + waited = wait_for_cached_response(organization_id, endpoint, cache_key) + if waited is not None: + return waited + return compute_fn() + try: + payload = compute_fn() + set_cached_response( + organization_id, + endpoint, + cache_key, + payload, + historical=historical, + version=version, + ) + return payload + finally: + release_read_lock(organization_id, endpoint, cache_key) diff --git a/app/services/usage/rollup_maintenance.py b/app/services/usage/rollup_maintenance.py new file mode 100644 index 00000000..4a58c31f --- /dev/null +++ b/app/services/usage/rollup_maintenance.py @@ -0,0 +1,620 @@ +"""Collision-safe usage-context backfill and non-blocking read indexes.""" + +from __future__ import annotations + +import json +from typing import Iterable, Sequence + +from loguru import logger +from sqlalchemy import text +from sqlalchemy.orm import Session + +_UUID_RE = ( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + "[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +_ZERO_UUID = "00000000-0000-0000-0000-000000000000" +_BATCH = 1000 +_ORPHAN_BATCH_LIMIT = 5000 + +_READ_INDEXES = ( + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_updated_at + ON llm_usage_daily (organization_id, updated_at DESC) + """, + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_section_date + ON llm_usage_daily (organization_id, product_section, usage_date) + """, + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_model_date + ON llm_usage_daily (organization_id, model, usage_date) + """, + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_date_call_import + ON llm_usage_daily (organization_id, usage_date, (context->>'call_import_id')) + WHERE context ? 'call_import_id' + """, + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_date_evaluation + ON llm_usage_daily (organization_id, usage_date, (context->>'evaluation_id')) + WHERE context ? 'evaluation_id' + """, + """ + CREATE INDEX {mode} IF NOT EXISTS ix_llm_usage_daily_org_date_resource + ON llm_usage_daily (organization_id, usage_date, (context->>'resource_id')) + WHERE context ? 'resource_id' + """, +) + +READ_INDEX_NAMES = ( + "ix_llm_usage_daily_org_updated_at", + "ix_llm_usage_daily_org_section_date", + "ix_llm_usage_daily_org_model_date", + "ix_llm_usage_daily_org_date_call_import", + "ix_llm_usage_daily_org_date_evaluation", + "ix_llm_usage_daily_org_date_resource", +) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _ensure_patch_table(db: Session) -> None: + db.execute( + text( + """ + CREATE TEMP TABLE IF NOT EXISTS usage_ctx_patch ( + src_id uuid PRIMARY KEY, + new_context jsonb NOT NULL + ) ON COMMIT DROP + """ + ) + ) + db.execute(text("TRUNCATE usage_ctx_patch")) + + +def merge_staged_usage_context(db: Session) -> None: + """Fold staged context rewrites into target buckets without losing rows or counts. + + ``uq_llm_usage_daily_bucket`` is keyed on + ``(organization_id, COALESCE(workspace_id, zero_uuid), product_section, + model, usage_date, usage_kind, context)``. Multiple staged rows may map onto + the same target bucket or onto an already-populated keeper row, while rows + on *different* dates/models that happen to share ``new_context`` remain + independent buckets and must all be updated. + """ + # Step 1: Fold any staged source rows into an already-existing keeper row + # that shares the exact full 7-tuple bucket key (pre-aggregating SUMs so + # 2+ staged rows hitting the same keeper never lose counts). + db.execute( + text( + """ + WITH colliding AS ( + SELECT + keeper.id AS keeper_id, + src.id AS src_id, + src.prompt_tokens, + src.completion_tokens, + src.cache_read_tokens, + src.cache_creation_tokens, + src.reasoning_tokens, + src.audio_seconds, + src.tts_characters, + src.call_count, + src.input_cost_micro_usd, + src.output_cost_micro_usd, + src.cache_read_cost_micro_usd, + src.cache_creation_cost_micro_usd, + src.reasoning_cost_micro_usd, + src.audio_cost_micro_usd, + src.tts_cost_micro_usd, + src.total_cost_micro_usd + FROM usage_ctx_patch AS patch + JOIN llm_usage_daily AS src ON src.id = patch.src_id + JOIN llm_usage_daily AS keeper + ON keeper.organization_id = src.organization_id + AND keeper.product_section = src.product_section + AND keeper.model = src.model + AND keeper.usage_date = src.usage_date + AND COALESCE(keeper.usage_kind, 'llm') = COALESCE(src.usage_kind, 'llm') + AND keeper.workspace_id IS NOT DISTINCT FROM src.workspace_id + AND keeper.context = patch.new_context + AND keeper.id <> src.id + ), + keeper_deltas AS ( + SELECT + keeper_id, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(cache_read_tokens)::bigint AS cache_read_tokens, + SUM(cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(reasoning_tokens)::bigint AS reasoning_tokens, + SUM(audio_seconds)::bigint AS audio_seconds, + SUM(tts_characters)::bigint AS tts_characters, + SUM(call_count)::bigint AS call_count, + SUM(input_cost_micro_usd)::bigint AS input_cost_micro_usd, + SUM(output_cost_micro_usd)::bigint AS output_cost_micro_usd, + SUM(cache_read_cost_micro_usd)::bigint AS cache_read_cost_micro_usd, + SUM(cache_creation_cost_micro_usd)::bigint AS cache_creation_cost_micro_usd, + SUM(reasoning_cost_micro_usd)::bigint AS reasoning_cost_micro_usd, + SUM(audio_cost_micro_usd)::bigint AS audio_cost_micro_usd, + SUM(tts_cost_micro_usd)::bigint AS tts_cost_micro_usd, + SUM(total_cost_micro_usd)::bigint AS total_cost_micro_usd + FROM colliding + GROUP BY keeper_id + ), + updated_keepers AS ( + UPDATE llm_usage_daily AS keeper + SET prompt_tokens = keeper.prompt_tokens + d.prompt_tokens, + completion_tokens = keeper.completion_tokens + d.completion_tokens, + cache_read_tokens = keeper.cache_read_tokens + d.cache_read_tokens, + cache_creation_tokens = keeper.cache_creation_tokens + d.cache_creation_tokens, + reasoning_tokens = keeper.reasoning_tokens + d.reasoning_tokens, + audio_seconds = keeper.audio_seconds + d.audio_seconds, + tts_characters = keeper.tts_characters + d.tts_characters, + call_count = keeper.call_count + d.call_count, + input_cost_micro_usd = keeper.input_cost_micro_usd + d.input_cost_micro_usd, + output_cost_micro_usd = keeper.output_cost_micro_usd + d.output_cost_micro_usd, + cache_read_cost_micro_usd = keeper.cache_read_cost_micro_usd + d.cache_read_cost_micro_usd, + cache_creation_cost_micro_usd = keeper.cache_creation_cost_micro_usd + d.cache_creation_cost_micro_usd, + reasoning_cost_micro_usd = keeper.reasoning_cost_micro_usd + d.reasoning_cost_micro_usd, + audio_cost_micro_usd = keeper.audio_cost_micro_usd + d.audio_cost_micro_usd, + tts_cost_micro_usd = keeper.tts_cost_micro_usd + d.tts_cost_micro_usd, + total_cost_micro_usd = keeper.total_cost_micro_usd + d.total_cost_micro_usd, + updated_at = now() + FROM keeper_deltas AS d + WHERE keeper.id = d.keeper_id + RETURNING keeper.id + ), + deleted_src AS ( + DELETE FROM llm_usage_daily AS src + USING colliding AS c + WHERE src.id = c.src_id + RETURNING src.id + ) + DELETE FROM usage_ctx_patch AS patch + USING deleted_src AS d + WHERE patch.src_id = d.id + """ + ) + ) + + # Step 2: Among remaining staged rows, merge any rows that collide with + # *each other* on the full 7-tuple bucket key into MIN(id::text)::uuid. + db.execute( + text( + f""" + WITH patch_groups AS ( + SELECT + MIN(src.id::text)::uuid AS keep_id, + SUM(src.prompt_tokens)::bigint AS prompt_tokens, + SUM(src.completion_tokens)::bigint AS completion_tokens, + SUM(src.cache_read_tokens)::bigint AS cache_read_tokens, + SUM(src.cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(src.reasoning_tokens)::bigint AS reasoning_tokens, + SUM(src.audio_seconds)::bigint AS audio_seconds, + SUM(src.tts_characters)::bigint AS tts_characters, + SUM(src.call_count)::bigint AS call_count, + SUM(src.input_cost_micro_usd)::bigint AS input_cost_micro_usd, + SUM(src.output_cost_micro_usd)::bigint AS output_cost_micro_usd, + SUM(src.cache_read_cost_micro_usd)::bigint AS cache_read_cost_micro_usd, + SUM(src.cache_creation_cost_micro_usd)::bigint AS cache_creation_cost_micro_usd, + SUM(src.reasoning_cost_micro_usd)::bigint AS reasoning_cost_micro_usd, + SUM(src.audio_cost_micro_usd)::bigint AS audio_cost_micro_usd, + SUM(src.tts_cost_micro_usd)::bigint AS tts_cost_micro_usd, + SUM(src.total_cost_micro_usd)::bigint AS total_cost_micro_usd + FROM usage_ctx_patch AS patch + JOIN llm_usage_daily AS src ON src.id = patch.src_id + GROUP BY + src.organization_id, + COALESCE(src.workspace_id, '{_ZERO_UUID}'::uuid), + src.product_section, + src.model, + src.usage_date, + COALESCE(src.usage_kind, 'llm'), + patch.new_context + HAVING COUNT(*) > 1 + ), + updated_self_keepers AS ( + UPDATE llm_usage_daily AS keeper + SET prompt_tokens = g.prompt_tokens, + completion_tokens = g.completion_tokens, + cache_read_tokens = g.cache_read_tokens, + cache_creation_tokens = g.cache_creation_tokens, + reasoning_tokens = g.reasoning_tokens, + audio_seconds = g.audio_seconds, + tts_characters = g.tts_characters, + call_count = g.call_count, + input_cost_micro_usd = g.input_cost_micro_usd, + output_cost_micro_usd = g.output_cost_micro_usd, + cache_read_cost_micro_usd = g.cache_read_cost_micro_usd, + cache_creation_cost_micro_usd = g.cache_creation_cost_micro_usd, + reasoning_cost_micro_usd = g.reasoning_cost_micro_usd, + audio_cost_micro_usd = g.audio_cost_micro_usd, + tts_cost_micro_usd = g.tts_cost_micro_usd, + total_cost_micro_usd = g.total_cost_micro_usd, + updated_at = now() + FROM patch_groups AS g + WHERE keeper.id = g.keep_id + RETURNING keeper.id + ), + dropped_peers AS ( + DELETE FROM llm_usage_daily AS src + USING usage_ctx_patch AS src_patch, + patch_groups AS g, + llm_usage_daily AS peer, + usage_ctx_patch AS peer_patch + WHERE src.id = src_patch.src_id + AND peer.id = g.keep_id + AND peer_patch.src_id = peer.id + AND src.id <> peer.id + AND src.organization_id = peer.organization_id + AND src.product_section = peer.product_section + AND src.model = peer.model + AND src.usage_date = peer.usage_date + AND COALESCE(src.usage_kind, 'llm') = COALESCE(peer.usage_kind, 'llm') + AND src.workspace_id IS NOT DISTINCT FROM peer.workspace_id + AND src_patch.new_context = peer_patch.new_context + RETURNING src.id + ) + DELETE FROM usage_ctx_patch AS patch + USING dropped_peers AS d + WHERE patch.src_id = d.id + """ + ) + ) + + # Step 3: Every remaining staged row now has a guaranteed-unique target key. + db.execute( + text( + """ + UPDATE llm_usage_daily AS src + SET context = patch.new_context, + updated_at = now() + FROM usage_ctx_patch AS patch + WHERE src.id = patch.src_id + """ + ) + ) + + +def stage_context_select(db: Session, select_sql: str, params: dict | None = None) -> None: + _ensure_patch_table(db) + db.execute( + text( + f"INSERT INTO usage_ctx_patch (src_id, new_context) {select_sql}" + ), + params or {}, + ) + merge_staged_usage_context(db) + + +def stage_context_pairs(db: Session, pairs: Sequence[tuple[str, dict]]) -> None: + if not pairs: + return + _ensure_patch_table(db) + payload = [ + {"src_id": src_id, "new_context": context} + for src_id, context in pairs + ] + db.execute( + text( + """ + INSERT INTO usage_ctx_patch (src_id, new_context) + SELECT CAST(x.src_id AS uuid), x.new_context + FROM jsonb_to_recordset(CAST(:rows AS jsonb)) + AS x(src_id text, new_context jsonb) + ON CONFLICT (src_id) DO UPDATE + SET new_context = EXCLUDED.new_context + """ + ), + {"rows": json.dumps(payload)}, + ) + merge_staged_usage_context(db) + + +def backfill_usage_context_keys(db: Session) -> None: + """Stamp call_import_id / evaluation_id without colliding on the bucket index.""" + stage_context_select( + db, + """ + SELECT u.id, + u.context || jsonb_build_object( + 'call_import_id', u.context->>'resource_id' + ) + FROM llm_usage_daily AS u + WHERE u.context->>'resource_type' = 'call_import' + AND NOT (u.context ? 'call_import_id') + AND COALESCE(u.context->>'resource_id', '') <> '' + """, + ) + stage_context_select( + db, + """ + SELECT u.id, + u.context || jsonb_build_object( + 'evaluation_id', u.context->>'resource_id' + ) + FROM llm_usage_daily AS u + WHERE u.context->>'resource_type' = 'call_import_evaluation' + AND NOT (u.context ? 'evaluation_id') + AND COALESCE(u.context->>'resource_id', '') <> '' + """, + ) + if _table_exists(db, "call_import_evaluations"): + stage_context_select( + db, + """ + SELECT u.id, + u.context || jsonb_build_object( + 'call_import_id', e.call_import_id::text + ) + FROM llm_usage_daily AS u + JOIN call_import_evaluations AS e + ON e.organization_id = u.organization_id + AND e.id = (u.context->>'evaluation_id')::uuid + WHERE NOT (u.context ? 'call_import_id') + AND u.context ? 'evaluation_id' + AND u.context->>'evaluation_id' ~ :uuid_re + """, + {"uuid_re": _UUID_RE}, + ) + if _table_exists(db, "call_import_rows") and _table_exists(db, "call_imports"): + stage_context_select( + db, + """ + SELECT u.id, + u.context || jsonb_build_object( + 'call_import_id', ci.id::text + ) + FROM llm_usage_daily AS u + JOIN call_import_rows AS cir + ON cir.id = (u.context->>'call_import_row_id')::uuid + JOIN call_imports AS ci + ON ci.id = cir.call_import_id + AND ci.organization_id = u.organization_id + WHERE NOT (u.context ? 'call_import_id') + AND u.context ? 'call_import_row_id' + AND u.context->>'call_import_row_id' ~ :uuid_re + """, + {"uuid_re": _UUID_RE}, + ) + if _table_exists(db, "call_import_evaluation_rows") and _table_exists( + db, "call_import_evaluations" + ): + stage_context_select( + db, + """ + SELECT u.id, + u.context + || jsonb_build_object('evaluation_id', e.id::text) + || jsonb_build_object('call_import_id', e.call_import_id::text) + FROM llm_usage_daily AS u + JOIN call_import_evaluation_rows AS er + ON er.id = (u.context->>'evaluation_row_id')::uuid + JOIN call_import_evaluations AS e + ON e.id = er.evaluation_id + AND e.organization_id = u.organization_id + WHERE u.context ? 'evaluation_row_id' + AND u.context->>'evaluation_row_id' ~ :uuid_re + AND ( + NOT (u.context ? 'evaluation_id') + OR NOT (u.context ? 'call_import_id') + ) + """, + {"uuid_re": _UUID_RE}, + ) + + +def _chunks(values: Sequence[str], size: int) -> Iterable[Sequence[str]]: + for start in range(0, len(values), size): + yield values[start : start + size] + + +_IMPORT_ORPHAN_SQL = """ +SELECT DISTINCT context->>'call_import_row_id' +FROM llm_usage_daily +WHERE context ? 'call_import_row_id' + AND NOT (context ? 'call_import_id') + AND context->>'call_import_row_id' ~ :uuid_re + AND context->>'call_import_row_id' > :cursor +ORDER BY 1 +LIMIT :limit +""" + +_EVAL_ORPHAN_SQL = """ +SELECT DISTINCT context->>'evaluation_row_id' +FROM llm_usage_daily +WHERE context ? 'evaluation_row_id' + AND ( + NOT (context ? 'evaluation_id') + OR NOT (context ? 'call_import_id') + ) + AND context->>'evaluation_row_id' ~ :uuid_re + AND context->>'evaluation_row_id' > :cursor +ORDER BY 1 +LIMIT :limit +""" + + +def _orphan_page(db: Session, sql: str, cursor: str) -> list[str]: + rows = db.execute( + text(sql), + {"uuid_re": _UUID_RE, "limit": _ORPHAN_BATCH_LIMIT, "cursor": cursor}, + ).all() + return [str(row[0]) for row in rows if row[0]] + + +def repair_orphaned_row_context(db: Session) -> None: + """Resolve row ids that live on shards into catalog context keys. + + Ids are walked in order. A page of deleted ids only moves the cursor + forward, so later live ids are still repaired. + """ + from app.db_sharding.pool_manager import db_pool_manager + + if not (db_pool_manager.sharding_enabled and db_pool_manager.router): + return + + import_cursor = "" + eval_cursor = "" + import_done = False + eval_done = False + while not (import_done and eval_done): + import_row_ids: list[str] = [] + eval_row_ids: list[str] = [] + if not import_done: + import_row_ids = _orphan_page(db, _IMPORT_ORPHAN_SQL, import_cursor) + if not import_row_ids or import_row_ids[-1] <= import_cursor: + import_done = True + import_row_ids = [] + else: + import_cursor = import_row_ids[-1] + if not eval_done: + eval_row_ids = _orphan_page(db, _EVAL_ORPHAN_SQL, eval_cursor) + if not eval_row_ids or eval_row_ids[-1] <= eval_cursor: + eval_done = True + eval_row_ids = [] + else: + eval_cursor = eval_row_ids[-1] + if not import_row_ids and not eval_row_ids: + return + + import_by_row: dict[str, str] = {} + eval_by_row: dict[str, str] = {} + for shard_id in db_pool_manager.router.shard_ids: + session = db_pool_manager.shard_session_factory(shard_id)() + try: + for batch in _chunks(import_row_ids, _BATCH): + found = session.execute( + text( + """ + SELECT id::text, call_import_id::text + FROM call_import_rows + WHERE id = ANY(CAST(:ids AS uuid[])) + """ + ), + {"ids": list(batch)}, + ).all() + import_by_row.update( + {row[0]: row[1] for row in found if row[0] and row[1]} + ) + for batch in _chunks(eval_row_ids, _BATCH): + found = session.execute( + text( + """ + SELECT id::text, evaluation_id::text + FROM call_import_evaluation_rows + WHERE id = ANY(CAST(:ids AS uuid[])) + """ + ), + {"ids": list(batch)}, + ).all() + eval_by_row.update( + {row[0]: row[1] for row in found if row[0] and row[1]} + ) + finally: + session.close() + + eval_to_import: dict[str, str] = {} + eval_ids = list({value for value in eval_by_row.values() if value}) + for batch in _chunks(eval_ids, _BATCH): + found = db.execute( + text( + """ + SELECT id::text, call_import_id::text + FROM call_import_evaluations + WHERE id = ANY(CAST(:ids AS uuid[])) + """ + ), + {"ids": list(batch)}, + ).all() + eval_to_import.update( + {row[0]: row[1] for row in found if row[0] and row[1]} + ) + + if import_by_row: + rows = db.execute( + text( + """ + SELECT id::text, context + FROM llm_usage_daily + WHERE context ? 'call_import_row_id' + AND NOT (context ? 'call_import_id') + AND context->>'call_import_row_id' = ANY(:ids) + """ + ), + {"ids": list(import_by_row)}, + ).all() + pairs = [] + for src_id, context in rows: + ctx = dict(context or {}) + import_id = import_by_row.get(str(ctx.get("call_import_row_id") or "")) + if not import_id: + continue + ctx["call_import_id"] = import_id + pairs.append((str(src_id), ctx)) + if pairs: + stage_context_pairs(db, pairs) + + if eval_by_row: + rows = db.execute( + text( + """ + SELECT id::text, context + FROM llm_usage_daily + WHERE context ? 'evaluation_row_id' + AND ( + NOT (context ? 'evaluation_id') + OR NOT (context ? 'call_import_id') + ) + AND context->>'evaluation_row_id' = ANY(:ids) + """ + ), + {"ids": list(eval_by_row)}, + ).all() + pairs = [] + for src_id, context in rows: + ctx = dict(context or {}) + eval_id = eval_by_row.get(str(ctx.get("evaluation_row_id") or "")) + if not eval_id: + continue + before = dict(ctx) + ctx.setdefault("evaluation_id", eval_id) + import_id = eval_to_import.get(eval_id) + if import_id: + ctx.setdefault("call_import_id", import_id) + if ctx != before: + pairs.append((str(src_id), ctx)) + if pairs: + stage_context_pairs(db, pairs) + + +def ensure_usage_read_indexes(db: Session) -> None: + """Build read indexes without holding the migration transaction open.""" + bind = db.get_bind() + db.commit() + if bind is not None and bind.dialect.name == "postgresql": + try: + with bind.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + for statement in _READ_INDEXES: + conn.execute(text(statement.format(mode="CONCURRENTLY"))) + return + except Exception as exc: + logger.warning("concurrent usage index build fell back: {}", exc) + for statement in _READ_INDEXES: + db.execute(text(statement.format(mode=""))) + db.commit() diff --git a/app/services/voice_agent/knowledge_base_inject_processor.py b/app/services/voice_agent/knowledge_base_inject_processor.py new file mode 100644 index 00000000..48f96e83 --- /dev/null +++ b/app/services/voice_agent/knowledge_base_inject_processor.py @@ -0,0 +1,153 @@ +"""Inject retrieved knowledge-base passages before each user turn (voice pipeline). + +Tool calling is optional and often unsupported by voice-bundle LLMs; this processor +runs vector search on final STT text and appends excerpts to the LLM context. +""" + +from __future__ import annotations + +import asyncio +import re +from typing import Any, Optional +from uuid import UUID + +from loguru import logger + +_GREETING_ONLY_RE = re.compile( + r"^(?:hi|hey|hello|howdy|good\s+(?:morning|afternoon|evening))[\s,!.?]*" + r"(?:how\s+are\s+you|what'?s\s+up)?[\s,!.?]*$", + re.IGNORECASE, +) + + +def _should_retrieve_on_transcript(text: str, *, min_query_chars: int) -> bool: + query = (text or "").strip() + if len(query) < min_query_chars: + return False + if _GREETING_ONLY_RE.match(query): + return False + return True + + +def create_knowledge_base_inject_processor( + context: Any, + *, + organization_id: str, + workspace_id: str, + agent_id: str, + kb_notifier: Any = None, + min_query_chars: int = 12, + role: str = "production", +): + """Return a FrameProcessor placed after STT and before the user context aggregator.""" + imports = None + + def _imports(): + nonlocal imports + if imports is None: + from efficientai.frames.frames import TranscriptionFrame + from efficientai.processors.frame_processor import FrameDirection, FrameProcessor + + imports = { + "TranscriptionFrame": TranscriptionFrame, + "FrameDirection": FrameDirection, + "FrameProcessor": FrameProcessor, + } + return imports + + imp = _imports() + FrameProcessor = imp["FrameProcessor"] + FrameDirection = imp["FrameDirection"] + TranscriptionFrame = imp["TranscriptionFrame"] + + org_uuid = UUID(organization_id) + ws_uuid = UUID(workspace_id) + agent_uuid = UUID(agent_id) + + class KnowledgeBaseInjectProcessor(FrameProcessor): + def __init__(self): + super().__init__() + self._last_query: Optional[str] = None + + def _inject_sync(self, query: str) -> tuple[int, Optional[float]]: + from app.database import SessionLocal + from app.services.knowledge_bases.retrieval import ( + format_search_results_for_tool, + search_knowledge_base, + ) + + db = SessionLocal() + try: + hits = search_knowledge_base( + db, + organization_id=org_uuid, + workspace_id=ws_uuid, + agent_id=agent_uuid, + query=query, + role=role, + ) + finally: + db.close() + if not hits: + return 0, None + top_score = hits[0].get("score") + block = format_search_results_for_tool(hits) + context.add_message( + { + "role": "system", + "content": ( + "Retrieved passages from attached knowledge bases. " + "The production agent's last message asked about topics below; " + "use these facts in your next assistant reply when they answer " + "that question (do not refuse solely because of your persona):\n\n" + f"{block}" + ), + } + ) + return len(hits), top_score if isinstance(top_score, (int, float)) else None + + async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + + if direction != FrameDirection.DOWNSTREAM: + await self.push_frame(frame, direction) + return + + if isinstance(frame, TranscriptionFrame): + query = (frame.text or "").strip() + if ( + _should_retrieve_on_transcript(query, min_query_chars=min_query_chars) + and query != self._last_query + ): + self._last_query = query + try: + hit_count, top_score = await asyncio.to_thread(self._inject_sync, query) + except Exception as exc: + logger.warning("[KnowledgeBase] auto-inject search failed: {}", exc) + hit_count, top_score = 0, None + else: + if hit_count: + logger.info( + "[KnowledgeBase] auto-inject agent_id={} query={!r} " + "hit_count={} top_score={}", + agent_id, + query, + hit_count, + top_score, + ) + if kb_notifier is not None: + await kb_notifier.on_tool_call( + query=query, + hit_count=hit_count, + channel="voice_rag_auto", + ) + else: + logger.debug( + "[KnowledgeBase] auto-inject skipped/no hits agent_id={} query={!r}", + agent_id, + query, + ) + + await self.push_frame(frame, direction) + + return KnowledgeBaseInjectProcessor() diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index e8c69a1a..089d5836 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -539,11 +539,17 @@ async def run_voice_bundle_fastapi( call_direction: str = "outbound", persona_speaks_via_tts: bool = False, tracing_task_kwargs: dict | None = None, + enable_knowledge_base_tools: bool = False, + enable_production_knowledge_tools: bool = False, ): """ Run the STT+LLM+TTS voice bundle pipeline over a FastAPI WebSocket. Records dual-track WAV via AudioRecorder and merges/uploads after the call. """ + use_kb_tools = False + kb_tool_calls_log: list[dict] = [] + kb_notifier = None + # Lazy load all efficientai dependencies imports = _get_imports() STT_PROVIDERS = _get_stt_providers() @@ -763,6 +769,51 @@ async def run_voice_bundle_fastapi( if system_instruction else "You are a helpful voice assistant. Keep responses concise and speakable." ) + kb_tools_enabled = enable_knowledge_base_tools or enable_production_knowledge_tools + if kb_tools_enabled and agent_id and organization_id and workspace_id: + from uuid import UUID + + from app.database import SessionLocal + from app.services.knowledge_bases.prompt import append_knowledge_base_instructions + from app.services.knowledge_bases.agent_links import ( + KB_ROLE_PRODUCTION, + KB_ROLE_TEST, + ) + from app.services.knowledge_bases.retrieval import agent_has_ready_knowledge + + # Telephony runs the production agent; the playground bundle plays the caller. + kb_role = KB_ROLE_PRODUCTION if telephony_mode else KB_ROLE_TEST + kb_db = SessionLocal() + try: + use_kb_tools = agent_has_ready_knowledge( + kb_db, agent_id=UUID(agent_id), role=kb_role + ) + finally: + kb_db.close() + if use_kb_tools: + base_instruction = append_knowledge_base_instructions( + base_instruction, + simulated_caller=not telephony_mode, + ) + from app.services.knowledge_bases.tool_call_logging import KnowledgeBaseVoiceNotifier + + kb_notifier = KnowledgeBaseVoiceNotifier( + agent_id=str(agent_id), + llm_model=llm_model, + tool_calls_log=kb_tool_calls_log, + ) + logger.info( + "[KnowledgeBase] voice_bundle tools enabled agent_id={} llm={}/{}", + agent_id, + llm_provider_value, + llm_model, + ) + elif kb_tools_enabled: + logger.info( + "[KnowledgeBase] voice_bundle KB tools requested but no ready documents " + "for agent_id={}", + agent_id, + ) # Sarvam TTS rejects text containing emojis / non-language characters, # so we inject a guardrail into the system instructions. @@ -787,6 +838,32 @@ async def run_voice_bundle_fastapi( context = imports["LLMContext"](messages) context_aggregator = imports["LLMContextAggregatorPair"](context) + kb_inject_processor = None + if use_kb_tools and agent_id and organization_id and workspace_id: + from app.services.knowledge_bases.voice_tools import register_voice_bundle_knowledge_tools + from app.services.voice_agent.knowledge_base_inject_processor import ( + create_knowledge_base_inject_processor, + ) + + register_voice_bundle_knowledge_tools( + llm, + context, + organization_id=str(organization_id), + workspace_id=str(workspace_id), + agent_id=str(agent_id), + kb_notifier=kb_notifier, + role=kb_role, + ) + kb_inject_processor = create_knowledge_base_inject_processor( + context, + organization_id=str(organization_id), + workspace_id=str(workspace_id), + agent_id=str(agent_id), + kb_notifier=kb_notifier, + role=kb_role, + ) + logger.info("[KnowledgeBase] voice_bundle auto-inject processor enabled") + from app.services.voice_agent.audio_recorder import get_audio_recorder_class from app.services.voice_agent.telephony_recording_paths import telephony_recording_temp_path @@ -844,6 +921,8 @@ async def on_silence_hangup(): if silence_hangup_processor: pipeline_processors.append(silence_hangup_processor) pipeline_processors.extend([user_recorder, stt]) + if kb_inject_processor is not None: + pipeline_processors.append(kb_inject_processor) if telephony_mode and call_short_id: from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor @@ -913,6 +992,8 @@ async def on_client_disconnected(transport, client): await task.cancel() else: rtvi = imports["RTVIProcessor"](config=imports["RTVIConfig"](config=[])) + if kb_notifier is not None: + kb_notifier.bind_rtvi(rtvi) rtvi_processors = [ws_transport.input()] if ambient_input_processor: @@ -923,6 +1004,10 @@ async def on_client_disconnected(transport, client): user_recorder, rtvi, stt, + ]) + if kb_inject_processor is not None: + rtvi_processors.append(kb_inject_processor) + rtvi_processors.extend([ context_aggregator.user(), llm, tts, @@ -1128,7 +1213,10 @@ async def on_client_disconnected(transport, client): "scenario_id": scenario_id, "transcription": transcript_text, "speaker_segments": conversation_turns if conversation_turns else None, + "knowledge_base_tools_enabled": bool(use_kb_tools), } + if kb_tool_calls_log: + metadata["knowledge_base_tool_calls"] = kb_tool_calls_log if not s3_key_result and not transcript_text: metadata["error"] = "No audio file was uploaded and no transcript captured" return metadata diff --git a/app/services/webrtc_bridge/test_agent_processor.py b/app/services/webrtc_bridge/test_agent_processor.py index 4986c0cf..d79d471b 100644 --- a/app/services/webrtc_bridge/test_agent_processor.py +++ b/app/services/webrtc_bridge/test_agent_processor.py @@ -552,6 +552,50 @@ def _sync_llm_call( ) if on_text_delta is not None: kwargs["on_text_delta"] = on_text_delta + if ( + on_text_delta is None + and self.config.db + and self.config.agent_id + and self.config.workspace_id + and self.config.organization_id + ): + from app.services.knowledge_bases.chat_tools import ( + generate_production_reply_with_knowledge_tools, + ) + from app.services.knowledge_bases.agent_links import KB_ROLE_TEST + from app.services.knowledge_bases.retrieval import agent_has_ready_knowledge + + agent_uuid = UUID(str(self.config.agent_id)) + org_uuid = UUID(str(self.config.organization_id)) + workspace_uuid = UUID(str(self.config.workspace_id)) + if agent_has_ready_knowledge(self.config.db, agent_id=agent_uuid, role=KB_ROLE_TEST): + + def _kb_call() -> Dict[str, Any]: + text, _kb_log = generate_production_reply_with_knowledge_tools( + self.config.db, + organization_id=org_uuid, + workspace_id=workspace_uuid, + agent_id=agent_uuid, + messages=messages, + llm_provider=self._llm_provider_enum(), + llm_model=self.config.llm_model, + llm_config=self._effective_llm_config( + allow_reasoning_override=allow_reasoning_override + ), + credential_id=UUID(str(credential_id)) if credential_id else None, + simulated_caller=True, + task_defaults={ + "temperature": kwargs.get("temperature", 0.7), + "max_tokens": kwargs.get("max_tokens"), + }, + ) + return {"text": text} + + if ctx is not None: + with llm_usage_context(ctx): + return _kb_call() + return _kb_call() + if ctx is not None: with llm_usage_context(ctx): return llm_service.generate_response(**kwargs) diff --git a/app/workers/catalog_cache.py b/app/workers/catalog_cache.py index 41b84ccb..023ba89d 100644 --- a/app/workers/catalog_cache.py +++ b/app/workers/catalog_cache.py @@ -1,12 +1,58 @@ -"""Short-lived in-process cache for catalog reads in bulk workers.""" +"""Short-lived in-process cache for catalog reads in bulk workers. + +One dict per worker process, shared by every thread in it. Entries expire by +their own TTL, but expiry alone never frees memory for keys that are not read +again (old version keys, finished evaluations and imports). Thread-pool +workers (``worker-imports``) are never recycled by ``worker_max_tasks_per_child``, +so the cache bounds itself: + +* **Sweep on write.** A store that comes at least ``CATALOG_CACHE_SWEEP_SECONDS`` + after the last sweep first drops every expired entry. No timer or thread: + an idle worker adds nothing, and its first write after idling cleans up. +* **Size cap.** At most ``CATALOG_CACHE_MAX_ENTRIES`` entries; the least + recently used is evicted first. An evicted entry is just reloaded. +""" from __future__ import annotations +import threading import time -from typing import Any, Callable, Dict, Tuple +from collections import OrderedDict +from typing import Any, Callable, Tuple + +from app.config import settings _TTL_SECONDS = 120.0 -_cache: Dict[Tuple[str, str], Tuple[float, Any]] = {} +_DEFAULT_MAX_ENTRIES = 2048 +_DEFAULT_SWEEP_SECONDS = 30.0 + +# (namespace, key) -> (expires_at, value), least recently used first. +_cache: "OrderedDict[Tuple[str, str], Tuple[float, Any]]" = OrderedDict() +_lock = threading.Lock() +_last_sweep = 0.0 + + +def _setting(name: str, default: float) -> float: + try: + return float(getattr(settings, name, default)) + except (TypeError, ValueError): + return default + + +def _max_entries() -> int: + return max(1, int(_setting("CATALOG_CACHE_MAX_ENTRIES", _DEFAULT_MAX_ENTRIES))) + + +def _sweep_seconds() -> float: + return max(0.0, _setting("CATALOG_CACHE_SWEEP_SECONDS", _DEFAULT_SWEEP_SECONDS)) + + +def _sweep_expired_locked(now: float) -> None: + global _last_sweep + expired = [key for key, (expires_at, _) in _cache.items() if expires_at <= now] + for key in expired: + del _cache[key] + _last_sweep = now def cached_catalog_fetch( @@ -18,13 +64,36 @@ def cached_catalog_fetch( ) -> Any: cache_key = (namespace, key) now = time.monotonic() - hit = _cache.get(cache_key) - if hit is not None and now - hit[0] < ttl_seconds: - return hit[1] + with _lock: + hit = _cache.get(cache_key) + if hit is not None and now < hit[0]: + _cache.move_to_end(cache_key) + return hit[1] + if ttl_seconds <= 0: + return loader() + + # Load outside the lock: it queries the catalog, and other threads must + # not wait on it. Two threads missing at once may both load; last wins. value = loader() - _cache[cache_key] = (now, value) + now = time.monotonic() + with _lock: + if now - _last_sweep >= _sweep_seconds(): + _sweep_expired_locked(now) + _cache[cache_key] = (now + ttl_seconds, value) + _cache.move_to_end(cache_key) + limit = _max_entries() + while len(_cache) > limit: + _cache.popitem(last=False) return value +def catalog_cache_size() -> int: + with _lock: + return len(_cache) + + def clear_worker_catalog_cache() -> None: - _cache.clear() + global _last_sweep + with _lock: + _cache.clear() + _last_sweep = 0.0 diff --git a/app/workers/celery_app.py b/app/workers/celery_app.py index 5a9c6042..805f6ea3 100644 --- a/app/workers/celery_app.py +++ b/app/workers/celery_app.py @@ -22,6 +22,8 @@ initiate_vobiz_outbound_call_task, finalize_telephony_recording_task, ) +from app.workers.tasks.ingest_knowledge_document import ingest_knowledge_document_task +from app.workers.tasks.run_harbor_simulation import run_harbor_simulation_task __all__ = [ "celery_app", @@ -36,4 +38,6 @@ "run_judge_alignment_task", "initiate_vobiz_outbound_call_task", "finalize_telephony_recording_task", + "ingest_knowledge_document_task", + "run_harbor_simulation_task", ] diff --git a/app/workers/celery_revoke.py b/app/workers/celery_revoke.py new file mode 100644 index 00000000..8f98bf72 --- /dev/null +++ b/app/workers/celery_revoke.py @@ -0,0 +1,57 @@ +"""Best-effort Celery task revocation from the API / bulk workers.""" + +from __future__ import annotations + +from typing import Iterable, List, Optional + +from loguru import logger + +_REVOKE_TASK_BATCH = 500 + + +def celery_revoke_use_terminate() -> bool: + """True only when workers use a pool that supports kill_job (e.g. prefork).""" + from app.config import settings + + return bool(getattr(settings, "CELERY_REVOKE_USE_TERMINATE", False)) + + +def revoke_celery_tasks( + task_ids: Iterable[str], + *, + terminate: Optional[bool] = None, +) -> int: + """Revoke task ids in batches. + + ``terminate=None`` (default) follows :func:`celery_revoke_use_terminate`. + ``terminate=False`` never sends SIGTERM. ``terminate=True`` forces terminate + (use only when callers know the target worker pool supports it). + """ + cleaned: List[str] = [tid for tid in task_ids if (tid or "").strip()] + if not cleaned: + return 0 + use_terminate = ( + celery_revoke_use_terminate() if terminate is None else bool(terminate) + ) + try: + from app.workers.celery_app import celery_app + + revoked = 0 + for start in range(0, len(cleaned), _REVOKE_TASK_BATCH): + chunk = cleaned[start : start + _REVOKE_TASK_BATCH] + if use_terminate: + celery_app.control.revoke( + chunk, terminate=True, signal="SIGTERM" + ) + else: + celery_app.control.revoke(chunk, terminate=False) + revoked += len(chunk) + if not use_terminate and terminate is None: + logger.debug( + "Revoked {} Celery task(s) with terminate=False (thread/solo pool)", + revoked, + ) + return revoked + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to revoke {} Celery task(s): {}", len(cleaned), exc) + return 0 diff --git a/app/workers/concurrency/diarization_dispatch.py b/app/workers/concurrency/diarization_dispatch.py index 90faf35d..1b5a7776 100644 --- a/app/workers/concurrency/diarization_dispatch.py +++ b/app/workers/concurrency/diarization_dispatch.py @@ -153,6 +153,36 @@ def pop_row_diarization_params(row_id: UUID | str) -> Optional[DiarizationRowPar return params +def _persist_diarization_row_fields( + catalog_db: Session, + call_import: CallImport, + row: CallImportRow, + patch: dict, +) -> None: + """Write diarisation dispatch fields on the shard that owns the row.""" + from app.db_sharding.row_ops import update_call_import_rows_on_shards + from app.db_sharding.sessions import is_sharding_enabled + + if not is_sharding_enabled(): + for key, value in patch.items(): + setattr(row, key, value) + catalog_db.commit() + return + + row_index = getattr(row, "row_index", None) + if row_index is None: + raise ValueError( + f"row_index required for sharded diarisation dispatch on row {row.id}" + ) + update_call_import_rows_on_shards( + catalog_db, + call_import.id, + [{"id": row.id, "row_index": int(row_index), **patch}], + ) + for key, value in patch.items(): + setattr(row, key, value) + + def build_diarization_params_from_request( *, stt_provider: Optional[str], @@ -208,35 +238,64 @@ def _try_dispatch_single_diarization_row( ): return "at_capacity" + slot_released = False try: - async_result = transcribe_call_import_row_task.apply_async( - args=( - str(row.id), - params.get("stt_provider"), - params.get("stt_model"), - params.get("credential_id"), - params.get("language"), - bool(params.get("overwrite_existing", False)), - None, - params.get("diarization_llm_provider"), - params.get("diarization_llm_model"), - params.get("diarization_llm_credential_id"), - params.get("diarization_prompt"), - params.get("mode") or "stt_llm", - ), - kwargs={"_eval_slot_task_id": reserved_task_id}, - queue=DIARIZATION_QUEUE, - task_id=reserved_task_id, + _persist_diarization_row_fields( + db, + call_import, + row, + { + "celery_task_id": reserved_task_id, + "diarised_transcript_error": None, + }, ) - except Exception: - release_eval_slot_for_celery_task(reserved_task_id) - raise - try: - row.celery_task_id = async_result.id - db.commit() + try: + async_result = transcribe_call_import_row_task.apply_async( + args=( + str(row.id), + params.get("stt_provider"), + params.get("stt_model"), + params.get("credential_id"), + params.get("language"), + bool(params.get("overwrite_existing", False)), + None, + params.get("diarization_llm_provider"), + params.get("diarization_llm_model"), + params.get("diarization_llm_credential_id"), + params.get("diarization_prompt"), + params.get("mode") or "stt_llm", + ), + kwargs={"_eval_slot_task_id": reserved_task_id}, + queue=DIARIZATION_QUEUE, + task_id=reserved_task_id, + ) + except Exception: + _persist_diarization_row_fields( + db, + call_import, + row, + {"celery_task_id": None}, + ) + release_eval_slot_for_celery_task(reserved_task_id) + slot_released = True + raise + + if async_result.id != reserved_task_id: + # The worker owns the row under ``_eval_slot_task_id``, so the row + # must keep the reserved id. Re-pointing it at the broker id would + # make every ownership check miss and report ``superseded``. + logger.warning( + "Diarisation task for row {} ignored the requested task id " + "(reserved={} broker={}); keeping the reserved id for " + "ownership checks.", + row.id, + reserved_task_id, + async_result.id, + ) except Exception: - release_eval_slot_for_celery_task(reserved_task_id) + if not slot_released: + release_eval_slot_for_celery_task(reserved_task_id) raise return "dispatched" diff --git a/app/workers/concurrency/dispatch_reconcile.py b/app/workers/concurrency/dispatch_reconcile.py index 47e6c838..d2bdbed6 100644 --- a/app/workers/concurrency/dispatch_reconcile.py +++ b/app/workers/concurrency/dispatch_reconcile.py @@ -175,3 +175,78 @@ def reconcile_orphaned_eval_dispatch_locks( if owns_session: shard_db.close() return total + + +def _reconcile_diarization_rows_on_session(db: Session) -> int: + """Clear stale diarisation locks and repair orphan ``running`` rows. + + Fair dispatch only enqueues rows that are ``pending`` with no + ``celery_task_id``. If we clear a finished task id but leave + ``diarised_transcript_status='running'`` (worker exited without a terminal + write), the row would never match the dispatcher — so we flip those back + to ``pending``. + + Rows that are ``running`` with ``celery_task_id IS NULL`` are similarly + stuck (nothing owns them); reset to ``pending`` so dispatch can retry. + """ + fixed = 0 + rows = ( + db.query(CallImportRow) + .filter( + CallImportRow.celery_task_id.isnot(None), + CallImportRow.diarised_transcript_status.in_(("pending", "running")), + ) + .limit(_RECONCILE_BATCH) + .all() + ) + for row in rows: + task_id = (row.celery_task_id or "").strip() + if not task_id or not _celery_task_is_finished(task_id): + continue + row.celery_task_id = None + if (row.diarised_transcript_status or "").lower() == "running": + row.diarised_transcript_status = "pending" + fixed += 1 + + orphans = ( + db.query(CallImportRow) + .filter( + CallImportRow.diarised_transcript_status == "running", + CallImportRow.celery_task_id.is_(None), + ) + .limit(_RECONCILE_BATCH) + .all() + ) + for row in orphans: + row.diarised_transcript_status = "pending" + fixed += 1 + + if fixed: + db.commit() + return fixed + + +def reconcile_orphaned_diarization_dispatch_locks(catalog_db: Session) -> int: + """Clear stale diarisation locks (worker restart, duplicate delivery). + + Fair diarisation dispatch only picks rows with ``diarised_transcript_status`` + pending and ``celery_task_id IS NULL``. Stale task ids and orphan + ``running`` states are repaired here so dispatch can make progress. + """ + from app.db_sharding.sessions import is_sharding_enabled + + if not is_sharding_enabled(): + return _reconcile_diarization_rows_on_session(catalog_db) + + from app.db_sharding.pool_manager import db_pool_manager + + router = db_pool_manager.router + assert router is not None + total = 0 + for shard_id in router.shard_ids: + shard_db = db_pool_manager.shard_session_factory(shard_id)() + try: + total += _reconcile_diarization_rows_on_session(shard_db) + finally: + shard_db.close() + return total diff --git a/app/workers/concurrency/eval_dispatch.py b/app/workers/concurrency/eval_dispatch.py index c99d1cb1..2c75ef1f 100644 --- a/app/workers/concurrency/eval_dispatch.py +++ b/app/workers/concurrency/eval_dispatch.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import timezone from typing import Callable, List, Literal, NamedTuple, Optional from uuid import UUID @@ -63,15 +64,25 @@ def _needs_transcribe_for_eval( *, transcribe_overwrite: bool, auto_transcribe: bool = True, + catalog_db: Session | None = None, ) -> bool: - if ( + from app.workers.tasks.evaluate_call_import_row_core import ( + evaluation_needs_diarised_reference, + ) + + transcript_source = ( (getattr(evaluation, "transcript_source", None) or "") .strip() .lower() - == "production" - ): + ) + needs_diarised_for_computed = False + if catalog_db is not None: + needs_diarised_for_computed = evaluation_needs_diarised_reference( + catalog_db, evaluation + ) + if transcript_source == "production" and not needs_diarised_for_computed: return False - if not auto_transcribe: + if not auto_transcribe and not needs_diarised_for_computed: return False transcribe_mode = ( getattr(evaluation, "transcribe_mode", None) or "stt_llm" @@ -90,8 +101,44 @@ def _needs_transcribe_for_eval( existing_dia = row_text_present( source_row, "diarised_transcript", "diarised_transcript_char_length" ) - needs_diarisation = not existing_dia or transcribe_overwrite - return bool(can_auto_transcribe and has_audio and needs_diarisation) + dia_status = ( + getattr(source_row, "diarised_transcript_status", None) or "" + ).strip().lower() + can_diarise = bool(can_auto_transcribe and has_audio) + # Overwrite runs re-diarise once per row. A ``completed`` transcript on its + # own is not proof the text belongs to *this* pass — it may predate the run + # (and the newly picked diariser model / prompt). Only the pass marker the + # worker writes when it finishes proves that, so stale transcripts are + # re-diarised instead of being scored as fresh. + if transcribe_overwrite: + if not existing_dia or dia_status != "completed": + return can_diarise + evaluation_id = getattr(evaluation, "id", None) + if evaluation_id is not None: + from app.workers.concurrency.fair_dispatch import row_diarised_in_pass + + in_pass = row_diarised_in_pass( + evaluation_id, getattr(source_row, "id", None) or "" + ) + if in_pass: + return False + # ``diarised_at`` is cleared whenever a pass resets the row, so any + # value left here came from a diarisation — fresh when it postdates the + # run start, stale (and therefore re-diarised) when it predates it. + # This keeps the answer independent of Redis availability, so a marker + # outage cannot either loop the dispatcher or score old text. + diarised_at = getattr(source_row, "diarised_at", None) + started_at = getattr(evaluation, "started_at", None) + if diarised_at is None: + return can_diarise + if started_at is None: + return False + if diarised_at.tzinfo is None: + diarised_at = diarised_at.replace(tzinfo=timezone.utc) + if started_at.tzinfo is None: + started_at = started_at.replace(tzinfo=timezone.utc) + return bool(diarised_at < started_at and can_diarise) + return bool(can_diarise and not existing_dia) def _diarisation_in_flight(source_row: CallImportRow) -> bool: @@ -105,14 +152,25 @@ def _diarisation_in_flight(source_row: CallImportRow) -> bool: def _needs_import_for_eval( source_row: CallImportRow, evaluation: CallImportEvaluation | None = None, + *, + catalog_db: Session | None = None, ) -> bool: """True when the eval pipeline must fetch a recording before later stages.""" + needs_diarised_for_computed = False + if evaluation is not None and catalog_db is not None: + from app.workers.tasks.evaluate_call_import_row_core import ( + evaluation_needs_diarised_reference, + ) + + needs_diarised_for_computed = evaluation_needs_diarised_reference( + catalog_db, evaluation + ) if evaluation is not None and ( (getattr(evaluation, "transcript_source", None) or "") .strip() .lower() == "production" - ): + ) and not needs_diarised_for_computed: return False if source_row.status in ( CallImportRowStatus.COMPLETED, @@ -145,7 +203,7 @@ def _fail_eval_row_for_import( commit_terminal_row_and_rollup, ) - previous_status = eval_row.status or "pending" + previous_status = getattr(eval_row, "status", None) or "pending" eval_row.status = "failed" eval_row.error_message = ( source_row.error_message or "Recording fetch failed" @@ -166,6 +224,47 @@ def _fail_eval_row_for_import( commit_shard_row_session(db) +def _fail_eval_row_for_diarisation( + db: Session, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow, + *, + catalog_db: Session | None = None, + evaluation: CallImportEvaluation | None = None, +) -> None: + """Terminalize an eval row whose diarisation failed and will not be retried. + + Leaving it pending makes the dispatcher reload the same row forever and + never reach later rows that are ready to score. + """ + from datetime import datetime, timezone + + from app.db_sharding.row_ops import commit_shard_row_session + from app.workers.tasks.evaluate_call_import_row_core import ( + commit_terminal_row_and_rollup, + ) + + previous_status = getattr(eval_row, "status", None) or "pending" + eval_row.status = "failed" + eval_row.error_message = ( + source_row.diarised_transcript_error or "Diarisation failed" + ) + eval_row.finished_at = datetime.now(timezone.utc) + eval_row.celery_task_id = None + if evaluation is not None: + commit_terminal_row_and_rollup( + db, + evaluation, + eval_row, + previous_row_status=previous_status, + catalog_db=( + catalog_db if catalog_db is not None and catalog_db is not db else None + ), + ) + else: + commit_shard_row_session(db) + + def source_row_import_blocks_eval(source_row: CallImportRow) -> bool: """True when import failure prevents the eval pipeline from continuing.""" if (source_row.recording_s3_key or "").strip(): @@ -176,12 +275,12 @@ def source_row_import_blocks_eval(source_row: CallImportRow) -> bool: def recover_eval_row_for_eval_chain(eval_row: CallImportEvaluationRow) -> None: """Undo a premature eval-row failure so the eval chain can continue.""" from app.workers.tasks.evaluate_call_import_row_core import ( - EVAL_CANCELLED_BY_USER_ERROR, + is_eval_row_user_cancelled, ) if eval_row.status != "failed": return - if (eval_row.error_message or "") == EVAL_CANCELLED_BY_USER_ERROR: + if is_eval_row_user_cancelled(eval_row): return eval_row.status = "pending" eval_row.error_message = None @@ -218,6 +317,7 @@ def build_eval_chain_transcribe_apply_async( reserved_task_id: str, restricted_metric_ids: Optional[List[str]] = None, transcribe_overwrite: bool = False, + emit_native_text: bool = False, ): """Build a Celery ``apply_async`` for eval-chain diarisation.""" from app.workers.tasks.transcribe_call_import_row import ( @@ -230,6 +330,8 @@ def build_eval_chain_transcribe_apply_async( kwargs = {"_eval_slot_task_id": reserved_task_id} if restricted_metric_ids: kwargs["eval_restricted_metric_ids"] = restricted_metric_ids + if emit_native_text: + kwargs["emit_native_text"] = True return transcribe_call_import_row_task.apply_async( args=( str(source_row.id), @@ -255,6 +357,100 @@ def build_eval_chain_transcribe_apply_async( ) +def _reset_source_row_for_eval_transcribe( + source_row: CallImportRow, + *, + reserved_task_id: str, + transcribe_overwrite: bool, +) -> None: + previous_task = (source_row.celery_task_id or "").strip() + if previous_task and previous_task != reserved_task_id: + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([previous_task]) + source_row.diarised_transcript_status = "pending" + source_row.diarised_transcript_error = None + if transcribe_overwrite: + source_row.diarised_transcript = None + # Drop the old stamp so dispatch can tell this pass's result from an + # earlier run's transcript. + source_row.diarised_at = None + source_row.celery_task_id = reserved_task_id + + +def _clear_transcribe_dispatch_task_ids( + db: Session, + source_row: CallImportRow, + eval_row: CallImportEvaluationRow, +) -> None: + source_row.celery_task_id = None + eval_row.celery_task_id = None + db.commit() + + +def _reserve_slot_enqueue_eval_transcribe( + *, + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow, + db: Session, + catalog_db: Session, + restricted_metric_ids: Optional[List[str]] = None, + transcribe_overwrite: bool = False, +) -> bool: + """Commit pending diarisation state before queueing the worker task.""" + from app.db_sharding.row_ops import shard_row_write_context + from app.workers.tasks.evaluate_call_import_row_core import ( + evaluation_needs_diarised_reference, + ) + + reserved_task_id = celery_uuid() + if not acquire_eval_slot( + workspace_id=evaluation.workspace_id, + organization_id=evaluation.organization_id, + celery_task_id=reserved_task_id, + evaluation_id=evaluation.id, + ): + return False + + slot_released = False + try: + with shard_row_write_context(db): + _reset_source_row_for_eval_transcribe( + source_row, + reserved_task_id=reserved_task_id, + transcribe_overwrite=transcribe_overwrite, + ) + eval_row.celery_task_id = None + db.commit() + + try: + async_result = build_eval_chain_transcribe_apply_async( + evaluation=evaluation, + eval_row=eval_row, + source_row=source_row, + reserved_task_id=reserved_task_id, + restricted_metric_ids=restricted_metric_ids, + transcribe_overwrite=transcribe_overwrite, + emit_native_text=evaluation_needs_diarised_reference( + catalog_db, evaluation + ), + ) + except Exception: + _clear_transcribe_dispatch_task_ids(db, source_row, eval_row) + release_eval_slot_for_celery_task(reserved_task_id) + slot_released = True + raise + + eval_row.celery_task_id = async_result.id + db.commit() + except Exception: + if not slot_released: + release_eval_slot_for_celery_task(reserved_task_id) + raise + return True + + def enqueue_eval_chain_transcribe_after_import( db: Session, *, @@ -270,6 +466,7 @@ def enqueue_eval_chain_transcribe_after_import( evaluation, source_row, transcribe_overwrite=transcribe_overwrite, + catalog_db=db, ): return False @@ -285,20 +482,32 @@ def enqueue_eval_chain_transcribe_after_import( return False with shard_row_write_context(db): - source_row.diarised_transcript_status = "pending" - source_row.diarised_transcript_error = None - source_row.celery_task_id = slot_task_id - eval_row.celery_task_id = None - db.flush() - - async_result = build_eval_chain_transcribe_apply_async( - evaluation=evaluation, - eval_row=eval_row, - source_row=source_row, + _reset_source_row_for_eval_transcribe( + source_row, reserved_task_id=slot_task_id, - restricted_metric_ids=restricted_metric_ids, transcribe_overwrite=transcribe_overwrite, ) + eval_row.celery_task_id = None + db.commit() + + from app.workers.tasks.evaluate_call_import_row_core import ( + evaluation_needs_diarised_reference, + ) + + try: + async_result = build_eval_chain_transcribe_apply_async( + evaluation=evaluation, + eval_row=eval_row, + source_row=source_row, + reserved_task_id=slot_task_id, + restricted_metric_ids=restricted_metric_ids, + transcribe_overwrite=transcribe_overwrite, + emit_native_text=evaluation_needs_diarised_reference(db, evaluation), + ) + except Exception: + _clear_transcribe_dispatch_task_ids(db, source_row, eval_row) + raise + eval_row.celery_task_id = async_result.id db.commit() return True @@ -480,7 +689,7 @@ def _try_dispatch_single_row( if not (source_row.recording_s3_key or "").strip(): return EvalDispatchOutcome("skip") - if _needs_import_for_eval(source_row, evaluation): + if _needs_import_for_eval(source_row, evaluation, catalog_db=catalog_db): from app.workers.concurrency.import_dispatch import ( _peek_authenticated_import_credit, ) @@ -523,6 +732,7 @@ def _enqueue_import(reserved_task_id: str): source_row, transcribe_overwrite=transcribe_overwrite, auto_transcribe=auto_transcribe, + catalog_db=catalog_db, ): from app.workers.tasks.evaluate_call_import_row_core import ( is_eval_row_user_cancelled, @@ -539,7 +749,7 @@ def _enqueue_import(reserved_task_id: str): ).strip().lower() from app.db_sharding.scatter_gather import row_text_present - if ( + diarisation_failed = ( dia_status == "failed" and not transcribe_overwrite and not row_text_present( @@ -547,33 +757,37 @@ def _enqueue_import(reserved_task_id: str): "diarised_transcript", "diarised_transcript_char_length", ) - ): + ) + transcript_source = ( + (getattr(evaluation, "transcript_source", None) or "diarised") + .strip() + .lower() + ) + # A failed diarisation with no stored transcript cannot be + # retried here. Diarised runs have nothing left to score, so + # close the row. Production runs can still score the LLM + # metrics; User WER/CER record a skip inside the worker. + if diarisation_failed and transcript_source != "production": + _fail_eval_row_for_diarisation( + mutate_db, + eval_row, + source_row, + catalog_db=catalog_db, + evaluation=evaluation, + ) return EvalDispatchOutcome("skip") - - def _enqueue_transcribe(reserved_task_id: str): - source_row.diarised_transcript_status = "pending" - source_row.diarised_transcript_error = None - if transcribe_overwrite: - source_row.diarised_transcript = None - source_row.celery_task_id = reserved_task_id - mutate_db.flush() - return build_eval_chain_transcribe_apply_async( + if not diarisation_failed: + if _reserve_slot_enqueue_eval_transcribe( evaluation=evaluation, eval_row=eval_row, source_row=source_row, - reserved_task_id=reserved_task_id, + db=mutate_db, + catalog_db=catalog_db, restricted_metric_ids=restricted_metric_ids, transcribe_overwrite=transcribe_overwrite, - ) - - if _reserve_slot_and_enqueue( - evaluation=evaluation, - eval_row=eval_row, - db=mutate_db, - enqueue_fn=_enqueue_transcribe, - ): - return EvalDispatchOutcome("dispatched") - return EvalDispatchOutcome("at_capacity") + ): + return EvalDispatchOutcome("dispatched") + return EvalDispatchOutcome("at_capacity") if row_needs_audio_phase( catalog_db, diff --git a/app/workers/concurrency/fair_diarization_dispatch.py b/app/workers/concurrency/fair_diarization_dispatch.py index 2499ce9c..30642548 100644 --- a/app/workers/concurrency/fair_diarization_dispatch.py +++ b/app/workers/concurrency/fair_diarization_dispatch.py @@ -1,265 +1,328 @@ -"""Round-robin fair dispatch for standalone call-import diarization.""" - -from __future__ import annotations - -from typing import List -from uuid import UUID - -import redis -from loguru import logger -from sqlalchemy.orm import Session - -from app.config import settings -from app.database import SessionLocal -from app.models.database import CallImport, CallImportRow -from app.workers.concurrency.diarization_dispatch import ( - _try_dispatch_single_diarization_row, - _MISSING_PARAMS_ERROR, - get_row_diarization_params, - pop_row_diarization_params, -) -from app.workers.concurrency.eval_dispatch import DIARIZATION_QUEUE -from app.workers.config import celery_app -from app.db_sharding.import_dispatch import ( - call_imports_with_pending_diarization as sharded_call_imports_diarize, - pending_diarization_row_for_call_import as sharded_pending_diarize_row, - pending_diarization_workspaces as sharded_pending_diarize_workspaces, -) -from app.db_sharding.sessions import is_sharding_enabled - -_RR_CURSOR_KEY = "diarisation:fair:rr_cursor" -_WS_CALL_IMPORT_RR_CURSOR_KEY_PREFIX = "diarisation:fair:rr_cursor:ws:" - -_redis_client: redis.Redis | None = None - - -def _get_redis() -> redis.Redis: - global _redis_client - if _redis_client is None: - _redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True) - return _redis_client - - -def _get_rr_cursor() -> int: - try: - raw = _get_redis().get(_RR_CURSOR_KEY) - return int(raw or 0) - except (redis.RedisError, ValueError, TypeError): - return 0 - - -def _set_rr_cursor(cursor: int) -> None: - try: - _get_redis().set(_RR_CURSOR_KEY, str(max(0, cursor))) - except redis.RedisError as exc: - logger.warning("Failed to persist diarization fair-dispatch RR cursor: {}", exc) - - -def _workspace_call_import_rr_cursor_key(workspace_id: UUID | str) -> str: - return f"{_WS_CALL_IMPORT_RR_CURSOR_KEY_PREFIX}{workspace_id}" - - -def _get_workspace_call_import_rr_cursor(workspace_id: UUID) -> int: - try: - raw = _get_redis().get(_workspace_call_import_rr_cursor_key(workspace_id)) - return int(raw or 0) - except (redis.RedisError, ValueError, TypeError): - return 0 - - -def _set_workspace_call_import_rr_cursor(workspace_id: UUID, cursor: int) -> None: - try: - _get_redis().set( - _workspace_call_import_rr_cursor_key(workspace_id), - str(max(0, cursor)), - ) - except redis.RedisError as exc: - logger.warning( - "Failed to persist workspace diarization RR cursor for {}: {}", - workspace_id, - exc, - ) - - -def _workspaces_with_pending_diarization(db: Session) -> List[UUID]: - if is_sharding_enabled(): - return sharded_pending_diarize_workspaces(db) - rows = ( - db.query(CallImport.workspace_id) - .join(CallImportRow, CallImportRow.call_import_id == CallImport.id) - .filter( - CallImportRow.diarised_transcript_status == "pending", - CallImportRow.celery_task_id.is_(None), - ) - .distinct() - .all() - ) - return sorted({row[0] for row in rows if row[0] is not None}) - - -def _call_imports_with_pending_diarization( - db: Session, - workspace_id: UUID, -) -> List[UUID]: - if is_sharding_enabled(): - return sharded_call_imports_diarize(db, workspace_id) - rows = ( - db.query(CallImport.id) - .join(CallImportRow, CallImportRow.call_import_id == CallImport.id) - .filter( - CallImport.workspace_id == workspace_id, - CallImportRow.diarised_transcript_status == "pending", - CallImportRow.celery_task_id.is_(None), - ) - .distinct() - .all() - ) - return sorted({row[0] for row in rows if row[0] is not None}) - - -def _pending_row_for_call_import( - db: Session, - call_import_id: UUID, -) -> tuple[CallImportRow, CallImport] | None: - if is_sharding_enabled(): - pending = sharded_pending_diarize_row(db, call_import_id) - return pending - row = ( - db.query(CallImportRow, CallImport) - .join(CallImport, CallImport.id == CallImportRow.call_import_id) - .filter( - CallImport.id == call_import_id, - CallImportRow.diarised_transcript_status == "pending", - CallImportRow.celery_task_id.is_(None), - ) - .order_by(CallImportRow.created_at.asc()) - .first() - ) - if row is None: - return None - return row[0], row[1] - - -def _dispatch_batch_for_workspace( - db: Session, - workspace_id: UUID, - *, - batch_size: int, -) -> int: - call_imports = _call_imports_with_pending_diarization(db, workspace_id) - if not call_imports: - return 0 - - cursor = _get_workspace_call_import_rr_cursor(workspace_id) % len(call_imports) - dispatched = 0 - skips = 0 - - while dispatched < batch_size and skips < len(call_imports): - call_import_id = call_imports[cursor] - pending = _pending_row_for_call_import(db, call_import_id) - if pending is None: - skips += 1 - cursor = (cursor + 1) % len(call_imports) - continue - - row, call_import = pending - params = get_row_diarization_params(row.id) - if not params: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = _MISSING_PARAMS_ERROR - db.commit() - skips += 1 - cursor = (cursor + 1) % len(call_imports) - continue - - result = _try_dispatch_single_diarization_row( - db=db, - row=row, - call_import=call_import, - params=params, - ) - if result == "dispatched": - pop_row_diarization_params(row.id) - dispatched += 1 - skips = 0 - elif result == "at_capacity": - _set_workspace_call_import_rr_cursor(workspace_id, cursor) - return dispatched - else: - skips += 1 - - cursor = (cursor + 1) % len(call_imports) - - _set_workspace_call_import_rr_cursor(workspace_id, cursor) - return dispatched - - -def schedule_fair_diarization_dispatch( - *, - max_workspace_turns: int = 1, - countdown: int = 0, -) -> None: - try: - dispatch_fair_diarization_rows_task.apply_async( - kwargs={"max_workspace_turns": max_workspace_turns}, - queue=DIARIZATION_QUEUE, - countdown=countdown, - ) - except Exception as exc: - logger.warning("Failed to schedule fair diarization dispatch: {}", exc) - - -@celery_app.task(name="dispatch_fair_diarization_rows", queue=DIARIZATION_QUEUE) -def dispatch_fair_diarization_rows_task(max_workspace_turns: int = 1) -> dict: - """Round-robin pending diarization rows across workspaces (batch K per turn).""" - db = SessionLocal() - try: - workspaces = _workspaces_with_pending_diarization(db) - if not workspaces: - return {"status": "ok", "dispatched": 0, "workspaces": 0} - - batch_size = max(1, int(settings.DIARIZATION_FAIR_DISPATCH_BATCH_SIZE)) - max_turns = max(1, int(max_workspace_turns)) - - cursor = _get_rr_cursor() % len(workspaces) - total_dispatched = 0 - turns_served = 0 - skips = 0 - - while turns_served < max_turns and skips < len(workspaces): - workspace_id = workspaces[cursor] - dispatched = _dispatch_batch_for_workspace( - db, - workspace_id, - batch_size=batch_size, - ) - if dispatched > 0: - total_dispatched += dispatched - turns_served += 1 - skips = 0 - cursor = (cursor + 1) % len(workspaces) - _set_rr_cursor(cursor) - else: - skips += 1 - cursor = (cursor + 1) % len(workspaces) - - return { - "status": "ok", - "dispatched": total_dispatched, - "workspaces": len(workspaces), - "turns_served": turns_served, - "batch_size": batch_size, - } - except Exception: - logger.exception("dispatch_fair_diarization_rows failed") - raise - finally: - db.close() - - -def finish_diarization_work_and_redispatch(celery_task_id: str) -> None: - """Release a slot and schedule one fair diarization workspace turn.""" - from app.workers.concurrency.limits import release_eval_slot_for_celery_task - - release_eval_slot_for_celery_task(celery_task_id) - schedule_fair_diarization_dispatch(max_workspace_turns=1) +"""Round-robin fair dispatch for standalone call-import diarization.""" + +from __future__ import annotations + +from typing import List, Optional +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy.orm import Session + +from app.config import settings +from app.database import SessionLocal +from app.models.database import CallImport, CallImportRow +from app.workers.concurrency.diarization_dispatch import ( + _MISSING_PARAMS_ERROR, + _persist_diarization_row_fields, + _try_dispatch_single_diarization_row, + get_row_diarization_params, + pop_row_diarization_params, +) +from app.workers.concurrency.eval_dispatch import IMPORTS_QUEUE +from app.workers.config import celery_app +from app.db_sharding.import_dispatch import ( + call_imports_with_pending_diarization as sharded_call_imports_diarize, + pending_diarization_row_for_call_import as sharded_pending_diarize_row, + pending_diarization_workspaces as sharded_pending_diarize_workspaces, +) +from app.db_sharding.sessions import is_sharding_enabled + +_RR_CURSOR_KEY = "diarisation:fair:rr_cursor" +_WS_CALL_IMPORT_RR_CURSOR_KEY_PREFIX = "diarisation:fair:rr_cursor:ws:" + +_redis_client: redis.Redis | None = None + + +def _get_redis() -> redis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis_client + + +def _get_rr_cursor() -> int: + try: + raw = _get_redis().get(_RR_CURSOR_KEY) + return int(raw or 0) + except (redis.RedisError, ValueError, TypeError): + return 0 + + +def _set_rr_cursor(cursor: int) -> None: + try: + _get_redis().set(_RR_CURSOR_KEY, str(max(0, cursor))) + except redis.RedisError as exc: + logger.warning("Failed to persist diarization fair-dispatch RR cursor: {}", exc) + + +def _workspace_call_import_rr_cursor_key(workspace_id: UUID | str) -> str: + return f"{_WS_CALL_IMPORT_RR_CURSOR_KEY_PREFIX}{workspace_id}" + + +def _get_workspace_call_import_rr_cursor(workspace_id: UUID) -> int: + try: + raw = _get_redis().get(_workspace_call_import_rr_cursor_key(workspace_id)) + return int(raw or 0) + except (redis.RedisError, ValueError, TypeError): + return 0 + + +def _set_workspace_call_import_rr_cursor(workspace_id: UUID, cursor: int) -> None: + try: + _get_redis().set( + _workspace_call_import_rr_cursor_key(workspace_id), + str(max(0, cursor)), + ) + except redis.RedisError as exc: + logger.warning( + "Failed to persist workspace diarization RR cursor for {}: {}", + workspace_id, + exc, + ) + + +def _workspaces_with_pending_diarization(db: Session) -> List[UUID]: + if is_sharding_enabled(): + return sharded_pending_diarize_workspaces(db) + rows = ( + db.query(CallImport.workspace_id) + .join(CallImportRow, CallImportRow.call_import_id == CallImport.id) + .filter( + CallImportRow.diarised_transcript_status == "pending", + CallImportRow.celery_task_id.is_(None), + ) + .distinct() + .all() + ) + return sorted({row[0] for row in rows if row[0] is not None}) + + +def _call_imports_with_pending_diarization( + db: Session, + workspace_id: UUID, +) -> List[UUID]: + if is_sharding_enabled(): + return sharded_call_imports_diarize(db, workspace_id) + rows = ( + db.query(CallImport.id) + .join(CallImportRow, CallImportRow.call_import_id == CallImport.id) + .filter( + CallImport.workspace_id == workspace_id, + CallImportRow.diarised_transcript_status == "pending", + CallImportRow.celery_task_id.is_(None), + ) + .distinct() + .all() + ) + return sorted({row[0] for row in rows if row[0] is not None}) + + +def _pending_row_for_call_import( + db: Session, + call_import_id: UUID, +) -> tuple[CallImportRow, CallImport] | None: + if is_sharding_enabled(): + pending = sharded_pending_diarize_row(db, call_import_id) + return pending + row = ( + db.query(CallImportRow, CallImport) + .join(CallImport, CallImport.id == CallImportRow.call_import_id) + .filter( + CallImport.id == call_import_id, + CallImportRow.diarised_transcript_status == "pending", + CallImportRow.celery_task_id.is_(None), + ) + .order_by(CallImportRow.created_at.asc()) + .first() + ) + if row is None: + return None + return row[0], row[1] + + +def _dispatch_batch_for_workspace( + db: Session, + workspace_id: UUID, + *, + batch_size: int, +) -> int: + call_imports = _call_imports_with_pending_diarization(db, workspace_id) + if not call_imports: + return 0 + + cursor = _get_workspace_call_import_rr_cursor(workspace_id) % len(call_imports) + dispatched = 0 + skips = 0 + + while dispatched < batch_size and skips < len(call_imports): + call_import_id = call_imports[cursor] + pending = _pending_row_for_call_import(db, call_import_id) + if pending is None: + skips += 1 + cursor = (cursor + 1) % len(call_imports) + continue + + row, call_import = pending + params = get_row_diarization_params(row.id) + if not params: + _persist_diarization_row_fields( + db, + call_import, + row, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": _MISSING_PARAMS_ERROR, + }, + ) + skips += 1 + cursor = (cursor + 1) % len(call_imports) + continue + + result = _try_dispatch_single_diarization_row( + db=db, + row=row, + call_import=call_import, + params=params, + ) + if result == "dispatched": + pop_row_diarization_params(row.id) + dispatched += 1 + skips = 0 + elif result == "at_capacity": + _set_workspace_call_import_rr_cursor(workspace_id, cursor) + return dispatched + else: + skips += 1 + + cursor = (cursor + 1) % len(call_imports) + + _set_workspace_call_import_rr_cursor(workspace_id, cursor) + return dispatched + + +def schedule_fair_diarization_dispatch( + *, + max_workspace_turns: int = 1, + countdown: int = 0, +) -> None: + try: + # Lightweight scheduler: run on ``imports`` alongside + # ``bulk_diarize_call_import`` so environments that only attach a + # worker to ``imports`` (and not ``diarization``) still drain pending + # rows. Heavy work stays on ``diarization`` via + # ``transcribe_call_import_row``. + dispatch_fair_diarization_rows_task.apply_async( + kwargs={"max_workspace_turns": max_workspace_turns}, + queue=IMPORTS_QUEUE, + countdown=countdown, + ) + except Exception as exc: + logger.warning("Failed to schedule fair diarization dispatch: {}", exc) + + +@celery_app.task(name="dispatch_fair_diarization_rows", queue=IMPORTS_QUEUE) +def dispatch_fair_diarization_rows_task(max_workspace_turns: int = 1) -> dict: + """Round-robin pending diarization rows across workspaces (batch K per turn).""" + db = SessionLocal() + try: + from app.workers.concurrency.dispatch_reconcile import ( + reconcile_orphaned_diarization_dispatch_locks, + ) + + reconciled = reconcile_orphaned_diarization_dispatch_locks(db) + if reconciled: + logger.info( + "Reconciled {} orphaned diarisation dispatch lock(s)", + reconciled, + ) + workspaces = _workspaces_with_pending_diarization(db) + if not workspaces: + logger.debug( + "dispatch_fair_diarization_rows: no pending diarisation rows " + "(pending+no task id) visible after reconcile={}", + reconciled, + ) + return {"status": "ok", "dispatched": 0, "workspaces": 0} + + batch_size = max(1, int(settings.DIARIZATION_FAIR_DISPATCH_BATCH_SIZE)) + max_turns = max(1, int(max_workspace_turns)) + + cursor = _get_rr_cursor() % len(workspaces) + total_dispatched = 0 + turns_served = 0 + skips = 0 + + while turns_served < max_turns and skips < len(workspaces): + workspace_id = workspaces[cursor] + dispatched = _dispatch_batch_for_workspace( + db, + workspace_id, + batch_size=batch_size, + ) + if dispatched > 0: + total_dispatched += dispatched + turns_served += 1 + skips = 0 + cursor = (cursor + 1) % len(workspaces) + _set_rr_cursor(cursor) + else: + skips += 1 + cursor = (cursor + 1) % len(workspaces) + + return { + "status": "ok", + "dispatched": total_dispatched, + "workspaces": len(workspaces), + "turns_served": turns_served, + "batch_size": batch_size, + } + except Exception: + logger.exception("dispatch_fair_diarization_rows failed") + raise + finally: + db.close() + + +def drain_after_diarization_enqueue( + queued: int, + *, + log_context: str = "", +) -> Optional[dict]: + """Run fair dispatch in-process right after rows are marked pending. + + Beat may tick ``dispatch_fair_diarization_rows`` before bulk enqueue + commits; draining here makes Redis params and pending rows visible before + transcribe tasks are enqueued. ``schedule_fair_diarization_dispatch`` still + schedules an async backup tick. + """ + if queued <= 0: + return None + turns = min(999, max(4, (queued + 74) // 75)) + label = log_context or "diarization enqueue" + try: + async_result = dispatch_fair_diarization_rows_task.apply( + kwargs={"max_workspace_turns": turns} + ) + if async_result.successful(): + return async_result.result + logger.warning( + "{}: in-process dispatch drain failed: {}", + label, + async_result.result, + ) + except Exception as exc: + logger.warning( + "{}: in-process dispatch drain crashed: {}", + label, + exc, + ) + return None + + +def finish_diarization_work_and_redispatch(celery_task_id: str) -> None: + """Release a slot and schedule one fair diarization workspace turn.""" + from app.workers.concurrency.limits import release_eval_slot_for_celery_task + + release_eval_slot_for_celery_task(celery_task_id) + schedule_fair_diarization_dispatch(max_workspace_turns=1) diff --git a/app/workers/concurrency/fair_dispatch.py b/app/workers/concurrency/fair_dispatch.py index e6b45a46..7e3cb0ab 100644 --- a/app/workers/concurrency/fair_dispatch.py +++ b/app/workers/concurrency/fair_dispatch.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from datetime import datetime, timezone from typing import List, Optional from uuid import UUID @@ -39,6 +40,10 @@ _WS_EVAL_RR_CURSOR_KEY_PREFIX = "eval:fair:rr_cursor:ws:" _RESTRICTED_ROW_KEY_PREFIX = "eval:restricted:row:" _TRANSCRIBE_OVERWRITE_KEY_PREFIX = "eval:transcribe_overwrite:" +# Source rows that finished diarisation in the evaluation's current pass. An +# overwrite run resets transcripts up front, so ``completed`` on its own does +# not prove the text belongs to this pass — this set does. +_DIA_PASS_KEY_PREFIX = "eval:dia_pass:" # A 400k metric retry is dispatched 75 rows at a time. The last row can sit # for more than a day, so the marker has to outlive that queue. _RESTRICTED_ROW_TTL_SECONDS = 72 * 60 * 60 @@ -94,16 +99,79 @@ def store_rows_restricted_metrics( return True +def _dia_pass_key(evaluation_id: UUID | str) -> str: + return f"{_DIA_PASS_KEY_PREFIX}{evaluation_id}" + + +def reset_diarisation_pass(evaluation_id: UUID | str) -> None: + """Forget which rows were diarised in earlier passes for this evaluation.""" + try: + _get_redis().delete(_dia_pass_key(evaluation_id)) + except redis.RedisError as exc: + logger.warning( + "Failed to reset diarisation pass for evaluation {}: {}", + evaluation_id, + exc, + ) + + +def mark_row_diarised_in_pass( + evaluation_id: UUID | str, + source_row_id: UUID | str, +) -> None: + """Record that ``source_row_id`` finished diarisation in this pass.""" + key = _dia_pass_key(evaluation_id) + try: + pipe = _get_redis().pipeline(transaction=False) + pipe.sadd(key, str(source_row_id)) + pipe.expire(key, _RESTRICTED_ROW_TTL_SECONDS) + pipe.execute() + except redis.RedisError as exc: + logger.warning( + "Failed to mark row {} diarised for evaluation {}: {}", + source_row_id, + evaluation_id, + exc, + ) + + +def row_diarised_in_pass( + evaluation_id: UUID | str, + source_row_id: UUID | str, +) -> Optional[bool]: + """True/False when Redis answers, ``None`` when the marker store is down.""" + try: + return bool( + _get_redis().sismember(_dia_pass_key(evaluation_id), str(source_row_id)) + ) + except redis.RedisError as exc: + logger.warning( + "Failed to read diarisation pass for evaluation {}: {}", + evaluation_id, + exc, + ) + return None + + def store_evaluation_transcribe_overwrite( evaluation_id: UUID | str, *, overwrite: bool, ) -> None: - if not overwrite: - return + """Persist the evaluation's overwrite intent and reset its diarisation pass. + + The flag is rewritten on every run/retry start: a retry that does *not* + ask for overwrite must clear a flag left behind by an earlier overwrite + run, otherwise dispatch would re-diarise rows nobody asked to redo. + """ key = f"{_TRANSCRIBE_OVERWRITE_KEY_PREFIX}{evaluation_id}" try: - _get_redis().setex(key, _RESTRICTED_ROW_TTL_SECONDS, "1") + client = _get_redis() + if not overwrite: + client.delete(key) + return + reset_diarisation_pass(evaluation_id) + client.setex(key, _RESTRICTED_ROW_TTL_SECONDS, "1") except redis.RedisError as exc: logger.warning( "Failed to store transcribe_overwrite for evaluation {}: {}", @@ -407,8 +475,14 @@ def _slice_registry_for(call_import_id: UUID) -> dict: dispatched += 1 evaluation_dispatched = True skips = 0 - if evaluation.status == "pending": - evaluation.status = "running" + # ``started_at`` is the run start shown in the UI and the + # freshness cutoff for overwrite re-diarisation. + missing_start = getattr(evaluation, "started_at", False) is None + if evaluation.status == "pending" or missing_start: + if evaluation.status == "pending": + evaluation.status = "running" + if missing_start: + evaluation.started_at = datetime.now(timezone.utc) db.commit() break if outcome.result in ("at_capacity", "credential_throttled"): diff --git a/app/workers/concurrency/fair_import_dispatch.py b/app/workers/concurrency/fair_import_dispatch.py index e2b8e25b..8407da8e 100644 --- a/app/workers/concurrency/fair_import_dispatch.py +++ b/app/workers/concurrency/fair_import_dispatch.py @@ -26,7 +26,9 @@ _RR_CURSOR_KEY = "import:fair:rr_cursor" _WS_CALL_IMPORT_RR_CURSOR_KEY_PREFIX = "import:fair:rr_cursor:ws:" _DISPATCH_DEDUPE_KEY = "import:fair:dispatch_dedupe" +_DISPATCH_BACKOFF_LEVEL_KEY = "import:fair:dispatch_backoff_level" _DISPATCH_AT_CAPACITY_BACKOFF_SECONDS = 15 +_DISPATCH_AT_CAPACITY_BACKOFF_MAX_SECONDS = 120 _redis_client: redis.Redis | None = None @@ -46,6 +48,22 @@ def _get_rr_cursor() -> int: return 0 +def _reset_dispatch_backoff_level() -> None: + try: + _get_redis().delete(_DISPATCH_BACKOFF_LEVEL_KEY) + except redis.RedisError: + pass + + +def _bump_dispatch_backoff_level() -> int: + try: + level = int(_get_redis().incr(_DISPATCH_BACKOFF_LEVEL_KEY)) + _get_redis().expire(_DISPATCH_BACKOFF_LEVEL_KEY, 3600) + return max(1, level) + except (redis.RedisError, ValueError, TypeError): + return 1 + + def _set_rr_cursor(cursor: int) -> None: try: _get_redis().set(_RR_CURSOR_KEY, str(max(0, cursor))) @@ -286,10 +304,24 @@ def dispatch_fair_import_rows_task(max_workspace_turns: int = 1) -> dict: skips += 1 cursor = (cursor + 1) % len(workspaces) + if total_dispatched > 0: + _reset_dispatch_backoff_level() + if hit_capacity and _workspaces_with_pending_imports(db): + from app.workers.concurrency.import_retry_backoff import ( + import_dispatch_at_capacity_countdown, + ) + + level = _bump_dispatch_backoff_level() + countdown = import_dispatch_at_capacity_countdown( + provider_wait_seconds=backoff_seconds, + level=level, + base_seconds=_DISPATCH_AT_CAPACITY_BACKOFF_SECONDS, + max_seconds=_DISPATCH_AT_CAPACITY_BACKOFF_MAX_SECONDS, + ) _schedule_import_dispatch_deduped( max_workspace_turns=1, - countdown=backoff_seconds or _DISPATCH_AT_CAPACITY_BACKOFF_SECONDS, + countdown=countdown, ) return { diff --git a/app/workers/concurrency/fair_platform_import_dispatch.py b/app/workers/concurrency/fair_platform_import_dispatch.py index 0c532561..dad4e990 100644 --- a/app/workers/concurrency/fair_platform_import_dispatch.py +++ b/app/workers/concurrency/fair_platform_import_dispatch.py @@ -21,7 +21,57 @@ _BOOTSTRAP_DEDUPE_KEY = "platform_import:discovery_bootstrap" _BOOTSTRAP_DEDUPE_SECONDS = 45 _DISPATCH_DEDUPE_KEY = "platform_import:row_dispatch" +_DISPATCH_BACKOFF_LEVEL_KEY = "platform_import:row_dispatch_backoff_level" _DISPATCH_DEDUPE_SECONDS = 5 +_DISPATCH_AT_CAPACITY_BACKOFF_SECONDS = 15 +_DISPATCH_AT_CAPACITY_BACKOFF_MAX_SECONDS = 120 + +_redis_client = None + + +def _get_redis(): + global _redis_client + if _redis_client is None: + import redis + + from app.config import settings + + _redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis_client + + +def _reset_platform_dispatch_backoff_level() -> None: + try: + import redis + + _get_redis().delete(_DISPATCH_BACKOFF_LEVEL_KEY) + except redis.RedisError: + pass + + +def _bump_platform_dispatch_backoff_level() -> int: + try: + import redis + + level = int(_get_redis().incr(_DISPATCH_BACKOFF_LEVEL_KEY)) + _get_redis().expire(_DISPATCH_BACKOFF_LEVEL_KEY, 3600) + return max(1, level) + except redis.RedisError: + return 1 + + +def _pending_row_exists_on_session(row_db: Session, call_import_id: UUID) -> bool: + return ( + row_db.query(CallImportRow.id) + .filter( + CallImportRow.call_import_id == call_import_id, + CallImportRow.status == CallImportRowStatus.PENDING, + CallImportRow.celery_task_id.is_(None), + ) + .limit(1) + .first() + is not None + ) def schedule_platform_discovery_bootstrap(*, countdown: int = 0) -> None: @@ -214,28 +264,21 @@ def _row_sessions_for_import(db: Session, call_import_id: UUID) -> list[Session] def schedule_fair_platform_import_dispatch(*, countdown: int = 0) -> None: - """Enqueue at most one row-dispatch task per few seconds. - - Every finished discovery day used to publish its own copy. Those copies - piled up even when there was nothing new to dispatch. - """ + """Enqueue at most one row-dispatch task per dedupe window.""" try: import redis - from app.config import settings - - client = redis.from_url(settings.REDIS_URL, decode_responses=True) - ttl = max(_DISPATCH_DEDUPE_SECONDS, countdown + 2) - if not client.set(_DISPATCH_DEDUPE_KEY, "1", nx=True, ex=ttl): + ttl = max(_DISPATCH_DEDUPE_SECONDS, int(countdown) + 2) + if not _get_redis().set(_DISPATCH_DEDUPE_KEY, "1", nx=True, ex=ttl): return - except Exception as exc: + except redis.RedisError as exc: logger.error("platform import dispatch dedupe failed: {}", exc) return try: dispatch_fair_platform_import_rows_task.apply_async( queue=PLATFORM_CONTROL_QUEUE, - countdown=countdown, + countdown=max(0, int(countdown)), ) except Exception as exc: logger.warning("Failed to schedule platform import dispatch: {}", exc) @@ -260,6 +303,7 @@ def dispatch_fair_platform_import_rows_task() -> dict: db = SessionLocal() dispatched = 0 dropped = 0 + hit_capacity = False try: imports = _active_platform_imports(db) platform_ids = {item.id for item in imports} @@ -296,6 +340,10 @@ def dispatch_fair_platform_import_rows_task() -> dict: claimed = True break if not claimed: + for row_db in sessions: + if _pending_row_exists_on_session(row_db, call_import_id): + hit_capacity = True + break break finally: for row_db in sessions: @@ -309,6 +357,21 @@ def dispatch_fair_platform_import_rows_task() -> dict: queue_depth(PLATFORM_CONTROL_QUEUE), lease_count("platform_import:lease:global"), ) + still_pending = _platform_import_ids_with_pending(db, imports) + if dispatched > 0: + _reset_platform_dispatch_backoff_level() + if still_pending and (dispatched == 0 or hit_capacity): + from app.workers.concurrency.import_retry_backoff import ( + import_dispatch_at_capacity_countdown, + ) + + level = _bump_platform_dispatch_backoff_level() + backoff = import_dispatch_at_capacity_countdown( + level=level, + base_seconds=_DISPATCH_AT_CAPACITY_BACKOFF_SECONDS, + max_seconds=_DISPATCH_AT_CAPACITY_BACKOFF_MAX_SECONDS, + ) + schedule_fair_platform_import_dispatch(countdown=backoff) return {"status": "ok", "dispatched": dispatched, "dropped_expired": dropped} finally: db.close() diff --git a/app/workers/concurrency/import_retry_backoff.py b/app/workers/concurrency/import_retry_backoff.py new file mode 100644 index 00000000..fc89704b --- /dev/null +++ b/app/workers/concurrency/import_retry_backoff.py @@ -0,0 +1,63 @@ +"""Shared retry countdown for call-import workers (recording fetch, upload, etc.).""" + +from __future__ import annotations + +from app.config import settings + + +def import_transient_retry_countdown( + exc: Exception | None, + *, + attempt: int, + base_seconds: int | None = None, + max_seconds: int | None = None, +) -> int: + """Return Celery ``countdown`` for a transient import failure. + + Honors ``retry_after_seconds`` on the exception when set (429 / credential + throttle). Otherwise uses exponential backoff from ``attempt`` (0 = first + retry), capped at ``max_seconds``. + """ + if exc is not None: + retry_after = getattr(exc, "retry_after_seconds", None) + if retry_after is not None: + try: + seconds = int(retry_after) + except (TypeError, ValueError): + seconds = 0 + if seconds > 0: + return max(1, seconds) + + base = max( + 1, + int( + base_seconds + if base_seconds is not None + else settings.TELEPHONY_IMPORT_BACKOFF_BASE_SECONDS + ), + ) + cap = max( + base, + int( + max_seconds + if max_seconds is not None + else settings.TELEPHONY_IMPORT_BACKOFF_MAX_SECONDS + ), + ) + level = max(0, int(attempt)) + return max(1, min(cap, base * (2**level))) + + +def import_dispatch_at_capacity_countdown( + *, + provider_wait_seconds: int = 0, + level: int, + base_seconds: int = 15, + max_seconds: int = 120, +) -> int: + """Backoff when fair import dispatch hits the global slot cap.""" + base = max(1, int(base_seconds)) + cap = max(base, int(max_seconds)) + lvl = max(1, int(level)) + exp = min(cap, base * (2 ** (lvl - 1))) + return max(int(provider_wait_seconds or 0), exp) diff --git a/app/workers/concurrency/limits.py b/app/workers/concurrency/limits.py index f1477054..5c32a1b3 100644 --- a/app/workers/concurrency/limits.py +++ b/app/workers/concurrency/limits.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from uuid import UUID import redis @@ -14,63 +15,68 @@ _redis_client: redis.Redis | None = None -_ACQUIRE_LUA = """ -local ws_key = KEYS[1] -local org_key = KEYS[2] -local glob_key = KEYS[3] -local task_key = KEYS[4] -local job_key = KEYS[5] -local ws_limit = tonumber(ARGV[1]) -local org_limit = tonumber(ARGV[2]) -local glob_limit = tonumber(ARGV[3]) -local ttl = tonumber(ARGV[4]) -local job_limit = tonumber(ARGV[5]) -local evaluation_id = ARGV[6] -local ws = tonumber(redis.call('GET', ws_key) or '0') -local org = tonumber(redis.call('GET', org_key) or '0') -local glob = tonumber(redis.call('GET', glob_key) or '0') -if ws >= ws_limit then return 0 end -if org >= org_limit then return 0 end -if glob >= glob_limit then return 0 end -if evaluation_id ~= '' and job_limit > 0 then - local job = tonumber(redis.call('GET', job_key) or '0') - if job >= job_limit then return 0 end +_ACQUIRE_LEASE_LUA = """ +local now = tonumber(ARGV[1]) +local ttl = tonumber(ARGV[2]) +local ws_limit = tonumber(ARGV[3]) +local org_limit = tonumber(ARGV[4]) +local glob_limit = tonumber(ARGV[5]) +local extra_limit = tonumber(ARGV[6]) +local task_id = ARGV[7] +local ws = ARGV[8] +local org = ARGV[9] +local extra_id = ARGV[10] +local ws_prefix = ARGV[11] +local org_prefix = ARGV[12] +local glob_key = ARGV[13] +local extra_prefix = ARGV[14] +local expiry = now + ttl +redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now) +redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) +redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now) +if extra_id ~= '' then + redis.call('ZREMRANGEBYSCORE', KEYS[4], '-inf', now) end -redis.call('INCR', ws_key) -redis.call('EXPIRE', ws_key, ttl) -redis.call('INCR', org_key) -redis.call('EXPIRE', org_key, ttl) -redis.call('INCR', glob_key) -redis.call('EXPIRE', glob_key, ttl) -if evaluation_id ~= '' and job_limit > 0 then - redis.call('INCR', job_key) - redis.call('EXPIRE', job_key, ttl) - redis.call('HSET', task_key, 'evaluation_id', evaluation_id) +if redis.call('ZCARD', KEYS[1]) >= ws_limit then return 0 end +if redis.call('ZCARD', KEYS[2]) >= org_limit then return 0 end +if redis.call('ZCARD', KEYS[3]) >= glob_limit then return 0 end +if extra_id ~= '' and extra_limit > 0 and redis.call('ZCARD', KEYS[4]) >= extra_limit then + return 0 end -redis.call('HSET', task_key, 'workspace_id', ARGV[7], 'organization_id', ARGV[8]) -redis.call('EXPIRE', task_key, ttl) +redis.call('ZADD', KEYS[1], expiry, task_id) +redis.call('ZADD', KEYS[2], expiry, task_id) +redis.call('ZADD', KEYS[3], expiry, task_id) +if extra_id ~= '' then + redis.call('ZADD', KEYS[4], expiry, task_id) +end +redis.call('HSET', KEYS[5], 'workspace_id', ws, 'organization_id', org, 'evaluation_id', extra_id) +redis.call('EXPIRE', KEYS[5], ttl + 120) return 1 """ -_RELEASE_LUA = """ -local ws_key = KEYS[1] -local org_key = KEYS[2] -local glob_key = KEYS[3] -local task_key = KEYS[4] -local job_key = KEYS[5] -if redis.call('EXISTS', task_key) == 0 then return 0 end -local evaluation_id = redis.call('HGET', task_key, 'evaluation_id') -redis.call('DEL', task_key) -local function decr_min_zero(key) - local v = tonumber(redis.call('DECR', key)) - if v < 0 then redis.call('SET', key, 0) end +_RELEASE_LEASE_LUA = """ +local meta = redis.call('HGETALL', KEYS[1]) +if #meta == 0 then return 0 end +local ws = nil +local org = nil +local evaluation_id = nil +for i = 1, #meta, 2 do + if meta[i] == 'workspace_id' then ws = meta[i + 1] end + if meta[i] == 'organization_id' then org = meta[i + 1] end + if meta[i] == 'evaluation_id' then evaluation_id = meta[i + 1] end end -decr_min_zero(ws_key) -decr_min_zero(org_key) -decr_min_zero(glob_key) -if evaluation_id then - decr_min_zero(job_key) +local task_id = ARGV[1] +local ws_prefix = ARGV[2] +local org_prefix = ARGV[3] +local glob_key = ARGV[4] +local job_prefix = ARGV[5] +if ws and ws ~= '' then redis.call('ZREM', ws_prefix .. ws, task_id) end +if org and org ~= '' then redis.call('ZREM', org_prefix .. org, task_id) end +redis.call('ZREM', glob_key, task_id) +if evaluation_id and evaluation_id ~= '' and job_prefix ~= '' then + redis.call('ZREM', job_prefix .. evaluation_id, task_id) end +redis.call('DEL', KEYS[1]) return 1 """ @@ -82,76 +88,184 @@ def _get_redis() -> redis.Redis: return _redis_client -def _workspace_key(workspace_id: UUID | str) -> str: - return f"eval:inflight:workspace:{workspace_id}" +def _eval_ws_lease_key(workspace_id: UUID | str) -> str: + return f"eval:lease:ws:{workspace_id}" -def _org_key(organization_id: UUID | str) -> str: - return f"eval:inflight:org:{organization_id}" +def _eval_org_lease_key(organization_id: UUID | str) -> str: + return f"eval:lease:org:{organization_id}" -def _global_key() -> str: - return "eval:inflight:global" +def _eval_global_lease_key() -> str: + return "eval:lease:global" -def _job_key(evaluation_id: UUID | str) -> str: - return f"eval:inflight:job:{evaluation_id}" +def _eval_job_lease_key(evaluation_id: UUID | str) -> str: + return f"eval:lease:job:{evaluation_id}" -def _task_key(celery_task_id: str) -> str: - return f"eval:slot:task:{celery_task_id}" +def _eval_meta_key(celery_task_id: str) -> str: + return f"eval:lease:meta:{celery_task_id}" -def _import_workspace_key(workspace_id: UUID | str) -> str: - return f"import:inflight:workspace:{workspace_id}" +def _import_ws_lease_key(workspace_id: UUID | str) -> str: + return f"import:lease:ws:{workspace_id}" -def _import_org_key(organization_id: UUID | str) -> str: - return f"import:inflight:org:{organization_id}" +def _import_org_lease_key(organization_id: UUID | str) -> str: + return f"import:lease:org:{organization_id}" -def _import_global_key() -> str: - return "import:inflight:global" +def _import_global_lease_key() -> str: + return "import:lease:global" -def _import_task_key(celery_task_id: str) -> str: - return f"import:slot:task:{celery_task_id}" +def _import_meta_key(celery_task_id: str) -> str: + return f"import:lease:meta:{celery_task_id}" -def acquire_eval_slot( +def _lease_inflight(client: redis.Redis, zset_key: str) -> int: + now = time.time() + try: + client.zremrangebyscore(zset_key, "-inf", now) + return int(client.zcard(zset_key) or 0) + except redis.RedisError: + return 0 + + +_UNUSED_LEASE_ZSET = "call_import:lease:_unused" + + +def _acquire_lease( *, + ws_zset: str, + org_zset: str, + glob_zset: str, + extra_zset: str | None, + meta_key: str, + celery_task_id: str, workspace_id: UUID | str, organization_id: UUID | str, - celery_task_id: str, - evaluation_id: UUID | str | None = None, + ws_limit: int, + org_limit: int, + glob_limit: int, + extra_limit: int, + extra_id: str, + ttl_seconds: int, + ws_prefix: str, + org_prefix: str, + glob_key_literal: str, + extra_prefix: str, + job_prefix: str, ) -> bool: - """Try to acquire one eval/transcribe in-flight slot. Returns False when at cap.""" - eval_id_str = str(evaluation_id) if evaluation_id else "" - job_key = _job_key(evaluation_id) if evaluation_id else _task_key(celery_task_id) try: client = _get_redis() + extra_key = extra_zset if extra_zset is not None else _UNUSED_LEASE_ZSET + keys = [ + ws_zset, + org_zset, + glob_zset, + extra_key, + meta_key, + ] acquired = client.eval( - _ACQUIRE_LUA, - 5, - _workspace_key(workspace_id), - _org_key(organization_id), - _global_key(), - _task_key(celery_task_id), - job_key, - str(settings.EVAL_WORKSPACE_INFLIGHT_LIMIT), - str(settings.EVAL_ORG_INFLIGHT_LIMIT), - str(settings.EVAL_GLOBAL_INFLIGHT_LIMIT), - str(_EVAL_SLOT_TTL_SECONDS), - str(settings.EVAL_JOB_INFLIGHT_LIMIT), - eval_id_str, + _ACQUIRE_LEASE_LUA, + len(keys), + *keys, + str(time.time()), + str(ttl_seconds), + str(ws_limit), + str(org_limit), + str(glob_limit), + str(extra_limit), + celery_task_id, str(workspace_id), str(organization_id), + extra_id, + ws_prefix, + org_prefix, + glob_key_literal, + extra_prefix, ) return bool(int(acquired or 0)) except redis.RedisError as exc: - logger.warning("Eval slot acquire failed (Redis error): {}", exc) - # Fail open so work is not permanently blocked by Redis hiccups. - return True + logger.warning("Slot lease acquire failed (Redis error): {}", exc) + return False + + +def _release_lease( + *, + meta_key: str, + celery_task_id: str, + ws_prefix: str, + org_prefix: str, + glob_key_literal: str, + job_prefix: str, +) -> None: + if not celery_task_id: + return + try: + client = _get_redis() + if not client.exists(meta_key): + return + client.eval( + _RELEASE_LEASE_LUA, + 1, + meta_key, + celery_task_id, + ws_prefix, + org_prefix, + glob_key_literal, + job_prefix, + ) + except redis.RedisError as exc: + logger.warning( + "Slot lease release failed for task {} (Redis error): {}", + celery_task_id, + exc, + ) + + +def acquire_eval_slot( + *, + workspace_id: UUID | str, + organization_id: UUID | str, + celery_task_id: str, + evaluation_id: UUID | str | None = None, +) -> bool: + """Try to acquire one eval/transcribe in-flight slot. Returns False when at cap.""" + eval_id_str = str(evaluation_id) if evaluation_id else "" + extra_zset = ( + _eval_job_lease_key(evaluation_id) + if evaluation_id and settings.EVAL_JOB_INFLIGHT_LIMIT > 0 + else None + ) + extra_limit = ( + int(settings.EVAL_JOB_INFLIGHT_LIMIT) + if evaluation_id and settings.EVAL_JOB_INFLIGHT_LIMIT > 0 + else 0 + ) + return _acquire_lease( + ws_zset=_eval_ws_lease_key(workspace_id), + org_zset=_eval_org_lease_key(organization_id), + glob_zset=_eval_global_lease_key(), + extra_zset=extra_zset, + meta_key=_eval_meta_key(celery_task_id), + celery_task_id=celery_task_id, + workspace_id=workspace_id, + organization_id=organization_id, + ws_limit=int(settings.EVAL_WORKSPACE_INFLIGHT_LIMIT), + org_limit=int(settings.EVAL_ORG_INFLIGHT_LIMIT), + glob_limit=int(settings.EVAL_GLOBAL_INFLIGHT_LIMIT), + extra_limit=extra_limit, + extra_id=eval_id_str if extra_zset else "", + ttl_seconds=_EVAL_SLOT_TTL_SECONDS, + ws_prefix="eval:lease:ws:", + org_prefix="eval:lease:org:", + glob_key_literal=_eval_global_lease_key(), + extra_prefix="eval:lease:job:", + job_prefix="eval:lease:job:", + ) def acquire_import_slot( @@ -161,65 +275,62 @@ def acquire_import_slot( celery_task_id: str, ) -> bool: """Try to acquire one recording-fetch in-flight slot. Returns False when at cap.""" - try: - client = _get_redis() - task_key = _import_task_key(celery_task_id) - acquired = client.eval( - _ACQUIRE_LUA, - 5, - _import_workspace_key(workspace_id), - _import_org_key(organization_id), - _import_global_key(), - task_key, - task_key, - str(settings.IMPORT_WORKSPACE_INFLIGHT_LIMIT), - str(settings.IMPORT_ORG_INFLIGHT_LIMIT), - str(settings.IMPORT_GLOBAL_INFLIGHT_LIMIT), - str(_IMPORT_SLOT_TTL_SECONDS), - "0", - "", - str(workspace_id), - str(organization_id), - ) - return bool(int(acquired or 0)) - except redis.RedisError as exc: - logger.warning("Import slot acquire failed (Redis error): {}", exc) - return True + return _acquire_lease( + ws_zset=_import_ws_lease_key(workspace_id), + org_zset=_import_org_lease_key(organization_id), + glob_zset=_import_global_lease_key(), + extra_zset=None, + meta_key=_import_meta_key(celery_task_id), + celery_task_id=celery_task_id, + workspace_id=workspace_id, + organization_id=organization_id, + ws_limit=int(settings.IMPORT_WORKSPACE_INFLIGHT_LIMIT), + org_limit=int(settings.IMPORT_ORG_INFLIGHT_LIMIT), + glob_limit=int(settings.IMPORT_GLOBAL_INFLIGHT_LIMIT), + extra_limit=0, + extra_id="", + ttl_seconds=_IMPORT_SLOT_TTL_SECONDS, + ws_prefix="import:lease:ws:", + org_prefix="import:lease:org:", + glob_key_literal=_import_global_lease_key(), + extra_prefix="", + job_prefix="", + ) def read_inflight_count(key: str) -> int: - """Read a single Redis in-flight counter (returns 0 on error).""" + """Read active lease count for a zset key (returns 0 on error).""" try: - return max(0, int(_get_redis().get(key) or 0)) - except (redis.RedisError, ValueError, TypeError): + return _lease_inflight(_get_redis(), key) + except redis.RedisError: return 0 def read_global_inflight() -> int: - return read_inflight_count(_global_key()) + return read_inflight_count(_eval_global_lease_key()) def read_import_global_inflight() -> int: - return read_inflight_count(_import_global_key()) + return read_inflight_count(_import_global_lease_key()) def read_org_inflight(organization_id: UUID | str) -> int: - return read_inflight_count(_org_key(organization_id)) + return read_inflight_count(_eval_org_lease_key(organization_id)) def read_workspace_inflight(workspace_id: UUID | str) -> int: - return read_inflight_count(_workspace_key(workspace_id)) + return read_inflight_count(_eval_ws_lease_key(workspace_id)) def read_job_inflight(evaluation_id: UUID | str) -> int: - return read_inflight_count(_job_key(evaluation_id)) + return read_inflight_count(_eval_job_lease_key(evaluation_id)) def slot_registered_for_task(celery_task_id: str) -> bool: try: client = _get_redis() - return bool(client.exists(_task_key(celery_task_id))) or bool( - client.exists(_import_task_key(celery_task_id)) + return bool(client.exists(_eval_meta_key(celery_task_id))) or bool( + client.exists(_import_meta_key(celery_task_id)) ) except redis.RedisError: return False @@ -227,65 +338,23 @@ def slot_registered_for_task(celery_task_id: str) -> bool: def release_eval_slot_for_celery_task(celery_task_id: str) -> None: """Release the slot held by a dispatched eval/transcribe Celery task.""" - if not celery_task_id: - return - task_key = _task_key(celery_task_id) - try: - client = _get_redis() - mapping = client.hgetall(task_key) - if not mapping: - return - workspace_id = mapping.get("workspace_id") - organization_id = mapping.get("organization_id") - evaluation_id = mapping.get("evaluation_id") - if not workspace_id or not organization_id: - client.delete(task_key) - return - job_key = _job_key(evaluation_id) if evaluation_id else task_key - client.eval( - _RELEASE_LUA, - 5, - _workspace_key(workspace_id), - _org_key(organization_id), - _global_key(), - task_key, - job_key, - ) - except redis.RedisError as exc: - logger.warning( - "Eval slot release failed for task {} (Redis error): {}", - celery_task_id, - exc, - ) + _release_lease( + meta_key=_eval_meta_key(celery_task_id), + celery_task_id=celery_task_id, + ws_prefix="eval:lease:ws:", + org_prefix="eval:lease:org:", + glob_key_literal=_eval_global_lease_key(), + job_prefix="eval:lease:job:", + ) def release_import_slot_for_celery_task(celery_task_id: str) -> None: """Release the slot held by a dispatched recording-fetch Celery task.""" - if not celery_task_id: - return - task_key = _import_task_key(celery_task_id) - try: - client = _get_redis() - mapping = client.hgetall(task_key) - if not mapping: - return - workspace_id = mapping.get("workspace_id") - organization_id = mapping.get("organization_id") - if not workspace_id or not organization_id: - client.delete(task_key) - return - client.eval( - _RELEASE_LUA, - 5, - _import_workspace_key(workspace_id), - _import_org_key(organization_id), - _import_global_key(), - task_key, - task_key, - ) - except redis.RedisError as exc: - logger.warning( - "Import slot release failed for task {} (Redis error): {}", - celery_task_id, - exc, - ) + _release_lease( + meta_key=_import_meta_key(celery_task_id), + celery_task_id=celery_task_id, + ws_prefix="import:lease:ws:", + org_prefix="import:lease:org:", + glob_key_literal=_import_global_lease_key(), + job_prefix="", + ) diff --git a/app/workers/config.py b/app/workers/config.py index 44c50347..a43df99b 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -1,247 +1,280 @@ -"""Celery application configuration and creation. - -Important: this module is imported via ``celery -A app.workers.celery_app`` -*before* any task module touches torch / numpy / librosa, so it is the -right place to enforce single-threaded BLAS/OMP/MKL. - -Why this matters: the default ``worker`` service runs the audio metric path -(``evaluate_call_import_row_audio`` → Praat / UTMOS / qualitative voice service) -on the ``audio-metrics`` queue. That path transitively loads torch, torchaudio, -librosa, transformers, and speechbrain into each prefork child. With Celery's -default prefork pool and ``--concurrency=N`` on an N-vCPU box, each child opens -up an OpenMP threadpool of size N, so the worker ends up with N×N native threads -fighting over shared OMP/MKL locks. After a handful of tasks one child -inevitably deadlocks inside ``pthread_cond_wait`` in libgomp, Celery -keeps thinking it's healthy, and the queue wedges. ``worker-imports`` (threads, -I/O + LLM only) should not load torch after the audio split. - -The mitigations applied here are the canonical Celery + PyTorch -hardening: - -* Cap BLAS/OMP/MKL threads to 1 *before* torch et al. get imported. -* ``worker_prefetch_multiplier = 1`` so a wedged child can hold at most - one task hostage instead of pre-reserving four. -* ``task_acks_late = True`` so a task killed by ``task_time_limit`` or - ``worker_max_tasks_per_child`` gets redelivered to a healthy child - instead of being silently lost. -* ``worker_max_tasks_per_child`` recycles children periodically so any - leaked torch threads / model memory get released. -""" - -from __future__ import annotations - -import os -from pathlib import Path - -# Single-threaded BLAS / OMP for every library that links against them. -# These must be set BEFORE numpy / torch / librosa / transformers / speechbrain -# are imported — once a library has created its OpenMP threadpool, changing -# the env var has no effect. Setting setdefault keeps any operator override -# (e.g. via the container env) intact. -for _var in ( - "OMP_NUM_THREADS", - "OPENBLAS_NUM_THREADS", - "MKL_NUM_THREADS", - "NUMEXPR_NUM_THREADS", - "VECLIB_MAXIMUM_THREADS", - "BLIS_NUM_THREADS", -): - os.environ.setdefault(_var, "1") - -# HuggingFace tokenizers warn (and occasionally deadlock) when forked after -# their fast-tokenizer threadpool has been created. Disable explicitly. -os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") - -from celery import Celery # noqa: E402 (env vars must be set first) -from celery.schedules import crontab # noqa: E402 -from loguru import logger # noqa: E402 - -from app.config import settings, load_config_from_file # noqa: E402 - -# Load config.yml if it exists (before using settings) -# This ensures the Celery worker has the same configuration as the main app -_config_path = Path(os.environ.get("EFFICIENTAI_CONFIG_PATH", "config.yml")) -if _config_path.exists(): - try: - load_config_from_file(str(_config_path)) - logger.info(f"✅ Celery worker loaded configuration from {_config_path}") - except Exception as e: - logger.warning(f"⚠️ Celery worker: Could not load config.yml: {e}") - -from app.services.billing.flexprice_service import log_startup_status # noqa: E402 - -log_startup_status(component="celery-worker") - -# Queues consumed by the dedicated call-import / evaluation worker. -PLATFORM_IMPORTS_WORKER_QUEUE = "platform-imports" -# Platform historical imports (discovery + hydrate) must run on a worker that -# consumes this queue — same pool as CSV import workers is fine. -IMPORTS_WORKER_QUEUES = ( - f"{PLATFORM_IMPORTS_WORKER_QUEUE},import-control,imports,diarization,eval-control,evaluations" -) -EVAL_CONTROL_QUEUE = "eval-control" -USAGE_WORKER_QUEUE = "usage" -PLATFORM_WORKER_QUEUE = "platform" -NOTIFICATIONS_WORKER_QUEUE = os.environ.get("NOTIFICATIONS_WORKER_QUEUE", "platform") -TRACES_WORKER_QUEUE = "traces" - - -def _usage_flush_beat_seconds() -> float: - raw = os.environ.get("USAGE_FLUSH_BEAT_SECONDS", "120") - try: - return max(30.0, float(raw)) - except (TypeError, ValueError): - return 120.0 - - -def _cron_dispatch_beat_seconds() -> float: - raw = os.environ.get("CRON_DISPATCH_INTERVAL_SECONDS", "30") - try: - return max(10.0, float(raw)) - except (TypeError, ValueError): - return 30.0 - - -def _platform_beat_schedule() -> dict: - """Periodic platform tasks — run from dedicated ``celery beat`` (single replica).""" - schedule = { - "flush-usage-counters": { - "task": "flush_usage_counters", - "schedule": _usage_flush_beat_seconds(), - }, - "dispatch-cron-jobs": { - "task": "dispatch_cron_jobs", - "schedule": _cron_dispatch_beat_seconds(), - }, - "evaluate-alerts": { - "task": "evaluate_alerts", - "schedule": crontab(minute="*/5"), - }, - "refresh-fx-rates": { - "task": "refresh_fx_rates", - "schedule": crontab(hour=6, minute=0), - }, - "prune-oss-usage-history": { - "task": "prune_oss_usage_history", - "schedule": crontab(hour=3, minute=0), - }, - "sweep-idle-traces": { - "task": "sweep_idle_traces", - "schedule": 30.0, - }, - "sweep-orphan-s3-batches": { - "task": "sweep_orphan_s3_batches", - "schedule": 900.0, - }, - } - if not getattr(settings, "CLICKHOUSE_URL", None): - schedule["sweep-staging-ingest"] = { - "task": "sweep_staging_ingest", - "schedule": 3600.0, - } - if settings.PLATFORM_IMPORT_FEATURE_ENABLED: - schedule["bootstrap-platform-import-discovery"] = { - "task": "discover_platform_slice", - "schedule": 90.0, - "kwargs": {"bootstrap": True}, - } - schedule["dispatch-fair-platform-import-rows"] = { - "task": "dispatch_fair_platform_import_rows", - "schedule": 45.0, - } - return schedule - - -# Create Celery app -celery_app = Celery( - "efficientai", - broker=settings.CELERY_BROKER_URL, - backend=settings.CELERY_RESULT_BACKEND, -) - -# Celery configuration -celery_app.conf.update( - task_serializer="json", - accept_content=["json"], - result_serializer="json", - timezone="UTC", - enable_utc=True, - task_track_started=True, - task_time_limit=30 * 60, # 30 minutes - task_soft_time_limit=25 * 60, # 25 minutes - # Long-running ML tasks (audio metrics, evaluations) tolerate prefork - # poorly. Prefetch=1 means a stuck child only blocks the one task it's - # currently working on; remaining tasks stay in the broker and get picked - # up by other healthy children. Default of 4 means a stuck child silently - # holds 4 reserved tasks hostage. - worker_prefetch_multiplier=1, - task_acks_late=True, - task_reject_on_worker_lost=True, - broker_transport_options={"visibility_timeout": 7200}, - # Recycle prefork children every N tasks. This frees torch model memory, - # any leaked OMP threads, and HuggingFace tokenizer state, and prevents - # slow drift from accumulating over hours of import processing. - worker_max_tasks_per_child=20, - beat_schedule=_platform_beat_schedule(), -) - -# Route call-import recording fetch to the imports queue (preferred by workers -# that consume ``imports,diarization,eval-control,evaluations``). Diarisation -# work (manual bulk transcribe and eval-chain transcribe) goes to the -# diarization queue. Bulk eval control (materialize, cancel, retry) goes to -# ``eval-control`` so operator actions are not head-of-line blocked by scoring. -# Eval fair dispatch and LLM scoring go to the evaluations queue. -celery_app.conf.task_routes = { - "process_call_import_row": {"queue": "imports"}, - "bulk_diarize_call_import": {"queue": "imports"}, - "bulk_delete_call_import_rows": {"queue": "imports"}, - "materialize_call_import_rows": {"queue": "imports"}, - "delete_call_import": {"queue": "import-control"}, - "discover_platform_slice": {"queue": "import-control"}, - "dispatch_fair_platform_import_rows": {"queue": "import-control"}, - "materialize_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, - "materialize_mapped_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, - "retry_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, - "cancel_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, - "export_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, - "evaluate_call_import_row": {"queue": "evaluations"}, - "evaluate_call_import_row_audio": {"queue": "audio-metrics"}, - "transcribe_call_import_row": {"queue": "diarization"}, - "dispatch_evaluation_rows": {"queue": "evaluations"}, - "dispatch_fair_eval_rows": {"queue": "evaluations"}, - "dispatch_fair_diarization_rows": {"queue": "diarization"}, - "dispatch_fair_import_rows": {"queue": "imports"}, - "generate_evaluation_tldr_insights": {"queue": "evaluations"}, - "generate_evaluation_user_insights": {"queue": "evaluations"}, - "generate_evaluation_metric_clusters": {"queue": "evaluations"}, - "generate_evaluator_result_metric_clusters": {"queue": "evaluations"}, - "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, - "evaluate_studio_run_item": {"queue": "evaluations"}, - "generate_agent_flowchart": {"queue": "celery"}, - "map_agent_flowchart_prompt_sections": {"queue": "celery"}, - "flush_usage_counters": {"queue": USAGE_WORKER_QUEUE}, - "recompute_usage_costs": {"queue": USAGE_WORKER_QUEUE}, - "evaluate_alerts": {"queue": PLATFORM_WORKER_QUEUE}, - "evaluate_alerts_for_org": {"queue": PLATFORM_WORKER_QUEUE}, - "send_alert_notifications": {"queue": NOTIFICATIONS_WORKER_QUEUE}, - "refresh_fx_rates": {"queue": PLATFORM_WORKER_QUEUE}, - "prune_oss_usage_history": {"queue": PLATFORM_WORKER_QUEUE}, - "dispatch_cron_jobs": {"queue": USAGE_WORKER_QUEUE}, - "run_cron_evaluator_job": {"queue": "celery"}, - "derive_trace_turns": {"queue": TRACES_WORKER_QUEUE}, - "sweep_idle_traces": {"queue": TRACES_WORKER_QUEUE}, - "process_staged_otlp": {"queue": TRACES_WORKER_QUEUE}, - "sweep_staging_ingest": {"queue": TRACES_WORKER_QUEUE}, - "close_and_offload_trace": {"queue": TRACES_WORKER_QUEUE}, - "process_s3_otlp_batch": {"queue": TRACES_WORKER_QUEUE}, - "sweep_orphan_s3_batches": {"queue": TRACES_WORKER_QUEUE}, -} - - -@celery_app.on_after_configure.connect -def _ensure_clickhouse_schema(**_kwargs): - from app.services.clickhouse.client import clickhouse_enabled - from app.services.clickhouse.schema import ensure_schema - - if clickhouse_enabled(): - ensure_schema() +"""Celery application configuration and creation. + +Important: this module is imported via ``celery -A app.workers.celery_app`` +*before* any task module touches torch / numpy / librosa, so it is the +right place to enforce single-threaded BLAS/OMP/MKL. + +Why this matters: the default ``worker`` service runs the audio metric path +(``evaluate_call_import_row_audio`` → Praat / UTMOS / qualitative voice service) +on the ``audio-metrics`` queue. That path transitively loads torch, torchaudio, +librosa, transformers, and speechbrain into each prefork child. With Celery's +default prefork pool and ``--concurrency=N`` on an N-vCPU box, each child opens +up an OpenMP threadpool of size N, so the worker ends up with N×N native threads +fighting over shared OMP/MKL locks. After a handful of tasks one child +inevitably deadlocks inside ``pthread_cond_wait`` in libgomp, Celery +keeps thinking it's healthy, and the queue wedges. ``worker-imports`` (threads, +I/O + LLM only) should not load torch after the audio split. + +The mitigations applied here are the canonical Celery + PyTorch +hardening: + +* Cap BLAS/OMP/MKL threads to 1 *before* torch et al. get imported. +* ``worker_prefetch_multiplier = 1`` so a wedged child can hold at most + one task hostage instead of pre-reserving four. +* ``task_acks_late = True`` so a task killed by ``task_time_limit`` or + ``worker_max_tasks_per_child`` gets redelivered to a healthy child + instead of being silently lost. +* ``worker_max_tasks_per_child`` recycles children periodically so any + leaked torch threads / model memory get released. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +# Single-threaded BLAS / OMP for every library that links against them. +# These must be set BEFORE numpy / torch / librosa / transformers / speechbrain +# are imported — once a library has created its OpenMP threadpool, changing +# the env var has no effect. Setting setdefault keeps any operator override +# (e.g. via the container env) intact. +for _var in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "BLIS_NUM_THREADS", +): + os.environ.setdefault(_var, "1") + +# HuggingFace tokenizers warn (and occasionally deadlock) when forked after +# their fast-tokenizer threadpool has been created. Disable explicitly. +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + +from celery import Celery # noqa: E402 (env vars must be set first) +from celery.schedules import crontab # noqa: E402 +from loguru import logger # noqa: E402 + +from app.config import settings, load_config_from_file # noqa: E402 + +# Load config.yml if it exists (before using settings) +# This ensures the Celery worker has the same configuration as the main app +_config_path = Path(os.environ.get("EFFICIENTAI_CONFIG_PATH", "config.yml")) +if _config_path.exists(): + try: + load_config_from_file(str(_config_path)) + logger.info(f"✅ Celery worker loaded configuration from {_config_path}") + except Exception as e: + logger.warning(f"⚠️ Celery worker: Could not load config.yml: {e}") + +from app.services.billing.flexprice_service import log_startup_status # noqa: E402 + +log_startup_status(component="celery-worker") + +# Queues consumed by the dedicated call-import / evaluation worker. +PLATFORM_IMPORTS_WORKER_QUEUE = "platform-imports" +# Platform historical imports (discovery + hydrate) must run on a worker that +# consumes this queue — same pool as CSV import workers is fine. +IMPORTS_WORKER_QUEUES = ( + f"{PLATFORM_IMPORTS_WORKER_QUEUE},import-control,imports,diarization,eval-control,evaluations" +) +EVAL_CONTROL_QUEUE = "eval-control" +USAGE_WORKER_QUEUE = "usage" +PLATFORM_WORKER_QUEUE = "platform" +NOTIFICATIONS_WORKER_QUEUE = os.environ.get("NOTIFICATIONS_WORKER_QUEUE", "platform") +TRACES_WORKER_QUEUE = "traces" +HARBOR_WORKER_QUEUE = "harbor-sim" + + +def _usage_flush_beat_seconds() -> float: + raw = os.environ.get("USAGE_FLUSH_BEAT_SECONDS", "120") + try: + return max(30.0, float(raw)) + except (TypeError, ValueError): + return 120.0 + + +def _cron_dispatch_beat_seconds() -> float: + raw = os.environ.get("CRON_DISPATCH_INTERVAL_SECONDS", "30") + try: + return max(10.0, float(raw)) + except (TypeError, ValueError): + return 30.0 + + +def _eval_progress_flush_beat_seconds() -> float: + try: + return max(5.0, float(getattr(settings, "EVAL_PROGRESS_FLUSH_SECONDS", 30) or 30)) + except (TypeError, ValueError): + return 30.0 + + +def _platform_beat_schedule() -> dict: + """Periodic platform tasks — run from dedicated ``celery beat`` (single replica).""" + schedule = { + "flush-usage-counters": { + "task": "flush_usage_counters", + "schedule": _usage_flush_beat_seconds(), + }, + "dispatch-cron-jobs": { + "task": "dispatch_cron_jobs", + "schedule": _cron_dispatch_beat_seconds(), + }, + "evaluate-alerts": { + "task": "evaluate_alerts", + "schedule": crontab(minute="*/5"), + }, + "refresh-fx-rates": { + "task": "refresh_fx_rates", + "schedule": crontab(hour=6, minute=0), + }, + "prune-oss-usage-history": { + "task": "prune_oss_usage_history", + "schedule": crontab(hour=3, minute=0), + }, + "sweep-idle-traces": { + "task": "sweep_idle_traces", + "schedule": 30.0, + }, + "sweep-orphan-s3-batches": { + "task": "sweep_orphan_s3_batches", + "schedule": 900.0, + }, + # Redis-buffered eval progress counters -> catalog (one batched + # UPDATE per tick) plus the periodic fenced recount sweep. + "flush-eval-progress-counters": { + "task": "flush_eval_progress_counters", + "schedule": _eval_progress_flush_beat_seconds(), + }, + "sweep-stale-running-evaluations": { + "task": "sweep_stale_running_evaluations", + "schedule": 86400.0, + }, + # Fair diarisation is only triggered after bulk enqueue + chain; + # periodic tick clears stale locks and drains the diarisation queue. + "dispatch-fair-diarization-rows": { + "task": "dispatch_fair_diarization_rows", + "schedule": 30.0, + "kwargs": {"max_workspace_turns": 4}, + }, + } + if not getattr(settings, "CLICKHOUSE_URL", None): + schedule["sweep-staging-ingest"] = { + "task": "sweep_staging_ingest", + "schedule": 3600.0, + } + if settings.PLATFORM_IMPORT_FEATURE_ENABLED: + schedule["bootstrap-platform-import-discovery"] = { + "task": "discover_platform_slice", + "schedule": 90.0, + "kwargs": {"bootstrap": True}, + } + schedule["dispatch-fair-platform-import-rows"] = { + "task": "dispatch_fair_platform_import_rows", + "schedule": 45.0, + } + return schedule + + +# Create Celery app +celery_app = Celery( + "efficientai", + broker=settings.CELERY_BROKER_URL, + backend=settings.CELERY_RESULT_BACKEND, +) + +# Celery configuration +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_track_started=True, + task_time_limit=30 * 60, # 30 minutes + task_soft_time_limit=25 * 60, # 25 minutes + # Long-running ML tasks (audio metrics, evaluations) tolerate prefork + # poorly. Prefetch=1 means a stuck child only blocks the one task it's + # currently working on; remaining tasks stay in the broker and get picked + # up by other healthy children. Default of 4 means a stuck child silently + # holds 4 reserved tasks hostage. + worker_prefetch_multiplier=1, + task_acks_late=True, + task_reject_on_worker_lost=True, + broker_transport_options={"visibility_timeout": 7200}, + # Recycle prefork children every N tasks. This frees torch model memory, + # any leaked OMP threads, and HuggingFace tokenizer state, and prevents + # slow drift from accumulating over hours of import processing. + worker_max_tasks_per_child=20, + beat_schedule=_platform_beat_schedule(), +) + +# Route call-import recording fetch to the imports queue (preferred by workers +# that consume ``imports,diarization,eval-control,evaluations``). Diarisation +# work (manual bulk transcribe and eval-chain transcribe) goes to the +# diarization queue. Bulk eval control (materialize, cancel, retry) goes to +# ``eval-control`` so operator actions are not head-of-line blocked by scoring. +# Eval fair dispatch and LLM scoring go to the evaluations queue. +celery_app.conf.task_routes = { + "process_call_import_row": {"queue": "imports"}, + "bulk_diarize_call_import": {"queue": "imports"}, + "bulk_delete_call_import_rows": {"queue": "imports"}, + "materialize_call_import_rows": {"queue": "imports"}, + "delete_call_import": {"queue": "import-control"}, + "discover_platform_slice": {"queue": "import-control"}, + "dispatch_fair_platform_import_rows": {"queue": "import-control"}, + "materialize_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "materialize_mapped_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "retry_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "cancel_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "export_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "evaluate_call_import_row": {"queue": "evaluations"}, + "evaluate_call_import_row_audio": {"queue": "audio-metrics"}, + "transcribe_call_import_row": {"queue": "diarization"}, + "dispatch_evaluation_rows": {"queue": "evaluations"}, + "dispatch_fair_eval_rows": {"queue": "evaluations"}, + # Scheduler-only; run on imports next to bulk_diarize (see fair_diarization_dispatch) + "dispatch_fair_diarization_rows": {"queue": "imports"}, + "dispatch_fair_import_rows": {"queue": "imports"}, + "generate_evaluation_tldr_insights": {"queue": "evaluations"}, + "generate_evaluation_user_insights": {"queue": "evaluations"}, + "generate_evaluation_metric_clusters": {"queue": "evaluations"}, + "generate_evaluator_result_metric_clusters": {"queue": "evaluations"}, + "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, + "evaluate_studio_run_item": {"queue": "evaluations"}, + # Harbor jobs need Python 3.12 + the harbor extra (worker-harbor service). + "run_harbor_simulation": {"queue": HARBOR_WORKER_QUEUE}, + "analyze_simulation_run_failures": {"queue": "evaluations"}, + "generate_agent_flowchart": {"queue": "celery"}, + "map_agent_flowchart_prompt_sections": {"queue": "celery"}, + "flush_usage_counters": {"queue": USAGE_WORKER_QUEUE}, + "recompute_usage_costs": {"queue": USAGE_WORKER_QUEUE}, + "evaluate_alerts": {"queue": PLATFORM_WORKER_QUEUE}, + "evaluate_alerts_for_org": {"queue": PLATFORM_WORKER_QUEUE}, + "send_alert_notifications": {"queue": NOTIFICATIONS_WORKER_QUEUE}, + "refresh_fx_rates": {"queue": PLATFORM_WORKER_QUEUE}, + "sweep_stale_running_evaluations": {"queue": PLATFORM_WORKER_QUEUE}, + "auto_abort_stale_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "finalize_call_import_evaluation": {"queue": EVAL_CONTROL_QUEUE}, + "flush_eval_progress_counters": {"queue": EVAL_CONTROL_QUEUE}, + "prune_oss_usage_history": {"queue": PLATFORM_WORKER_QUEUE}, + "dispatch_cron_jobs": {"queue": USAGE_WORKER_QUEUE}, + "run_cron_evaluator_job": {"queue": "celery"}, + "derive_trace_turns": {"queue": TRACES_WORKER_QUEUE}, + "sweep_idle_traces": {"queue": TRACES_WORKER_QUEUE}, + "process_staged_otlp": {"queue": TRACES_WORKER_QUEUE}, + "sweep_staging_ingest": {"queue": TRACES_WORKER_QUEUE}, + "close_and_offload_trace": {"queue": TRACES_WORKER_QUEUE}, + "process_s3_otlp_batch": {"queue": TRACES_WORKER_QUEUE}, + "sweep_orphan_s3_batches": {"queue": TRACES_WORKER_QUEUE}, +} + + +@celery_app.on_after_configure.connect +def _ensure_clickhouse_schema(**_kwargs): + from app.services.clickhouse.client import clickhouse_enabled + from app.services.clickhouse.schema import ensure_schema + + if clickhouse_enabled(): + ensure_schema() diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index 44b07b5b..3569dcb6 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -1,150 +1,158 @@ -"""Celery task modules - import to register tasks with the app.""" - -from app.workers.config import celery_app - -# Import task modules to register tasks with Celery -from . import process_evaluation -from . import process_evaluator_result -from . import run_evaluator -from . import tts_comparison -from . import tts_report -from . import run_prompt_optimization -from . import process_call_import_row -from . import evaluate_call_import_row -from . import evaluate_call_import_row_audio -from . import transcribe_call_import_row -from . import run_judge_alignment -from . import generate_evaluation_user_insights -from . import generate_evaluation_tldr_insights -from . import generate_evaluation_metric_clusters -from . import generate_evaluator_result_metric_clusters -from . import generate_evaluation_prompt_improvements -from . import agent_flowchart_jobs -from . import initiate_vobiz_outbound -from . import finalize_telephony_recording -from . import evaluate_studio_run_item -from . import call_import_bulk_ops -from . import flush_usage_counters -from . import recompute_usage_costs -from . import prune_oss_usage_history -from . import dispatch_cron_jobs -from . import evaluate_alerts -from . import evaluate_alerts_for_org -from . import send_alert_notifications -from . import refresh_fx_rates -from . import run_cron_evaluator_job -from . import trace_tasks -from app.workers.concurrency import eval_dispatch -from app.workers.concurrency import fair_dispatch -from app.workers.concurrency import fair_diarization_dispatch -from app.workers.concurrency import fair_import_dispatch -from app.workers.concurrency import fair_platform_import_dispatch - -__all__ = [ - "celery_app", - "process_evaluation_task", - "process_evaluator_result_task", - "run_evaluator_task", - "generate_tts_comparison_task", - "evaluate_tts_comparison_task", - "generate_tts_report_pdf_task", - "run_prompt_optimization_task", - "process_call_import_row_task", - "evaluate_call_import_row_task", - "evaluate_call_import_row_audio_task", - "transcribe_call_import_row_task", - "run_judge_alignment_task", - "generate_evaluation_user_insights_task", - "generate_evaluation_tldr_insights_task", - "generate_evaluation_metric_clusters_task", - "generate_evaluation_prompt_improvements_task", - "generate_agent_flowchart_task", - "map_agent_flowchart_prompt_sections_task", - "initiate_vobiz_outbound_call_task", - "finalize_telephony_recording_task", - "dispatch_evaluation_rows_task", - "dispatch_fair_eval_rows_task", - "dispatch_fair_diarization_rows_task", - "evaluate_studio_run_item_task", - "dispatch_fair_import_rows_task", - "dispatch_fair_platform_import_rows_task", - "discover_platform_slice_task", - "bulk_diarize_call_import_task", - "bulk_delete_call_import_rows_task", - "materialize_call_import_rows_task", - "delete_call_import_task", - "materialize_call_import_evaluation_task", -] - -process_evaluation_task = process_evaluation.process_evaluation_task -process_evaluator_result_task = process_evaluator_result.process_evaluator_result_task -run_evaluator_task = run_evaluator.run_evaluator_task -generate_tts_comparison_task = tts_comparison.generate_tts_comparison_task -evaluate_tts_comparison_task = tts_comparison.evaluate_tts_comparison_task -generate_tts_report_pdf_task = tts_report.generate_tts_report_pdf_task -run_prompt_optimization_task = run_prompt_optimization.run_prompt_optimization_task -process_call_import_row_task = process_call_import_row.process_call_import_row_task -evaluate_call_import_row_task = evaluate_call_import_row.evaluate_call_import_row_task -evaluate_call_import_row_audio_task = ( - evaluate_call_import_row_audio.evaluate_call_import_row_audio_task -) -transcribe_call_import_row_task = ( - transcribe_call_import_row.transcribe_call_import_row_task -) -run_judge_alignment_task = run_judge_alignment.run_judge_alignment_task -generate_evaluation_user_insights_task = ( - generate_evaluation_user_insights.generate_evaluation_user_insights_task -) -generate_evaluation_tldr_insights_task = ( - generate_evaluation_tldr_insights.generate_evaluation_tldr_insights_task -) -generate_evaluation_metric_clusters_task = ( - generate_evaluation_metric_clusters.generate_evaluation_metric_clusters_task -) -generate_evaluator_result_metric_clusters_task = ( - generate_evaluator_result_metric_clusters.generate_evaluator_result_metric_clusters_task -) -generate_evaluation_prompt_improvements_task = ( - generate_evaluation_prompt_improvements.generate_evaluation_prompt_improvements_task -) -generate_agent_flowchart_task = agent_flowchart_jobs.generate_agent_flowchart_task -map_agent_flowchart_prompt_sections_task = ( - agent_flowchart_jobs.map_agent_flowchart_prompt_sections_task -) -initiate_vobiz_outbound_call_task = initiate_vobiz_outbound.initiate_vobiz_outbound_call_task -finalize_telephony_recording_task = ( - finalize_telephony_recording.finalize_telephony_recording_task -) -dispatch_evaluation_rows_task = eval_dispatch.dispatch_evaluation_rows_task -dispatch_fair_eval_rows_task = fair_dispatch.dispatch_fair_eval_rows_task -dispatch_fair_diarization_rows_task = ( - fair_diarization_dispatch.dispatch_fair_diarization_rows_task -) -dispatch_fair_import_rows_task = ( - fair_import_dispatch.dispatch_fair_import_rows_task -) -dispatch_fair_platform_import_rows_task = ( - fair_platform_import_dispatch.dispatch_fair_platform_import_rows_task -) -discover_platform_slice_task = fair_platform_import_dispatch.discover_platform_slice_task -bulk_diarize_call_import_task = call_import_bulk_ops.bulk_diarize_call_import_task -bulk_delete_call_import_rows_task = ( - call_import_bulk_ops.bulk_delete_call_import_rows_task -) -materialize_call_import_rows_task = ( - call_import_bulk_ops.materialize_call_import_rows_task -) -delete_call_import_task = call_import_bulk_ops.delete_call_import_task -evaluate_studio_run_item_task = ( - evaluate_studio_run_item.evaluate_studio_run_item_task -) -materialize_call_import_evaluation_task = ( - call_import_bulk_ops.materialize_call_import_evaluation_task -) -flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task -recompute_usage_costs_task = recompute_usage_costs.recompute_usage_costs_task -prune_oss_usage_history_task = prune_oss_usage_history.prune_oss_usage_history_task -dispatch_cron_jobs_task = dispatch_cron_jobs.dispatch_cron_jobs_task -evaluate_alerts_task = evaluate_alerts.evaluate_alerts_task -refresh_fx_rates_task = refresh_fx_rates.refresh_fx_rates_task -run_cron_evaluator_job_task = run_cron_evaluator_job.run_cron_evaluator_job_task +"""Celery task modules - import to register tasks with the app.""" + +from app.workers.config import celery_app + +# Import task modules to register tasks with Celery +from . import process_evaluation +from . import process_evaluator_result +from . import run_evaluator +from . import tts_comparison +from . import tts_report +from . import run_prompt_optimization +from . import process_call_import_row +from . import platform_call_import_hydrate # noqa: F401 — fail fast at worker boot +from . import evaluate_call_import_row +from . import evaluate_call_import_row_audio +from . import sweep_stale_evaluations +from . import finalize_call_import_evaluation +from . import flush_eval_progress +from . import transcribe_call_import_row +from . import run_judge_alignment +from . import generate_evaluation_user_insights +from . import generate_evaluation_tldr_insights +from . import generate_evaluation_metric_clusters +from . import generate_evaluator_result_metric_clusters +from . import generate_evaluation_prompt_improvements +from . import agent_flowchart_jobs +from . import initiate_vobiz_outbound +from . import finalize_telephony_recording +from . import evaluate_studio_run_item +from . import call_import_bulk_ops +from . import flush_usage_counters +from . import recompute_usage_costs +from . import prune_oss_usage_history +from . import dispatch_cron_jobs +from . import evaluate_alerts +from . import evaluate_alerts_for_org +from . import send_alert_notifications +from . import refresh_fx_rates +from . import run_cron_evaluator_job +from . import trace_tasks +from . import ingest_knowledge_document +from . import run_harbor_simulation +from . import analyze_simulation_run_failures +from app.workers.concurrency import eval_dispatch +from app.workers.concurrency import fair_dispatch +from app.workers.concurrency import fair_diarization_dispatch +from app.workers.concurrency import fair_import_dispatch +from app.workers.concurrency import fair_platform_import_dispatch + +__all__ = [ + "celery_app", + "process_evaluation_task", + "process_evaluator_result_task", + "run_evaluator_task", + "generate_tts_comparison_task", + "evaluate_tts_comparison_task", + "generate_tts_report_pdf_task", + "run_prompt_optimization_task", + "process_call_import_row_task", + "evaluate_call_import_row_task", + "evaluate_call_import_row_audio_task", + "transcribe_call_import_row_task", + "run_judge_alignment_task", + "generate_evaluation_user_insights_task", + "generate_evaluation_tldr_insights_task", + "generate_evaluation_metric_clusters_task", + "generate_evaluation_prompt_improvements_task", + "generate_agent_flowchart_task", + "map_agent_flowchart_prompt_sections_task", + "initiate_vobiz_outbound_call_task", + "finalize_telephony_recording_task", + "dispatch_evaluation_rows_task", + "dispatch_fair_eval_rows_task", + "dispatch_fair_diarization_rows_task", + "evaluate_studio_run_item_task", + "dispatch_fair_import_rows_task", + "dispatch_fair_platform_import_rows_task", + "discover_platform_slice_task", + "bulk_diarize_call_import_task", + "bulk_delete_call_import_rows_task", + "materialize_call_import_rows_task", + "delete_call_import_task", + "materialize_call_import_evaluation_task", +] + +process_evaluation_task = process_evaluation.process_evaluation_task +process_evaluator_result_task = process_evaluator_result.process_evaluator_result_task +run_evaluator_task = run_evaluator.run_evaluator_task +generate_tts_comparison_task = tts_comparison.generate_tts_comparison_task +evaluate_tts_comparison_task = tts_comparison.evaluate_tts_comparison_task +generate_tts_report_pdf_task = tts_report.generate_tts_report_pdf_task +run_prompt_optimization_task = run_prompt_optimization.run_prompt_optimization_task +process_call_import_row_task = process_call_import_row.process_call_import_row_task +evaluate_call_import_row_task = evaluate_call_import_row.evaluate_call_import_row_task +evaluate_call_import_row_audio_task = ( + evaluate_call_import_row_audio.evaluate_call_import_row_audio_task +) +transcribe_call_import_row_task = ( + transcribe_call_import_row.transcribe_call_import_row_task +) +run_judge_alignment_task = run_judge_alignment.run_judge_alignment_task +generate_evaluation_user_insights_task = ( + generate_evaluation_user_insights.generate_evaluation_user_insights_task +) +generate_evaluation_tldr_insights_task = ( + generate_evaluation_tldr_insights.generate_evaluation_tldr_insights_task +) +generate_evaluation_metric_clusters_task = ( + generate_evaluation_metric_clusters.generate_evaluation_metric_clusters_task +) +generate_evaluator_result_metric_clusters_task = ( + generate_evaluator_result_metric_clusters.generate_evaluator_result_metric_clusters_task +) +generate_evaluation_prompt_improvements_task = ( + generate_evaluation_prompt_improvements.generate_evaluation_prompt_improvements_task +) +generate_agent_flowchart_task = agent_flowchart_jobs.generate_agent_flowchart_task +map_agent_flowchart_prompt_sections_task = ( + agent_flowchart_jobs.map_agent_flowchart_prompt_sections_task +) +initiate_vobiz_outbound_call_task = initiate_vobiz_outbound.initiate_vobiz_outbound_call_task +finalize_telephony_recording_task = ( + finalize_telephony_recording.finalize_telephony_recording_task +) +dispatch_evaluation_rows_task = eval_dispatch.dispatch_evaluation_rows_task +dispatch_fair_eval_rows_task = fair_dispatch.dispatch_fair_eval_rows_task +dispatch_fair_diarization_rows_task = ( + fair_diarization_dispatch.dispatch_fair_diarization_rows_task +) +dispatch_fair_import_rows_task = ( + fair_import_dispatch.dispatch_fair_import_rows_task +) +dispatch_fair_platform_import_rows_task = ( + fair_platform_import_dispatch.dispatch_fair_platform_import_rows_task +) +discover_platform_slice_task = fair_platform_import_dispatch.discover_platform_slice_task +bulk_diarize_call_import_task = call_import_bulk_ops.bulk_diarize_call_import_task +bulk_delete_call_import_rows_task = ( + call_import_bulk_ops.bulk_delete_call_import_rows_task +) +materialize_call_import_rows_task = ( + call_import_bulk_ops.materialize_call_import_rows_task +) +delete_call_import_task = call_import_bulk_ops.delete_call_import_task +evaluate_studio_run_item_task = ( + evaluate_studio_run_item.evaluate_studio_run_item_task +) +materialize_call_import_evaluation_task = ( + call_import_bulk_ops.materialize_call_import_evaluation_task +) +flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task +recompute_usage_costs_task = recompute_usage_costs.recompute_usage_costs_task +prune_oss_usage_history_task = prune_oss_usage_history.prune_oss_usage_history_task +dispatch_cron_jobs_task = dispatch_cron_jobs.dispatch_cron_jobs_task +evaluate_alerts_task = evaluate_alerts.evaluate_alerts_task +refresh_fx_rates_task = refresh_fx_rates.refresh_fx_rates_task +run_cron_evaluator_job_task = run_cron_evaluator_job.run_cron_evaluator_job_task +ingest_knowledge_document_task = ingest_knowledge_document.ingest_knowledge_document_task diff --git a/app/workers/tasks/analyze_simulation_run_failures.py b/app/workers/tasks/analyze_simulation_run_failures.py new file mode 100644 index 00000000..656dca14 --- /dev/null +++ b/app/workers/tasks/analyze_simulation_run_failures.py @@ -0,0 +1,194 @@ +"""Celery task: failure clusters + prompt suggestions for a finished simulation run.""" + +from __future__ import annotations + +import uuid +from typing import Any +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.database import SessionLocal +from app.models.database import SimulationRun +from app.services.call_import_metric_clusters import ( + metric_clusters_raw_is_cancelled, + metric_clusters_state_to_db, +) +from app.workers.config import celery_app + +# Key in ``SimulationRun.metric_clusters`` naming the analysis that owns it. +ANALYSIS_ID_KEY = "analysis_id" + + +def new_failure_analysis_state() -> dict[str, Any]: + """Fresh ``metric_clusters`` header for a queued analysis, with its ownership ID.""" + return {"status": "running", ANALYSIS_ID_KEY: uuid.uuid4().hex} + + +def _owned_by(raw: Any, analysis_id: str | None) -> bool: + # Tasks queued before ownership IDs existed carry none; let them through. + if analysis_id is None: + return True + return isinstance(raw, dict) and raw.get(ANALYSIS_ID_KEY) == analysis_id + + +def _lock_run(db: Session, run_id: UUID) -> SimulationRun | None: + """Re-read the run under a row lock so an ownership check and its write are atomic.""" + return ( + db.query(SimulationRun) + .filter(SimulationRun.id == run_id) + .populate_existing() + .with_for_update() + .first() + ) + + +@celery_app.task(name="analyze_simulation_run_failures", bind=True, max_retries=0) +def analyze_simulation_run_failures_task( + self, + simulation_run_id: str, + *, + provider: str | None = None, + model: str | None = None, + credential_id: str | None = None, + max_llm_calls: int | None = None, + analysis_id: str | None = None, +): + from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request + from app.services.harbor.failure_analysis import ( + failed_prompt_improvements, + generate_simulation_metric_clusters, + generate_simulation_prompt_improvements, + ) + + db = SessionLocal() + run_uuid = UUID(simulation_run_id) + run: SimulationRun | None = None + + def _with_analysis_id(raw: dict[str, Any]) -> dict[str, Any]: + return {**raw, ANALYSIS_ID_KEY: analysis_id} if analysis_id else raw + + try: + run = db.query(SimulationRun).filter(SimulationRun.id == run_uuid).first() + if run is None: + logger.error("Simulation failure analysis: run {} not found", simulation_run_id) + return + if not _owned_by(run.metric_clusters, analysis_id): + logger.info("Simulation failure analysis: run {} was superseded, skipping", run.run_id) + return + if metric_clusters_raw_is_cancelled(run.metric_clusters): + logger.info("Simulation failure analysis: run {} cancelled, skipping", run.run_id) + return + + credential_uuid = UUID(credential_id) if credential_id else None + provider_enum, model_str = get_llm_provider_and_model_for_request( + run.organization_id, db, provider, model, credential_uuid + ) + + def _reload_cancelled() -> bool: + """True when cancelled, or when a newer analysis has taken over this run.""" + db.expire(run, ["metric_clusters"]) + db.refresh(run) + raw = run.metric_clusters + return not _owned_by(raw, analysis_id) or metric_clusters_raw_is_cancelled(raw) + + def on_progress(update: dict) -> None: + locked = _lock_run(db, run_uuid) + if ( + locked is None + or not _owned_by(locked.metric_clusters, analysis_id) + or metric_clusters_raw_is_cancelled(locked.metric_clusters) + ): + db.rollback() + return + locked.metric_clusters = { + **(locked.metric_clusters or {}), + "status": "running", + "progress": update, + } + flag_modified(locked, "metric_clusters") + db.commit() + + state = generate_simulation_metric_clusters( + db, + run, + provider=provider_enum, + model=model_str, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + is_cancelled=_reload_cancelled, + credential_id=credential_uuid, + ) + + run = _lock_run(db, run_uuid) + if run is None or not _owned_by(run.metric_clusters, analysis_id): + db.rollback() + logger.info( + "Simulation failure analysis for {} superseded; dropping clusters", + simulation_run_id, + ) + return + run.metric_clusters = _with_analysis_id(metric_clusters_state_to_db(state)) + flag_modified(run, "metric_clusters") + if state.status != "completed": + run.prompt_improvements = None + db.commit() + logger.info( + "Simulation failure analysis for run {} ended with clusters status={}", + run.run_id, + state.status, + ) + return + + run.prompt_improvements = {"status": "running"} + db.commit() + try: + prompt_improvements = generate_simulation_prompt_improvements( + db, + run, + state, + provider=provider_enum.value, + model=model_str, + credential_id=credential_uuid, + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Simulation prompt suggestions failed for run {}: {}", run.run_id, exc) + prompt_improvements = failed_prompt_improvements(str(exc)) + + run = _lock_run(db, run_uuid) + if run is None or not _owned_by(run.metric_clusters, analysis_id): + db.rollback() + logger.info( + "Simulation failure analysis for {} superseded; dropping prompt suggestions", + simulation_run_id, + ) + return + run.prompt_improvements = prompt_improvements + flag_modified(run, "prompt_improvements") + db.commit() + logger.info("Simulation failure analysis completed for run {}", run.run_id) + except Exception as exc: # noqa: BLE001 + logger.exception("Simulation failure analysis failed for run {}: {}", simulation_run_id, exc) + try: + db.rollback() + run = _lock_run(db, run_uuid) + if run is None or not _owned_by(run.metric_clusters, analysis_id): + db.rollback() + return + prior = run.metric_clusters if isinstance(run.metric_clusters, dict) else {} + run.metric_clusters = { + **prior, + "status": "failed", + "error_message": str(exc)[:500], + } + flag_modified(run, "metric_clusters") + db.commit() + except Exception: # noqa: BLE001 + logger.exception( + "Could not record failure for simulation run {}", simulation_run_id + ) + db.rollback() + finally: + db.close() diff --git a/app/workers/tasks/call_import_bulk_ops.py b/app/workers/tasks/call_import_bulk_ops.py index aba30b04..1128322b 100644 --- a/app/workers/tasks/call_import_bulk_ops.py +++ b/app/workers/tasks/call_import_bulk_ops.py @@ -19,6 +19,9 @@ execute_evaluation_retry, materialize_and_enqueue_evaluation, ) +from app.workers.concurrency.fair_diarization_dispatch import ( + drain_after_diarization_enqueue, +) from app.workers.config import celery_app @@ -57,11 +60,18 @@ def bulk_diarize_call_import_task( payload, requested_row_ids=parsed_row_ids, ) - return { + out = { "queued": result.queued, "skipped_rows": result.skipped_rows, "skipped_reason_counts": result.skipped_reason_counts, } + dispatch_summary = drain_after_diarization_enqueue( + result.queued, + log_context=f"bulk_diarize_call_import import={call_import_id}", + ) + if dispatch_summary is not None: + out["post_enqueue_dispatch"] = dispatch_summary + return out except ValueError as exc: logger.warning( "bulk_diarize_call_import_task validation failed for {}: {}", @@ -227,6 +237,8 @@ def retry_call_import_evaluation_task( ), include_completed=bool(payload_dict.get("include_completed", False)), transcribe_overwrite=bool(payload_dict.get("transcribe_overwrite", False)), + retry_search_term=str(payload_dict.get("retry_search_term") or ""), + retry_status_filter=payload_dict.get("retry_status_filter"), ) finally: db.close() @@ -302,7 +314,8 @@ def bulk_delete_call_import_rows_task( self, call_import_id: str, organization_id: str, - row_ids: List[str], + row_ids: List[str] | None, + row_filter: dict | None = None, ) -> dict: """Delete many import rows and their blob recordings off the API thread.""" del self @@ -323,12 +336,33 @@ def bulk_delete_call_import_rows_task( ) return {"deleted": 0} - deleted = execute_bulk_row_delete( - db, - UUID(organization_id), - call_import, - [UUID(rid) for rid in row_ids], - ) + if row_filter: + from app.models.schemas import CallImportRowSelectionFilter + from app.services.call_imports.bulk_ops import ( + execute_bulk_row_delete_by_filter, + ) + from app.services.call_imports.row_selection import ( + normalize_row_selection_filter, + ) + + parsed = CallImportRowSelectionFilter.model_validate(row_filter) + search_term, diarised_status = normalize_row_selection_filter(parsed) + deleted = execute_bulk_row_delete_by_filter( + db, + UUID(organization_id), + call_import, + search_term=search_term, + diarised_status_filter=diarised_status, + ) + else: + from app.services.call_imports.bulk_ops import execute_bulk_row_delete + + deleted = execute_bulk_row_delete( + db, + UUID(organization_id), + call_import, + [UUID(rid) for rid in (row_ids or [])], + ) return {"deleted": deleted} finally: db.close() diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index 7e6f38b2..7d910f4c 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -1,756 +1,799 @@ -"""Celery task: evaluate one CallImport row against selected metrics. - -Metric routing mirrors ``process_evaluator_result``: - -* Audio-only metrics run on the ``audio-metrics`` queue via - :func:`app.workers.tasks.evaluate_call_import_row_audio.evaluate_call_import_row_audio_task` - (``worker`` service). This task handles LLM / comparison metrics only. -* When chained after the audio phase, ``_skip_audio=True`` merges scores - already persisted on the row. -""" - -from __future__ import annotations - -import json -from types import SimpleNamespace -from typing import Any, List, Optional -from uuid import UUID - -from loguru import logger -from sqlalchemy.orm import Session - -from app.database import SessionLocal -from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - CallImportRow, - Metric, -) -from app.workers.config import celery_app -from app.workers.tasks.evaluate_call_import_row_core import ( - as_json_dict, - build_all_columns_block, - build_llm_config_buckets, - bucket_needs_comparison_pair, - build_parent_groups, - categorize_metrics, - load_enabled_metrics, - metric_text_references_production, - now_utc, - parse_restricted_metric_uuids, - rollup_parent, - commit_terminal_row_and_rollup, - was_cancelled_externally, -) -from app.workers.tasks.helpers.llm_evaluation import ( - evaluate_with_llm, - flatten_metric_groups, - handle_llm_evaluation_error, -) - -# Re-export core helpers for tests and API route mirrors. -_now = now_utc -_as_json_dict = as_json_dict -_was_cancelled_externally = was_cancelled_externally -_build_all_columns_block = build_all_columns_block -_categorize_metrics = categorize_metrics -_rollup_parent = rollup_parent -_metric_text_references_production = metric_text_references_production - - -_commit_terminal_row_and_rollup = commit_terminal_row_and_rollup - - -def _rollup_terminal( - row_db: Session, - catalog_db, - evaluation: CallImportEvaluation, - eval_row: CallImportEvaluationRow, - *, - previous_row_status: str, -) -> None: - cat = catalog_db if catalog_db is not row_db else None - _commit_terminal_row_and_rollup( - row_db, - evaluation, - eval_row, - previous_row_status=previous_row_status, - catalog_db=cat, - ) - - -def _persist_eval_sessions(row_db, catalog_db) -> None: - row_db.commit() - if catalog_db is not row_db: - catalog_db.commit() - - -def _run_llm_scoring( - *, - eval_row_id: UUID, - organization_id: UUID, - transcript: str, - production_transcript: str, - diarised_transcript: str, - llm_config_buckets: dict, - comparison_metric_ids: set[str], - all_columns_block: str | None, - ai_providers: list, - llm_provider: str | None, - llm_model: str | None, - llm_credential_id: str | None, - llm_config: dict | None, - discover_new_metrics: bool, - running_discovered_metrics: list, - transcript_unavailable: bool, - missing_label: str, -) -> dict[str, Any]: - """Score LLM metrics without holding a long-lived DB session.""" - result_id = f"call-import-eval:{eval_row_id}" - metric_scores: dict[str, dict[str, Any]] = {} - evaluation_failed = transcript_unavailable - primary_error_message: str | None = ( - f"No {missing_label} transcript for this row; transcript-based " - "metrics could not be scored." - if transcript_unavailable - else None - ) - - if not llm_config_buckets: - return { - "metric_scores": metric_scores, - "evaluation_failed": evaluation_failed, - "primary_error_message": primary_error_message, - } - - metric_discovery_emitted = False - - for config, groups in llm_config_buckets.items(): - provider, model, llm_config_key, credential_id = config - llm_cfg = json.loads(llm_config_key) if llm_config_key else None - evaluator_obj = None - if provider and model: - evaluator_obj = SimpleNamespace( - llm_provider=provider, - llm_model=model, - llm_config=llm_cfg, - llm_credential_id=credential_id, - custom_prompt=None, - ) - bucket_metrics = flatten_metric_groups(groups) - emit_metric_discovery = discover_new_metrics and not metric_discovery_emitted - bucket_comparison_pair: tuple[str, str] | None = None - if bucket_needs_comparison_pair( - groups, - production_transcript=production_transcript, - diarised_transcript=diarised_transcript, - comparison_metric_ids=comparison_metric_ids, - ): - bucket_comparison_pair = ( - production_transcript, - diarised_transcript, - ) - bucket_has_transcript_metrics = any( - str(metric.id) not in comparison_metric_ids for metric in bucket_metrics - ) - bucket_transcription = ( - transcript if bucket_has_transcript_metrics else "" - ) - try: - llm_db = SessionLocal() - try: - llm_scores, _eval_time = evaluate_with_llm( - transcription=bucket_transcription, - llm_metrics=bucket_metrics, - ai_providers=ai_providers, - organization_id=organization_id, - result_id=result_id, - db=llm_db, - evaluator=evaluator_obj, - agent=None, - persona=None, - scenario=None, - all_columns_block=all_columns_block, - comparison_pair=bucket_comparison_pair, - discover_new_metrics=emit_metric_discovery, - running_discovered_metrics=( - running_discovered_metrics - if emit_metric_discovery - else None - ), - metric_groups=groups, - ) - finally: - llm_db.close() - if emit_metric_discovery: - metric_discovery_emitted = True - metric_scores.update(llm_scores) - except Exception as exc: # noqa: BLE001 - logger.exception( - "[CallImportEval {}] LLM evaluation failed for " - "provider={} model={}", - eval_row_id, - provider, - model, - ) - metric_scores.update(handle_llm_evaluation_error(bucket_metrics, exc)) - evaluation_failed = True - primary_error_message = str(exc) - - return { - "metric_scores": metric_scores, - "evaluation_failed": evaluation_failed, - "primary_error_message": primary_error_message, - } - - -# Per-task time limits keep a wedged LLM evaluation from holding a worker -# child hostage for the global 30 min fallback. -@celery_app.task( - name="evaluate_call_import_row", - bind=True, - max_retries=2, - time_limit=10 * 60, - soft_time_limit=8 * 60, -) -def evaluate_call_import_row_task( - self, - eval_row_id: str, - restricted_metric_ids: Optional[List[str]] = None, - _eval_slot_task_id: Optional[str] = None, - _skip_audio: bool = False, - _predecessor_task_id: Optional[str] = None, -): - """Evaluate one row using the appropriate library per metric type. - - When ``restricted_metric_ids`` is set, this is a **metric-subset** - pass: only those metrics are recomputed, and the resulting scores - are merged into the row's existing ``metric_scores`` dict so other - metrics' previously-computed values are preserved. Used by the - "Re-run metrics" UI; the create-evaluation flow always leaves this - None for a full row evaluation. - """ - slot_task_id = _eval_slot_task_id or self.request.id - scoring_inputs: dict[str, Any] | None = None - restricted_metric_uuids: list[UUID] | None = None - usage_ctx_token = None - release_slot = True - owns_execution = False - try: - from app.db_sharding.row_ops import ( - close_row_sessions, - locate_call_import_evaluation_row, - ) - - row_db = catalog_db = None - save_shard_id: str | None = None - try: - row_uuid = UUID(eval_row_id) - try: - row_db, catalog_db, eval_row, source_row, save_shard_id = ( - locate_call_import_evaluation_row(row_uuid) - ) - except LookupError: - logger.warning("CallImportEvaluationRow {} not found", eval_row_id) - return {"status": "skipped", "reason": "row_not_found"} - - from app.workers.tasks.evaluate_call_import_row_core import ( - load_evaluation_for_worker, - ) - - evaluation = load_evaluation_for_worker( - catalog_db, eval_row.evaluation_id - ) - if not evaluation: - logger.warning( - "CallImportEvaluation {} missing", eval_row.evaluation_id - ) - eval_row.status = "failed" - eval_row.error_message = "Evaluation parent not found" - row_db.commit() - return {"status": "failed", "reason": "evaluation_missing"} - - from app.services.usage.call_import_context import ( - call_import_evaluation_usage_context, - ) - from app.services.usage.context import set_usage_context - - usage_ctx_token = set_usage_context( - call_import_evaluation_usage_context( - organization_id=evaluation.organization_id, - workspace_id=evaluation.workspace_id, - evaluation_id=evaluation.id, - call_import_id=evaluation.call_import_id, - ) - ) - - from app.workers.tasks.evaluate_call_import_row_core import ( - claim_eval_row_for_task, - ) - - if not claim_eval_row_for_task( - row_db, - eval_row, - self.request.id, - predecessor_task_id=_predecessor_task_id, - ): - logger.info( - "Skipping eval row {} already claimed or finished", - eval_row_id, - ) - return {"status": "skipped", "reason": "already_claimed"} - - from app.workers.tasks.evaluate_call_import_row_core import ( - acquire_eval_execution, - ) - - if not acquire_eval_execution(self.request.id): - release_slot = False - from celery.exceptions import MaxRetriesExceededError - - try: - self.retry(countdown=30, max_retries=40) - except MaxRetriesExceededError: - release_slot = True - return {"status": "skipped", "reason": "already_claimed"} - owns_execution = True - - previous_row_status = "running" - if evaluation.status == "pending": - evaluation.status = "running" - evaluation.started_at = evaluation.started_at or _now() - catalog_db.commit() - - production_transcript = (source_row.transcript or "").strip() - diarised_transcript = (source_row.diarised_transcript or "").strip() - eval_source = ( - (evaluation.transcript_source or "diarised").strip().lower() - ) - if eval_source == "production": - transcript = production_transcript - missing_label = "production" - else: - transcript = diarised_transcript - missing_label = "diarised" - recording_s3_key = (source_row.recording_s3_key or "").strip() or None - has_audio = recording_s3_key is not None - - restricted_metric_uuids = parse_restricted_metric_uuids( - restricted_metric_ids - ) - if ( - restricted_metric_ids is not None - and restricted_metric_uuids is not None - ): - selected_raw = { - str(x) for x in (evaluation.selected_metric_ids or []) - } - if restricted_metric_uuids and not any( - str(mid) in selected_raw for mid in restricted_metric_uuids - ): - eval_row.status = "completed" - eval_row.error_message = None - eval_row.finished_at = _now() - _rollup_terminal( - row_db, - catalog_db, - evaluation, - eval_row, - previous_row_status=previous_row_status, - ) - return { - "status": "skipped", - "reason": "restricted_metric_ids_no_match", - } - - metrics = load_enabled_metrics( - catalog_db, evaluation, restricted_metric_ids=restricted_metric_ids - ) - if not metrics: - eval_row.status = "failed" - eval_row.error_message = ( - "No enabled metrics selected for this evaluation" - ) - eval_row.finished_at = _now() - _rollup_terminal( - row_db, - catalog_db, - evaluation, - eval_row, - previous_row_status=previous_row_status, - ) - return {"status": "failed", "reason": "no_metrics"} - - raw_columns = ( - source_row.raw_columns - if isinstance(source_row.raw_columns, dict) - else {} - ) - parent_import = getattr(source_row, "call_import", None) - if parent_import is None and source_row.call_import_id: - from app.models.database import CallImport - - parent_import = ( - catalog_db.query(CallImport) - .filter(CallImport.id == source_row.call_import_id) - .first() - ) - custom_column_mapping = ( - parent_import.custom_column_mapping - if parent_import is not None - and isinstance( - getattr(parent_import, "custom_column_mapping", None), dict - ) - else {} - ) - - ( - transcript_metrics, - audio_metrics, - comparison_metrics, - metric_scores, - ) = _categorize_metrics( - metrics, - has_audio, - has_production_transcript=bool(production_transcript), - has_diarised_transcript=bool(diarised_transcript), - ) - - if _skip_audio: - existing_scores = ( - eval_row.metric_scores - if isinstance(eval_row.metric_scores, dict) - else {} - ) - metric_scores = {**existing_scores, **metric_scores} - audio_metrics = [] - elif audio_metrics: - logger.warning( - "[CallImportEval {}] Audio metrics present but task was " - "dispatched without audio phase — skipping audio bucket", - eval_row.id, - ) - audio_metrics = [] - - all_columns_block = _build_all_columns_block( - raw_columns, custom_column_mapping - ) - - if ( - not transcript_metrics - and not audio_metrics - and not comparison_metrics - ): - eval_row.status = "failed" - eval_row.error_message = ( - "Selected metrics could not be evaluated on this row " - "(missing audio, missing one of the two transcripts " - "required for a comparison metric, or no enabled " - "metrics matched)." - ) - eval_row.metric_scores = _as_json_dict(metric_scores) - eval_row.finished_at = _now() - _rollup_terminal( - row_db, - catalog_db, - evaluation, - eval_row, - previous_row_status=previous_row_status, - ) - return {"status": "failed", "reason": "no_evaluable_metrics"} - - transcript_unavailable = bool(transcript_metrics) and not transcript - if ( - transcript_unavailable - and not audio_metrics - and not comparison_metrics - ): - logger.warning( - "[CallImportEval {}] Skipping LLM metrics: {} transcript " - "is empty", - eval_row.id, - missing_label, - ) - empty_msg = f"No {missing_label} transcript for this row" - err = RuntimeError(empty_msg) - metric_scores.update( - handle_llm_evaluation_error(transcript_metrics, err) - ) - eval_row.status = "failed" - eval_row.error_message = ( - f"{empty_msg}; LLM-evaluated metrics could not be scored." - ) - eval_row.metric_scores = _as_json_dict(metric_scores) - eval_row.finished_at = _now() - _rollup_terminal( - row_db, - catalog_db, - evaluation, - eval_row, - previous_row_status=previous_row_status, - ) - return {"status": "failed", "reason": "missing_transcript"} - - if transcript_unavailable: - logger.warning( - "[CallImportEval {}] Skipping transcript-LLM metrics: {} " - "transcript is empty (audio/column metrics still scored)", - eval_row.id, - missing_label, - ) - empty_msg = f"No {missing_label} transcript for this row" - err = RuntimeError(empty_msg) - metric_scores.update( - handle_llm_evaluation_error(transcript_metrics, err) - ) - transcript_metrics = [] - - from app.workers.tasks.evaluate_call_import_row_core import ( - load_active_ai_providers, - ) - - ai_providers = load_active_ai_providers( - catalog_db, evaluation.organization_id - ) - - running_discovered_metrics: list = [] - running_discovered_by_parent: dict[UUID, list] = {} - llm_metrics_for_scoring: list[Metric] = list(comparison_metrics) - if transcript: - llm_metrics_for_scoring = transcript_metrics + comparison_metrics - - llm_config_buckets: dict = {} - comparison_metric_ids = {str(m.id) for m in comparison_metrics} - if llm_metrics_for_scoring: - if bool(getattr(evaluation, "discover_new_metrics", False)): - from app.api.v1.routes.call_import_evaluations import ( - _get_running_discovered_metrics, - ) - - alias_map_metrics = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - running_discovered_metrics = _get_running_discovered_metrics( - catalog_db, - evaluation.id, - organization_id=evaluation.organization_id, - alias_map=alias_map_metrics, - from_cache=True, - ) - from app.api.v1.routes.call_import_evaluations import ( - _alias_map_for_parent, - _get_running_discovered_labels, - ) - - parents_by_id, children_by_parent, _ = build_parent_groups( - catalog_db, llm_metrics_for_scoring - ) - for parent_id in children_by_parent: - parent_metric = parents_by_id.get(parent_id) - if parent_metric is None: - continue - if not bool(getattr(parent_metric, "allow_discovery", False)): - continue - if (parent_metric.selection_mode or "").lower() not in { - "single_choice", - "multi_label", - }: - continue - running_discovered_by_parent[parent_id] = ( - _get_running_discovered_labels( - catalog_db, - evaluation.id, - parent_metric.id, - organization_id=evaluation.organization_id, - alias_map=_alias_map_for_parent( - evaluation, parent_metric.id - ), - from_cache=True, - ) - ) - - overrides = ( - evaluation.metric_llm_overrides - if isinstance(evaluation.metric_llm_overrides, dict) - else {} - ) - llm_config_buckets = build_llm_config_buckets( - catalog_db, - llm_metrics_for_scoring, - overrides=overrides, - run_provider=(evaluation.llm_provider or "").strip() or None, - run_model=(evaluation.llm_model or "").strip() or None, - run_llm_config=( - evaluation.llm_config - if isinstance(evaluation.llm_config, dict) - else None - ), - run_credential_id=( - str(evaluation.llm_credential_id) - if evaluation.llm_credential_id - else None - ), - running_discovered_by_parent=running_discovered_by_parent, - ) - - scoring_inputs = { - "eval_row_id": eval_row.id, - "organization_id": evaluation.organization_id, - "transcript": transcript, - "production_transcript": production_transcript, - "diarised_transcript": diarised_transcript, - "llm_config_buckets": llm_config_buckets, - "comparison_metric_ids": comparison_metric_ids, - "all_columns_block": all_columns_block, - "ai_providers": ai_providers, - "llm_provider": evaluation.llm_provider, - "llm_model": evaluation.llm_model, - "llm_credential_id": ( - str(evaluation.llm_credential_id) - if evaluation.llm_credential_id - else None - ), - "llm_config": evaluation.llm_config, - "discover_new_metrics": bool( - getattr(evaluation, "discover_new_metrics", False) - ), - "running_discovered_metrics": running_discovered_metrics, - "transcript_unavailable": transcript_unavailable, - "missing_label": missing_label, - "pre_llm_metric_scores": dict(metric_scores), - } - finally: - if row_db is not None: - close_row_sessions(row_db, catalog_db) - - pre_llm_metric_scores = scoring_inputs.pop("pre_llm_metric_scores") - llm_result = _run_llm_scoring(**scoring_inputs) - metric_scores = { - **pre_llm_metric_scores, - **llm_result["metric_scores"], - } - evaluation_failed = llm_result["evaluation_failed"] - primary_error_message = llm_result["primary_error_message"] - - row_db = catalog_db = None - try: - from app.db_sharding.row_ops import ( - close_row_sessions, - locate_call_import_evaluation_row, - ) - - try: - row_db, catalog_db, eval_row, _source_row, _ = ( - locate_call_import_evaluation_row( - UUID(eval_row_id), - shard_id=save_shard_id, - ) - ) - except LookupError: - logger.warning( - "[CallImportEval {}] Row not found on any shard — skipping", - eval_row_id, - ) - return {"status": "skipped", "reason": "row_not_found"} - from app.workers.tasks.evaluate_call_import_row_core import ( - load_evaluation_for_worker, - ) - - evaluation = load_evaluation_for_worker( - catalog_db, eval_row.evaluation_id - ) - if not evaluation: - return {"status": "failed", "reason": "evaluation_missing"} - - if _was_cancelled_externally(row_db, eval_row): - logger.info( - "[CallImportEval {}] Skipping terminal write; " - "row was cancelled by user", - eval_row.id, - ) - try: - _rollup_parent( - catalog_db, - evaluation, - previous_row_status="running", - new_row_status="failed", - ) - catalog_db.commit() - except Exception: # noqa: BLE001 — rollup is best-effort here - catalog_db.rollback() - return { - "status": "cancelled", - "eval_row_id": eval_row_id, - } - - if evaluation_failed: - eval_row.status = "failed" - eval_row.error_message = ( - primary_error_message - or "Evaluation failed for one or more metrics" - ) - else: - eval_row.status = "completed" - eval_row.error_message = None - - from app.api.v1.routes.call_import_evaluations import ( - normalize_scores_with_aliases, - ) - - normalize_scores_with_aliases( - metric_scores, evaluation, catalog_db, evaluation.organization_id - ) - - new_scores = _as_json_dict(metric_scores) - if restricted_metric_uuids: - existing = ( - eval_row.metric_scores - if isinstance(eval_row.metric_scores, dict) - else {} - ) - merged = dict(existing) - for key, value in new_scores.items(): - merged[key] = value - eval_row.metric_scores = merged - else: - eval_row.metric_scores = new_scores - eval_row.finished_at = _now() - _rollup_terminal( - row_db, - catalog_db, - evaluation, - eval_row, - previous_row_status="running", - ) - - return { - "status": eval_row.status, - "eval_row_id": eval_row_id, - "metrics": len(eval_row.metric_scores or {}), - "subset_retry": bool(restricted_metric_uuids), - } - finally: - if row_db is not None: - close_row_sessions(row_db, catalog_db) - finally: - if usage_ctx_token is not None: - from app.services.usage.context import reset_usage_context - - reset_usage_context(usage_ctx_token) - if owns_execution: - from app.workers.tasks.evaluate_call_import_row_core import ( - release_eval_execution, - ) - - release_eval_execution(self.request.id) - if release_slot: - from app.workers.concurrency.fair_dispatch import ( - finish_eval_work_and_redispatch, - ) - - finish_eval_work_and_redispatch( - slot_task_id, - restricted_metric_ids=restricted_metric_ids, - ) +"""Celery task: evaluate one CallImport row against selected metrics. + +Metric routing mirrors ``process_evaluator_result``: + +* Audio-only metrics run on the ``audio-metrics`` queue via + :func:`app.workers.tasks.evaluate_call_import_row_audio.evaluate_call_import_row_audio_task` + (``worker`` service). This task handles LLM / comparison metrics only. +* When chained after the audio phase, ``_skip_audio=True`` merges scores + already persisted on the row. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any, List, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.database import SessionLocal +from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + CallImportRow, + Metric, +) +from app.workers.config import celery_app +from app.workers.tasks.evaluate_call_import_row_core import ( + as_json_dict, + build_all_columns_block, + build_llm_config_buckets, + bucket_needs_comparison_pair, + build_parent_groups, + categorize_metrics, + load_call_import_eval_context, + load_enabled_metrics, + metric_text_references_production, + now_utc, + parse_restricted_metric_uuids, + mark_evaluation_running_if_pending, + rollup_parent, + commit_terminal_row_and_rollup, + was_cancelled_externally, +) +from app.workers.tasks.helpers.llm_evaluation import ( + evaluate_with_llm, + flatten_metric_groups, + handle_llm_evaluation_error, +) + +# Re-export core helpers for tests and API route mirrors. +_now = now_utc +_as_json_dict = as_json_dict +_was_cancelled_externally = was_cancelled_externally +_build_all_columns_block = build_all_columns_block +_categorize_metrics = categorize_metrics +_rollup_parent = rollup_parent +_metric_text_references_production = metric_text_references_production + + +_commit_terminal_row_and_rollup = commit_terminal_row_and_rollup + + +def _rollup_terminal( + row_db: Session, + catalog_db, + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + *, + previous_row_status: str, +) -> None: + cat = catalog_db if catalog_db is not row_db else None + _commit_terminal_row_and_rollup( + row_db, + evaluation, + eval_row, + previous_row_status=previous_row_status, + catalog_db=cat, + ) + + +def _persist_eval_sessions(row_db, catalog_db) -> None: + # Only the shard holds changes here. The worker reads the parent from a + # cached snapshot, so the catalog session has nothing to commit; with no + # query run it never opened a transaction and closing it is free. + del catalog_db + row_db.commit() + + +def _run_llm_scoring( + *, + eval_row_id: UUID, + organization_id: UUID, + transcript: str, + production_transcript: str, + diarised_transcript: str, + llm_config_buckets: dict, + comparison_metric_ids: set[str], + all_columns_block: str | None, + ai_providers: list, + llm_provider: str | None, + llm_model: str | None, + llm_credential_id: str | None, + llm_config: dict | None, + discover_new_metrics: bool, + running_discovered_metrics: list, + transcript_unavailable: bool, + missing_label: str, +) -> dict[str, Any]: + """Score LLM metrics without holding a long-lived DB session.""" + result_id = f"call-import-eval:{eval_row_id}" + metric_scores: dict[str, dict[str, Any]] = {} + evaluation_failed = transcript_unavailable + primary_error_message: str | None = ( + f"No {missing_label} transcript for this row; transcript-based " + "metrics could not be scored." + if transcript_unavailable + else None + ) + + if not llm_config_buckets: + return { + "metric_scores": metric_scores, + "evaluation_failed": evaluation_failed, + "primary_error_message": primary_error_message, + } + + metric_discovery_emitted = False + + for config, groups in llm_config_buckets.items(): + provider, model, llm_config_key, credential_id = config + llm_cfg = json.loads(llm_config_key) if llm_config_key else None + evaluator_obj = None + if provider and model: + evaluator_obj = SimpleNamespace( + llm_provider=provider, + llm_model=model, + llm_config=llm_cfg, + llm_credential_id=credential_id, + custom_prompt=None, + ) + bucket_metrics = flatten_metric_groups(groups) + emit_metric_discovery = discover_new_metrics and not metric_discovery_emitted + bucket_comparison_pair: tuple[str, str] | None = None + if bucket_needs_comparison_pair( + groups, + production_transcript=production_transcript, + diarised_transcript=diarised_transcript, + comparison_metric_ids=comparison_metric_ids, + ): + bucket_comparison_pair = ( + production_transcript, + diarised_transcript, + ) + bucket_has_transcript_metrics = any( + str(metric.id) not in comparison_metric_ids for metric in bucket_metrics + ) + bucket_transcription = ( + transcript if bucket_has_transcript_metrics else "" + ) + try: + llm_db = SessionLocal() + try: + llm_scores, _eval_time = evaluate_with_llm( + transcription=bucket_transcription, + llm_metrics=bucket_metrics, + ai_providers=ai_providers, + organization_id=organization_id, + result_id=result_id, + db=llm_db, + evaluator=evaluator_obj, + agent=None, + persona=None, + scenario=None, + all_columns_block=all_columns_block, + comparison_pair=bucket_comparison_pair, + discover_new_metrics=emit_metric_discovery, + running_discovered_metrics=( + running_discovered_metrics + if emit_metric_discovery + else None + ), + metric_groups=groups, + ) + finally: + llm_db.close() + if emit_metric_discovery: + metric_discovery_emitted = True + metric_scores.update(llm_scores) + except Exception as exc: # noqa: BLE001 + logger.exception( + "[CallImportEval {}] LLM evaluation failed for " + "provider={} model={}", + eval_row_id, + provider, + model, + ) + metric_scores.update(handle_llm_evaluation_error(bucket_metrics, exc)) + evaluation_failed = True + primary_error_message = str(exc) + + return { + "metric_scores": metric_scores, + "evaluation_failed": evaluation_failed, + "primary_error_message": primary_error_message, + } + + +# Per-task time limits keep a wedged LLM evaluation from holding a worker +# child hostage for the global 30 min fallback. +@celery_app.task( + name="evaluate_call_import_row", + bind=True, + max_retries=2, + time_limit=10 * 60, + soft_time_limit=8 * 60, +) +def evaluate_call_import_row_task( + self, + eval_row_id: str, + restricted_metric_ids: Optional[List[str]] = None, + _eval_slot_task_id: Optional[str] = None, + _skip_audio: bool = False, + _predecessor_task_id: Optional[str] = None, +): + """Evaluate one row using the appropriate library per metric type. + + When ``restricted_metric_ids`` is set, this is a **metric-subset** + pass: only those metrics are recomputed, and the resulting scores + are merged into the row's existing ``metric_scores`` dict so other + metrics' previously-computed values are preserved. Used by the + "Re-run metrics" UI; the create-evaluation flow always leaves this + None for a full row evaluation. + """ + slot_task_id = _eval_slot_task_id or self.request.id + scoring_inputs: dict[str, Any] | None = None + restricted_metric_uuids: list[UUID] | None = None + usage_ctx_token = None + release_slot = True + owns_execution = False + try: + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + ) + + row_db = catalog_db = None + save_shard_id: str | None = None + try: + row_uuid = UUID(eval_row_id) + try: + row_db, catalog_db, eval_row, source_row, save_shard_id = ( + locate_call_import_evaluation_row(row_uuid) + ) + except LookupError: + logger.warning("CallImportEvaluationRow {} not found", eval_row_id) + return {"status": "skipped", "reason": "row_not_found"} + + from app.workers.tasks.evaluate_call_import_row_core import ( + load_evaluation_snapshot, + ) + + evaluation = load_evaluation_snapshot( + catalog_db, eval_row.evaluation_id + ) + if not evaluation: + logger.warning( + "CallImportEvaluation {} missing", eval_row.evaluation_id + ) + eval_row.status = "failed" + eval_row.error_message = "Evaluation parent not found" + row_db.commit() + return {"status": "failed", "reason": "evaluation_missing"} + + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + ) + from app.services.usage.context import set_usage_context + + usage_ctx_token = set_usage_context( + call_import_evaluation_usage_context( + organization_id=evaluation.organization_id, + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) + ) + + from app.workers.tasks.evaluate_call_import_row_core import ( + claim_eval_row_for_task, + ) + + if not claim_eval_row_for_task( + row_db, + eval_row, + self.request.id, + predecessor_task_id=_predecessor_task_id, + ): + logger.info( + "Skipping eval row {} already claimed or finished", + eval_row_id, + ) + return {"status": "skipped", "reason": "already_claimed"} + + from app.workers.tasks.evaluate_call_import_row_core import ( + acquire_eval_execution, + ) + + if not acquire_eval_execution(self.request.id): + release_slot = False + from celery.exceptions import MaxRetriesExceededError + + from app.workers.concurrency.import_retry_backoff import ( + import_transient_retry_countdown, + ) + + attempt = int(getattr(self.request, "retries", 0) or 0) + slot_countdown = import_transient_retry_countdown( + None, + attempt=attempt, + base_seconds=15, + max_seconds=120, + ) + try: + self.retry(countdown=slot_countdown, max_retries=40) + except MaxRetriesExceededError: + release_slot = True + return {"status": "skipped", "reason": "already_claimed"} + owns_execution = True + + previous_row_status = "running" + mark_evaluation_running_if_pending(catalog_db, evaluation) + + production_transcript = (source_row.transcript or "").strip() + diarised_transcript = (source_row.diarised_transcript or "").strip() + eval_source = ( + (evaluation.transcript_source or "diarised").strip().lower() + ) + if eval_source == "production": + transcript = production_transcript + missing_label = "production" + else: + transcript = diarised_transcript + missing_label = "diarised" + recording_s3_key = (source_row.recording_s3_key or "").strip() or None + has_audio = recording_s3_key is not None + + restricted_metric_uuids = parse_restricted_metric_uuids( + restricted_metric_ids + ) + if ( + restricted_metric_ids is not None + and restricted_metric_uuids is not None + ): + selected_raw = { + str(x) for x in (evaluation.selected_metric_ids or []) + } + if restricted_metric_uuids and not any( + str(mid) in selected_raw for mid in restricted_metric_uuids + ): + eval_row.status = "completed" + eval_row.error_message = None + eval_row.finished_at = _now() + _rollup_terminal( + row_db, + catalog_db, + evaluation, + eval_row, + previous_row_status=previous_row_status, + ) + return { + "status": "skipped", + "reason": "restricted_metric_ids_no_match", + } + + metrics = load_enabled_metrics( + catalog_db, evaluation, restricted_metric_ids=restricted_metric_ids + ) + if not metrics: + eval_row.status = "failed" + eval_row.error_message = ( + "No enabled metrics selected for this evaluation" + ) + eval_row.finished_at = _now() + _rollup_terminal( + row_db, + catalog_db, + evaluation, + eval_row, + previous_row_status=previous_row_status, + ) + return {"status": "failed", "reason": "no_metrics"} + + raw_columns = ( + source_row.raw_columns + if isinstance(source_row.raw_columns, dict) + else {} + ) + custom_column_mapping, speaker_aliases = load_call_import_eval_context( + catalog_db, source_row.call_import_id + ) + + categorized = _categorize_metrics( + metrics, + has_audio, + has_production_transcript=bool(production_transcript), + has_diarised_transcript=bool(diarised_transcript) + or ( + isinstance(source_row.diarised_segments, list) + and bool(source_row.diarised_segments) + ), + ) + transcript_metrics = categorized.transcript_metrics + audio_metrics = categorized.audio_metrics + comparison_metrics = categorized.comparison_metrics + computed_metrics = categorized.computed_metrics + metric_scores = dict(categorized.skipped_scores) + + from app.workers.tasks.evaluate_call_import_row_core import ( + run_computed_metrics_for_row, + ) + + run_computed_metrics_for_row( + source_row=source_row, + computed_metrics=computed_metrics, + metric_scores=metric_scores, + speaker_aliases=speaker_aliases, + ) + + if _skip_audio: + existing_scores = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + metric_scores = {**existing_scores, **metric_scores} + audio_metrics = [] + elif audio_metrics: + logger.warning( + "[CallImportEval {}] Audio metrics present but task was " + "dispatched without audio phase — skipping audio bucket", + eval_row.id, + ) + audio_metrics = [] + + all_columns_block = _build_all_columns_block( + raw_columns, custom_column_mapping + ) + + if ( + not transcript_metrics + and not audio_metrics + and not comparison_metrics + and not computed_metrics + ): + eval_row.status = "failed" + eval_row.error_message = ( + "Selected metrics could not be evaluated on this row " + "(missing audio, missing one of the two transcripts " + "required for a comparison metric, or no enabled " + "metrics matched)." + ) + eval_row.metric_scores = _as_json_dict(metric_scores) + eval_row.finished_at = _now() + _rollup_terminal( + row_db, + catalog_db, + evaluation, + eval_row, + previous_row_status=previous_row_status, + ) + return {"status": "failed", "reason": "no_evaluable_metrics"} + + transcript_unavailable = bool(transcript_metrics) and not transcript + if ( + transcript_unavailable + and not audio_metrics + and not comparison_metrics + and not computed_metrics + ): + logger.warning( + "[CallImportEval {}] Skipping LLM metrics: {} transcript " + "is empty", + eval_row.id, + missing_label, + ) + empty_msg = f"No {missing_label} transcript for this row" + err = RuntimeError(empty_msg) + metric_scores.update( + handle_llm_evaluation_error(transcript_metrics, err) + ) + eval_row.status = "failed" + eval_row.error_message = ( + f"{empty_msg}; LLM-evaluated metrics could not be scored." + ) + eval_row.metric_scores = _as_json_dict(metric_scores) + eval_row.finished_at = _now() + _rollup_terminal( + row_db, + catalog_db, + evaluation, + eval_row, + previous_row_status=previous_row_status, + ) + return {"status": "failed", "reason": "missing_transcript"} + + if transcript_unavailable: + logger.warning( + "[CallImportEval {}] Skipping transcript-LLM metrics: {} " + "transcript is empty (audio/column metrics still scored)", + eval_row.id, + missing_label, + ) + empty_msg = f"No {missing_label} transcript for this row" + err = RuntimeError(empty_msg) + metric_scores.update( + handle_llm_evaluation_error(transcript_metrics, err) + ) + transcript_metrics = [] + + from app.workers.tasks.evaluate_call_import_row_core import ( + load_active_ai_providers, + ) + + ai_providers = load_active_ai_providers( + catalog_db, evaluation.organization_id + ) + + running_discovered_metrics: list = [] + running_discovered_by_parent: dict[UUID, list] = {} + llm_metrics_for_scoring: list[Metric] = list(comparison_metrics) + if transcript and (transcript_metrics or comparison_metrics): + llm_metrics_for_scoring = transcript_metrics + comparison_metrics + + llm_config_buckets: dict = {} + comparison_metric_ids = {str(m.id) for m in comparison_metrics} + if llm_metrics_for_scoring: + if bool(getattr(evaluation, "discover_new_metrics", False)): + from app.api.v1.routes.call_import_evaluations import ( + _get_running_discovered_metrics, + ) + + alias_map_metrics = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + running_discovered_metrics = _get_running_discovered_metrics( + catalog_db, + evaluation.id, + organization_id=evaluation.organization_id, + alias_map=alias_map_metrics, + from_cache=True, + ) + from app.api.v1.routes.call_import_evaluations import ( + _alias_map_for_parent, + _get_running_discovered_labels, + ) + + parents_by_id, children_by_parent, _ = build_parent_groups( + catalog_db, llm_metrics_for_scoring + ) + for parent_id in children_by_parent: + parent_metric = parents_by_id.get(parent_id) + if parent_metric is None: + continue + if not bool(getattr(parent_metric, "allow_discovery", False)): + continue + if (parent_metric.selection_mode or "").lower() not in { + "single_choice", + "multi_label", + }: + continue + running_discovered_by_parent[parent_id] = ( + _get_running_discovered_labels( + catalog_db, + evaluation.id, + parent_metric.id, + organization_id=evaluation.organization_id, + alias_map=_alias_map_for_parent( + evaluation, parent_metric.id + ), + from_cache=True, + ) + ) + + overrides = ( + evaluation.metric_llm_overrides + if isinstance(evaluation.metric_llm_overrides, dict) + else {} + ) + llm_config_buckets = build_llm_config_buckets( + catalog_db, + llm_metrics_for_scoring, + overrides=overrides, + run_provider=(evaluation.llm_provider or "").strip() or None, + run_model=(evaluation.llm_model or "").strip() or None, + run_llm_config=( + evaluation.llm_config + if isinstance(evaluation.llm_config, dict) + else None + ), + run_credential_id=( + str(evaluation.llm_credential_id) + if evaluation.llm_credential_id + else None + ), + running_discovered_by_parent=running_discovered_by_parent, + ) + + # Computed metrics (User WER/CER) finish before LLM calls. Persist them + # while the row is still running so the UI shows scores during LLM work + # and cancel mid-flight does not discard deterministic results. + if llm_config_buckets and metric_scores: + existing_scores = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + eval_row.metric_scores = { + **existing_scores, + **_as_json_dict(metric_scores), + } + _persist_eval_sessions(row_db, catalog_db) + + if _was_cancelled_externally(row_db, eval_row): + logger.info( + "[CallImportEval {}] Row cancelled before LLM phase; " + "keeping computed scores already persisted", + eval_row.id, + ) + try: + # The cancel path already moved this row to failed and recounted + # the parent; recording running -> failed here would double-count. + catalog_db.commit() + except Exception: # noqa: BLE001 + catalog_db.rollback() + return { + "status": "cancelled", + "eval_row_id": eval_row_id, + } + + scoring_inputs = { + "eval_row_id": eval_row.id, + "organization_id": evaluation.organization_id, + "transcript": transcript, + "production_transcript": production_transcript, + "diarised_transcript": diarised_transcript, + "llm_config_buckets": llm_config_buckets, + "comparison_metric_ids": comparison_metric_ids, + "all_columns_block": all_columns_block, + "ai_providers": ai_providers, + "llm_provider": evaluation.llm_provider, + "llm_model": evaluation.llm_model, + "llm_credential_id": ( + str(evaluation.llm_credential_id) + if evaluation.llm_credential_id + else None + ), + "llm_config": evaluation.llm_config, + "discover_new_metrics": bool( + getattr(evaluation, "discover_new_metrics", False) + ), + "running_discovered_metrics": running_discovered_metrics, + "transcript_unavailable": transcript_unavailable, + "missing_label": missing_label, + "pre_llm_metric_scores": dict(metric_scores), + } + finally: + if row_db is not None: + close_row_sessions(row_db, catalog_db) + + pre_llm_metric_scores = scoring_inputs.pop("pre_llm_metric_scores") + llm_result = _run_llm_scoring(**scoring_inputs) + metric_scores = { + **pre_llm_metric_scores, + **llm_result["metric_scores"], + } + evaluation_failed = llm_result["evaluation_failed"] + primary_error_message = llm_result["primary_error_message"] + + row_db = catalog_db = None + try: + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + ) + + try: + row_db, catalog_db, eval_row, _source_row, _ = ( + locate_call_import_evaluation_row( + UUID(eval_row_id), + shard_id=save_shard_id, + ) + ) + except LookupError: + logger.warning( + "[CallImportEval {}] Row not found on any shard — skipping", + eval_row_id, + ) + return {"status": "skipped", "reason": "row_not_found"} + from app.workers.tasks.evaluate_call_import_row_core import ( + load_evaluation_snapshot, + ) + + evaluation = load_evaluation_snapshot( + catalog_db, eval_row.evaluation_id + ) + if not evaluation: + return {"status": "failed", "reason": "evaluation_missing"} + + if _was_cancelled_externally(row_db, eval_row): + logger.info( + "[CallImportEval {}] Skipping terminal write; " + "row was cancelled by user", + eval_row.id, + ) + try: + # The cancel path already moved this row to failed and recounted + # the parent; recording running -> failed here would double-count. + catalog_db.commit() + except Exception: # noqa: BLE001 — rollup is best-effort here + catalog_db.rollback() + return { + "status": "cancelled", + "eval_row_id": eval_row_id, + } + + if evaluation_failed: + eval_row.status = "failed" + eval_row.error_message = ( + primary_error_message + or "Evaluation failed for one or more metrics" + ) + else: + eval_row.status = "completed" + eval_row.error_message = None + + from app.api.v1.routes.call_import_evaluations import ( + normalize_scores_with_aliases, + ) + + normalize_scores_with_aliases( + metric_scores, evaluation, catalog_db, evaluation.organization_id + ) + + new_scores = _as_json_dict(metric_scores) + if restricted_metric_uuids: + existing = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + merged = dict(existing) + for key, value in new_scores.items(): + merged[key] = value + eval_row.metric_scores = merged + else: + eval_row.metric_scores = new_scores + eval_row.finished_at = _now() + _rollup_terminal( + row_db, + catalog_db, + evaluation, + eval_row, + previous_row_status="running", + ) + + return { + "status": eval_row.status, + "eval_row_id": eval_row_id, + "metrics": len(eval_row.metric_scores or {}), + "subset_retry": bool(restricted_metric_uuids), + } + finally: + if row_db is not None: + close_row_sessions(row_db, catalog_db) + finally: + if usage_ctx_token is not None: + from app.services.usage.context import reset_usage_context + + reset_usage_context(usage_ctx_token) + if owns_execution: + from app.workers.tasks.evaluate_call_import_row_core import ( + release_eval_execution, + ) + + release_eval_execution(self.request.id) + if release_slot: + from app.workers.concurrency.fair_dispatch import ( + finish_eval_work_and_redispatch, + ) + + finish_eval_work_and_redispatch( + slot_task_id, + restricted_metric_ids=restricted_metric_ids, + ) diff --git a/app/workers/tasks/evaluate_call_import_row_audio.py b/app/workers/tasks/evaluate_call_import_row_audio.py index c7eddd86..d4db97ff 100644 --- a/app/workers/tasks/evaluate_call_import_row_audio.py +++ b/app/workers/tasks/evaluate_call_import_row_audio.py @@ -18,9 +18,9 @@ as_json_dict, categorize_row_metrics, load_enabled_metrics, + mark_evaluation_running_if_pending, now_utc, parse_restricted_metric_uuids, - rollup_parent, claim_eval_row_for_task, commit_terminal_row_and_rollup, row_needs_llm_phase, @@ -39,12 +39,6 @@ def _rollup_terminal(row_db, catalog_db, evaluation, eval_row, *, previous_row_s ) -def _persist_sessions(row_db, catalog_db) -> None: - row_db.commit() - if catalog_db is not row_db: - catalog_db.commit() - - _CHAIN_TTL_SECONDS = 7200 _PUBLISH_LOCK_SECONDS = 60 _PUBLISH_ATTEMPTS = 8 @@ -247,10 +241,10 @@ def evaluate_call_import_row_audio_task( _shard_id, ): from app.workers.tasks.evaluate_call_import_row_core import ( - load_evaluation_for_worker, + load_evaluation_snapshot, ) - evaluation = load_evaluation_for_worker( + evaluation = load_evaluation_snapshot( catalog_db, eval_row.evaluation_id ) if not evaluation: @@ -348,10 +342,7 @@ def evaluate_call_import_row_audio_task( return {"status": "skipped", "reason": "already_claimed"} previous_row_status = "running" - if evaluation.status == "pending": - evaluation.status = "running" - evaluation.started_at = evaluation.started_at or now_utc() - catalog_db.commit() + mark_evaluation_running_if_pending(catalog_db, evaluation) restricted_uuids = parse_restricted_metric_uuids(restricted_metric_ids) if restricted_metric_ids is not None and restricted_uuids is not None: @@ -390,12 +381,11 @@ def evaluate_call_import_row_audio_task( ) return {"status": "failed", "reason": "no_metrics"} - ( - _transcript_metrics, - audio_metrics, - _comparison_metrics, - metric_scores, - ) = categorize_row_metrics(catalog_db, evaluation, source_row, metrics) + categorized = categorize_row_metrics( + catalog_db, evaluation, source_row, metrics + ) + audio_metrics = categorized.audio_metrics + metric_scores = dict(categorized.skipped_scores) recording_s3_key = (source_row.recording_s3_key or "").strip() or None result_id = f"call-import-eval:{eval_row.id}" @@ -459,10 +449,10 @@ def evaluate_call_import_row_audio_task( ) rebind_evaluation_row_session(origin_row_db, row_db, catalog_db) from app.workers.tasks.evaluate_call_import_row_core import ( - load_evaluation_for_worker, + load_evaluation_snapshot, ) - evaluation = load_evaluation_for_worker( + evaluation = load_evaluation_snapshot( catalog_db, eval_row.evaluation_id ) if evaluation is None: @@ -470,12 +460,8 @@ def evaluate_call_import_row_audio_task( if was_cancelled_externally(row_db, eval_row): try: - rollup_parent( - catalog_db, - evaluation, - previous_row_status="running", - new_row_status="failed", - ) + # The cancel path already moved this row to failed and recounted + # the parent; recording running -> failed here would double-count. catalog_db.commit() except Exception: # noqa: BLE001 catalog_db.rollback() diff --git a/app/workers/tasks/evaluate_call_import_row_core.py b/app/workers/tasks/evaluate_call_import_row_core.py index d780f32b..de3ee2a2 100644 --- a/app/workers/tasks/evaluate_call_import_row_core.py +++ b/app/workers/tasks/evaluate_call_import_row_core.py @@ -6,10 +6,10 @@ import os import tempfile from datetime import datetime, timezone -from typing import Any, List, Optional +from typing import Any, List, NamedTuple, Optional from uuid import UUID -from sqlalchemy import and_, case, func, or_, update +from sqlalchemy import and_, func, or_, update from sqlalchemy.orm import Session, load_only from app.models.database import ( @@ -23,9 +23,19 @@ from app.workers.tasks.helpers.score_utils import get_metric_type_value EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" +EVAL_AUTO_ABORTED_ERROR: str = "Automatically aborted after 24 hours" +AUTO_ABORTED_STATUS: str = "auto_aborted" _BILLING_META_KEY = "_billing" + +class CategorizedMetrics(NamedTuple): + transcript_metrics: list[Metric] + audio_metrics: list[Metric] + comparison_metrics: list[Metric] + computed_metrics: list[Metric] + skipped_scores: dict[str, dict[str, Any]] + _ALL_COLUMNS_BLOCK_MAX_CHARS = 16_000 _ALL_COLUMNS_CELL_MAX_CHARS = 4_000 @@ -161,10 +171,9 @@ def was_cancelled_externally(db, eval_row: CallImportEvaluationRow) -> bool: def is_eval_row_user_cancelled(eval_row: CallImportEvaluationRow) -> bool: """True when the row was aborted by the operator.""" - return ( - (eval_row.status or "").lower() == "failed" - and (eval_row.error_message or "") == EVAL_CANCELLED_BY_USER_ERROR - ) + return (eval_row.status or "").lower() == "failed" and ( + eval_row.error_message or "" + ) in {EVAL_CANCELLED_BY_USER_ERROR, EVAL_AUTO_ABORTED_ERROR} def metric_text_references_production( @@ -229,22 +238,82 @@ def build_all_columns_block( return "\n".join(lines) +def load_call_import_eval_context( + catalog_db: Session, + call_import_id: UUID | None, +) -> tuple[dict[str, Any], Any]: + """Import-level ``(custom_column_mapping, speaker_aliases)`` for scoring. + + Static for the life of a run, so it is cached per worker process + (``cached_catalog_fetch``, 120 s) instead of read on every row. Each call + gets its own copy so worker threads never share mutable JSON. + """ + if call_import_id is None: + return {}, None + import copy + + from app.workers.catalog_cache import cached_catalog_fetch + + cached = cached_catalog_fetch( + "call_import_eval_context", + str(call_import_id), + lambda: _query_call_import_eval_context(catalog_db, call_import_id), + ) + return copy.deepcopy(cached) + + +def _query_call_import_eval_context( + catalog_db: Session, + call_import_id: UUID | str, +) -> tuple[dict[str, Any], Any]: + """Narrow typed select (uses the primary-key index). + + Avoids an ORM lazy-load of ``CallImport``. Pre-migration-114 databases + have no ``speaker_aliases`` column; those still score LLM metrics and + computed metrics fall back to default speaker aliases. + """ + from sqlalchemy import select + from sqlalchemy.exc import ProgrammingError + + from app.models.database import CallImport + + table = CallImport.__table__ + cid = call_import_id if isinstance(call_import_id, UUID) else UUID(str(call_import_id)) + try: + row = catalog_db.execute( + select(table.c.custom_column_mapping, table.c.speaker_aliases).where( + table.c.id == cid + ) + ).one_or_none() + except ProgrammingError: + catalog_db.rollback() + row = catalog_db.execute( + select(table.c.custom_column_mapping).where(table.c.id == cid) + ).one_or_none() + if row is None: + return {}, None + mapping = row[0] if isinstance(row[0], dict) else {} + return mapping, None + if row is None: + return {}, None + mapping = row[0] if isinstance(row[0], dict) else {} + return mapping, row[1] + + def categorize_metrics( metrics: list[Metric], has_audio: bool, has_production_transcript: bool = False, has_diarised_transcript: bool = False, -) -> tuple[ - list[Metric], - list[Metric], - list[Metric], - dict[str, dict[str, Any]], -]: +) -> CategorizedMetrics: transcript_metrics: list[Metric] = [] audio_metrics: list[Metric] = [] comparison_metrics: list[Metric] = [] + computed_metrics: list[Metric] = [] skipped_scores: dict[str, dict[str, Any]] = {} + from app.services.metrics.computed_metrics import is_computed_metric + for m in metrics: normalized = (m.name or "").strip().lower() if normalized in AUDIO_ONLY_METRIC_NAMES: @@ -259,6 +328,24 @@ def categorize_metrics( } continue + if is_computed_metric(m): + missing: list[str] = [] + if not has_production_transcript: + missing.append("production") + if not has_diarised_transcript: + missing.append("diarised") + if missing: + skipped_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "skipped": "comparison_missing_transcript", + "missing_transcripts": missing, + } + continue + computed_metrics.append(m) + continue + is_explicit_compare = bool(getattr(m, "compare_transcripts", False)) is_standalone = not getattr(m, "parent_metric_id", None) is_auto_compare = ( @@ -268,7 +355,7 @@ def categorize_metrics( and metric_text_references_production(m) ) if is_explicit_compare or is_auto_compare: - missing: list[str] = [] + missing = [] if not has_production_transcript: missing.append("production") if not has_diarised_transcript: @@ -287,11 +374,12 @@ def categorize_metrics( transcript_metrics.append(m) - return ( - transcript_metrics, - audio_metrics, - comparison_metrics, - skipped_scores, + return CategorizedMetrics( + transcript_metrics=transcript_metrics, + audio_metrics=audio_metrics, + comparison_metrics=comparison_metrics, + computed_metrics=computed_metrics, + skipped_scores=skipped_scores, ) @@ -313,6 +401,11 @@ def build_parent_groups( .filter(Metric.id.in_(list(children_by_parent.keys()))) .all() ) + # Scoring reads name/description after this session commits and closes. + # Leave the parents detached with their columns already loaded; otherwise + # the commit expires them and the later read raises DetachedInstanceError. + for row in rows: + db.expunge(row) parents_by_id = {row.id: row for row in rows} orphaned: list[UUID] = [] for pid in list(children_by_parent.keys()): @@ -488,61 +581,121 @@ def counter_deltas_for_status_transition( return completed_delta, failed_delta +def _count_evaluation_rows( + db: Session, + evaluation_id: UUID, +) -> tuple[int, int, int, int]: + """(total, completed, failed, in_progress) from the row tables.""" + from app.db_sharding.scatter_gather import aggregate_evaluation_row_counts_detailed + + return aggregate_evaluation_row_counts_detailed(db, evaluation_id) + + def reconcile_evaluation_counters( db: Session, evaluation: CallImportEvaluation, ) -> None: - """Sync parent counters from child rows using aggregate queries.""" - from app.db_sharding.sessions import is_sharding_enabled + """Sync parent counters from child rows (fenced recount). - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import aggregate_evaluation_row_counts + Every caller (abort, retry, row delete, finalize) discards buffered Redis + deltas instead of double-applying them. + """ + recount_evaluation_counters(db, evaluation) - total, completed, failed = aggregate_evaluation_row_counts(db, evaluation.id) - else: - counts = ( - db.query( - func.count().label("total"), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "completed", 1), - else_=0, - ) - ), - 0, - ).label("completed"), - func.coalesce( - func.sum( - case( - (CallImportEvaluationRow.status == "failed", 1), - else_=0, - ) - ), - 0, - ).label("failed"), - ) - .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) - .one() - ) - total = int(counts.total or 0) - completed = int(counts.completed or 0) - failed = int(counts.failed or 0) - db.execute( +def _bump_progress_counter_gen(db: Session, evaluation_id: UUID) -> tuple[int, int] | None: + """Increment the catalog fence gen; returns ``(new_gen, flush_seq)``. + + The UPDATE takes the parent row lock until the caller commits, so a flush + that drained under the old gen blocks and then matches 0 rows. + """ + stmt = ( update(CallImportEvaluation) - .where(CallImportEvaluation.id == evaluation.id) - .values( - total_rows=total, - completed_rows=completed, - failed_rows=failed, + .where(CallImportEvaluation.id == evaluation_id) + .values(progress_counter_gen=CallImportEvaluation.progress_counter_gen + 1) + .returning( + CallImportEvaluation.progress_counter_gen, + CallImportEvaluation.progress_flush_seq, ) ) - db.flush() - db.refresh(evaluation) - from app.services.call_imports.progress_counters import clear_eval_progress_redis + row = db.execute(stmt).first() + if row is None: + return None + return int(row[0] or 0), int(row[1] or 0) + + +def recount_evaluation_counters( + db: Session, + evaluation: CallImportEvaluation, +) -> int: + """Fenced shard recount. Returns rows still pending or running. + + Order matters: + + 1. Bump ``progress_counter_gen`` in the catalog (source of truth for gen, + survives Redis loss; holds the parent row lock until commit). + 2. Fence Redis: adopt the new gen, drop buffered and pending deltas. They + describe rows already committed on the shards, so the count below + includes them. + 3. Count the shards (total / completed / failed / in_progress). + 4. Write absolute counters, guarded by the new gen so an older, + overlapping recount can never overwrite a newer one. + 5. Set Redis ``remaining`` from the shard in-progress count. + """ + import redis as redis_lib + from loguru import logger + + from app.services.call_imports import eval_progress_buffer as buf + + evaluation_id = evaluation.id + token = buf.acquire_lock(evaluation_id) + try: + bumped = _bump_progress_counter_gen(db, evaluation_id) + if bumped is None: + return 0 + new_gen, catalog_seq = bumped + fenced = True + try: + buf.fence(evaluation_id, gen=new_gen, catalog_seq=catalog_seq) + except redis_lib.RedisError as exc: + fenced = False + logger.warning( + "eval counter fence failed for {} (gen guard still applies): {}", + evaluation_id, + exc, + ) + + total, completed, failed, in_progress = _count_evaluation_rows(db, evaluation_id) + db.execute( + update(CallImportEvaluation) + .where( + CallImportEvaluation.id == evaluation_id, + CallImportEvaluation.progress_counter_gen == new_gen, + ) + .values( + total_rows=total, + completed_rows=completed, + failed_rows=failed, + ) + ) + db.flush() + db.refresh(evaluation) - clear_eval_progress_redis(evaluation.id) + if fenced: + try: + buf.set_remaining_after_count( + evaluation_id, gen=new_gen, in_progress=in_progress + ) + except redis_lib.RedisError as exc: + logger.warning( + "eval counter remaining reset failed for {}: {}", evaluation_id, exc + ) + from app.services.call_imports.progress_counters import clear_eval_progress_redis + + clear_eval_progress_redis(evaluation_id) + return in_progress + finally: + buf.release_lock(evaluation_id, token) def _should_confirm_eval_finished( @@ -592,8 +745,14 @@ def _apply_parent_status_from_counters( completed = int(evaluation.completed_rows or 0) failed = int(evaluation.failed_rows or 0) in_progress = total - completed - failed + status = (evaluation.status or "").strip().lower() - if (evaluation.status or "").strip().lower() == "cancelled": + if status == AUTO_ABORTED_STATUS: + if in_progress <= 0: + evaluation.finished_at = now_utc() + return + + if status == "cancelled": if in_progress > 0: return evaluation.finished_at = now_utc() @@ -671,6 +830,137 @@ def load_evaluation_for_worker( ) +_SNAPSHOT_CACHE_TTL_SECONDS = 600.0 + + +def _snapshot_cache_ttl_seconds() -> float: + """Backstop only: settings changes bump the version key and reload at once.""" + from app.config import settings + + try: + return max( + 1.0, + float( + getattr(settings, "EVAL_SNAPSHOT_CACHE_TTL_SECONDS", _SNAPSHOT_CACHE_TTL_SECONDS) + ), + ) + except (TypeError, ValueError): + return _SNAPSHOT_CACHE_TTL_SECONDS + + +class EvaluationSnapshot: + """Read-only copy of the parent evaluation's worker columns. + + Not an ORM instance: nothing assigned to it is ever flushed. Code that + must write the parent (status flip, Redis-down rollup) uses Core updates + or reloads an attached object instead. + """ + + __slots__ = tuple(column.key for column in _EVALUATION_WORKER_COLUMNS) + + def __init__(self, **values: Any) -> None: + for name in self.__slots__: + setattr(self, name, values.get(name)) + + def __repr__(self) -> str: + return f"EvaluationSnapshot(id={getattr(self, 'id', None)!r})" + + +def _query_evaluation_snapshot( + db: Session, + evaluation_id: UUID, +) -> EvaluationSnapshot | None: + row = ( + db.query(*_EVALUATION_WORKER_COLUMNS) + .filter(CallImportEvaluation.id == evaluation_id) + .first() + ) + if row is None: + return None + return EvaluationSnapshot(**dict(row._mapping)) + + +def load_evaluation_snapshot( + db: Session, + evaluation_id: UUID, +) -> EvaluationSnapshot | None: + """Parent evaluation settings for scoring, cached per worker process. + + Keyed by the Redis settings version (``eval:settings_ver:{id}``), which + the API bumps on any settings change, so a retry with a new model or an + alias edit is picked up on the next row. When Redis is down the version + is unknown, so this reads the catalog directly instead of trusting the + cache. Every caller gets its own deep copy (worker threads share the + process cache). + """ + import copy + + from app.services.call_imports.eval_settings_version import read_settings_version + from app.workers.catalog_cache import cached_catalog_fetch + + version = read_settings_version(evaluation_id) + if version is None: + return _query_evaluation_snapshot(db, evaluation_id) + cached = cached_catalog_fetch( + "evaluation_worker", + f"{evaluation_id}:{version}", + lambda: _query_evaluation_snapshot(db, evaluation_id), + ttl_seconds=_snapshot_cache_ttl_seconds(), + ) + return copy.deepcopy(cached) + + +def mark_evaluation_running_if_pending( + catalog_db: Session, + evaluation: EvaluationSnapshot | CallImportEvaluation, +) -> None: + """Make sure the parent is ``running`` and has a ``started_at``. + + Materialize and fair dispatch normally do both before any row runs; this + covers a worker that starts before those commits land, and runs created + before ``started_at`` was stamped there. Each guarded UPDATE is a no-op + when another worker got there first, and the version bump makes cached + snapshots elsewhere reload instead of retrying. No SQL at all when the + snapshot already shows a started run. + """ + status = (getattr(evaluation, "status", None) or "").strip().lower() + needs_flip = status == "pending" + needs_start = getattr(evaluation, "started_at", None) is None and status == "running" + if not (needs_flip or needs_start): + return + from app.services.call_imports.eval_settings_version import bump_settings_version + + now = now_utc() + if needs_flip: + catalog_db.execute( + update(CallImportEvaluation) + .where( + CallImportEvaluation.id == evaluation.id, + CallImportEvaluation.status == "pending", + ) + .values( + status="running", + started_at=func.coalesce(CallImportEvaluation.started_at, now), + ) + .execution_options(synchronize_session=False) + ) + evaluation.status = "running" + else: + catalog_db.execute( + update(CallImportEvaluation) + .where( + CallImportEvaluation.id == evaluation.id, + CallImportEvaluation.started_at.is_(None), + ) + .values(started_at=now) + .execution_options(synchronize_session=False) + ) + catalog_db.commit() + bump_settings_version([evaluation.id]) + if getattr(evaluation, "started_at", None) is None: + evaluation.started_at = now + + def commit_terminal_row_and_rollup( row_db: Session, evaluation: CallImportEvaluation, @@ -684,35 +974,77 @@ def commit_terminal_row_and_rollup( from app.db_sharding.row_ops import shard_row_write_context parent_db = catalog_db if catalog_db is not None and catalog_db is not row_db else row_db + replay_bill_only = False # Shard FK points at the catalogue evaluation; skip it for this write. write_ctx = ( shard_row_write_context(row_db) if parent_db is not row_db else nullcontext() ) with write_ctx: - transition = row_db.execute( - update(CallImportEvaluationRow) - .where( - CallImportEvaluationRow.id == eval_row.id, - CallImportEvaluationRow.status.in_(("pending", "running")), + target_status = eval_row.status + with row_db.no_autoflush: + transition = row_db.execute( + update(CallImportEvaluationRow) + .where( + CallImportEvaluationRow.id == eval_row.id, + CallImportEvaluationRow.status.in_(("pending", "running")), + ) + .values(status=target_status) ) - .values(status=eval_row.status) + current = ( + row_db.query(CallImportEvaluationRow.status) + .filter(CallImportEvaluationRow.id == eval_row.id) + .scalar() ) - if transition.rowcount != 1: - row_db.rollback() - return - if (eval_row.status or "").lower() == "completed": - source_row = ( - row_db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - maybe_bill_completed_eval_row_flexprice( - row_db, - evaluation, - eval_row, - source_row=source_row, - ) - row_db.commit() + target = (target_status or "").lower() + current_norm = (current or "").lower() + bill_after_commit = None + if transition.rowcount not in (1, -1): + if current_norm == target and target == "completed": + replay_bill_only = True + source_row = ( + row_db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + if source_row is not None: + bill_after_commit = (evaluation, eval_row, source_row) + elif current_norm == target: + return + else: + row_db.rollback() + return + else: + if current_norm != target: + row_db.rollback() + return + bill_after_commit = None + if (eval_row.status or "").lower() == "completed": + source_row = ( + row_db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + if source_row is not None: + bill_after_commit = (evaluation, eval_row, source_row) + row_db.flush() + row_db.commit() + row_db.refresh(eval_row) + if bill_after_commit is not None: + evaluation, eval_row, source_row = bill_after_commit + from app.services.call_imports.eval_flexprice_billing import ( + maybe_enqueue_recording_minutes_for_completed_row, + ) + + maybe_enqueue_recording_minutes_for_completed_row( + row_db, + evaluation, + eval_row, + source_row=source_row, + resolve_audio_seconds=resolve_eval_row_audio_seconds, + billable_minutes_fn=billable_audio_minutes, + ) + if replay_bill_only: + return if (eval_row.status or "").lower() == "completed": from app.services.call_imports.discovered_running_cache import ( record_row_discoveries, @@ -723,10 +1055,17 @@ def commit_terminal_row_and_rollup( eval_row.id, eval_row.metric_scores, ) - if parent_db is not row_db: - evaluation = load_evaluation_for_worker(parent_db, evaluation.id) - if evaluation is None: - return + if record_buffered_transition( + evaluation.id, + previous_row_status=previous_row_status, + new_row_status=eval_row.status, + ): + return + # Redis-down fallback writes the parent directly, so it needs an attached + # ORM object (the worker holds a read-only snapshot). + evaluation = load_evaluation_for_worker(parent_db, evaluation.id) + if evaluation is None: + return rollup_parent( parent_db, evaluation, @@ -736,6 +1075,126 @@ def commit_terminal_row_and_rollup( parent_db.commit() +def record_buffered_transition( + evaluation_id: UUID, + *, + previous_row_status: str | None, + new_row_status: str | None, +) -> bool: + """Buffer one row transition in Redis instead of writing the catalog. + + Returns ``False`` when Redis is down; the caller then runs the direct + catalog rollup. Triggers finalize when ``remaining`` + reaches 0, when this is the only task still in flight (covers a lost + decrement), or when the counter hash had to be recreated (``uninit``). + """ + from app.services.call_imports import eval_progress_buffer as buf + + if new_row_status is None: + return False + import redis as redis_lib + from loguru import logger + + completed_delta, failed_delta = counter_deltas_for_status_transition( + previous_row_status, new_row_status + ) + remaining_delta = buf.remaining_delta_for_transition( + previous_row_status, new_row_status + ) + if not (completed_delta or failed_delta or remaining_delta): + return True + try: + result = buf.record_transition( + evaluation_id, + completed_delta=completed_delta, + failed_delta=failed_delta, + remaining_delta=remaining_delta, + ) + except redis_lib.RedisError as exc: + logger.warning( + "eval counter buffer unavailable for {}, using direct catalog write: {}", + evaluation_id, + exc, + ) + return False + + if result.remaining <= 0 or result.created: + buf.request_finalize(evaluation_id) + elif remaining_delta < 0: + from app.workers.concurrency.limits import read_job_inflight + + if read_job_inflight(evaluation_id) == 1: + buf.request_finalize(evaluation_id) + return True + + +_TERMINAL_EVAL_STATUSES = frozenset({"completed", "failed", "partial"}) + + +def finalize_evaluation_progress(db: Session, evaluation_id: UUID) -> str: + """Fenced recount, then the terminal tail once no row is in flight. + + Returns a short outcome label: ``missing``, ``in_progress`` or the final + evaluation status. + """ + from app.services.call_imports import eval_progress_buffer as buf + + evaluation = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == evaluation_id) + .first() + ) + if evaluation is None: + buf.delete(evaluation_id) + return "missing" + + previous_status = (evaluation.status or "").strip().lower() + in_progress = recount_evaluation_counters(db, evaluation) + if in_progress > 0: + db.commit() + return "in_progress" + + previous_finished_at = evaluation.finished_at + _apply_parent_status_from_counters(evaluation) + if ( + previous_status in _TERMINAL_EVAL_STATUSES + and previous_finished_at is not None + and (evaluation.status or "").strip().lower() == previous_status + ): + # Re-finalizing an already closed run (late trigger, sweep): keep + # the original close time. + evaluation.finished_at = previous_finished_at + db.flush() + + from app.services.call_imports.eval_flexprice_billing import ( + maybe_finalize_evaluation_flexprice_billing, + ) + from app.services.call_imports.metric_aggregate_cache import ( + maybe_refresh_aggregate_cache_after_row, + ) + + maybe_finalize_evaluation_flexprice_billing(db, evaluation) + maybe_refresh_aggregate_cache_after_row( + db, + evaluation, + new_row_status="completed", + ) + if (evaluation.status or "").lower() in _TERMINAL_EVAL_STATUSES: + from app.models.database import CallImport + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + call_import = ( + db.query(CallImport) + .filter(CallImport.id == evaluation.call_import_id) + .first() + ) + if call_import is not None: + rollup_call_import_batch_status(db, call_import) + db.commit() + buf.delete(evaluation_id) + return str(evaluation.status or "") + + def rollup_parent( db: Session, evaluation: CallImportEvaluation, @@ -806,6 +1265,17 @@ def rollup_parent( _apply_parent_status_from_counters(evaluation) + if previous_row_status is not None and new_row_status is not None: + from app.services.call_imports.metric_aggregate_cache import ( + maybe_refresh_aggregate_cache_after_row, + ) + + maybe_refresh_aggregate_cache_after_row( + db, + evaluation, + new_row_status=new_row_status, + ) + total = int(evaluation.total_rows or 0) completed = int(evaluation.completed_rows or 0) failed = int(evaluation.failed_rows or 0) @@ -813,32 +1283,11 @@ def rollup_parent( if in_progress > 0: return - already_billed = int(getattr(evaluation, "billed_completed_rows", 0) or 0) - delta = completed - already_billed - if delta > 0: - evaluation = ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == evaluation_id) - .with_for_update() - .one() - ) - from app.services.billing.flexprice_service import ( - record_call_import_evaluation_completed, - ) + from app.services.call_imports.eval_flexprice_billing import ( + maybe_finalize_evaluation_flexprice_billing, + ) - metric_count = len(evaluation.selected_metric_ids or []) - billing_accepted = record_call_import_evaluation_completed( - evaluation.organization_id, - evaluation.id, - workspace_id=evaluation.workspace_id, - call_import_id=evaluation.call_import_id, - rows_billed=delta, - completed_total=completed, - total_rows=int(evaluation.total_rows or 0), - metric_count=metric_count, - ) - if billing_accepted: - evaluation.billed_completed_rows = completed + maybe_finalize_evaluation_flexprice_billing(db, evaluation) def parse_restricted_metric_uuids( @@ -861,7 +1310,7 @@ def load_enabled_metrics( *, restricted_metric_ids: Optional[List[str]] = None, ) -> list[Metric]: - metric_ids_raw = evaluation.selected_metric_ids or [] + metric_ids_raw = getattr(evaluation, "selected_metric_ids", None) or [] metric_ids: list[UUID] = [] for item in metric_ids_raw: try: @@ -902,6 +1351,19 @@ def _load() -> list[Metric]: return list(cached_catalog_fetch("enabled_metrics", cache_key, _load)) +def evaluation_needs_diarised_reference( + db: Session, + evaluation: CallImportEvaluation, +) -> bool: + from app.services.metrics.computed_metrics import ( + metrics_require_transcript_agreement, + ) + + return metrics_require_transcript_agreement( + load_enabled_metrics(db, evaluation) + ) + + def load_active_ai_providers(db: Session, organization_id: UUID) -> list: """Active org providers, cached so each scoring call does not re-read them.""" from app.models.database import AIProvider @@ -925,17 +1387,60 @@ def _load() -> list: ) +def run_computed_metrics_for_row( + *, + source_row: CallImportRow, + computed_metrics: list[Metric], + metric_scores: dict[str, dict[str, Any]], + speaker_aliases: Any = None, +) -> None: + """Score deterministic computed metrics into ``metric_scores``.""" + if not computed_metrics: + return + + from app.services.metrics.computed_metrics import ( + ComputedMetricContext, + run_computed_scorer, + ) + + segments = ( + source_row.diarised_segments + if isinstance(source_row.diarised_segments, list) + else None + ) + ctx = ComputedMetricContext( + production_transcript=(source_row.transcript or "").strip(), + diarised_transcript=(source_row.diarised_transcript or "").strip(), + diarised_segments=segments, + diarised_speaker_swap=bool( + getattr(source_row, "diarised_speaker_swap", False) + ), + speaker_aliases=speaker_aliases, + diarised_native_transcript=( + getattr(source_row, "diarised_native_transcript", None) or "" + ).strip(), + ) + + for metric in computed_metrics: + mid = str(metric.id) + result = run_computed_scorer(metric, ctx) + if "skipped" in result: + metric_scores[mid] = { + "value": None, + "type": get_metric_type_value(metric), + "metric_name": metric.name, + "skipped": result["skipped"], + } + continue + metric_scores[mid] = result + + def categorize_row_metrics( db: Session, evaluation: CallImportEvaluation, source_row: CallImportRow, metrics: list[Metric], -) -> tuple[ - list[Metric], - list[Metric], - list[Metric], - dict[str, dict[str, Any]], -]: +) -> CategorizedMetrics: from app.db_sharding.scatter_gather import row_text_present production_transcript = row_text_present( @@ -944,13 +1449,16 @@ def categorize_row_metrics( diarised_transcript = row_text_present( source_row, "diarised_transcript", "diarised_transcript_char_length" ) + has_diarised_segments = isinstance(source_row.diarised_segments, list) and bool( + source_row.diarised_segments + ) recording_s3_key = (source_row.recording_s3_key or "").strip() or None has_audio = recording_s3_key is not None return categorize_metrics( metrics, has_audio, has_production_transcript=bool(production_transcript), - has_diarised_transcript=bool(diarised_transcript), + has_diarised_transcript=bool(diarised_transcript) or has_diarised_segments, ) @@ -966,10 +1474,10 @@ def row_needs_audio_phase( ) if not metrics: return False - _, audio_metrics, _, _ = categorize_row_metrics( + categorized = categorize_row_metrics( db, evaluation, source_row, metrics ) - return bool(audio_metrics) + return bool(categorized.audio_metrics) def row_needs_llm_phase( @@ -984,10 +1492,14 @@ def row_needs_llm_phase( ) if not metrics: return False - transcript_metrics, _audio, comparison_metrics, _ = categorize_row_metrics( + categorized = categorize_row_metrics( db, evaluation, source_row, metrics ) - return bool(transcript_metrics or comparison_metrics) + return bool( + categorized.transcript_metrics + or categorized.comparison_metrics + or categorized.computed_metrics + ) def cache_eval_row_audio_seconds( @@ -1058,50 +1570,16 @@ def resolve_eval_row_audio_seconds( ) if source_row is None: return 0 - recording_s3_key = (source_row.recording_s3_key or "").strip() - if not recording_s3_key: - return 0 - seconds = probe_recording_audio_seconds(recording_s3_key) - if seconds > 0: + stored = getattr(source_row, "recording_duration_seconds", None) + if stored is not None and int(stored) > 0: + seconds = int(stored) cache_eval_row_audio_seconds(eval_row, seconds) - return seconds - + return seconds + key = (getattr(source_row, "recording_s3_key", None) or "").strip() + if key: + seconds = probe_recording_audio_seconds(key) + if seconds > 0: + cache_eval_row_audio_seconds(eval_row, seconds) + return seconds + return 0 -def maybe_bill_completed_eval_row_flexprice( - db: Session, - evaluation: CallImportEvaluation, - eval_row: CallImportEvaluationRow, - *, - source_row: CallImportRow | None = None, -) -> None: - if (eval_row.status or "").lower() != "completed": - return - from app.services.billing.flexprice_service import ( - record_call_import_recording_minutes_billed, - ) - - if source_row is None: - source_row = ( - db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - if source_row is None: - return - if not (source_row.recording_s3_key or "").strip(): - return - - audio_seconds = resolve_eval_row_audio_seconds(db, eval_row, source_row) - minutes = billable_audio_minutes(audio_seconds) - if minutes <= 0: - return - - record_call_import_recording_minutes_billed( - evaluation.organization_id, - eval_row.id, - workspace_id=eval_row.workspace_id or evaluation.workspace_id, - evaluation_id=evaluation.id, - call_import_id=evaluation.call_import_id, - audio_seconds=audio_seconds, - billable_minutes=minutes, - ) diff --git a/app/workers/tasks/evaluate_studio_run_item.py b/app/workers/tasks/evaluate_studio_run_item.py index 2caa7224..a17579bb 100644 --- a/app/workers/tasks/evaluate_studio_run_item.py +++ b/app/workers/tasks/evaluate_studio_run_item.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import datetime, timezone +from types import SimpleNamespace from typing import Any from uuid import UUID @@ -18,12 +19,13 @@ ) from app.services.metric_studio.metric_selection import load_studio_run_metrics from app.services.metric_studio.run_rollup import rollup_metric_studio_run -from app.services.metric_studio.source_resolver import resolve_source +from app.services.metric_studio.source_resolver import ResolvedCallSample, resolve_source from app.workers.config import celery_app from app.workers.tasks.evaluate_call_import_row_core import ( bucket_needs_comparison_pair, build_llm_config_buckets, categorize_metrics, + run_computed_metrics_for_row, ) from app.workers.tasks.helpers.audio_evaluation import ( evaluate_audio_metrics, @@ -36,6 +38,22 @@ ) +def computed_metric_source_from_sample(sample: ResolvedCallSample) -> SimpleNamespace: + """Inputs User WER / User CER use for an imported call. + + Segments carry ``text_native``. The rendered transcript drops that + text, and custom speaker names live on the import's aliases. + """ + segments = sample.diarised_segments if isinstance(sample.diarised_segments, list) else None + return SimpleNamespace( + transcript=(sample.transcript or "").strip(), + diarised_transcript=(sample.diarised_transcript or "").strip(), + diarised_segments=segments, + diarised_speaker_swap=bool(sample.diarised_speaker_swap), + diarised_native_transcript=(sample.diarised_native_transcript or "").strip(), + ) + + def _now_utc() -> datetime: return datetime.now(timezone.utc) @@ -114,16 +132,29 @@ def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: metrics = load_studio_run_metrics(db, run.organization_id, metric_ids) has_audio = bool(sample.audio_s3_key) has_production = bool((sample.transcript or "").strip()) - has_diarised = bool((sample.diarised_transcript or "").strip()) + has_diarised = bool((sample.diarised_transcript or "").strip()) or ( + isinstance(sample.diarised_segments, list) and bool(sample.diarised_segments) + ) - transcript_metrics, audio_metrics, comparison_metrics, skipped = categorize_metrics( + categorized = categorize_metrics( metrics, has_audio, has_production_transcript=has_production, has_diarised_transcript=has_diarised, ) + transcript_metrics = categorized.transcript_metrics + audio_metrics = categorized.audio_metrics + comparison_metrics = categorized.comparison_metrics + skipped = categorized.skipped_scores + computed_metrics = categorized.computed_metrics llm_metrics = transcript_metrics + comparison_metrics metric_scores: dict[str, Any] = dict(skipped) + run_computed_metrics_for_row( + source_row=computed_metric_source_from_sample(sample), + computed_metrics=computed_metrics, + metric_scores=metric_scores, + speaker_aliases=sample.speaker_aliases, + ) if not transcript and not has_audio: result_row.status = "failed" diff --git a/app/workers/tasks/finalize_call_import_evaluation.py b/app/workers/tasks/finalize_call_import_evaluation.py new file mode 100644 index 00000000..4baa97a6 --- /dev/null +++ b/app/workers/tasks/finalize_call_import_evaluation.py @@ -0,0 +1,47 @@ +"""Close an evaluation run once no rows are pending or running. + +Enqueued (deduplicated) by the worker hot path when the Redis ``remaining`` +counter reaches 0, by the progress flusher, and by the periodic catalog sweep. +Always does a fenced shard recount first, so it never trusts buffered +counters to decide that a run is finished. +""" + +from __future__ import annotations + +from uuid import UUID + +from loguru import logger + +from app.workers.config import EVAL_CONTROL_QUEUE, celery_app + + +@celery_app.task( + name="finalize_call_import_evaluation", + queue=EVAL_CONTROL_QUEUE, +) +def finalize_call_import_evaluation_task(evaluation_id: str) -> dict: + from app.database import SessionLocal + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + from app.workers.tasks.evaluate_call_import_row_core import ( + finalize_evaluation_progress, + ) + + evaluation_uuid = UUID(str(evaluation_id)) + bulk_op = get_evaluation_bulk_operation(evaluation_uuid) + if bulk_op: + # The bulk op ends with its own fenced recount + status rollup; the + # periodic sweep re-checks anything it leaves running. + return {"evaluation_id": str(evaluation_uuid), "skipped": bulk_op} + + db = SessionLocal() + try: + outcome = finalize_evaluation_progress(db, evaluation_uuid) + except Exception: + db.rollback() + logger.exception("finalize_call_import_evaluation failed for {}", evaluation_uuid) + raise + finally: + db.close() + return {"evaluation_id": str(evaluation_uuid), "outcome": outcome} diff --git a/app/workers/tasks/flush_eval_progress.py b/app/workers/tasks/flush_eval_progress.py new file mode 100644 index 00000000..e9429c02 --- /dev/null +++ b/app/workers/tasks/flush_eval_progress.py @@ -0,0 +1,324 @@ +"""Flush Redis-buffered evaluation progress counters into the catalog. + +Each tick: + +1. Drain every dirty evaluation into its Redis pending slot (or reuse a + pending slot left by a failed/crashed earlier tick, same ``seq``). +2. Apply all pending slots in one ``UPDATE ... FROM (VALUES ...)`` guarded by + ``progress_counter_gen = gen AND progress_flush_seq < seq``, one commit. +3. Commit succeeded: acknowledge every slot, whatever the row count. A 0-row + match means a recount superseded the batch or it was already applied; + both are correct outcomes and must not be restored. + Commit raised: leave the slots; the next tick re-sends the same payload. +4. Safety nets: request finalize when ``remaining <= 0``, when the hash had + to be recreated (``uninit``), or when Redis' gen is ahead of the catalog + (a recount's transaction rolled back). + +Every ``EVAL_PROGRESS_RECOUNT_SECONDS`` it also sweeps running evaluations +from the *catalog* (not the dirty set) and requests a finalize/recount for +any not recounted recently, which catches idle runs whose last decrement or +finalize trigger was lost. +""" + +from __future__ import annotations + +import time +from typing import Iterable +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy import and_, case, or_, text, update +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import CallImportEvaluation +from app.services.call_imports import eval_progress_buffer as buf +from app.workers.config import EVAL_CONTROL_QUEUE, celery_app + +_FLUSHER_LOCK_KEY = "eval:ctr:flusher" +_SWEEP_GATE_KEY = "eval:ctr:sweep" +_VALUES_CHUNK = 500 +_SWEEP_LIMIT = 500 + + +def _apply_batch_postgres(db: Session, batch: list[tuple[UUID, buf.DrainResult]]) -> set[UUID]: + matched: set[UUID] = set() + for start in range(0, len(batch), _VALUES_CHUNK): + chunk = batch[start : start + _VALUES_CHUNK] + params: dict = {} + rows_sql: list[str] = [] + for i, (eid, drained) in enumerate(chunk): + rows_sql.append( + f"(CAST(:id{i} AS uuid), CAST(:dc{i} AS integer), CAST(:df{i} AS integer), " + f"CAST(:gen{i} AS integer), CAST(:seq{i} AS bigint))" + ) + params[f"id{i}"] = str(eid) + params[f"dc{i}"] = drained.completed_delta + params[f"df{i}"] = drained.failed_delta + params[f"gen{i}"] = drained.gen + params[f"seq{i}"] = drained.seq + statement = text( + f""" + UPDATE call_import_evaluations AS e SET + completed_rows = GREATEST(0, e.completed_rows + v.dc), + failed_rows = GREATEST(0, e.failed_rows + v.df), + progress_flush_seq = v.seq, + updated_at = now() + FROM (VALUES {", ".join(rows_sql)}) AS v(id, dc, df, gen, seq) + WHERE e.id = v.id + AND e.progress_counter_gen = v.gen + AND e.progress_flush_seq < v.seq + RETURNING e.id + """ + ) + for (eid,) in db.execute(statement, params): + matched.add(eid if isinstance(eid, UUID) else UUID(str(eid))) + return matched + + +def _non_negative(expr): + return case((expr < 0, 0), else_=expr) + + +def _apply_batch_generic(db: Session, batch: list[tuple[UUID, buf.DrainResult]]) -> set[UUID]: + """Same guarded update, one statement per evaluation (SQLite / tests).""" + matched: set[UUID] = set() + for eid, drained in batch: + result = db.execute( + update(CallImportEvaluation) + .where( + CallImportEvaluation.id == eid, + CallImportEvaluation.progress_counter_gen == drained.gen, + CallImportEvaluation.progress_flush_seq < drained.seq, + ) + .values( + completed_rows=_non_negative( + CallImportEvaluation.completed_rows + drained.completed_delta + ), + failed_rows=_non_negative( + CallImportEvaluation.failed_rows + drained.failed_delta + ), + progress_flush_seq=drained.seq, + ) + .execution_options(synchronize_session=False) + ) + if result.rowcount: + matched.add(eid) + return matched + + +def apply_flush_batch(db: Session, batch: list[tuple[UUID, buf.DrainResult]]) -> set[UUID]: + """Apply drained slots; returns evaluation ids whose row matched.""" + if not batch: + return set() + bind = db.get_bind() + if bind is not None and bind.dialect.name == "postgresql": + return _apply_batch_postgres(db, batch) + return _apply_batch_generic(db, batch) + + +def _parse_ids(raw_ids: Iterable[str]) -> list[UUID]: + parsed: list[UUID] = [] + for raw in raw_ids: + try: + parsed.append(UUID(str(raw))) + except (TypeError, ValueError): + logger.warning("dropping malformed eval counter dirty member {!r}", raw) + try: + buf._client().srem(buf.DIRTY_SET_KEY, raw) + except redis.RedisError: + pass + return parsed + + +def flush_eval_progress_counters(db: Session) -> dict: + """One flush tick. Safe to run concurrently with workers and recounts.""" + stats = {"drained": 0, "applied": 0, "stale": 0, "finalize": 0, "failed": False} + try: + dirty = _parse_ids(buf.dirty_evaluation_ids()) + except redis.RedisError as exc: + logger.warning("eval progress flush skipped (redis unavailable): {}", exc) + return stats + + batch: list[tuple[UUID, buf.DrainResult]] = [] + tokens: dict[UUID, str] = {} + finalize: set[UUID] = set() + try: + for eid in dirty: + token = buf.acquire_lock(eid) + if token is None: + continue # a fenced recount owns this evaluation right now + tokens[eid] = token + try: + drained = buf.drain(eid) + except redis.RedisError as exc: + logger.warning("eval counter drain failed for {}: {}", eid, exc) + continue + if drained.kind == "uninit": + finalize.add(eid) + continue + if drained.kind == "pending": + batch.append((eid, drained)) + if drained.kind in ("pending", "empty") and drained.remaining <= 0: + finalize.add(eid) + stats["drained"] = len(batch) + + if batch: + try: + matched = apply_flush_batch(db, batch) + db.commit() + except Exception as exc: # noqa: BLE001 + db.rollback() + stats["failed"] = True + # Pending slots stay in Redis; the next tick re-sends them + # with the same seq, which the catalog guard makes idempotent. + logger.warning( + "eval progress catalog flush failed for {} evaluations: {}", + len(batch), + exc, + ) + return stats + + stats["applied"] = len(matched) + unmatched = [(eid, drained) for eid, drained in batch if eid not in matched] + stats["stale"] = len(unmatched) + _inspect_unmatched(db, unmatched, finalize) + for eid, drained in batch: + try: + buf.ack(eid, drained.seq) + except redis.RedisError as exc: + # Re-sent next tick; the seq guard makes that a no-op. + logger.debug("eval counter ack failed for {}: {}", eid, exc) + finally: + for eid, token in tokens.items(): + buf.release_lock(eid, token) + + for eid in finalize: + if buf.request_finalize(eid): + stats["finalize"] += 1 + return stats + + +def _inspect_unmatched( + db: Session, + unmatched: list[tuple[UUID, buf.DrainResult]], + finalize: set[UUID], +) -> None: + """Classify batches whose guarded UPDATE matched no row. + + * row gone: evaluation deleted, drop the Redis state. + * catalog gen < batch gen: Redis was fenced by a recount whose catalog + transaction rolled back. Request a recount so both sides agree again. + * otherwise (newer gen, or seq already applied): stale, ack and drop. + """ + if not unmatched: + return + ids = [eid for eid, _ in unmatched] + catalog = { + row[0]: (int(row[1] or 0), int(row[2] or 0)) + for row in db.query( + CallImportEvaluation.id, + CallImportEvaluation.progress_counter_gen, + CallImportEvaluation.progress_flush_seq, + ) + .filter(CallImportEvaluation.id.in_(ids)) + .all() + } + db.rollback() + for eid, drained in unmatched: + current = catalog.get(eid) + if current is None: + buf.delete(eid) + continue + catalog_gen, _catalog_seq = current + if catalog_gen < drained.gen: + logger.warning( + "eval counter gen ahead of catalog for {} (redis {} > catalog {}); recounting", + eid, + drained.gen, + catalog_gen, + ) + finalize.add(eid) + + +def sweep_running_evaluations(db: Session, *, now: float | None = None) -> int: + """Request a fenced recount for running evaluations not recounted lately. + + Selects from the catalog, so it also finds idle evaluations that will + never be marked dirty again (lost last decrement / lost finalize). + """ + interval = max(30, int(getattr(settings, "EVAL_PROGRESS_RECOUNT_SECONDS", 300) or 300)) + moment = now if now is not None else time.time() + rows = ( + db.query(CallImportEvaluation.id) + .filter( + or_( + CallImportEvaluation.status.in_(("running", "cancelled")), + and_( + CallImportEvaluation.status == "auto_aborted", + CallImportEvaluation.finished_at.is_(None), + ), + ) + ) + .order_by(CallImportEvaluation.updated_at.asc()) + .limit(_SWEEP_LIMIT) + .all() + ) + db.rollback() + requested = 0 + for (eid,) in rows: + snapshot = buf.read_buffered(eid) + if snapshot.recount_at is not None and moment - snapshot.recount_at < interval: + continue + if buf.request_finalize(eid): + requested += 1 + if rows: + # Eval dispatch has no beat of its own; it runs when something asks. + # A lost request (e.g. one that ran while a bulk-op lock was held) + # would leave pending rows of a running evaluation idle forever. + try: + from app.workers.concurrency.fair_dispatch import schedule_fair_dispatch + + schedule_fair_dispatch(max_workspace_turns=999) + except Exception as exc: # noqa: BLE001 + logger.warning("eval dispatch safety-net schedule failed: {}", exc) + return requested + + +def _sweep_due() -> bool: + interval = max(30, int(getattr(settings, "EVAL_PROGRESS_RECOUNT_SECONDS", 300) or 300)) + try: + return bool(buf._client().set(_SWEEP_GATE_KEY, "1", nx=True, ex=interval)) + except redis.RedisError: + return False + + +@celery_app.task( + name="flush_eval_progress_counters", + queue=EVAL_CONTROL_QUEUE, + ignore_result=True, +) +def flush_eval_progress_counters_task() -> dict: + from app.database import SessionLocal + + try: + client = buf._client() + if not client.set(_FLUSHER_LOCK_KEY, "1", nx=True, ex=120): + return {"skipped": "flusher_running"} + except redis.RedisError as exc: + logger.warning("eval progress flush skipped (redis unavailable): {}", exc) + return {"skipped": "redis_unavailable"} + + db = SessionLocal() + try: + stats = flush_eval_progress_counters(db) + if _sweep_due(): + stats["swept"] = sweep_running_evaluations(db) + return stats + finally: + db.close() + try: + buf._client().delete(_FLUSHER_LOCK_KEY) + except redis.RedisError: + pass diff --git a/app/workers/tasks/flush_usage_counters.py b/app/workers/tasks/flush_usage_counters.py index 3dc158d0..b11a90bc 100644 --- a/app/workers/tasks/flush_usage_counters.py +++ b/app/workers/tasks/flush_usage_counters.py @@ -10,9 +10,29 @@ @celery_app.task(name="flush_usage_counters") def flush_usage_counters_task() -> dict: + from app.services.billing.flexprice_buffer import flush_buffered_flexprice_events + from app.services.call_imports.eval_flexprice_billing import ( + republish_sealed_recording_batches, + ) from app.services.usage.llm_usage import flush_all_usage_to_catalog + try: + republish_sealed_recording_batches() + except Exception as exc: + logger.warning("Flexprice sealed batch republish failed: {}", exc) + flexprice_stats = flush_buffered_flexprice_events() flushed = flush_all_usage_to_catalog(SessionLocal) if flushed: logger.info("Flushed {} LLM usage rollup buckets", flushed) - return {"flushed_buckets": flushed} + if flexprice_stats.get("flushed"): + logger.info( + "Flushed {} buffered Flexprice events ({} failed, {} pending)", + flexprice_stats["flushed"], + flexprice_stats.get("failed", 0), + flexprice_stats.get("pending", 0), + ) + return { + "flushed_buckets": flushed, + "flexprice_flushed": flexprice_stats.get("flushed", 0), + "flexprice_pending": flexprice_stats.get("pending", 0), + } diff --git a/app/workers/tasks/generate_evaluation_metric_clusters.py b/app/workers/tasks/generate_evaluation_metric_clusters.py index 8cb720df..36b6b0cc 100644 --- a/app/workers/tasks/generate_evaluation_metric_clusters.py +++ b/app/workers/tasks/generate_evaluation_metric_clusters.py @@ -8,21 +8,40 @@ from loguru import logger from sqlalchemy.orm.attributes import flag_modified +try: # Celery raises this inside the task body on the soft limit + from celery.exceptions import SoftTimeLimitExceeded +except Exception: # pragma: no cover - celery is always installed at runtime + class SoftTimeLimitExceeded(Exception): # type: ignore[no-redef] + pass + from app.database import SessionLocal -from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - CallImportRow, -) +from app.models.database import CallImportEvaluation from app.services.call_import_metric_clusters import ( - filter_completed_row_pairs, - generate_metric_clusters, + ClusterOutputTruncated, metric_clusters_raw_is_cancelled, metric_clusters_state_to_db, ) +from app.services.call_import_metric_clusters_wave import ( + process_metric_clusters_wave_step, + resume_metric_clusters_job_header, + start_metric_clusters_job_header, +) +from app.services.evaluation_analysis_checkpoints import ( + analysis_header_matches_run, + classify_task_run, + new_job_run_id, + normalize_transcript_source, + stamp_job_heartbeat, +) from app.workers.config import celery_app +def _reload_clusters_cancelled(db, evaluation: CallImportEvaluation) -> bool: + db.expire(evaluation, ["metric_clusters"]) + db.refresh(evaluation) + return metric_clusters_raw_is_cancelled(evaluation.metric_clusters) + + @celery_app.task(name="generate_evaluation_metric_clusters", bind=True, max_retries=0) def generate_evaluation_metric_clusters_task( self, @@ -33,10 +52,14 @@ def generate_evaluation_metric_clusters_task( credential_id: str | None = None, max_llm_calls: int | None = None, evaluation_row_ids: list[str] | None = None, + transcript_source: str | None = None, + resume: bool = False, + job_run_id: str | None = None, + **_ignored_kwargs: object, ): - from app.api.v1.routes.call_import_evaluations import ( - _child_names_by_parent, - _compute_metric_aggregates, + del max_llm_calls, evaluation_row_ids, _ignored_kwargs + from app.api.v1.routes.call_import_evaluations import _child_names_by_parent + from app.api.v1.routes.call_import_evaluations.clusters import ( _metrics_for_clustering, ) from app.services.metric_failure_policy import effective_policies @@ -44,6 +67,12 @@ def generate_evaluation_metric_clusters_task( db = SessionLocal() evaluation: CallImportEvaluation | None = None + run_id: UUID | None = None + # Needed by the soft-time-limit re-enqueue path, which can fire before the + # in-try assignments have run. + llm_provider: str | None = provider + llm_model: str | None = model + source: str | None = transcript_source try: evaluation = ( db.query(CallImportEvaluation) @@ -54,37 +83,56 @@ def generate_evaluation_metric_clusters_task( logger.error("Metric clusters: evaluation {} not found", evaluation_id) return - if metric_clusters_raw_is_cancelled(evaluation.metric_clusters): + if _reload_clusters_cancelled(db, evaluation): logger.info( "Metric clusters: evaluation {} already cancelled, skipping", evaluation_id, ) return + raw_for_llm = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + claim_state, _claimed_run = classify_task_run(raw_for_llm, job_run_id) + if claim_state in ("stale", "unclaimed"): + logger.info( + "Metric clusters: {} task ignored; task run is {} and header is {}", + claim_state, + job_run_id, + raw_for_llm.get("job_run_id"), + ) + return + llm_provider = (provider or "").strip() or ( + raw_for_llm.get("provider") + if isinstance(raw_for_llm.get("provider"), str) + else None + ) + llm_model = (model or "").strip() or ( + raw_for_llm.get("model") if isinstance(raw_for_llm.get("model"), str) else None + ) + provider_enum, model_str = get_llm_provider_and_model_for_request( evaluation.organization_id, db, - provider, - model, + llm_provider, + llm_model, UUID(credential_id) if credential_id else None, ) + # The re-enqueue path must carry a concrete provider/model pair. + llm_provider = provider_enum.value if hasattr(provider_enum, "value") else str(provider_enum) + llm_model = model_str - from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.scatter_gather import load_evaluation_score_rows_for_aggregate + from app.services.call_imports.metric_aggregate_cache import ( + evaluation_metric_aggregates_for_analysis, + ) - rows = load_evaluation_row_pairs(db, evaluation.id) - completed_pairs = [ - (eval_row, source_row) - for eval_row, source_row in rows - if eval_row.status == "completed" - ] - if evaluation_row_ids: - completed_pairs = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in evaluation_row_ids], - ) - eval_rows = [eval_row for eval_row, _source_row in rows] + score_rows = load_evaluation_score_rows_for_aggregate(db, evaluation.id) + aggregates = evaluation_metric_aggregates_for_analysis(db, evaluation) + eval_rows = score_rows metrics = _metrics_for_clustering(db, evaluation, eval_rows) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) parent_ids = [ m.id for m in metrics @@ -101,19 +149,86 @@ def generate_evaluation_metric_clusters_task( child_names_by_parent=child_names_by_parent, ) + raw = evaluation.metric_clusters if isinstance(evaluation.metric_clusters, dict) else {} + source = normalize_transcript_source( + transcript_source or raw.get("transcript_source") or evaluation.transcript_source + ) + + claim_state, claimed_run = classify_task_run(raw, job_run_id) + if claim_state in ("stale", "unclaimed"): + logger.info( + "Metric clusters: {} task ignored; task run is {} and header is {}", + claim_state, + job_run_id, + raw.get("job_run_id"), + ) + return + if claim_state == "owned" and claimed_run is not None: + run_id = claimed_run + if resume or raw.get("status") != "running": + evaluation.metric_clusters = resume_metric_clusters_job_header( + db, + evaluation, + raw, + transcript_source=source, + provider=provider_enum.value, + model=model_str, + ) + flag_modified(evaluation, "metric_clusters") + db.commit() + else: + run_id = new_job_run_id() + merged_raw = {**raw} if isinstance(raw, dict) else {} + evaluation.metric_clusters = start_metric_clusters_job_header( + evaluation, + db, + job_run_id=run_id, + transcript_source=source, + provider=provider_enum.value, + model=model_str, + metric_clusters_raw=merged_raw, + ) + flag_modified(evaluation, "metric_clusters") + db.commit() + def _reload_cancelled() -> bool: db.expire(evaluation, ["metric_clusters"]) db.refresh(evaluation) - return metric_clusters_raw_is_cancelled(evaluation.metric_clusters) + current = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + if metric_clusters_raw_is_cancelled(current): + return True + if not analysis_header_matches_run(current, run_id): + logger.info( + "Metric clusters: task for run {} stopped; header is {}", + run_id, + current.get("job_run_id"), + ) + return True + return False def on_progress(update: dict) -> None: if _reload_cancelled(): return - evaluation.metric_clusters = { - **(evaluation.metric_clusters or {}), + current = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + if metric_clusters_raw_is_cancelled(current): + return + evaluation.metric_clusters = stamp_job_heartbeat({ + **current, "status": "running", "progress": update, - } + }) + if isinstance(update.get("completed_llm_calls"), (int, float)): + evaluation.metric_clusters["llm_calls_used"] = int( + update["completed_llm_calls"] + ) flag_modified(evaluation, "metric_clusters") db.commit() @@ -121,33 +236,180 @@ def on_progress(update: dict) -> None: from app.services.usage.context import llm_usage_context with llm_usage_context(usage_context_for_evaluation(evaluation)): - state = generate_metric_clusters( + action, terminal_state = process_metric_clusters_wave_step( db, evaluation, evaluation.organization_id, provider_enum, model_str, - completed_row_pairs=completed_pairs, metrics=metrics, policies=policies, + job_run_id=run_id, on_progress=on_progress, - max_llm_calls=max_llm_calls, is_cancelled=_reload_cancelled, ) if _reload_cancelled(): return - persisted = metric_clusters_state_to_db(state) - if isinstance(evaluation.metric_clusters, dict): - evaluation.metric_clusters = { - **persisted, - "celery_task_id": None, - } - else: + if terminal_state is not None: + if terminal_state.status == "cancelled": + return + persisted = metric_clusters_state_to_db(terminal_state) + persisted["job_run_id"] = str(run_id) + persisted["transcript_source"] = source + persisted["celery_task_id"] = None + if isinstance(evaluation.metric_clusters, dict): + cursor = evaluation.metric_clusters.get("cursor") + if cursor: + persisted["cursor"] = cursor + if evaluation.metric_clusters.get("discovered_problems") is not None: + persisted["discovered_problems"] = evaluation.metric_clusters.get( + "discovered_problems" + ) evaluation.metric_clusters = persisted + flag_modified(evaluation, "metric_clusters") + db.commit() + return + + if _reload_cancelled(): + return + flag_modified(evaluation, "metric_clusters") db.commit() + + if action == "continue": + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": evaluation_id, + "provider": llm_provider, + "model": llm_model, + "credential_id": credential_id, + "transcript_source": source, + "resume": True, + "job_run_id": str(run_id), + }, + queue="imports", + ) + # Cancel could have landed while we were enqueuing; revoke the + # follow-up task instead of letting it resurrect the job. + db.expire(evaluation, ["metric_clusters"]) + db.refresh(evaluation) + if metric_clusters_raw_is_cancelled(evaluation.metric_clusters) or ( + isinstance(evaluation.metric_clusters, dict) + and not analysis_header_matches_run(evaluation.metric_clusters, run_id) + ): + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([async_result.id]) + return + if isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["status"] = "running" + evaluation.metric_clusters["celery_task_id"] = async_result.id + stamp_job_heartbeat(evaluation.metric_clusters) + flag_modified(evaluation, "metric_clusters") + db.commit() + except SoftTimeLimitExceeded: + # A single checkpoint page can outlast the 25 minute soft limit. The + # extraction sub-batches are already checkpointed, so hand the next + # step to a fresh task instead of failing the whole job. + logger.warning( + "Metric clusters hit the soft time limit for evaluation {}; " + "re-enqueuing from the last checkpoint", + evaluation_id, + ) + try: + db.rollback() + evaluation = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == UUID(evaluation_id)) + .first() + ) + if evaluation is None or metric_clusters_raw_is_cancelled( + evaluation.metric_clusters + ): + return + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + if run_id is not None and not analysis_header_matches_run(prior, run_id): + return + header_run_id = prior.get("job_run_id") + if not header_run_id: + logger.warning( + "Metric clusters soft time limit for evaluation {} with no " + "job_run_id; leaving it resumable instead of re-enqueuing", + evaluation_id, + ) + return + evaluation.metric_clusters = stamp_job_heartbeat( + {**prior, "status": "running", "celery_task_id": None} + ) + flag_modified(evaluation, "metric_clusters") + db.commit() + except Exception: # noqa: BLE001 + db.rollback() + return + try: + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": evaluation_id, + "provider": llm_provider, + "model": llm_model, + "credential_id": credential_id, + "transcript_source": source, + "resume": True, + "job_run_id": str(header_run_id), + }, + queue="imports", + ) + if isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["status"] = "running" + evaluation.metric_clusters["celery_task_id"] = async_result.id + flag_modified(evaluation, "metric_clusters") + db.commit() + except Exception: # noqa: BLE001 + logger.exception( + "Failed to re-enqueue metric clusters after soft time limit " + "for evaluation {}", + evaluation_id, + ) + except ClusterOutputTruncated as exc: + logger.exception( + "Metric clusters output truncated for evaluation {}: {}", + evaluation_id, + exc, + ) + try: + db.rollback() + evaluation = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == UUID(evaluation_id)) + .first() + ) + if evaluation is not None and not metric_clusters_raw_is_cancelled( + evaluation.metric_clusters + ): + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + if run_id is None or analysis_header_matches_run(prior, run_id): + evaluation.metric_clusters = { + **prior, + "status": "failed", + "error_message": str(exc) or "Cluster output was truncated.", + "generated_at": datetime.now(timezone.utc).isoformat(), + "celery_task_id": None, + } + flag_modified(evaluation, "metric_clusters") + db.commit() + except Exception: # noqa: BLE001 + db.rollback() + raise except Exception as exc: # noqa: BLE001 logger.exception( "Metric clusters generation failed for evaluation {}: {}", @@ -163,7 +425,17 @@ def on_progress(update: dict) -> None: if evaluation is not None and not metric_clusters_raw_is_cancelled( evaluation.metric_clusters ): + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + if run_id is not None and not analysis_header_matches_run( + prior, run_id + ): + return evaluation.metric_clusters = { + **prior, "status": "failed", "error_message": str(exc), "generated_at": datetime.now(timezone.utc).isoformat(), diff --git a/app/workers/tasks/generate_evaluation_prompt_improvements.py b/app/workers/tasks/generate_evaluation_prompt_improvements.py index 9bbe1f3c..1004fa6d 100644 --- a/app/workers/tasks/generate_evaluation_prompt_improvements.py +++ b/app/workers/tasks/generate_evaluation_prompt_improvements.py @@ -9,20 +9,30 @@ from sqlalchemy.orm.attributes import flag_modified from app.database import SessionLocal -from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - PromptPartial, -) +from app.models.database import CallImportEvaluation, PromptPartial from app.services.call_import_metric_clusters import metric_clusters_state_from_raw from app.services.call_import_prompt_improvements import ( - generate_prompt_improvements, is_imported_agent, prompt_improvements_state_to_db, ) +from app.services.call_import_prompt_improvements_wave import ( + process_prompt_improvements_wave_step, +) +from app.services.evaluation_analysis_checkpoints import ( + analysis_header_matches_run, + classify_task_run, + new_job_run_id, + prompt_improvements_raw_is_cancelled, +) from app.workers.config import celery_app +def _reload_prompt_cancelled(db, evaluation: CallImportEvaluation) -> bool: + db.expire(evaluation, ["prompt_improvements"]) + db.refresh(evaluation) + return prompt_improvements_raw_is_cancelled(evaluation.prompt_improvements) + + @celery_app.task(name="generate_evaluation_prompt_improvements", bind=True, max_retries=0) def generate_evaluation_prompt_improvements_task( self, @@ -32,17 +42,23 @@ def generate_evaluation_prompt_improvements_task( provider: str | None = None, model: str | None = None, credential_id: str | None = None, + resume: bool = False, + job_run_id: str | None = None, + **_ignored_kwargs: object, ): - from app.api.v1.routes.call_import_evaluations import ( - _aggregate_to_dict, - _compute_metric_aggregates, - _period_deltas_from_evaluation, - _resolve_baseline_evaluation, - ) - + del _ignored_kwargs db = SessionLocal() evaluation: CallImportEvaluation | None = None + run_id: UUID | None = None try: + from app.api.v1.routes.call_import_evaluations.aggregate import ( + _aggregate_to_dict, + _compute_metric_aggregates, + _period_deltas_from_evaluation, + _resolve_baseline_evaluation, + ) + from app.models.database import CallImportEvaluationRow + evaluation = ( db.query(CallImportEvaluation) .filter(CallImportEvaluation.id == UUID(evaluation_id)) @@ -52,8 +68,7 @@ def generate_evaluation_prompt_improvements_task( logger.error("Prompt improvements: evaluation {} not found", evaluation_id) return - raw = evaluation.prompt_improvements - if isinstance(raw, dict) and raw.get("status") == "cancelled": + if _reload_prompt_cancelled(db, evaluation): logger.info( "Prompt improvements: evaluation {} cancelled, skipping", evaluation_id, @@ -107,27 +122,172 @@ def generate_evaluation_prompt_improvements_task( eval_rows, ) + raw = ( + evaluation.prompt_improvements + if isinstance(evaluation.prompt_improvements, dict) + else {} + ) + claim_state, claimed_run = classify_task_run(raw, job_run_id) + if claim_state in ("stale", "unclaimed"): + logger.info( + "Prompt improvements: {} task ignored; task run is {} and header is {}", + claim_state, + job_run_id, + raw.get("job_run_id"), + ) + return + total_groups = len(clusters_state.groups or []) + provider_value = (provider or "").strip() or ( + raw.get("provider") if isinstance(raw.get("provider"), str) else None + ) + model_value = (model or "").strip() or ( + raw.get("model") if isinstance(raw.get("model"), str) else None + ) + + if claim_state == "owned" and claimed_run is not None: + run_id = claimed_run + evaluation.prompt_improvements = { + **raw, + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent.name, + "provider": provider_value, + "model": model_value, + "error_message": None, + "celery_task_id": None, + "completed_group_metric_ids": raw.get("completed_group_metric_ids") or [], + "total_groups": total_groups, + "progress": raw.get("progress") + or { + "completed_groups": 0, + "total_groups": total_groups, + "completed_rows": 0, + "total_rows": total_groups, + "completed_batches": 0, + "total_batches": total_groups, + "completed_llm_calls": 0, + "total_llm_calls": total_groups, + "phase": "rows", + }, + } + else: + run_id = new_job_run_id() + prior_suggestions = ( + raw.get("suggestions") if isinstance(raw.get("suggestions"), list) else [] + ) + evaluation.prompt_improvements = { + **raw, + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent.name, + "job_run_id": str(run_id), + "completed_group_metric_ids": [], + "completed_group_count": 0, + "total_groups": total_groups, + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "provider": provider_value, + "model": model_value, + "error_message": None, + "celery_task_id": None, + "suggestions": prior_suggestions if raw.get("status") == "completed" else [], + "progress": { + "completed_groups": 0, + "total_groups": total_groups, + "completed_rows": 0, + "total_rows": total_groups, + "completed_batches": 0, + "total_batches": total_groups, + "completed_llm_calls": 0, + "total_llm_calls": total_groups, + "phase": "rows", + }, + } + flag_modified(evaluation, "prompt_improvements") + db.commit() + + def _reload_cancelled() -> bool: + db.expire(evaluation, ["prompt_improvements"]) + db.refresh(evaluation) + current = ( + evaluation.prompt_improvements + if isinstance(evaluation.prompt_improvements, dict) + else {} + ) + if prompt_improvements_raw_is_cancelled(current): + return True + if not analysis_header_matches_run(current, run_id): + logger.info( + "Prompt improvements: task for run {} stopped; header is {}", + run_id, + current.get("job_run_id"), + ) + return True + return False + + def on_progress(update: dict) -> None: + if _reload_cancelled(): + return + current = ( + evaluation.prompt_improvements + if isinstance(evaluation.prompt_improvements, dict) + else {} + ) + if prompt_improvements_raw_is_cancelled(current): + return + evaluation.prompt_improvements = { + **current, + "status": "running", + "progress": update, + } + flag_modified(evaluation, "prompt_improvements") + db.commit() + from app.services.usage.call_import_context import usage_context_for_evaluation from app.services.usage.context import llm_usage_context with llm_usage_context(usage_context_for_evaluation(evaluation)): - state = generate_prompt_improvements( - evaluation=evaluation, - imported_agent=imported_agent, - clusters_state=clusters_state, - organization_id=evaluation.organization_id, - db=db, - provider=provider, - model=model, + action, terminal_state = process_prompt_improvements_wave_step( + db, + evaluation, + imported_agent, + clusters_state, + evaluation.organization_id, + job_run_id=run_id, + provider=provider_value, + model=model_value, credential_id=UUID(credential_id) if credential_id else None, period_deltas=period_deltas, + on_progress=on_progress, + is_cancelled=_reload_cancelled, ) - payload = prompt_improvements_state_to_db(state) + + if _reload_cancelled(): + return + + if terminal_state is not None: + if terminal_state.status == "cancelled": + return + payload = prompt_improvements_state_to_db(terminal_state) payload["compose_base_prompt"] = imported_agent.content or "" + payload["job_run_id"] = str(run_id) + payload["celery_task_id"] = None + payload["completed_group_metric_ids"] = sorted( + { + str(x) + for x in ((evaluation.prompt_improvements or {}).get( + "completed_group_metric_ids" + ) or []) + if x + } + ) + payload["completed_group_count"] = int( + (evaluation.prompt_improvements or {}).get("completed_group_count") or 0 + ) evaluation.prompt_improvements = payload flag_modified(evaluation, "prompt_improvements") db.commit() - if state.status == "completed": + if terminal_state.status == "completed": from app.services.billing.flexprice_service import ( record_call_import_prompt_improvements_generated, ) @@ -142,8 +302,48 @@ def generate_evaluation_prompt_improvements_task( logger.info( "Prompt improvements completed for evaluation {} ({} suggestions)", evaluation_id, - len(state.suggestions), + len(terminal_state.suggestions), ) + return + + if _reload_cancelled(): + return + + flag_modified(evaluation, "prompt_improvements") + db.commit() + + if action == "continue": + async_result = generate_evaluation_prompt_improvements_task.apply_async( + kwargs={ + "evaluation_id": evaluation_id, + "imported_agent_id": imported_agent_id, + "provider": provider_value, + "model": model_value, + "credential_id": credential_id, + "resume": True, + "job_run_id": str(run_id), + }, + queue="imports", + ) + db.expire(evaluation, ["prompt_improvements"]) + db.refresh(evaluation) + if prompt_improvements_raw_is_cancelled(evaluation.prompt_improvements): + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([async_result.id]) + return + if isinstance(evaluation.prompt_improvements, dict): + if not analysis_header_matches_run( + evaluation.prompt_improvements, run_id + ): + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([async_result.id]) + return + evaluation.prompt_improvements["celery_task_id"] = async_result.id + evaluation.prompt_improvements["status"] = "running" + flag_modified(evaluation, "prompt_improvements") + db.commit() except Exception as exc: logger.exception( "Prompt improvements failed for evaluation {}: {}", @@ -151,19 +351,32 @@ def generate_evaluation_prompt_improvements_task( exc, ) if evaluation is not None: - prior = ( + try: + db.expire(evaluation, ["prompt_improvements"]) + db.refresh(evaluation) + except Exception: # noqa: BLE001 + db.rollback() + if not prompt_improvements_raw_is_cancelled( evaluation.prompt_improvements - if isinstance(evaluation.prompt_improvements, dict) - else {} - ) - evaluation.prompt_improvements = { - **prior, - "status": "failed", - "imported_agent_id": imported_agent_id, - "error_message": str(exc), - "generated_at": datetime.now(timezone.utc).isoformat(), - } - flag_modified(evaluation, "prompt_improvements") - db.commit() + ): + prior = ( + evaluation.prompt_improvements + if isinstance(evaluation.prompt_improvements, dict) + else {} + ) + if run_id is not None and not analysis_header_matches_run( + prior, run_id + ): + return + evaluation.prompt_improvements = { + **prior, + "status": "failed", + "imported_agent_id": imported_agent_id, + "error_message": str(exc), + "generated_at": datetime.now(timezone.utc).isoformat(), + "celery_task_id": None, + } + flag_modified(evaluation, "prompt_improvements") + db.commit() finally: db.close() diff --git a/app/workers/tasks/generate_evaluation_user_insights.py b/app/workers/tasks/generate_evaluation_user_insights.py index fb1e10a2..ae0e2876 100644 --- a/app/workers/tasks/generate_evaluation_user_insights.py +++ b/app/workers/tasks/generate_evaluation_user_insights.py @@ -9,18 +9,29 @@ from sqlalchemy.orm.attributes import flag_modified from app.database import SessionLocal -from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - CallImportRow, +from app.models.database import CallImportEvaluation +from app.services.call_import_user_insights import user_insights_state_to_db +from app.services.call_import_user_insights_wave import ( + process_user_insights_wave_step, + start_user_insights_job_header, ) -from app.services.call_import_user_insights import ( - generate_user_insights, - user_insights_state_to_db, +from app.services.evaluation_analysis_checkpoints import ( + analysis_header_matches_run, + classify_task_run, + new_job_run_id, + normalize_transcript_source, + stamp_job_heartbeat, + user_insights_raw_is_cancelled, ) from app.workers.config import celery_app +def _refresh_user_insights_cancelled(db, evaluation: CallImportEvaluation) -> bool: + db.expire(evaluation, ["user_insights"]) + db.refresh(evaluation) + return user_insights_raw_is_cancelled(evaluation.user_insights) + + @celery_app.task(name="generate_evaluation_user_insights", bind=True, max_retries=0) def generate_evaluation_user_insights_task( self, @@ -29,15 +40,18 @@ def generate_evaluation_user_insights_task( provider: str | None = None, model: str | None = None, max_llm_calls: int | None = None, + job_run_id: str | None = None, + **_ignored_kwargs: object, ): + del max_llm_calls, _ignored_kwargs from app.api.v1.routes.call_import_evaluations import ( - _compute_metric_aggregates, _metrics_for_ids, _serialize_selected_metric_ids, ) from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request db = SessionLocal() + run_id = None try: evaluation = ( db.query(CallImportEvaluation) @@ -48,36 +62,117 @@ def generate_evaluation_user_insights_task( logger.error("User insights: evaluation {} not found", evaluation_id) return + if _refresh_user_insights_cancelled(db, evaluation): + logger.info( + "User insights: evaluation {} cancelled, skipping", + evaluation_id, + ) + return + + raw_for_llm = ( + evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + ) + claim_state, _claimed_run = classify_task_run(raw_for_llm, job_run_id) + if claim_state in ("stale", "unclaimed"): + logger.info( + "User insights: {} task ignored; task run is {} and header is {}", + claim_state, + job_run_id, + raw_for_llm.get("job_run_id"), + ) + return + llm_provider = (provider or "").strip() or ( + raw_for_llm.get("provider") + if isinstance(raw_for_llm.get("provider"), str) + else None + ) + llm_model = (model or "").strip() or ( + raw_for_llm.get("model") if isinstance(raw_for_llm.get("model"), str) else None + ) + provider_enum, model_str = get_llm_provider_and_model_for_request( evaluation.organization_id, db, - provider, - model, + llm_provider, + llm_model, ) - from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.services.call_imports.metric_aggregate_cache import ( + evaluation_metric_aggregates_for_analysis, + ) - rows = load_evaluation_row_pairs(db, evaluation.id) - completed_pairs = [ - (eval_row, source_row) - for eval_row, source_row in rows - if eval_row.status == "completed" - ] selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) - aggregate = _compute_metric_aggregates( - db, evaluation, [er for er, _ in rows] + aggregate = evaluation_metric_aggregates_for_analysis(db, evaluation) + + raw = evaluation.user_insights if isinstance(evaluation.user_insights, dict) else {} + source = normalize_transcript_source( + raw.get("transcript_source") or evaluation.transcript_source ) - def on_progress(completed: int, total: int) -> None: - evaluation.user_insights = { - **(evaluation.user_insights or {}), + claim_state, claimed_run = classify_task_run(raw, job_run_id) + if claim_state in ("stale", "unclaimed"): + logger.info( + "User insights: {} task ignored; task run is {} and header is {}", + claim_state, + job_run_id, + raw.get("job_run_id"), + ) + return + if claim_state == "owned" and claimed_run is not None: + run_id = claimed_run + else: + run_id = new_job_run_id() + prior_insights = raw.get("insights") if isinstance(raw.get("insights"), list) else [] + evaluation.user_insights = start_user_insights_job_header( + evaluation, + db, + job_run_id=run_id, + transcript_source=source, + provider=provider_enum.value, + model=model_str, + prior_insights=prior_insights if raw.get("status") == "completed" else [], + ) + flag_modified(evaluation, "user_insights") + db.commit() + + def _reload_cancelled() -> bool: + db.expire(evaluation, ["user_insights"]) + db.refresh(evaluation) + current = ( + evaluation.user_insights + if isinstance(evaluation.user_insights, dict) + else {} + ) + if user_insights_raw_is_cancelled(current): + return True + if not analysis_header_matches_run(current, run_id): + logger.info( + "User insights: task for run {} stopped; header is {}", + run_id, + current.get("job_run_id"), + ) + return True + return False + + def on_progress(update: dict) -> None: + if _reload_cancelled(): + return + current = ( + evaluation.user_insights + if isinstance(evaluation.user_insights, dict) + else {} + ) + if user_insights_raw_is_cancelled(current): + return + merged = stamp_job_heartbeat({ + **current, "status": "running", - "progress": { - "completed_llm_calls": completed, - "total_llm_calls": total, - }, - } + "progress": update, + }) + if isinstance(update.get("completed_llm_calls"), (int, float)): + merged["llm_calls_used"] = int(update["completed_llm_calls"]) + evaluation.user_insights = merged flag_modified(evaluation, "user_insights") db.commit() @@ -85,22 +180,38 @@ def on_progress(completed: int, total: int) -> None: from app.services.usage.context import llm_usage_context with llm_usage_context(usage_context_for_evaluation(evaluation)): - state = generate_user_insights( + action, terminal_state = process_user_insights_wave_step( db, evaluation, evaluation.organization_id, provider_enum, model_str, - completed_row_pairs=completed_pairs, metrics=metrics, aggregate=aggregate, + transcript_source=source, + job_run_id=run_id, on_progress=on_progress, - max_llm_calls=max_llm_calls, + is_cancelled=_reload_cancelled, ) - evaluation.user_insights = user_insights_state_to_db(state) + + if _reload_cancelled(): + return + + if terminal_state is not None: + if terminal_state.status == "cancelled": + return + payload = user_insights_state_to_db(terminal_state) + payload["job_run_id"] = str(run_id) + payload["transcript_source"] = source + payload["celery_task_id"] = None + if isinstance(evaluation.user_insights, dict): + cursor = evaluation.user_insights.get("cursor") + if cursor: + payload["cursor"] = cursor + evaluation.user_insights = payload flag_modified(evaluation, "user_insights") db.commit() - if state.status == "completed": + if terminal_state.status == "completed": from app.services.billing.flexprice_service import ( record_call_import_user_insights_generated, ) @@ -111,6 +222,50 @@ def on_progress(completed: int, total: int) -> None: workspace_id=evaluation.workspace_id, evaluation_id=evaluation.id, ) + return + + if _reload_cancelled(): + return + + flag_modified(evaluation, "user_insights") + db.commit() + + if action == "continue": + if _reload_cancelled(): + return + async_result = generate_evaluation_user_insights_task.apply_async( + kwargs={ + "evaluation_id": evaluation_id, + "provider": llm_provider, + "model": llm_model, + "job_run_id": str(run_id), + }, + queue="imports", + ) + if _reload_cancelled(): + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([async_result.id]) + return + header = ( + evaluation.user_insights + if isinstance(evaluation.user_insights, dict) + else {} + ) + if user_insights_raw_is_cancelled(header) or not analysis_header_matches_run( + header, run_id + ): + from app.workers.celery_revoke import revoke_celery_tasks + + revoke_celery_tasks([async_result.id]) + return + evaluation.user_insights = { + **header, + "status": "running", + "celery_task_id": async_result.id, + } + flag_modified(evaluation, "user_insights") + db.commit() except Exception as exc: # noqa: BLE001 logger.exception( "User insights generation failed for evaluation {}: {}", @@ -123,11 +278,24 @@ def on_progress(completed: int, total: int) -> None: .filter(CallImportEvaluation.id == UUID(evaluation_id)) .first() ) - if evaluation is not None: + if evaluation is not None and not user_insights_raw_is_cancelled( + evaluation.user_insights + ): + prior = ( + evaluation.user_insights + if isinstance(evaluation.user_insights, dict) + else {} + ) + if run_id is not None and not analysis_header_matches_run( + prior, run_id + ): + return evaluation.user_insights = { + **prior, "status": "failed", "error_message": str(exc), "generated_at": datetime.now(timezone.utc).isoformat(), + "celery_task_id": None, } flag_modified(evaluation, "user_insights") db.commit() diff --git a/app/workers/tasks/helpers/llm_diarisation.py b/app/workers/tasks/helpers/llm_diarisation.py index 7156faff..ced35fe4 100644 --- a/app/workers/tasks/helpers/llm_diarisation.py +++ b/app/workers/tasks/helpers/llm_diarisation.py @@ -48,7 +48,6 @@ import json import re -from pathlib import Path from typing import Any, Dict, List, Optional from uuid import UUID @@ -56,13 +55,16 @@ from sqlalchemy.orm import Session from app.models.enums import ModelProvider +from app.services.ai.llm_call_scope import with_llm_call_scope from app.services.ai.llm_service import llm_service +from app.services.import_eval_llm_output_budgets import ( + DIARISER_COMPACT_OUTPUT_FIRST, + DIARISER_COMPACT_OUTPUT_RETRY, +) from .json_utils import repair_truncated_json -_DEBUG_LOG_PATH = Path(__file__).resolve().parents[4] / "debug-bfc313.log" - DEFAULT_DIARIZATION_PROMPT = ( "You are a transcript diariser. Split the following raw call " @@ -127,6 +129,79 @@ ) +def _build_native_diarization_prompt() -> str: + """Default prompt plus a ``text_native`` field. Used only when an eval asks for it.""" + prompt = DEFAULT_DIARIZATION_PROMPT + rule4_tail = ( + "If the operator's " + "instructions explicitly request a different script (e.g. " + '"transcribe in Devanagari") or an actual translation into ' + "another language, those instructions take precedence over " + "this rule (per rule 3).\n" + ) + native_rule4 = ( + rule4_tail + + "Additionally, for every turn emit `text_native`: the same words " + "in the script they were spoken in (Devanagari for Hindi, and the " + "matching script for other languages; English words stay in Latin). " + "Verbatim, no translation. If the turn is English-only, `text_native` " + "equals `text`.\n" + ) + example = ( + "Each entry must have exactly the keys `speaker` " + '("agent" or "user") and `text` (string). Example:\n' + '{"turns": [{"speaker": "agent", "text": "Hello, this ' + 'is Acme support."}, {"speaker": "user", "text": "Hi, ' + 'I have a billing question."}]}' + ) + native_example = ( + "Each entry must have exactly the keys `speaker` " + '("agent" or "user"), `text` (romanised string), and ' + "`text_native` (original script; equals `text` when the turn is " + "English-only). Example:\n" + '{"turns": [{"speaker": "agent", "text": "Hello, this ' + 'is Acme support.", "text_native": "Hello, this is Acme support."}, ' + '{"speaker": "user", "text": "Main aap ki kya sahayata kar sakthi hu", ' + '"text_native": "मैं आपकी क्या सहायता कर सकती हूं"}]}' + ) + if rule4_tail not in prompt or example not in prompt: + raise RuntimeError("Could not build the native diarisation prompt") + prompt = prompt.replace(rule4_tail, native_rule4, 1) + prompt = prompt.replace( + "6. Use exactly the keys `speaker` and `text` in each entry.", + "6. Use exactly the keys `speaker`, `text`, and `text_native` in each entry.", + 1, + ) + return prompt.replace(example, native_example, 1) + + +DEFAULT_DIARIZATION_PROMPT_WITH_NATIVE = _build_native_diarization_prompt() + +# Extra output when every turn also carries ``text_native``. +_NATIVE_OUTPUT_TOKEN_MULTIPLIER = 1.9 + + +def resolve_diarisation_system_prompt( + custom_prompt: Optional[str], + *, + emit_native_text: bool = False, +) -> str: + """Operator prompt wins. Native text is opt-in and only for the default prompt.""" + custom = (custom_prompt or "").strip() + if custom: + return custom + if emit_native_text: + return DEFAULT_DIARIZATION_PROMPT_WITH_NATIVE + return DEFAULT_DIARIZATION_PROMPT + + +def _native_emission_active( + custom_prompt: Optional[str], + emit_native_text: bool, +) -> bool: + return bool(emit_native_text) and not (custom_prompt or "").strip() + + # How much of the prompt + transcript we'll feed to the LLM. We don't # enforce a hard token cap (provider-specific) but we do clip the # transcript so a 1 MB CSV cell can't blow up the prompt window. @@ -154,6 +229,20 @@ ("gemini-1.5-flash", 65_536), ) +# Gemini 3.x and GLM use a smaller two-step output budget (8k then 16k). +_GEMINI_3_DIARISER_RE = re.compile( + r"(?:^|[/-])gemini-3(?:\.\d+)?(?:[-.]|$)", re.IGNORECASE +) + + +def _diariser_uses_compact_output_budget(llm_model: str) -> bool: + name = (llm_model or "").strip().lower() + if _GEMINI_3_DIARISER_RE.search(name): + return True + if "glm" in name: + return True + return False + def _diarisation_output_token_ceiling(llm_model: str) -> int: """Provider/model-specific max ``max_tokens`` for diariser completions.""" @@ -170,6 +259,7 @@ def _estimate_diarisation_max_tokens( text_length: int = 0, audio_bytes: int = 0, llm_model: str = "", + emit_native_text: bool = False, ) -> int: """Scale the diariser output budget to the input payload size.""" ceiling = _diarisation_output_token_ceiling(llm_model) @@ -182,6 +272,8 @@ def _estimate_diarisation_max_tokens( estimated = int(estimated_chars * 0.45) + 512 else: estimated = _DIARISATION_MIN_MAX_TOKENS + if emit_native_text: + estimated = int(estimated * _NATIVE_OUTPUT_TOKEN_MULTIPLIER) return min(ceiling, max(_DIARISATION_MIN_MAX_TOKENS, estimated)) @@ -197,6 +289,8 @@ def _generate_diarisation_response( config: Dict[str, Any], content_length: int = 0, audio_bytes: int = 0, + emit_native_text: bool = False, + allow_budget_retry: bool = True, ) -> Dict[str, Any]: """Call the diariser LLM with a scaled ``max_tokens`` and one retry.""" ceiling = _diarisation_output_token_ceiling(llm_model) @@ -204,9 +298,20 @@ def _generate_diarisation_response( text_length=content_length, audio_bytes=audio_bytes, llm_model=llm_model, + emit_native_text=emit_native_text, ) - retry_budget = min(ceiling, base_budget * 2) - budgets = [base_budget] if retry_budget <= base_budget else [base_budget, retry_budget] + if _diariser_uses_compact_output_budget(llm_model): + budgets = [DIARISER_COMPACT_OUTPUT_FIRST] + if ( + allow_budget_retry + and DIARISER_COMPACT_OUTPUT_RETRY > DIARISER_COMPACT_OUTPUT_FIRST + ): + budgets.append(DIARISER_COMPACT_OUTPUT_RETRY) + else: + retry_budget = min(ceiling, base_budget * 2) + budgets = [base_budget] + if allow_budget_retry and retry_budget > base_budget: + budgets.append(retry_budget) response: Dict[str, Any] = {} for attempt_idx, max_tokens in enumerate(budgets): @@ -234,6 +339,63 @@ def _generate_diarisation_response( return response +def _generate_turns( + *, + build_messages, + custom_prompt: Optional[str], + emit_native_text: bool, + provider_enum: ModelProvider, + llm_model: str, + organization_id: UUID, + db: Session, + temperature: float, + credential_id: Optional[UUID], + config: Dict[str, Any], + content_length: int = 0, + audio_bytes: int = 0, +) -> List[Dict[str, Any]]: + """Run the diariser. A truncated native-text response retries once without it.""" + emitting = _native_emission_active(custom_prompt, emit_native_text) + system_prompt = resolve_diarisation_system_prompt( + custom_prompt, + emit_native_text=emit_native_text, + ) + + def _call(prompt: str, *, native: bool, allow_budget_retry: bool) -> Dict[str, Any]: + return _generate_diarisation_response( + messages=build_messages(prompt), + provider_enum=provider_enum, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=temperature, + credential_id=credential_id, + config=config, + content_length=content_length, + audio_bytes=audio_bytes, + emit_native_text=native, + allow_budget_retry=allow_budget_retry, + ) + + response = _call( + system_prompt, + native=emitting, + allow_budget_retry=not emitting, + ) + if emitting and response.get("truncated"): + logger.warning( + "Diariser response truncated while emitting text_native " + "(model={}); retrying once without native text.", + llm_model, + ) + response = _call( + DEFAULT_DIARIZATION_PROMPT, + native=False, + allow_budget_retry=True, + ) + return _parse_turns_from_response(response) + + # Synonym keys we accept from over-creative LLMs that don't follow the # canonical ``{"speaker": ..., "text": ...}`` schema verbatim. Without # this leniency a model that emits ``{"speaker": "agent", "utterance": @@ -266,6 +428,12 @@ def _generate_diarisation_response( "participant", "label", ) +_NATIVE_TEXT_KEYS: tuple[str, ...] = ( + "text_native", + "native", + "original_script", + "text_original", +) def _first_nonempty_str(entry: Dict[str, Any], keys: tuple[str, ...]) -> str: @@ -465,6 +633,7 @@ def diarize_transcript_with_llm( custom_prompt: Optional[str] = None, credential_id: Optional[UUID] = None, temperature: float = 0.0, + emit_native_text: bool = False, ) -> List[Dict[str, Any]]: """Ask an LLM to split ``transcript`` into ``agent`` / ``user`` turns. @@ -517,8 +686,6 @@ def diarize_transcript_with_llm( ) cleaned = cleaned[:_MAX_TRANSCRIPT_CHARS] - system_prompt = (custom_prompt or "").strip() or DEFAULT_DIARIZATION_PROMPT - try: provider_enum = ModelProvider(llm_provider.lower()) except ValueError as exc: @@ -542,19 +709,20 @@ def diarize_transcript_with_llm( # The downstream parser (``_extract_json_array``) handles both the # wrapped object and a bare array via its bracket-scan fallback, # so a model that ignores the wrapper still flows through. - messages = [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": ( - "Diarise the following transcript. Return ONLY a JSON " - "object with a single key `turns` whose value is the " - "array of turn objects described in the system prompt " - "— no prose, no markdown fence. Transcript:\n\n" - f"{cleaned}" - ), - }, - ] + def _build_messages(system_prompt: str) -> List[Dict[str, Any]]: + return [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": ( + "Diarise the following transcript. Return ONLY a JSON " + "object with a single key `turns` whose value is the " + "array of turn objects described in the system prompt " + "— no prose, no markdown fence. Transcript:\n\n" + f"{cleaned}" + ), + }, + ] # ``response_format={"type": "json_object"}`` is forwarded to # providers that support it (OpenAI JSON mode, Gemini's @@ -565,8 +733,10 @@ def diarize_transcript_with_llm( # other providers. config = {"response_format": {"type": "json_object"}} - response = _generate_diarisation_response( - messages=messages, + turns = _generate_turns( + build_messages=_build_messages, + custom_prompt=custom_prompt, + emit_native_text=emit_native_text, provider_enum=provider_enum, llm_model=llm_model, organization_id=organization_id, @@ -577,8 +747,6 @@ def diarize_transcript_with_llm( content_length=len(cleaned), ) - turns = _parse_turns_from_response(response) - # Latin-script compliance check (text path). # # ``DEFAULT_DIARIZATION_PROMPT`` rule 4 instructs the model to @@ -670,14 +838,17 @@ def _parse_turns_from_response(response: Dict[str, Any]) -> List[Dict[str, Any]] if not text: continue raw_speaker = _normalise_speaker_label(raw_speaker_value) - turns.append( - { - "speaker": raw_speaker, - "text": text, - "start": round(cursor, 3), - "end": round(cursor + 1.0, 3), - } - ) + turn: Dict[str, Any] = { + "speaker": raw_speaker, + "text": text, + "start": round(cursor, 3), + "end": round(cursor + 1.0, 3), + } + if isinstance(entry, dict): + native_text = _first_nonempty_str(entry, _NATIVE_TEXT_KEYS) + if native_text: + turn["text_native"] = native_text + turns.append(turn) cursor += 1.0 if not turns: @@ -711,6 +882,8 @@ def _parse_turns_from_response(response: Dict[str, Any]) -> List[Dict[str, Any]] # Provider support is intentionally narrow — only providers whose chat # completions API accepts inline audio content parts: # * OpenAI ``gpt-4o-audio-preview`` family → ``{"type": "input_audio"}`` +# * OpenRouter (OpenAI-compatible chat completions, including routed +# Gemini models) → same ``input_audio`` shape # * Google Gemini 1.5 / 2.0+ → ``{"type": "image_url", # "image_url": "data:audio/;base64,..."}`` (LiteLLM normalises this # into Gemini's native ``inline_data`` shape). @@ -742,7 +915,13 @@ def _parse_turns_from_response(response: Dict[str, Any]) -> List[Dict[str, Any]] # how to shape its content parts; until then we surface a clean error. # ``custom`` is allowed separately when the org credential routes via # the LLM gateway with a pinned ``gateway_model`` (OpenAI-compatible). -_AUDIO_SUPPORTED_PROVIDERS: frozenset[str] = frozenset({"openai", "google"}) +_AUDIO_SUPPORTED_PROVIDERS: frozenset[str] = frozenset( + {"openai", "google", "openrouter"} +) +# Chat providers that attach audio via OpenAI-style ``input_audio`` parts. +_INPUT_AUDIO_CONTENT_PROVIDERS: frozenset[str] = frozenset( + {"openai", "openrouter", "custom"} +) def _custom_gateway_audio_allowed( @@ -756,14 +935,14 @@ def _custom_gateway_audio_allowed( if provider_value != "custom": return False - from app.services.credentials import resolve_ai_provider + from app.services.credentials import resolve_ai_provider_for_call try: provider_enum = ModelProvider(provider_value) except ValueError: return False - ai_provider_row = resolve_ai_provider( + ai_provider_row = resolve_ai_provider_for_call( provider_enum.value, db, organization_id, @@ -846,9 +1025,9 @@ def _resolve_provider_api_key( # path of a module that's also used by the lighter text-only # diariser. from app.core.encryption import decrypt_api_key - from app.services.credentials import resolve_ai_provider + from app.services.credentials import resolve_ai_provider_for_call - ai_provider = resolve_ai_provider( + ai_provider = resolve_ai_provider_for_call( provider, db, organization_id, credential_id=credential_id ) if ai_provider is None: @@ -977,9 +1156,14 @@ def _build_audio_messages( LiteLLM's content-part vocabulary differs slightly per provider: - * **OpenAI** accepts a first-class ``{"type": "input_audio", + * **OpenAI / OpenRouter / Custom gateway** accept + ``{"type": "input_audio", "input_audio": {"data": , "format": "wav|mp3|..."}}`` content - part on ``gpt-4o-audio-*`` models. + parts (OpenRouter documents the same shape for routed multimodal + models such as ``google/gemini-2.5-flash``). + * **OpenAI-only note:** ``gpt-4o-audio-*`` models on the OpenAI + provider use this part shape; other OpenAI chat models may reject + audio. * **Gemini (Files API path, preferred)** accepts a ``{"type": "file", "file": {"file_id": "", "format": ""}}`` part whose ``file_id`` is the URI @@ -1027,7 +1211,7 @@ def _build_audio_messages( "expected answer for empty / non-speech audio." ) - if provider_value == "openai" or provider_value == "custom": + if provider_value in _INPUT_AUDIO_CONTENT_PROVIDERS: audio_part: Dict[str, Any] = { "type": "input_audio", "input_audio": {"data": audio_b64, "format": openai_format}, @@ -1059,6 +1243,7 @@ def _build_audio_messages( ] +@with_llm_call_scope def diarize_audio_with_llm( audio_bytes: bytes, *, @@ -1071,6 +1256,7 @@ def diarize_audio_with_llm( temperature: float = 0.0, audio_file_key: Optional[str] = None, mime_type: Optional[str] = None, + emit_native_text: bool = False, ) -> List[Dict[str, Any]]: """Ask a multimodal LLM to transcribe + diarise raw audio in one pass. @@ -1087,9 +1273,10 @@ def diarize_audio_with_llm( ``s3_service.download_file_by_key``). Empty input short- circuits to ``[]`` so callers don't have to special-case it. llm_provider: - Must be one of :data:`_AUDIO_SUPPORTED_PROVIDERS` (``openai`` - or ``google`` at the moment). Other providers raise a typed - error instead of attempting an unsupported call. + Must be one of :data:`_AUDIO_SUPPORTED_PROVIDERS` (``openai``, + ``google``, or ``openrouter``), or ``custom`` when + :func:`_custom_gateway_audio_allowed` passes. Other providers + raise a typed error instead of attempting an unsupported call. llm_model: Concrete model name. The caller is responsible for picking a model that actually accepts audio input (e.g. @@ -1123,8 +1310,10 @@ def diarize_audio_with_llm( raise LLMDiarisationError( f"Provider '{llm_provider}' is not supported in LLM-only " "mode yet. Pick an OpenAI gpt-4o-audio model, a Google " - "Gemini model that accepts audio input, or a Custom " - "gateway credential with a gateway model configured." + "Gemini model that accepts audio input, an OpenRouter " + "model with audio input (e.g. google/gemini-2.5-flash), " + "or a Custom gateway credential with a gateway model " + "configured." ) try: @@ -1142,8 +1331,6 @@ def diarize_audio_with_llm( # at call sites so worker boot time isn't polluted. import base64 as _b64 - system_prompt = (custom_prompt or "").strip() or DEFAULT_DIARIZATION_PROMPT - # ---- Gemini Files API fast-path ------------------------------------- # For non-trivial recordings on Gemini we upload to the Files API # first and reference the resulting URI in the prompt. This is @@ -1156,9 +1343,9 @@ def diarize_audio_with_llm( resolve_effective_routing, routing_context_from_ai_provider, ) - from app.services.credentials import resolve_ai_provider + from app.services.credentials import resolve_ai_provider_for_call - _ai_provider_row = resolve_ai_provider( + _ai_provider_row = resolve_ai_provider_for_call( provider_enum.value, db, organization_id, @@ -1172,14 +1359,6 @@ def diarize_audio_with_llm( _, _effective_routing = resolve_effective_routing( organization_id, db, _cred_ctx ) - # #region agent log - try: - import json as _json, time as _time - with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as _f: - _f.write(_json.dumps({"sessionId": "bfc313", "hypothesisId": "B", "location": "llm_diarisation.py:diarize_audio_with_llm", "message": "audio diariser credential routing", "data": {"credential_id": str(credential_id) if credential_id else None, "provider_row_id": str(getattr(_ai_provider_row, "id", None)), "routing_mode": getattr(_ai_provider_row, "routing_mode", None), "effective_routing": _effective_routing, "audio_bytes": len(audio_bytes), "files_api_threshold": _GEMINI_FILES_API_MIN_BYTES}, "timestamp": int(_time.time() * 1000)}) + "\n") - except Exception: - pass - # #endregion gemini_file_id: Optional[str] = None # Gemini Files API uploads go directly to Google and bind the file to @@ -1211,14 +1390,6 @@ def diarize_audio_with_llm( len(audio_bytes), resolved_mime, ) - # #region agent log - try: - import json as _json, time as _time - with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as _f: - _f.write(_json.dumps({"sessionId": "bfc313", "runId": "post-fix", "hypothesisId": "C", "location": "llm_diarisation.py:diarize_audio_with_llm", "message": "audio transport chosen", "data": {"effective_routing": _effective_routing, "gemini_file_id_used": bool(gemini_file_id), "gemini_file_id_prefix": (gemini_file_id or "")[:80], "uses_inline_base64": gemini_file_id is None, "files_api_skipped_for_gateway": _effective_routing != "direct" and provider_value == "google" and len(audio_bytes) >= _GEMINI_FILES_API_MIN_BYTES}, "timestamp": int(_time.time() * 1000)}) + "\n") - except Exception: - pass - # #endregion # Only build the (expensive) base64 payload when we actually need # it for the wire — for the OpenAI path always, and for the Gemini @@ -1227,14 +1398,15 @@ def diarize_audio_with_llm( if provider_value in {"openai", "custom"} or gemini_file_id is None: audio_b64 = _b64.b64encode(audio_bytes).decode("ascii") - messages = _build_audio_messages( - provider_value=provider_value, - audio_b64=audio_b64, - mime_type=resolved_mime, - openai_format=openai_format, - system_prompt=system_prompt, - gemini_file_id=gemini_file_id, - ) + def _build_messages(system_prompt: str) -> List[Dict[str, Any]]: + return _build_audio_messages( + provider_value=provider_value, + audio_b64=audio_b64, + mime_type=resolved_mime, + openai_format=openai_format, + system_prompt=system_prompt, + gemini_file_id=gemini_file_id, + ) # ``response_format`` is forwarded for OpenAI / Gemini and dropped # by other providers via ``litellm.drop_params=True`` — same @@ -1242,8 +1414,10 @@ def diarize_audio_with_llm( config = {"response_format": {"type": "json_object"}} try: - response = _generate_diarisation_response( - messages=messages, + turns = _generate_turns( + build_messages=_build_messages, + custom_prompt=custom_prompt, + emit_native_text=emit_native_text, provider_enum=provider_enum, llm_model=llm_model, organization_id=organization_id, @@ -1253,6 +1427,8 @@ def diarize_audio_with_llm( config=config, audio_bytes=len(audio_bytes), ) + except LLMDiarisationError: + raise except Exception as exc: # LiteLLM surfaces "model does not support audio input" as a # generic RuntimeError. Translate it to LLMDiarisationError so @@ -1270,8 +1446,6 @@ def diarize_audio_with_llm( ) ) from exc - turns = _parse_turns_from_response(response) - # Latin-script compliance check (audio / LLM-only path). # # Mirrors the text-diariser's check: detect non-Latin output and diff --git a/app/workers/tasks/helpers/llm_evaluation.py b/app/workers/tasks/helpers/llm_evaluation.py index 1c781ad7..3e6e8bd1 100644 --- a/app/workers/tasks/helpers/llm_evaluation.py +++ b/app/workers/tasks/helpers/llm_evaluation.py @@ -29,6 +29,12 @@ def flatten_metric_groups(metric_groups: list[MetricPromptGroup]) -> list: from loguru import logger from app.models.database import ModelProvider +from app.services.metrics.option_labels import ( + canonical_enum_option, + coerce_metric_option_label, + enum_numeric_matches_ambiguous, + enum_option_labels_from_config, +) from .json_utils import repair_truncated_json from .score_utils import ( @@ -70,10 +76,7 @@ def _classification_metric_error_entry( def _get_enum_options(metric) -> list[str]: """Return the list of allowed enum option labels for an enum custom metric.""" cfg = getattr(metric, "custom_config", None) or {} - options = cfg.get("options") if isinstance(cfg, dict) else None - if not isinstance(options, list): - return [] - return [str(o).strip() for o in options if str(o).strip()] + return enum_option_labels_from_config(cfg) def _get_number_range(metric) -> Optional[dict]: @@ -138,17 +141,26 @@ def _normalize_enum_value(raw: Any, options: list[str]) -> Optional[str]: if raw is None or not options: return None if isinstance(raw, dict): - for k in ("value", "label", "choice", "answer"): - if k in raw: - raw = raw[k] - break - text = str(raw).strip() + coerced = coerce_metric_option_label(raw) + if coerced is not None: + text = coerced + else: + for k in ("value", "label", "choice", "answer"): + if k in raw: + raw = raw[k] + break + text = str(raw).strip() + else: + coerced = coerce_metric_option_label(raw) + text = coerced if coerced is not None else str(raw).strip() if not text: return None + canonical = canonical_enum_option(text, options) + if canonical is not None: + return canonical + if enum_numeric_matches_ambiguous(text, options): + return None text_lower = text.lower() - for opt in options: - if opt.lower() == text_lower: - return opt # Lenient fallback: substring containment in either direction. for opt in options: if opt.lower() in text_lower or text_lower in opt.lower(): diff --git a/app/workers/tasks/ingest_knowledge_document.py b/app/workers/tasks/ingest_knowledge_document.py new file mode 100644 index 00000000..c82a7cd9 --- /dev/null +++ b/app/workers/tasks/ingest_knowledge_document.py @@ -0,0 +1,23 @@ +"""Celery task to ingest a knowledge base document.""" + +from __future__ import annotations + +from uuid import UUID + +from app.database import SessionLocal +from app.services.knowledge_bases.ingest import ingest_knowledge_document +from app.workers.config import celery_app + + +@celery_app.task(name="ingest_knowledge_document", bind=True, max_retries=2) +def ingest_knowledge_document_task(self, document_id: str): + db = SessionLocal() + try: + ingest_knowledge_document(db, document_id=UUID(document_id)) + except Exception as exc: + db.rollback() + if self.request.retries < self.max_retries: + raise self.retry(exc=exc, countdown=30) from exc + raise + finally: + db.close() diff --git a/app/workers/tasks/platform_call_import_hydrate.py b/app/workers/tasks/platform_call_import_hydrate.py index 549c727c..c73223c1 100644 --- a/app/workers/tasks/platform_call_import_hydrate.py +++ b/app/workers/tasks/platform_call_import_hydrate.py @@ -68,17 +68,6 @@ def _extension_for_content_type(content_type: str) -> str: return mapping.get(ct, "mp3") -def _retryable_status(exc: Exception) -> int | None: - from app.services.call_imports.platform_discovery import list_failure_disposition - - kind, _wait = list_failure_disposition(exc) - if kind == "auth": - return 401 - if kind == "retry": - return 503 - return None - - def _hand_off( *, row, @@ -143,7 +132,6 @@ def hydrate_platform_call_import_row( slot_task_id: str, filters: Dict[str, Any], ) -> Dict[str, Any]: - from app.services.call_imports.debounced_rollup import maybe_record_import_row_transition from app.services.storage.s3_service import s3_service from app.workers.tasks.process_call_import_row import ( _extension_for_content_type as ext_fn, @@ -233,24 +221,18 @@ def hydrate_platform_call_import_row( normalized = adapter.hydrate(item) clear_platform_auth_failures(integration.id) except Exception as exc: - status_code = _retryable_status(exc) - if status_code in (401, 403): + from app.services.call_imports.platform_discovery import list_failure_disposition + + kind, wait_seconds = list_failure_disposition(exc) + if kind == "auth": record_platform_auth_failure(integration.id) - elif status_code == 429 or (status_code is not None and status_code >= 500): - retry_after = 1 - response = getattr(exc, "response", None) - header = getattr(response, "headers", None) - if header is not None: - try: - retry_after = int(header.get("Retry-After") or 1) - except (TypeError, ValueError): - retry_after = 1 + elif kind == "retry": return _hand_off( row=row, db=db, row_id=row_id, slot_task_id=slot_task_id, - wait_seconds=retry_after, + wait_seconds=wait_seconds, reason="provider_retry", ) logger.warning("platform hydrate failed row {}: {}", row_id, exc) @@ -294,13 +276,17 @@ def hydrate_platform_call_import_row( row.recording_s3_key = None row.status = CallImportRowStatus.COMPLETED row.error_message = None - if not _safe_commit(db, row_id=row_id, context="platform_chat_completed"): - return {"status": "skipped", "reason": "row_deleted"} - maybe_record_import_row_transition( + from app.services.call_imports.debounced_rollup import ( + record_import_progress_before_row_commit, + ) + + record_import_progress_before_row_commit( call_import.id, previous_status=previous_status, new_status=row.status, ) + if not _safe_commit(db, row_id=row_id, context="platform_chat_completed"): + return {"status": "skipped", "reason": "row_deleted"} call_import.last_progress_at = datetime.now(timezone.utc) _rollup_parent_on_catalog(catalog_db, row_db, call_import) _safe_commit(db, row_id=row_id, context="platform_chat_rollup") @@ -348,14 +334,16 @@ def hydrate_platform_call_import_row( try: stream, content_type = adapter.open_audio_stream(normalized) except Exception as exc: - status_code = _retryable_status(exc) - if status_code == 429 or (status_code is not None and status_code >= 500): + from app.services.call_imports.platform_discovery import list_failure_disposition + + kind, wait_seconds = list_failure_disposition(exc) + if kind == "retry": return _hand_off( row=row, db=db, row_id=row_id, slot_task_id=slot_task_id, - wait_seconds=1, + wait_seconds=wait_seconds, reason="provider_retry", ) return _mark_row_failed_for_fetch( @@ -428,8 +416,13 @@ def hydrate_platform_call_import_row( context="platform_empty_audio", reason="recording_empty", ) - s3_service.upload_file_by_key_from_path( - tmp.name, key, content_type=content_type or "audio/mpeg" + from app.services.call_imports.recording_storage import upload_local_recording_file + + size_bytes, duration_seconds = upload_local_recording_file( + tmp.name, + key=key, + content_type=content_type or "audio/mpeg", + size_bytes=total, ) finally: tmp.close() @@ -446,17 +439,22 @@ def hydrate_platform_call_import_row( previous_status = row.status row.recording_s3_key = key row.recording_content_type = content_type - row.recording_size_bytes = total + row.recording_size_bytes = size_bytes + row.recording_duration_seconds = duration_seconds if duration_seconds > 0 else None row.status = CallImportRowStatus.COMPLETED row.error_message = None - if not _safe_commit(db, row_id=row_id, context="platform_completed"): - return {"status": "skipped", "reason": "row_deleted"} + from app.services.call_imports.debounced_rollup import ( + record_import_progress_before_row_commit, + ) - maybe_record_import_row_transition( + record_import_progress_before_row_commit( call_import.id, previous_status=previous_status, new_status=row.status, ) + if not _safe_commit(db, row_id=row_id, context="platform_completed"): + return {"status": "skipped", "reason": "row_deleted"} + call_import.last_progress_at = datetime.now(timezone.utc) _rollup_parent_on_catalog(catalog_db, row_db, call_import) _safe_commit(db, row_id=row_id, context="platform_rollup") diff --git a/app/workers/tasks/process_call_import_row.py b/app/workers/tasks/process_call_import_row.py index 50899fb7..1f2ff9b5 100644 --- a/app/workers/tasks/process_call_import_row.py +++ b/app/workers/tasks/process_call_import_row.py @@ -24,6 +24,8 @@ from __future__ import annotations import mimetypes +import os +import tempfile from typing import Optional, Tuple from uuid import UUID @@ -33,10 +35,41 @@ from app.workers.config import celery_app -_RETRYABLE_COUNTDOWN_SECONDS = 60 _CREDENTIAL_AUTH_FAIL_ATTEMPT_THRESHOLD = 2 +def _fetch_recording_to_local_file( + recording_url: str, + *, + telephony_client, + use_credentialed_download: bool, +) -> Tuple[str, str]: + """Return (temp_path, content_type). Caller must unlink temp_path.""" + tmp_fd, tmp_path = tempfile.mkstemp(suffix=".audio") + os.close(tmp_fd) + try: + if use_credentialed_download: + audio_bytes, content_type = telephony_client.download_recording( + recording_url + ) + with open(tmp_path, "wb") as handle: + handle.write(audio_bytes) + return tmp_path, content_type or "audio/mpeg" + + from app.services.telephony.recording_download import ( + download_public_recording_to_path, + ) + + content_type = download_public_recording_to_path(recording_url, tmp_path) + return tmp_path, content_type + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + def _is_credentialed_auth_rejection(exc: Exception) -> bool: message = str(exc).lower() return "rejected credentials" in message and ("401" in message or "403" in message) @@ -70,26 +103,34 @@ def _mark_row_failed_for_fetch( ): from app.models.enums import CallImportRowStatus + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = message + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context=context): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context=f"rollup_{context}"): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": reason} -def _retry_countdown_for_error(exc: Exception) -> int: - retry_after = getattr(exc, "retry_after_seconds", None) - if retry_after is not None: - try: - seconds = int(retry_after) - except (TypeError, ValueError): - seconds = 0 - if seconds > 0: - return max(1, seconds) - return _RETRYABLE_COUNTDOWN_SECONDS +def _retry_countdown_for_error(exc: Exception, *, attempt: int = 0) -> int: + from app.workers.concurrency.import_retry_backoff import ( + import_transient_retry_countdown, + ) + + return import_transient_retry_countdown(exc, attempt=attempt) def _schedule_transient_retry( @@ -123,7 +164,10 @@ def _schedule_transient_retry( row.error_message = f"Transient: {exc}" if not _safe_commit(db, row_id=row_id, context=context): return {"status": "skipped", "reason": "row_deleted"} - raise self.retry(exc=exc, countdown=_retry_countdown_for_error(exc)) + attempt = int(getattr(getattr(self, "request", None), "retries", 0) or 0) + raise self.retry( + exc=exc, countdown=_retry_countdown_for_error(exc, attempt=attempt) + ) def _consume_authenticated_import_credit(client) -> Optional[Exception]: @@ -145,6 +189,26 @@ def _consume_authenticated_import_credit(client) -> Optional[Exception]: ) +def _persist_session_changes(db) -> None: + """Commit in production; flush inside nested test transactions (savepoints).""" + from app.db_sharding.row_ops import commit_shard_row_session + from app.db_sharding.sessions import is_sharding_enabled + + trans = db.get_transaction() + nested_test_tx = ( + trans is not None + and trans.parent is not None + and getattr(trans.parent, "nested", False) + ) + if nested_test_tx: + db.flush() + return + if is_sharding_enabled(): + commit_shard_row_session(db) + else: + db.commit() + + def _safe_commit( db, *, @@ -152,14 +216,8 @@ def _safe_commit( context: str = "update", ) -> bool: """Commit pending ORM changes; return False when the row/import was deleted.""" - from app.db_sharding.row_ops import commit_shard_row_session - from app.db_sharding.sessions import is_sharding_enabled - try: - if is_sharding_enabled(): - commit_shard_row_session(db) - else: - db.commit() + _persist_session_changes(db) return True except StaleDataError: db.rollback() @@ -277,15 +335,41 @@ def _rollup_parent_status(db, call_import, *, force: bool = False) -> None: schedule_debounced_import_rollup(db, call_import, force=force) +def _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + *, + previous_row_status, + new_status, +) -> None: + from app.db_sharding.sessions import is_sharding_enabled + from app.services.call_imports.debounced_rollup import ( + record_import_progress_before_row_commit, + ) + + target = catalog_db if is_sharding_enabled() and catalog_db is not row_db else row_db + record_import_progress_before_row_commit( + call_import.id, + previous_status=previous_row_status, + new_status=new_status, + db=target, + ) + + def _rollup_parent_on_catalog( - catalog_db, row_db, call_import, *, force: bool = False + catalog_db, + row_db, + call_import, + *, + force: bool = False, ) -> None: from app.db_sharding.sessions import is_sharding_enabled target = catalog_db if is_sharding_enabled() and catalog_db is not row_db else row_db _rollup_parent_status(target, call_import, force=force) if target is not row_db: - target.commit() + _persist_session_changes(target) @celery_app.task(name="process_call_import_row", bind=True, max_retries=3) @@ -454,11 +538,23 @@ def process_call_import_row_task( ) except Exception as exc: logger.exception("Failed to build provider client for row {}", row_id) + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = f"Provider client error: {exc}" + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="provider_client_error"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context="rollup_provider_client_error"): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": "provider_client_error"} @@ -479,22 +575,46 @@ def process_call_import_row_task( "Cannot fetch recording: row has no recording URL." ) logger.warning("{} (row {})", msg, row_id) + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = msg + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="no_recording_source"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit( db, row_id=row_id, context="rollup_no_recording_source" ): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": "no_recording_source"} + previous_status = row.status row.status = CallImportRowStatus.COMPLETED row.error_message = None + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="mark_completed_no_recording"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit( db, row_id=row_id, context="rollup_completed_no_recording" ): @@ -514,14 +634,17 @@ def process_call_import_row_task( # ------------------------------------------------------------------ if direct_url_mode: _release_fetch_sessions(row_db, catalog_db) - audio_bytes: Optional[bytes] = None + local_recording_path: Optional[str] = None content_type: Optional[str] = None direct_failure: Optional[Exception] = None direct_was_transient = False try: - fetched = download_public_recording(original_csv_url) - audio_bytes, content_type = fetched + local_recording_path, content_type = _fetch_recording_to_local_file( + original_csv_url, + telephony_client=client, + use_credentialed_download=False, + ) except NON_RETRYABLE_ERRORS as exc: direct_failure = exc logger.warning( @@ -545,7 +668,7 @@ def process_call_import_row_task( "Direct URL download failed (unexpected) for row {}", row_id ) - if audio_bytes is None: + if local_recording_path is None: if direct_was_transient: terminal_message = _terminal_transient_failure_message( self, row, direct_failure @@ -565,18 +688,36 @@ def process_call_import_row_task( row.error_message = f"Transient: {direct_failure}" if not _safe_commit(db, row_id=row_id, context="direct_url_transient"): return {"status": "skipped", "reason": "row_deleted"} + attempt = int( + getattr(getattr(self, "request", None), "retries", 0) or 0 + ) raise self.retry( - exc=direct_failure, countdown=_RETRYABLE_COUNTDOWN_SECONDS + exc=direct_failure, + countdown=_retry_countdown_for_error( + direct_failure, attempt=attempt + ), ) + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = ( f"recording URL: {direct_failure}" if direct_failure is not None else "Recording fetch failed" ) + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="direct_url_failed"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context="rollup_direct_url_failed"): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": "non_retryable_provider_error"} @@ -585,11 +726,9 @@ def process_call_import_row_task( else: # ------------------------------------------------------------------ # Credentialed import — download from the CSV-supplied URL only. - # Exotel and Plivo URLs require HTTP Basic auth; other providers - # use public download (e.g. presigned links). # ------------------------------------------------------------------ - audio_bytes: Optional[bytes] = None - content_type: Optional[str] = None + local_recording_path = None + content_type = None download_failure: Optional[Exception] = None download_was_transient = False use_credentialed_download = _use_credentialed_recording_download( @@ -613,13 +752,11 @@ def process_call_import_row_task( _release_fetch_sessions(row_db, catalog_db) try: - if use_credentialed_download: - fetched: Tuple[bytes, str] = client.download_recording( - original_csv_url - ) - else: - fetched = download_public_recording(original_csv_url) - audio_bytes, content_type = fetched + local_recording_path, content_type = _fetch_recording_to_local_file( + original_csv_url, + telephony_client=client, + use_credentialed_download=use_credentialed_download, + ) except NON_RETRYABLE_ERRORS as exc: download_failure = exc logger.warning( @@ -643,7 +780,7 @@ def process_call_import_row_task( "Recording download failed (unexpected) for row {}", row_id ) - if audio_bytes is None: + if local_recording_path is None: if download_was_transient: return _schedule_transient_retry( self, @@ -656,15 +793,27 @@ def process_call_import_row_task( row_db=row_db, call_import=call_import, ) + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = ( f"recording URL: {download_failure}" if download_failure is not None else "Recording fetch failed" ) + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="non_retryable_provider_error"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context="rollup_non_retryable_provider_error"): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": "non_retryable_provider_error"} @@ -675,11 +824,23 @@ def process_call_import_row_task( or "Cloud blob storage is not enabled or not configured" ) logger.error("Cloud blob storage unavailable for row {}: {}", row_id, err) + previous_status = row.status row.status = CallImportRowStatus.FAILED row.error_message = f"Cloud blob storage unavailable: {err}" + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="s3_unavailable"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context="rollup_s3_unavailable"): return {"status": "skipped", "reason": "row_deleted"} return {"status": "failed", "reason": "s3_unavailable"} @@ -693,7 +854,21 @@ def process_call_import_row_task( _release_fetch_sessions(row_db, catalog_db) try: - s3_service.upload_file_by_key(audio_bytes, key, content_type=content_type) + from app.services.call_imports.recording_storage import ( + upload_local_recording_file, + ) + + try: + size_bytes, duration_seconds = upload_local_recording_file( + local_recording_path, + key=key, + content_type=content_type or "audio/mpeg", + ) + finally: + try: + os.unlink(local_recording_path) + except OSError: + pass except Exception as exc: logger.exception("Failed to upload recording to S3 for row {}", row_id) terminal_message = _terminal_transient_failure_message(self, row, exc) @@ -713,7 +888,11 @@ def process_call_import_row_task( row.status = CallImportRowStatus.PENDING if not _safe_commit(db, row_id=row_id, context="s3_upload_retry"): return {"status": "skipped", "reason": "row_deleted"} - raise self.retry(exc=exc, countdown=_RETRYABLE_COUNTDOWN_SECONDS) + attempt = int(getattr(getattr(self, "request", None), "retries", 0) or 0) + raise self.retry( + exc=exc, + countdown=_retry_countdown_for_error(exc, attempt=attempt), + ) skip_reason = _row_or_import_gone( db, @@ -731,14 +910,26 @@ def process_call_import_row_task( row.recording_s3_key = key row.recording_content_type = content_type - row.recording_size_bytes = len(audio_bytes) + row.recording_size_bytes = size_bytes + row.recording_duration_seconds = duration_seconds if duration_seconds > 0 else None previous_import_status = row.status row.status = CallImportRowStatus.COMPLETED row.error_message = None + _record_import_progress_before_row_commit( + catalog_db, + row_db, + call_import, + previous_row_status=previous_import_status, + new_status=row.status, + ) if not _safe_commit(db, row_id=row_id, context="mark_completed"): return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) + _rollup_parent_on_catalog( + catalog_db, + row_db, + call_import, + ) if not _safe_commit(db, row_id=row_id, context="rollup_completed"): return {"status": "skipped", "reason": "row_deleted"} @@ -784,7 +975,7 @@ def process_call_import_row_task( "status": "completed", "row_id": row_id, "s3_key": key, - "size_bytes": len(audio_bytes), + "size_bytes": size_bytes, } except StaleDataError: db.rollback() @@ -808,7 +999,7 @@ def process_call_import_row_task( source_row_import_blocks_eval, ) from app.workers.tasks.evaluate_call_import_row_core import ( - EVAL_CANCELLED_BY_USER_ERROR, + is_eval_row_user_cancelled, ) user_cancelled = False @@ -822,11 +1013,7 @@ def process_call_import_row_task( pass else: try: - user_cancelled = ( - (eval_row.status or "").lower() == "failed" - and (eval_row.error_message or "") - == EVAL_CANCELLED_BY_USER_ERROR - ) + user_cancelled = is_eval_row_user_cancelled(eval_row) if user_cancelled: pass elif source_row_import_blocks_eval(source_row): diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index cbccf3c3..315d1126 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -216,11 +216,21 @@ def _generate_call_analysis(transcription, ai_providers, organization_id, result def _categorize_metrics(enabled_metrics, has_audio): """Split metrics into LLM-evaluable and audio-only categories.""" + from app.services.metrics.computed_metrics import is_computed_metric + llm_metrics = [] audio_metrics = [] skipped_scores = {} for m in enabled_metrics: + if is_computed_metric(m): + skipped_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "skipped": "not_applicable_surface", + } + continue if m.name.lower() in AUDIO_ONLY_METRIC_NAMES: if has_audio: audio_metrics.append(m) @@ -280,6 +290,139 @@ def _evaluate_llm_metrics_grouped( return scores, eval_time +def select_metrics_for_evaluation( + db, + *, + organization_id, + agent, + metric_ids=None, + workspace_id=None, +) -> list: + """Active, surface-enabled org metrics, narrowed to ``metric_ids`` when given. + + Include metrics that have the agent's surface in their enabled_surfaces so users can + restrict metrics to specific surfaces from the metrics page. Legacy rows (created + before the surfaces column existed, or via fixtures that don't set the field) keep + the original behavior: an enabled=True metric with an empty enabled_surfaces list is + treated as agent-enabled. Categorization parents in ``metric_ids`` are stored as a + single ID and expanded to child labels. + """ + from app.models.database import Metric + from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation + from app.services.metrics.surfaces import ( + metric_enabled_for_eval_surface, + metric_eval_surface_for_call_medium, + ) + + enabled_metrics = db.query(Metric).filter( + Metric.organization_id == organization_id, + Metric.enabled == True, + or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), + ).all() + eval_surface = metric_eval_surface_for_call_medium(agent.call_medium if agent else None) + enabled_metrics = [ + m for m in enabled_metrics + if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES + and metric_enabled_for_eval_surface(m.enabled_surfaces, eval_surface) + ] + + selected_metric_ids = {str(mid) for mid in (metric_ids or [])} + if selected_metric_ids: + expanded_ids = expand_metric_ids_for_evaluation( + db, + organization_id, + list(selected_metric_ids), + ) or set() + enabled_metrics = [m for m in enabled_metrics if str(m.id) in expanded_ids] + if workspace_id is not None: + enabled_metrics = [ + m + for m in enabled_metrics + if m.workspace_id is None or m.workspace_id == workspace_id + ] + return enabled_metrics + + +def evaluate_transcript_metrics( + db, + *, + result, + agent, + persona, + scenario, + metric_ids=None, +) -> tuple[dict, list]: + """Score a text-only result's transcript; returns ``(metric_scores, metrics)``. + + Used by Harbor simulation trials, which have no audio and no Evaluator row. + Also attaches call analysis to ``result.call_data``. The caller persists. + """ + from app.models.database import AIProvider + + ai_providers = db.query(AIProvider).filter( + AIProvider.organization_id == result.organization_id, + AIProvider.is_active == True, + ).all() + metrics = select_metrics_for_evaluation( + db, + organization_id=result.organization_id, + agent=agent, + metric_ids=metric_ids, + workspace_id=getattr(result, "workspace_id", None), + ) + from app.services.metrics.evaluation_scope import applicable_metrics + + agent_id = getattr(agent, "id", None) if agent is not None else None + scenario_id = getattr(scenario, "id", None) if scenario is not None else None + metrics = applicable_metrics( + metrics, agent_id=agent_id, scenario_id=scenario_id, db=db + ) + llm_metrics, _audio_metrics, metric_scores = _categorize_metrics(metrics, has_audio=False) + transcription = result.transcription or "" + from app.services.harbor.tool_checks import format_tool_calls_for_judge + + call_data_in = result.call_data if isinstance(result.call_data, dict) else {} + tool_section = format_tool_calls_for_judge(call_data_in.get("tool_calls")) + judge_transcription = f"{transcription}\n\n{tool_section}" if tool_section else transcription + if llm_metrics and transcription: + try: + llm_scores, _ = _evaluate_llm_metrics_grouped( + transcription=judge_transcription, + llm_metrics=llm_metrics, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + evaluator=None, + agent=agent, + persona=persona, + scenario=scenario, + ) + metric_scores.update(llm_scores) + except Exception as llm_err: + logger.error( + f"[EvaluatorResult {result.result_id}] LLM evaluation failed: {llm_err}", + exc_info=True, + ) + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) + + if transcription: + call_analysis = _generate_call_analysis( + transcription=transcription, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + agent=agent, + scenario=scenario, + ) + if call_analysis: + call_data = dict(result.call_data) if isinstance(result.call_data, dict) else {} + call_data["call_analysis"] = call_analysis + result.call_data = call_data + return _make_json_serializable(metric_scores), metrics + + _PERMANENT_VAPI_ENDED_REASONS = frozenset( { "call.in-progress.error-assistant-did-not-receive-customer-audio", @@ -733,46 +876,23 @@ def process_evaluator_result_task(self, result_id: str): db.commit() # Step 2: Load and categorize metrics - # Include metrics that have "agent" in their enabled_surfaces so users can - # restrict metrics to specific surfaces from the metrics page. Legacy rows - # (created before the surfaces column existed, or via fixtures that don't - # set the field) keep the original behavior: an enabled=True metric with - # an empty enabled_surfaces list is treated as agent-enabled. - enabled_metrics = db.query(Metric).filter( - Metric.organization_id == result.organization_id, - Metric.enabled == True, - or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), - ).all() - from app.services.metrics.surfaces import ( - metric_enabled_for_eval_surface, - metric_eval_surface_for_call_medium, + enabled_metrics = select_metrics_for_evaluation( + db, + organization_id=result.organization_id, + agent=agent, + metric_ids=getattr(evaluator, "metric_ids", None), + workspace_id=getattr(result, "workspace_id", None), ) - - eval_surface = metric_eval_surface_for_call_medium( - agent.call_medium if agent else None + from app.services.metrics.evaluation_scope import applicable_metrics + + agent_id = getattr(agent, "id", None) if agent is not None else None + scenario_id = getattr(scenario, "id", None) if scenario is not None else None + enabled_metrics = applicable_metrics( + enabled_metrics, + agent_id=agent_id, + scenario_id=scenario_id, + db=db, ) - enabled_metrics = [ - m for m in enabled_metrics - if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES - and metric_enabled_for_eval_surface(m.enabled_surfaces, eval_surface) - ] - - # Explicit metric selection on the evaluator/suite. Categorization - # parents are stored as a single ID and expanded to child labels here. - from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation - - selected_metric_ids = { - str(mid) for mid in (getattr(evaluator, "metric_ids", None) or []) - } - if selected_metric_ids: - expanded_ids = expand_metric_ids_for_evaluation( - db, - result.organization_id, - list(selected_metric_ids), - ) or set() - enabled_metrics = [ - m for m in enabled_metrics if str(m.id) in expanded_ids - ] has_audio = bool(result.audio_s3_key) llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) diff --git a/app/workers/tasks/run_harbor_simulation.py b/app/workers/tasks/run_harbor_simulation.py new file mode 100644 index 00000000..9e240155 --- /dev/null +++ b/app/workers/tasks/run_harbor_simulation.py @@ -0,0 +1,210 @@ +"""Celery task: run a Harbor simulation job (scenarios × personas × attempts).""" + +import os +import signal +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal +from app.models.database import Agent, SimulationRun, SimulationRunStatus +from app.workers.config import celery_app + +_POLL_SECONDS = 5 +_CANCEL_GRACE_SECONDS = 30 +_LOG_TAIL_CHARS = 2000 +_PROJECT_ROOT = Path(__file__).resolve().parents[3] + + +def _log_tail(path: Path) -> str: + try: + return path.read_text(errors="replace")[-_LOG_TAIL_CHARS:] + except OSError: + return "" + + +def _stop_process(proc: subprocess.Popen) -> None: + """SIGINT lets Harbor cancel trials and write job results; SIGKILL after a grace period.""" + proc.send_signal(signal.SIGINT) + try: + proc.wait(timeout=_CANCEL_GRACE_SECONDS) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def _queue_failure_analysis(db, run: SimulationRun) -> None: + """Cluster trials that missed their success rules once a run completes.""" + summary = run.summary if isinstance(run.summary, dict) else {} + if run.status != SimulationRunStatus.COMPLETED.value: + return + if (summary.get("n_passed") or 0) >= (summary.get("n_completed") or 0): + return + from app.workers.tasks.analyze_simulation_run_failures import ( + ANALYSIS_ID_KEY, + analyze_simulation_run_failures_task, + new_failure_analysis_state, + ) + + try: + run.metric_clusters = new_failure_analysis_state() + db.commit() + analyze_simulation_run_failures_task.delay( + str(run.id), analysis_id=run.metric_clusters[ANALYSIS_ID_KEY] + ) + except Exception as exc: + logger.warning(f"[SimulationRun {run.run_id}] Could not queue failure analysis: {exc}") + run.metric_clusters = {"status": "failed", "error_message": "Could not queue failure analysis"} + db.commit() + + +@celery_app.task( + name="run_harbor_simulation", + bind=True, + time_limit=6 * 60 * 60, + soft_time_limit=6 * 60 * 60 - 120, +) +def run_harbor_simulation_task(self, simulation_run_id: str): + from app.services.harbor.job_runner import harbor_job_dir, harbor_job_name + from app.services.harbor.result_ingest import ( + archive_run_dir, + assign_attempt_indexes, + compute_summary, + fail_unfinished_trials, + read_harbor_job_stats, + refresh_run_progress, + trial_results, + ) + from app.services.harbor.task_exporter import export_tasks, run_work_dir + from app.workers.tasks.process_evaluator_result import select_metrics_for_evaluation + + db = SessionLocal() + run_uuid = UUID(simulation_run_id) + try: + run = db.query(SimulationRun).filter(SimulationRun.id == run_uuid).first() + if not run: + logger.error(f"[SimulationRun {simulation_run_id}] Not found") + return {"error": "Simulation run not found"} + # acks_late + long runs can redeliver this task; only a queued run may start. + if run.status != SimulationRunStatus.QUEUED.value: + logger.warning( + f"[SimulationRun {run.run_id}] Status is {run.status}, not starting again" + ) + return {"status": run.status} + + run.status = SimulationRunStatus.PREPARING.value + run.celery_task_id = self.request.id + run.started_at = datetime.now(timezone.utc) + db.commit() + + cancelled = False + returncode = None + proc: subprocess.Popen | None = None + log_path = run_work_dir(run) / "harbor.log" + try: + agent = db.query(Agent).filter(Agent.id == run.agent_id).one() + metrics = select_metrics_for_evaluation( + db, + organization_id=run.organization_id, + agent=agent, + metric_ids=run.metric_ids, + workspace_id=run.workspace_id, + ) + tasks = export_tasks(db, run=run, agent=agent, metrics=metrics) + run.expected_trials = len(tasks) * run.n_attempts + run.harbor_job_name = harbor_job_name(run) + run.status = SimulationRunStatus.RUNNING.value + db.commit() + logger.info( + f"[SimulationRun {run.run_id}] Exported {len(tasks)} tasks; " + f"starting Harbor job ({run.expected_trials} trials)" + ) + + from app.workers.config import _config_path + + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join( + filter(None, [str(_PROJECT_ROOT), os.environ.get("PYTHONPATH")]) + ), + # Same config.yml this worker loaded (job_runner loads it before any DB use). + "EFFICIENTAI_CONFIG_PATH": str(_config_path.resolve()), + } + with open(log_path, "w") as log_file: + proc = subprocess.Popen( + [sys.executable, "-m", "app.services.harbor.job_runner", str(run.id)], + cwd=_PROJECT_ROOT, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + while proc.poll() is None: + time.sleep(_POLL_SECONDS) + db.refresh(run) + refresh_run_progress(db, run) + db.commit() + if run.cancel_requested and not cancelled: + logger.info(f"[SimulationRun {run.run_id}] Cancelling Harbor job") + cancelled = True + _stop_process(proc) + returncode = proc.returncode + + run.status = SimulationRunStatus.INGESTING.value + db.commit() + + results = trial_results(db, run.id) + assign_attempt_indexes(results) + fail_unfinished_trials( + results, + "Trial cancelled" if cancelled else "Trial did not finish (see Harbor job log)", + ) + summary = compute_summary(db, run, results) + summary["harbor"] = read_harbor_job_stats(harbor_job_dir(run)) + run.summary = summary + refresh_run_progress(db, run) + if run.failed_trials and not run.error_message: + preview = (summary.get("failed_trial_errors") or [{}])[0].get("error_message") + if preview: + run.error_message = ( + f"{run.failed_trials} trial(s) failed. First error: {preview[:2000]}" + ) + db.commit() + + try: + run.artifacts_s3_key = archive_run_dir(run, run_work_dir(run)) + except Exception as archive_err: + logger.warning( + f"[SimulationRun {run.run_id}] Artifact upload failed: {archive_err}" + ) + + if cancelled: + run.status = SimulationRunStatus.CANCELLED.value + elif returncode == 0: + run.status = SimulationRunStatus.COMPLETED.value + else: + run.status = SimulationRunStatus.FAILED.value + run.error_message = f"Harbor exited with code {returncode}:\n{_log_tail(log_path)}" + except Exception as exc: + if proc is not None and proc.poll() is None: + _stop_process(proc) + db.rollback() + logger.error(f"[SimulationRun {simulation_run_id}] Failed: {exc}", exc_info=True) + run = db.query(SimulationRun).filter(SimulationRun.id == run_uuid).first() + run.status = SimulationRunStatus.FAILED.value + run.error_message = str(exc) + + run.finished_at = datetime.now(timezone.utc) + db.commit() + logger.info( + f"[SimulationRun {run.run_id}] {run.status}: " + f"{run.completed_trials}/{run.expected_trials} completed, {run.failed_trials} failed" + ) + _queue_failure_analysis(db, run) + return {"simulation_run_id": simulation_run_id, "status": run.status} + finally: + db.close() diff --git a/app/workers/tasks/sweep_stale_evaluations.py b/app/workers/tasks/sweep_stale_evaluations.py new file mode 100644 index 00000000..a751e5b8 --- /dev/null +++ b/app/workers/tasks/sweep_stale_evaluations.py @@ -0,0 +1,169 @@ +"""Abort evaluations that have been running for 24 hours.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from itertools import groupby +from uuid import UUID + +from loguru import logger +from sqlalchemy import func, or_ +from sqlalchemy.orm import Session + +from app.models.database import CallImportEvaluation +from app.workers.config import EVAL_CONTROL_QUEUE, PLATFORM_WORKER_QUEUE, celery_app +from app.workers.tasks.evaluate_call_import_row_core import ( + AUTO_ABORTED_STATUS, + EVAL_AUTO_ABORTED_ERROR, +) + +STALE_RUNNING_HOURS = 24 + + +def _as_utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def stale_cutoff(now: datetime | None = None) -> datetime: + moment = _as_utc(now or datetime.now(timezone.utc)) + return moment - timedelta(hours=STALE_RUNNING_HOURS) + + +def list_stale_running_evaluation_ids( + db: Session, + *, + now: datetime | None = None, +) -> list[tuple[UUID, UUID]]: + """Parent ids for running evaluations older than 24 hours, any workspace.""" + cutoff = stale_cutoff(now) + age = func.coalesce( + CallImportEvaluation.started_at, + CallImportEvaluation.created_at, + ) + rows = ( + db.query(CallImportEvaluation.organization_id, CallImportEvaluation.id) + .filter( + CallImportEvaluation.status == "running", + or_(age.is_(None), age <= cutoff), + ) + .order_by(CallImportEvaluation.organization_id, CallImportEvaluation.id) + .all() + ) + return [ + (org_id, evaluation_id) + for org_id, evaluation_id in rows + if org_id is not None + ] + + +def evaluation_is_stale_running( + db: Session, + evaluation_id: UUID, + *, + now: datetime | None = None, +) -> bool: + row = ( + db.query( + CallImportEvaluation.status, + CallImportEvaluation.started_at, + CallImportEvaluation.created_at, + ) + .filter(CallImportEvaluation.id == evaluation_id) + .one_or_none() + ) + if row is None: + return False + status, started_at, created_at = row + if (status or "").strip().lower() != "running": + return False + age = started_at or created_at + if age is None: + return True + return _as_utc(age) <= stale_cutoff(now) + + +def enqueue_stale_evaluation_aborts( + db: Session, + *, + now: datetime | None = None, +) -> int: + from app.services.call_imports.evaluation_bulk_op import ( + clear_evaluation_bulk_operation, + try_set_evaluation_bulk_operation, + ) + + enqueued = 0 + for org_id, group in groupby( + list_stale_running_evaluation_ids(db, now=now), + key=lambda item: item[0], + ): + org_count = 0 + for _, evaluation_id in group: + if not try_set_evaluation_bulk_operation(evaluation_id, "abort"): + continue + try: + auto_abort_stale_evaluation_task.delay(str(evaluation_id)) + except Exception: + clear_evaluation_bulk_operation(evaluation_id) + logger.exception( + "Failed to enqueue auto-abort for evaluation {}", + evaluation_id, + ) + continue + org_count += 1 + if org_count: + logger.info( + "Enqueued auto-abort for {} evaluations in organization {}", + org_count, + org_id, + ) + enqueued += org_count + return enqueued + + +@celery_app.task( + name="sweep_stale_running_evaluations", + queue=PLATFORM_WORKER_QUEUE, +) +def sweep_stale_running_evaluations_task() -> int: + from app.database import SessionLocal + + db = SessionLocal() + try: + return enqueue_stale_evaluation_aborts(db) + finally: + db.close() + + +@celery_app.task( + name="auto_abort_stale_evaluation", + queue=EVAL_CONTROL_QUEUE, +) +def auto_abort_stale_evaluation_task(evaluation_id: str) -> dict: + from app.database import SessionLocal + from app.services.call_imports.bulk_ops import execute_evaluation_cancel + from app.services.call_imports.evaluation_bulk_op import ( + clear_evaluation_bulk_operation, + ) + + evaluation_uuid = UUID(evaluation_id) + db = SessionLocal() + try: + if not evaluation_is_stale_running(db, evaluation_uuid): + return { + "evaluation_id": evaluation_id, + "cancelled": 0, + "skipped": True, + } + return execute_evaluation_cancel( + db, + evaluation_uuid, + mode="abort", + row_error_message=EVAL_AUTO_ABORTED_ERROR, + sticky_status=AUTO_ABORTED_STATUS, + ) + finally: + clear_evaluation_bulk_operation(evaluation_uuid) + db.close() diff --git a/app/workers/tasks/transcribe_call_import_row.py b/app/workers/tasks/transcribe_call_import_row.py index 525a7a48..9f773279 100644 --- a/app/workers/tasks/transcribe_call_import_row.py +++ b/app/workers/tasks/transcribe_call_import_row.py @@ -1,1064 +1,1288 @@ -"""Celery task: diarise one CallImport row's audio recording. - -The call-import pipeline historically relied on the uploader to provide -transcripts via a CSV column. This task lets users fill that gap (or -add a second transcript next to one already supplied by the CSV) by -running the existing :class:`TranscriptionService` over the row's S3 -recording and storing the result on ``CallImportRow.diarised_transcript`` -— a *separate* column from the CSV-supplied production transcript so -the two values coexist and can be evaluated / exported side by side. - -Diarisation is a **two-stage LLM-augmented** pipeline: - - 1. ``TranscriptionService.transcribe`` produces plain text (no - speaker segmentation — we explicitly pass - ``enable_speaker_diarization=False`` because pyannote is no - longer in the loop). - 2. :func:`app.workers.tasks.helpers.llm_diarisation.diarize_transcript_with_llm` - hands that plain text to a chat model along with a caller-supplied - prompt and asks it to emit structured ``agent`` / ``user`` turns. - -The row stores: - - * Structured turns on ``diarised_segments`` (one entry per turn - returned by the LLM, mapped through the first-speaker-is-agent - heuristic in :func:`_segments_to_user_agent_turns` so reviewers - keep the existing per-row "Swap user ↔ agent" affordance). - * A rendered ``: `` line per turn on - ``diarised_transcript`` so the existing UI / CSV-export code that - reads the plain-text column keeps working without changes. - * ``diarised_llm_provider`` / ``diarised_llm_model`` / ``diarised_prompt`` - so reviewers can see exactly which model + instructions produced - the turns. - -If the LLM diariser fails (bad JSON, unknown provider, …) the worker -records a typed error on the row and surfaces it in the UI — there is -no silent fallback, because the operator explicitly opted into LLM- -based diarisation by picking the model in the modal. - -Provider/model for the STT step are caller-controlled — the user picks -them from the UI. We accept anything ``TranscriptionService.transcribe`` -already supports (OpenAI, Deepgram, ElevenLabs, Sarvam, Smallest, plus -local Whisper as the unconditional fallback inside that service). - -When ``run_eval_row_id`` is set the task fires the per-row evaluation -worker in a chord-like fashion: this lets the Run Evaluation modal -auto-transcribe missing transcripts and then evaluate without an extra -round-trip from the API. We do this directly (rather than via Celery's -``chord`` primitive) so the eval row only kicks off after *its own* -transcription has finished — partial failures in one row's transcription -no longer block evaluation for siblings whose transcripts succeeded. -""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from uuid import UUID - -from loguru import logger - -from app.database import SessionLocal -from app.workers.config import celery_app - - -def _now() -> datetime: - return datetime.now(timezone.utc) - - -def _segments_to_user_agent_turns( - speaker_segments: Optional[List[Dict[str, Any]]], -) -> List[Dict[str, Any]]: - """Map pyannote-labelled segments to canonical ``agent`` / ``user`` turns. - - ``TranscriptionService.transcribe(enable_speaker_diarization=True)`` - returns ``[{ "speaker": "Speaker 1", "text": "...", "start": float, - "end": float }, ...]`` — generic speaker labels with no semantic - role. We pick the canonical role for each raw label using a - deterministic, easy-to-reason-about rule: the raw speaker who - appears **earliest in time** is the ``agent``; the other becomes - the ``user``. Most call-import recordings are agent-initiated - (outbound IVR / outbound dialer / inbound greeting) so the agent - almost always speaks first; when the heuristic is wrong reviewers - flip ``diarised_speaker_swap`` from the row detail panel without - re-running the worker. - - For recordings with more than two speakers (rare, but legal — e.g. - conference calls) any extras keep their raw ``speaker_N`` label so - they're still visible in the rendered transcript without being - silently merged into ``user``. - - Each returned entry preserves ``raw_speaker`` so the swap endpoint - can rebuild the rendered string from this list alone, without - needing to re-read the original ``speaker_segments`` snapshot. - """ - if not speaker_segments: - return [] - - earliest_by_label: Dict[str, float] = {} - for seg in speaker_segments: - raw = (seg.get("speaker") or "").strip() - if not raw: - continue - start = seg.get("start") - try: - start_val = float(start) if start is not None else float("inf") - except (TypeError, ValueError): - start_val = float("inf") - if raw not in earliest_by_label or start_val < earliest_by_label[raw]: - earliest_by_label[raw] = start_val - - if not earliest_by_label: - return [] - - # Sort raw labels by first-appearance time, tie-breaking on label - # text so the mapping is deterministic across calls with the same - # transcript. The first-appearance label becomes "agent"; the next - # becomes "user"; anything beyond that keeps a generic numbered - # label so we never silently lose a speaker. - ordered_labels = sorted( - earliest_by_label.keys(), - key=lambda lbl: (earliest_by_label[lbl], lbl), - ) - role_map: Dict[str, str] = {} - for idx, raw_label in enumerate(ordered_labels): - if idx == 0: - role_map[raw_label] = "agent" - elif idx == 1: - role_map[raw_label] = "user" - else: - role_map[raw_label] = f"speaker_{idx + 1}" - - turns: List[Dict[str, Any]] = [] - for seg in speaker_segments: - raw = (seg.get("speaker") or "").strip() - text = (seg.get("text") or "").strip() - if not raw or not text: - continue - try: - start_val = float(seg.get("start") or 0.0) - except (TypeError, ValueError): - start_val = 0.0 - try: - end_val = float(seg.get("end") or 0.0) - except (TypeError, ValueError): - end_val = 0.0 - turns.append( - { - "speaker": role_map.get(raw, raw.lower().replace(" ", "_")), - "text": text, - "start": round(start_val, 3), - "end": round(end_val, 3), - "raw_speaker": raw, - } - ) - return turns - - -def _render_turns_as_text( - turns: Optional[List[Dict[str, Any]]], - *, - swap: bool = False, -) -> str: - """Render structured turns as ``: `` lines. - - The frontend's ``TranscriptView`` parses this exact shape into chat - bubbles (see ``frontend/src/pages/callImports/components/TranscriptView.tsx``). - When ``swap`` is True, ``agent`` and ``user`` are swapped — but only - those two labels, so ``speaker_3``+ on multi-party calls keep their - canonical role through a swap. - """ - if not turns: - return "" - out: List[str] = [] - for turn in turns: - speaker = (turn.get("speaker") or "").strip() - text = (turn.get("text") or "").strip() - if not speaker or not text: - continue - if swap: - if speaker == "agent": - speaker = "user" - elif speaker == "user": - speaker = "agent" - out.append(f"{speaker}: {text}") - return "\n".join(out) - - -def _plain_text_from_turns(turns: Optional[List[Dict[str, Any]]]) -> str: - """Join turn text without speaker prefixes (single-speaker fallback).""" - if not turns: - return "" - parts: List[str] = [] - for turn in turns: - text = (turn.get("text") or "").strip() - if text: - parts.append(text) - return " ".join(parts).strip() - - -# Sentinel error message stamped on rows that the operator cancelled -# via ``POST /v1/call-imports/{id}/rows/{row_id}/cancel-diarisation``. -# Kept in sync with :data:`app.api.v1.routes.call_imports.CANCELLED_BY_USER_ERROR` -# — duplicated here so the worker doesn't have to import the route -# module (which would pull FastAPI / Pydantic into the worker boot -# path). Touching either copy means touching both. -_CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" - - -def _was_cancelled_externally(db, row) -> bool: - """Re-read ``row`` from the DB and return True if it was cancelled. - - Used by every terminal-status write in this task so an external - cancel (which flips the row to ``failed`` + sets - :data:`_CANCELLED_BY_USER_ERROR`) WINS THE RACE against a worker - that's already past its slowest operation (LLM call / S3 download) - and is about to overwrite the row with its own terminal state. - - Why ``db.expire`` + ``refresh``: the worker's SQLAlchemy session - cached the row when it pulled it for the run, so a parallel - update from the FastAPI process is invisible until we explicitly - re-read. We expire just the two columns we care about so other - in-flight modifications on the same row aren't clobbered. - """ - try: - db.expire( - row, - [ - "diarised_transcript_status", - "diarised_transcript_error", - ], - ) - db.refresh( - row, - attribute_names=[ - "diarised_transcript_status", - "diarised_transcript_error", - ], - ) - except Exception: # noqa: BLE001 — refresh is best-effort - return False - return ( - (row.diarised_transcript_status or "").lower() == "failed" - and (row.diarised_transcript_error or "") == _CANCELLED_BY_USER_ERROR - ) - - -def _compact_diarisation_error(text: str, *, max_chars: int = 280) -> str: - """Keep per-row diarisation errors readable in the UI.""" - cleaned = (text or "").strip() - for sep in ("\nDetails:", "\nTraceback (most recent call last):"): - if sep in cleaned: - cleaned = cleaned.split(sep, 1)[0].strip() - if "does not accept audio input via Chat Completions" in cleaned: - provider_idx = cleaned.find("Provider error:") - if provider_idx >= 0: - cleaned = cleaned[:provider_idx].strip() - cleaned = " ".join(cleaned.split()) - if len(cleaned) > max_chars: - cleaned = cleaned[: max_chars - 1].rstrip() + "…" - return cleaned - - -def _summarize_exc(exc: BaseException, *, max_chars: int = 240) -> str: - """Pull a human-friendly one-liner out of an STT-client exception. - - Most provider SDKs surface the actually-useful detail in the *root* - cause's ``str(...)`` (Deepgram: ``DeepgramApiError: Project does - not have access to the requested model. (Status: 403)``; OpenAI: - ``BadRequestError: ...``; Sarvam: the wrapped HTTP body). We walk - the ``__cause__`` chain, prefer the deepest message, prefix the - exception class name so the operator knows the shape, and clamp - the result so the transcript_error column / UI banner stay - readable. - """ - - cur: BaseException | None = exc - last: BaseException = exc - while cur is not None: - last = cur - cur = cur.__cause__ or cur.__context__ - if cur is exc: - break - - text = str(last) or last.__class__.__name__ - text = " ".join(text.split()) - label = last.__class__.__name__ - if label and not text.lower().startswith(label.lower()): - text = f"{label}: {text}" - return _compact_diarisation_error(text, max_chars=max_chars) - - -def _apply_eval_chain_transcribe_cleanup(run_eval_row_id: str) -> None: - """Clear eval-row dispatch state after eval-chain transcribe completes.""" - from app.db_sharding.row_ops import ( - close_row_sessions, - locate_call_import_evaluation_row, - ) - from app.models.database import CallImportEvaluation - from app.workers.tasks.evaluate_call_import_row_core import ( - commit_terminal_row_and_rollup, - ) - - try: - row_db, catalog_db, eval_row, source_row, _ = ( - locate_call_import_evaluation_row(UUID(run_eval_row_id)) - ) - except LookupError: - return - try: - eval_row.celery_task_id = None - previous_status = eval_row.status or "pending" - marked_failed = False - if ( - (source_row.diarised_transcript_status or "").lower() == "failed" - and eval_row.status == "pending" - ): - eval_row.status = "failed" - eval_row.error_message = ( - source_row.diarised_transcript_error or "Diarisation failed" - ) - eval_row.finished_at = _now() - marked_failed = True - - if marked_failed: - parent_db = catalog_db if catalog_db is not None else row_db - evaluation = ( - parent_db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == eval_row.evaluation_id) - .first() - ) - if evaluation is not None: - commit_terminal_row_and_rollup( - row_db, - evaluation, - eval_row, - previous_row_status=previous_status, - catalog_db=( - catalog_db - if catalog_db is not None and catalog_db is not row_db - else None - ), - ) - else: - row_db.commit() - else: - row_db.commit() - finally: - close_row_sessions(row_db, catalog_db) - - -def _was_cancelled_by_row_id(row_id: str | UUID) -> bool: - """Re-read the row in a short-lived session during slow I/O.""" - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, catalog_db, row, _ = locate_call_import_row(row_id) - except LookupError: - return False - try: - return _was_cancelled_externally(row_db, row) - finally: - close_row_sessions(row_db, catalog_db) - - -def _persist_diarization_failure( - row_id: str | UUID, - error_message: str, - *, - reason: str, -) -> dict[str, Any]: - """Write a terminal diarisation failure without holding a long session.""" - from app.models.database import CallImportRow - - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, catalog_db, row, _ = locate_call_import_row(row_id) - except LookupError: - return {"status": "skipped", "reason": "row_not_found"} - try: - if _was_cancelled_externally(row_db, row): - return {"status": "cancelled", "reason": "cancelled_by_user"} - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = error_message - row_db.commit() - return {"status": "failed", "reason": reason} - finally: - close_row_sessions(row_db, catalog_db) - - -def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: - """STT / S3 / LLM diarisation without a long-lived DB session.""" - from app.models.enums import ModelProvider - from app.services.usage.call_import_context import call_import_row_usage_context - from app.services.usage.context import llm_usage_context - from app.workers.tasks.helpers.llm_diarisation import ( - LLMDiarisationError, - diarize_audio_with_llm, - diarize_transcript_with_llm, - ) - - row_id = ctx["row_id"] - normalised_mode = ctx["normalised_mode"] - recording_key = ctx["recording_key"] - organization_id = ctx["organization_id"] - llm_provider_value = ctx["llm_provider_value"] - llm_model_value = ctx["llm_model_value"] - llm_credential_uuid = ctx["llm_credential_uuid"] - effective_prompt = ctx["effective_prompt"] - - evaluation_id = ctx.get("evaluation_id") - call_import_id = ctx.get("call_import_id") - if not call_import_id: - raise ValueError("call_import_id missing from transcribe pipeline context") - - with llm_usage_context( - call_import_row_usage_context( - organization_id=organization_id, - workspace_id=ctx.get("workspace_id"), - call_import_id=call_import_id, - evaluation_id=evaluation_id, - ) - ): - return _run_diarization_pipeline_inner( - ctx, - row_id=row_id, - normalised_mode=normalised_mode, - recording_key=recording_key, - organization_id=organization_id, - llm_provider_value=llm_provider_value, - llm_model_value=llm_model_value, - llm_credential_uuid=llm_credential_uuid, - effective_prompt=effective_prompt, - ModelProvider=ModelProvider, - LLMDiarisationError=LLMDiarisationError, - diarize_audio_with_llm=diarize_audio_with_llm, - diarize_transcript_with_llm=diarize_transcript_with_llm, - ) - - -def _run_diarization_pipeline_inner( - ctx: dict[str, Any], - *, - row_id, - normalised_mode, - recording_key, - organization_id, - llm_provider_value, - llm_model_value, - llm_credential_uuid, - effective_prompt, - ModelProvider, - LLMDiarisationError, - diarize_audio_with_llm, - diarize_transcript_with_llm, -) -> dict[str, Any]: - plain_text: Optional[str] = None - raw_turns: Optional[List[Dict[str, Any]]] = None - - if normalised_mode == "stt_llm": - from app.services.ai.transcription_service import transcription_service - - provider_enum = ModelProvider(ctx["stt_provider"]) - stt_model = ctx["stt_model"] - credential_uuid = ctx["credential_uuid"] - language = ctx["language"] - - try: - stt_db = SessionLocal() - try: - result = transcription_service.transcribe( - audio_file_key=recording_key, - stt_provider=provider_enum, - stt_model=stt_model, - organization_id=organization_id, - db=stt_db, - language=language, - enable_speaker_diarization=False, - credential_id=credential_uuid, - ) - finally: - stt_db.close() - except Exception as exc: # noqa: BLE001 - want message on the row - logger.exception( - "transcribe_call_import_row failed for row {}", row_id - ) - if _was_cancelled_by_row_id(row_id): - logger.info( - "Row {} was cancelled by the user mid-flight; " - "preserving cancelled state instead of writing " - "STT failure.", - row_id, - ) - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - _summarize_exc(exc), - reason="transcription_error", - ) - - plain_text = (result.get("transcript") or "").strip() or None - if not plain_text: - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - "Transcription returned an empty result.", - reason="empty_transcript", - ) - - try: - llm_db = SessionLocal() - try: - raw_turns = diarize_transcript_with_llm( - plain_text, - llm_provider=llm_provider_value, - llm_model=llm_model_value, - organization_id=organization_id, - db=llm_db, - custom_prompt=effective_prompt, - credential_id=llm_credential_uuid, - ) - finally: - llm_db.close() - except LLMDiarisationError as exc: - logger.warning( - "LLM diarisation failed for row {}: {}", row_id, exc - ) - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - _compact_diarisation_error(str(exc)), - reason="llm_diarisation_error", - ) - except Exception as exc: # noqa: BLE001 — same surfacing as STT - logger.exception( - "LLM diarisation crashed for row {}", row_id - ) - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - _summarize_exc(exc), - reason="llm_diarisation_error", - ) - else: - from app.services.storage.s3_service import s3_service - - try: - audio_bytes = s3_service.download_file_by_key(recording_key) - except Exception as exc: # noqa: BLE001 — surfaced to the row - logger.exception( - "Failed to fetch recording for LLM-only diarise on row {}", - row_id, - ) - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - "Failed to download recording from storage: " - f"{_summarize_exc(exc)}", - reason="recording_download_error", - ) - - try: - llm_db = SessionLocal() - try: - raw_turns = diarize_audio_with_llm( - audio_bytes, - llm_provider=llm_provider_value, - llm_model=llm_model_value, - organization_id=organization_id, - db=llm_db, - custom_prompt=effective_prompt, - credential_id=llm_credential_uuid, - audio_file_key=recording_key, - ) - finally: - llm_db.close() - except LLMDiarisationError as exc: - logger.warning( - "LLM-only diarisation failed for row {}: {}", row_id, exc - ) - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - _compact_diarisation_error(str(exc)), - reason="llm_only_diarisation_error", - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "LLM-only diarisation crashed for row {}", row_id - ) - if _was_cancelled_by_row_id(row_id): - return {"status": "cancelled", "reason": "cancelled_by_user"} - return _persist_diarization_failure( - row_id, - _summarize_exc(exc), - reason="llm_only_diarisation_error", - ) - - if not raw_turns: - logger.info( - "transcribe_call_import_row: row {} LLM-only diariser " - "returned no turns (no audible speech detected); " - "marking diarisation completed with empty transcript.", - row_id, - ) - if _was_cancelled_by_row_id(row_id): - return { - "status": "cancelled", - "reason": "cancelled_by_user", - } - return { - "status": "success", - "reason": "no_speech_detected", - "plain_text": None, - "raw_turns": [], - } - - return { - "status": "success", - "reason": "pipeline_complete", - "plain_text": plain_text, - "raw_turns": raw_turns, - } - - -def _finalize_diarization_row( - row_id: str | UUID, - *, - pipeline_ctx: dict[str, Any], - work_result: dict[str, Any], -) -> dict[str, Any]: - """Persist diarisation results after slow I/O completes.""" - from app.models.database import CallImportRow - from app.models.enums import ModelProvider - - normalised_mode = pipeline_ctx["normalised_mode"] - llm_provider_value = pipeline_ctx["llm_provider_value"] - llm_model_value = pipeline_ctx["llm_model_value"] - effective_prompt = pipeline_ctx["effective_prompt"] - stt_model = pipeline_ctx.get("stt_model") - provider_enum: Optional[ModelProvider] = None - if pipeline_ctx.get("stt_provider"): - provider_enum = ModelProvider(pipeline_ctx["stt_provider"]) - - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, catalog_db, row, _ = locate_call_import_row(row_id) - except LookupError: - return {"status": "skipped", "reason": "row_not_found"} - try: - if work_result.get("reason") == "no_speech_detected": - if _was_cancelled_externally(row_db, row): - return {"status": "cancelled", "reason": "cancelled_by_user"} - row.diarised_transcript = "" - row.diarised_segments = [] - row.diarised_speaker_swap = False - row.transcribe_mode = normalised_mode - row.diarised_transcript_status = "completed" - row.diarised_transcript_error = ( - "No audible speech detected in the recording; " - "diariser returned no turns." - ) - row.diarised_prompt = effective_prompt - row.diarised_at = _now() - row_db.commit() - return { - "status": "completed", - "row_id": str(row_id), - "mode": normalised_mode, - "reason": "no_speech_detected", - "turn_count": 0, - "characters": 0, - } - - plain_text = work_result.get("plain_text") - raw_turns = work_result.get("raw_turns") or [] - turns = _segments_to_user_agent_turns(raw_turns) - - distinct_roles = {turn.get("speaker") for turn in turns} - has_real_diarisation = len(distinct_roles - {None, ""}) >= 2 - rendered_turns = ( - _render_turns_as_text(turns, swap=False) - if turns and has_real_diarisation - else "" - ) - - transcript_to_store = ( - rendered_turns or plain_text or _plain_text_from_turns(turns) or "" - ).strip() - - if not transcript_to_store: - logger.info( - "transcribe_call_import_row: row {} diariser produced no " - "storable transcript; marking completed with empty transcript.", - row_id, - ) - if _was_cancelled_externally(row_db, row): - return { - "status": "cancelled", - "reason": "cancelled_by_user", - } - row.diarised_transcript = "" - row.diarised_segments = turns if has_real_diarisation else None - row.diarised_speaker_swap = False - row.transcribe_mode = normalised_mode - row.diarised_transcript_status = "completed" - row.diarised_transcript_error = ( - "Diariser returned no storable transcript text." - ) - row.diarised_prompt = effective_prompt - row.diarised_at = _now() - row_db.commit() - return { - "status": "completed", - "row_id": str(row_id), - "mode": normalised_mode, - "reason": "empty_transcript", - "turn_count": len(turns) if has_real_diarisation else 0, - "characters": 0, - } - - if _was_cancelled_externally(row_db, row): - logger.info( - "Row {} was cancelled by the user mid-flight; " - "skipping success write and preserving cancelled state.", - row_id, - ) - return {"status": "cancelled", "reason": "cancelled_by_user"} - - row.diarised_transcript = transcript_to_store - row.diarised_segments = turns if has_real_diarisation else None - row.diarised_speaker_swap = False - if normalised_mode == "stt_llm" and provider_enum is not None: - row.diarised_transcript_provider = provider_enum.value - row.diarised_transcript_model = stt_model - row.transcribe_mode = normalised_mode - row.diarised_transcript_status = "completed" - row.diarised_transcript_error = None - row.diarised_prompt = effective_prompt - row.diarised_at = _now() - row_db.commit() - - return { - "status": "completed", - "row_id": str(row_id), - "mode": normalised_mode, - "provider": provider_enum.value if provider_enum else "llm_only", - "model": stt_model if normalised_mode == "stt_llm" else llm_model_value, - "llm_provider": llm_provider_value, - "llm_model": llm_model_value, - "characters": len(transcript_to_store), - "turn_count": len(turns) if has_real_diarisation else 0, - } - finally: - close_row_sessions(row_db, catalog_db) - - -@celery_app.task( - name="transcribe_call_import_row", - bind=True, - max_retries=2, - time_limit=15 * 60, - soft_time_limit=12 * 60, -) -def transcribe_call_import_row_task( - self, - row_id: str, - stt_provider: Optional[str] = None, - stt_model: Optional[str] = None, - credential_id: Optional[str] = None, - language: Optional[str] = None, - overwrite_existing: bool = False, - run_eval_row_id: Optional[str] = None, - diarization_llm_provider: Optional[str] = None, - diarization_llm_model: Optional[str] = None, - diarization_llm_credential_id: Optional[str] = None, - diarization_prompt: Optional[str] = None, - mode: str = "stt_llm", - eval_restricted_metric_ids: Optional[List[str]] = None, - _eval_slot_task_id: Optional[str] = None, -): - """Diarise a single row's recording. - - Skips rows with no S3 recording, or with an existing diarised - transcript when ``overwrite_existing`` is False. Errors are recorded - on the row (``diarised_transcript_status='failed'`` + - ``diarised_transcript_error``) so the UI can surface a per-row - diagnostic without polling Celery directly. The CSV-supplied - production ``transcript`` is never touched by this task. - - Two pipeline shapes: - - * ``mode="stt_llm"`` (default, backward-compat) — STT produces plain - text, then ``diarization_llm_*`` splits it into structured - ``agent`` / ``user`` turns. ``stt_provider`` / ``stt_model`` are - required in this mode. - * ``mode="llm_only"`` — STT is skipped; the audio bytes are sent - directly to a multimodal ``diarization_llm_*`` model along with - ``diarization_prompt`` for a single-pass transcribe + diarise. - The STT kwargs are ignored. - - The ``diarization_llm_*`` kwargs identify the chat model that - produces the structured turns. They are required in both modes: - the worker fails the row when either ``diarization_llm_provider`` - or ``diarization_llm_model`` is missing. - """ - - from app.models.database import CallImportRow - from app.models.enums import ModelProvider - from app.workers.tasks.helpers.llm_diarisation import ( - DEFAULT_DIARIZATION_PROMPT, - ) - - evaluation_id_for_dispatch: Optional[str] = None - slot_task_id = _eval_slot_task_id or self.request.id - pipeline_ctx: dict[str, Any] | None = None - try: - from app.db_sharding.row_ops import ( - close_row_sessions, - locate_call_import_evaluation_row, - locate_call_import_row, - ) - - row_db = catalog_db = None - eval_row_for_chain = None - try: - if run_eval_row_id: - try: - _er_db, _cat, eval_row_for_chain, _, _ = ( - locate_call_import_evaluation_row(UUID(run_eval_row_id)) - ) - from app.workers.tasks.evaluate_call_import_row_core import ( - was_cancelled_externally, - ) - - if was_cancelled_externally(_er_db, eval_row_for_chain): - close_row_sessions(_er_db, _cat) - return { - "status": "skipped", - "reason": "eval_cancelled_by_user", - } - evaluation_id_for_dispatch = str( - eval_row_for_chain.evaluation_id - ) - close_row_sessions(_er_db, _cat) - eval_row_for_chain = None - except LookupError: - pass - - row_db, catalog_db, row, _shard_id = locate_call_import_row(row_id) - row_uuid = row.id - if row is None: - logger.warning( - "transcribe_call_import_row: row {} not found, skipping", - row_id, - ) - return {"status": "skipped", "reason": "row_not_found"} - - if ( - (row.diarised_transcript_status or "").lower() == "failed" - and (row.diarised_transcript_error or "") == _CANCELLED_BY_USER_ERROR - ): - logger.info( - "transcribe_call_import_row: row {} already cancelled by " - "user (likely a redelivery after SIGTERM revoke); skipping", - row_id, - ) - return {"status": "skipped", "reason": "cancelled_by_user"} - - existing_diarised = (row.diarised_transcript or "").strip() - if existing_diarised and not overwrite_existing: - row.diarised_transcript_status = "completed" - row_db.commit() - return { - "status": "skipped", - "reason": "transcript_present", - "row_id": row_id, - } - - recording_key = (row.recording_s3_key or "").strip() - if not recording_key: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = ( - "No recording available for this row; cannot diarise." - ) - row_db.commit() - return {"status": "skipped", "reason": "no_recording"} - - normalised_mode = (mode or "stt_llm").strip().lower() - if normalised_mode not in {"stt_llm", "llm_only"}: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = ( - f"Unknown diarisation mode '{mode}'. Expected " - "'stt_llm' or 'llm_only'." - ) - row_db.commit() - return {"status": "failed", "reason": "unknown_mode"} - - provider_enum: Optional[ModelProvider] = None - if normalised_mode == "stt_llm": - if not stt_provider or not stt_model: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = ( - "STT provider/model not configured. Pick an STT " - "model in the Diarise modal." - ) - row_db.commit() - return {"status": "failed", "reason": "missing_stt"} - try: - provider_enum = ModelProvider(stt_provider.lower()) - except ValueError: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = ( - f"Unknown STT provider '{stt_provider}'." - ) - row_db.commit() - return {"status": "failed", "reason": "unknown_provider"} - - llm_provider_value = (diarization_llm_provider or "").strip() - llm_model_value = (diarization_llm_model or "").strip() - if not llm_provider_value or not llm_model_value: - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = ( - "Diarisation LLM provider/model not configured. Pick " - "a chat model in the Diarise modal." - ) - row_db.commit() - return {"status": "failed", "reason": "missing_llm_diariser"} - - row.diarised_transcript_status = "running" - row.diarised_transcript_error = None - if normalised_mode == "stt_llm" and provider_enum is not None: - row.diarised_transcript_provider = provider_enum.value - row.diarised_transcript_model = stt_model - else: - row.diarised_transcript_provider = "llm_only" - row.diarised_transcript_model = llm_model_value - row.diarised_llm_provider = llm_provider_value - row.diarised_llm_model = llm_model_value - try: - llm_credential_uuid = ( - UUID(diarization_llm_credential_id) - if diarization_llm_credential_id - else None - ) - except (TypeError, ValueError): - llm_credential_uuid = None - row.diarised_llm_credential_id = llm_credential_uuid - row.celery_task_id = self.request.id - - effective_prompt = (diarization_prompt or "").strip() or ( - DEFAULT_DIARIZATION_PROMPT - ) - row.diarised_prompt = effective_prompt - - try: - credential_uuid = UUID(credential_id) if credential_id else None - except (TypeError, ValueError): - credential_uuid = None - - row_db.commit() - - pipeline_ctx = { - "row_id": row_id, - "normalised_mode": normalised_mode, - "recording_key": recording_key, - "organization_id": row.organization_id, - "workspace_id": getattr(row, "workspace_id", None), - "call_import_id": row.call_import_id, - "evaluation_id": ( - UUID(evaluation_id_for_dispatch) - if evaluation_id_for_dispatch - else None - ), - "evaluation_row_id": ( - UUID(run_eval_row_id) if run_eval_row_id else None - ), - "stt_provider": provider_enum.value if provider_enum else None, - "stt_model": stt_model, - "credential_uuid": credential_uuid, - "language": language, - "llm_provider_value": llm_provider_value, - "llm_model_value": llm_model_value, - "llm_credential_uuid": llm_credential_uuid, - "effective_prompt": effective_prompt, - } - finally: - if row_db is not None: - close_row_sessions(row_db, catalog_db) - - if pipeline_ctx is None: - return {"status": "skipped", "reason": "setup_incomplete"} - - work_result = _run_diarization_pipeline(pipeline_ctx) - if work_result["status"] in {"failed", "cancelled"}: - return work_result - - return _finalize_diarization_row( - row_id, - pipeline_ctx=pipeline_ctx, - work_result=work_result, - ) - except Exception as exc: # noqa: BLE001 — terminal row state + no retry loop - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - logger.exception( - "transcribe_call_import_row crashed for row {}", row_id - ) - try: - row_db, catalog_db, row, _ = locate_call_import_row(row_id) - except LookupError: - return {"status": "failed", "reason": "unexpected_error"} - try: - if _was_cancelled_externally(row_db, row): - return {"status": "cancelled", "reason": "cancelled_by_user"} - if (row.diarised_transcript_status or "").lower() == "running": - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = _summarize_exc(exc) - row_db.commit() - except Exception: - logger.exception( - "Failed to persist unexpected-error state for row {}", - row_id, - ) - try: - row_db.rollback() - except Exception: - pass - finally: - close_row_sessions(row_db, catalog_db) - return {"status": "failed", "reason": "unexpected_error"} - finally: - if run_eval_row_id: - try: - _apply_eval_chain_transcribe_cleanup(run_eval_row_id) - except Exception: - logger.exception( - "Failed to finalize eval-chain transcribe cleanup for row {}", - run_eval_row_id, - ) - if run_eval_row_id: - from app.workers.concurrency.fair_dispatch import ( - finish_eval_work_and_redispatch, - ) - - finish_eval_work_and_redispatch( - slot_task_id, - restricted_metric_ids=eval_restricted_metric_ids, - ) - else: - from app.workers.concurrency.limits import slot_registered_for_task - - if slot_registered_for_task(slot_task_id): - from app.workers.concurrency.fair_diarization_dispatch import ( - finish_diarization_work_and_redispatch, - ) - - finish_diarization_work_and_redispatch(slot_task_id) +"""Celery task: diarise one CallImport row's audio recording. + +The call-import pipeline historically relied on the uploader to provide +transcripts via a CSV column. This task lets users fill that gap (or +add a second transcript next to one already supplied by the CSV) by +running the existing :class:`TranscriptionService` over the row's S3 +recording and storing the result on ``CallImportRow.diarised_transcript`` +— a *separate* column from the CSV-supplied production transcript so +the two values coexist and can be evaluated / exported side by side. + +Diarisation is a **two-stage LLM-augmented** pipeline: + + 1. ``TranscriptionService.transcribe`` produces plain text (no + speaker segmentation — we explicitly pass + ``enable_speaker_diarization=False`` because pyannote is no + longer in the loop). + 2. :func:`app.workers.tasks.helpers.llm_diarisation.diarize_transcript_with_llm` + hands that plain text to a chat model along with a caller-supplied + prompt and asks it to emit structured ``agent`` / ``user`` turns. + +The row stores: + + * Structured turns on ``diarised_segments`` (one entry per turn + returned by the LLM, mapped through the first-speaker-is-agent + heuristic in :func:`_segments_to_user_agent_turns` so reviewers + keep the existing per-row "Swap user ↔ agent" affordance). + * A rendered ``: `` line per turn on + ``diarised_transcript`` so the existing UI / CSV-export code that + reads the plain-text column keeps working without changes. + * ``diarised_llm_provider`` / ``diarised_llm_model`` / ``diarised_prompt`` + so reviewers can see exactly which model + instructions produced + the turns. + +If the LLM diariser fails (bad JSON, unknown provider, …) the worker +records a typed error on the row and surfaces it in the UI — there is +no silent fallback, because the operator explicitly opted into LLM- +based diarisation by picking the model in the modal. + +Provider/model for the STT step are caller-controlled — the user picks +them from the UI. We accept anything ``TranscriptionService.transcribe`` +already supports (OpenAI, Deepgram, ElevenLabs, Sarvam, Smallest, plus +local Whisper as the unconditional fallback inside that service). + +When ``run_eval_row_id`` is set the task fires the per-row evaluation +worker in a chord-like fashion: this lets the Run Evaluation modal +auto-transcribe missing transcripts and then evaluate without an extra +round-trip from the API. We do this directly (rather than via Celery's +``chord`` primitive) so the eval row only kicks off after *its own* +transcription has finished — partial failures in one row's transcription +no longer block evaluation for siblings whose transcripts succeeded. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal +from app.workers.config import celery_app + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _segments_to_user_agent_turns( + speaker_segments: Optional[List[Dict[str, Any]]], + *, + keep_native: bool = False, +) -> List[Dict[str, Any]]: + """Map pyannote-labelled segments to canonical ``agent`` / ``user`` turns. + + ``TranscriptionService.transcribe(enable_speaker_diarization=True)`` + returns ``[{ "speaker": "Speaker 1", "text": "...", "start": float, + "end": float }, ...]`` — generic speaker labels with no semantic + role. We pick the canonical role for each raw label using a + deterministic, easy-to-reason-about rule: the raw speaker who + appears **earliest in time** is the ``agent``; the other becomes + the ``user``. Most call-import recordings are agent-initiated + (outbound IVR / outbound dialer / inbound greeting) so the agent + almost always speaks first; when the heuristic is wrong reviewers + flip ``diarised_speaker_swap`` from the row detail panel without + re-running the worker. + + For recordings with more than two speakers (rare, but legal — e.g. + conference calls) any extras keep their raw ``speaker_N`` label so + they're still visible in the rendered transcript without being + silently merged into ``user``. + + Each returned entry preserves ``raw_speaker`` so the swap endpoint + can rebuild the rendered string from this list alone, without + needing to re-read the original ``speaker_segments`` snapshot. + """ + if not speaker_segments: + return [] + + earliest_by_label: Dict[str, float] = {} + for seg in speaker_segments: + raw = (seg.get("speaker") or "").strip() + if not raw: + continue + start = seg.get("start") + try: + start_val = float(start) if start is not None else float("inf") + except (TypeError, ValueError): + start_val = float("inf") + if raw not in earliest_by_label or start_val < earliest_by_label[raw]: + earliest_by_label[raw] = start_val + + if not earliest_by_label: + return [] + + # Sort raw labels by first-appearance time, tie-breaking on label + # text so the mapping is deterministic across calls with the same + # transcript. The first-appearance label becomes "agent"; the next + # becomes "user"; anything beyond that keeps a generic numbered + # label so we never silently lose a speaker. + ordered_labels = sorted( + earliest_by_label.keys(), + key=lambda lbl: (earliest_by_label[lbl], lbl), + ) + role_map: Dict[str, str] = {} + for idx, raw_label in enumerate(ordered_labels): + if idx == 0: + role_map[raw_label] = "agent" + elif idx == 1: + role_map[raw_label] = "user" + else: + role_map[raw_label] = f"speaker_{idx + 1}" + + turns: List[Dict[str, Any]] = [] + for seg in speaker_segments: + raw = (seg.get("speaker") or "").strip() + text = (seg.get("text") or "").strip() + if not raw or not text: + continue + try: + start_val = float(seg.get("start") or 0.0) + except (TypeError, ValueError): + start_val = 0.0 + try: + end_val = float(seg.get("end") or 0.0) + except (TypeError, ValueError): + end_val = 0.0 + turn: Dict[str, Any] = { + "speaker": role_map.get(raw, raw.lower().replace(" ", "_")), + "text": text, + "start": round(start_val, 3), + "end": round(end_val, 3), + "raw_speaker": raw, + } + if keep_native: + native_text = (seg.get("text_native") or "").strip() + if native_text: + turn["text_native"] = native_text + turns.append(turn) + return turns + + +def _render_turns_as_text( + turns: Optional[List[Dict[str, Any]]], + *, + swap: bool = False, +) -> str: + """Render structured turns as ``: `` lines. + + The frontend's ``TranscriptView`` parses this exact shape into chat + bubbles (see ``frontend/src/pages/callImports/components/TranscriptView.tsx``). + When ``swap`` is True, ``agent`` and ``user`` are swapped — but only + those two labels, so ``speaker_3``+ on multi-party calls keep their + canonical role through a swap. + """ + if not turns: + return "" + out: List[str] = [] + for turn in turns: + speaker = (turn.get("speaker") or "").strip() + text = (turn.get("text") or "").strip() + if not speaker or not text: + continue + if swap: + if speaker == "agent": + speaker = "user" + elif speaker == "user": + speaker = "agent" + out.append(f"{speaker}: {text}") + return "\n".join(out) + + +def _plain_text_from_turns(turns: Optional[List[Dict[str, Any]]]) -> str: + """Join turn text without speaker prefixes (single-speaker fallback).""" + if not turns: + return "" + parts: List[str] = [] + for turn in turns: + text = (turn.get("text") or "").strip() + if text: + parts.append(text) + return " ".join(parts).strip() + + +# Sentinel error message stamped on rows that the operator cancelled +# via ``POST /v1/call-imports/{id}/rows/{row_id}/cancel-diarisation``. +# Kept in sync with :data:`app.api.v1.routes.call_imports.CANCELLED_BY_USER_ERROR` +# — duplicated here so the worker doesn't have to import the route +# module (which would pull FastAPI / Pydantic into the worker boot +# path). Touching either copy means touching both. +_CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" + + +def _was_cancelled_externally(db, row) -> bool: + """Re-read ``row`` from the DB and return True if it was cancelled. + + Used by every terminal-status write in this task so an external + cancel (which flips the row to ``failed`` + sets + :data:`_CANCELLED_BY_USER_ERROR`) WINS THE RACE against a worker + that's already past its slowest operation (LLM call / S3 download) + and is about to overwrite the row with its own terminal state. + + Why ``db.expire`` + ``refresh``: the worker's SQLAlchemy session + cached the row when it pulled it for the run, so a parallel + update from the FastAPI process is invisible until we explicitly + re-read. We expire just the two columns we care about so other + in-flight modifications on the same row aren't clobbered. + """ + try: + db.expire( + row, + [ + "diarised_transcript_status", + "diarised_transcript_error", + ], + ) + db.refresh( + row, + attribute_names=[ + "diarised_transcript_status", + "diarised_transcript_error", + ], + ) + except Exception: # noqa: BLE001 — refresh is best-effort + return False + return ( + (row.diarised_transcript_status or "").lower() == "failed" + and (row.diarised_transcript_error or "") == _CANCELLED_BY_USER_ERROR + ) + + +def _compact_diarisation_error(text: str, *, max_chars: int = 280) -> str: + """Keep per-row diarisation errors readable in the UI.""" + cleaned = (text or "").strip() + for sep in ("\nDetails:", "\nTraceback (most recent call last):"): + if sep in cleaned: + cleaned = cleaned.split(sep, 1)[0].strip() + if "does not accept audio input via Chat Completions" in cleaned: + provider_idx = cleaned.find("Provider error:") + if provider_idx >= 0: + cleaned = cleaned[:provider_idx].strip() + cleaned = " ".join(cleaned.split()) + if len(cleaned) > max_chars: + cleaned = cleaned[: max_chars - 1].rstrip() + "…" + return cleaned + + +def _summarize_exc(exc: BaseException, *, max_chars: int = 240) -> str: + """Pull a human-friendly one-liner out of an STT-client exception. + + Most provider SDKs surface the actually-useful detail in the *root* + cause's ``str(...)`` (Deepgram: ``DeepgramApiError: Project does + not have access to the requested model. (Status: 403)``; OpenAI: + ``BadRequestError: ...``; Sarvam: the wrapped HTTP body). We walk + the ``__cause__`` chain, prefer the deepest message, prefix the + exception class name so the operator knows the shape, and clamp + the result so the transcript_error column / UI banner stay + readable. + """ + + cur: BaseException | None = exc + last: BaseException = exc + while cur is not None: + last = cur + cur = cur.__cause__ or cur.__context__ + if cur is exc: + break + + text = str(last) or last.__class__.__name__ + text = " ".join(text.split()) + label = last.__class__.__name__ + if label and not text.lower().startswith(label.lower()): + text = f"{label}: {text}" + return _compact_diarisation_error(text, max_chars=max_chars) + + +def _owner_task_id(task_id: str | None) -> str: + return (task_id or "").strip() + + +def _diarisation_update_if_owned( + row_db, + row_id: str | UUID, + task_id: str | None, + values: dict, + *, + require_pending: bool = False, + row=None, +) -> bool: + """Apply row updates only when this Celery task still owns the row. + + Ownership needs a task id to compare against. Direct ``.run()`` calls + (tests, repair scripts) carry none, so they fall back to writing the + loaded ``row`` object directly — the pre-ownership behaviour — instead of + reporting ``superseded`` for work nobody else could own. + + Terminal statuses (``completed`` / ``failed``) always clear + ``celery_task_id`` so fair dispatch can enqueue future work and rows + are never left ``pending`` with a stale lock after a successful run. + """ + from sqlalchemy import update + + from app.models.database import CallImportRow + + cleaned = _owner_task_id(task_id) + values_to_apply = dict(values) + status_val = values_to_apply.get("diarised_transcript_status") + if isinstance(status_val, str) and status_val.lower() in ("completed", "failed"): + values_to_apply["celery_task_id"] = None + + if not cleaned: + if row is None: + return False + # No Celery id means nobody can own the row: this is a direct + # ``.run()`` call (test, repair script) where the caller already holds + # the session and row, so write through instead of reporting + # ``superseded``. + for column, value in values_to_apply.items(): + setattr(row, column, value) + row_db.commit() + return True + rid = row_id if isinstance(row_id, UUID) else UUID(str(row_id)) + filters = [ + CallImportRow.id == rid, + CallImportRow.celery_task_id == cleaned, + ] + if require_pending: + filters.append(CallImportRow.diarised_transcript_status == "pending") + result = row_db.execute( + update(CallImportRow).where(*filters).values(**values_to_apply) + ) + row_db.commit() + return result.rowcount == 1 + + +def _claim_diarisation_running( + row_db, + row_id: str | UUID, + task_id: str | None, + values: dict, + *, + row=None, +) -> bool: + return _diarisation_update_if_owned( + row_db, + row_id, + task_id, + { + "diarised_transcript_status": "running", + "diarised_transcript_error": None, + **values, + }, + require_pending=True, + row=row, + ) + + +def _apply_eval_chain_transcribe_cleanup(run_eval_row_id: str) -> None: + """Clear eval-row dispatch state after eval-chain transcribe completes.""" + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + ) + from app.models.database import CallImportEvaluation + from app.workers.tasks.evaluate_call_import_row_core import ( + commit_terminal_row_and_rollup, + ) + + try: + row_db, catalog_db, eval_row, source_row, _ = ( + locate_call_import_evaluation_row(UUID(run_eval_row_id)) + ) + except LookupError: + return + try: + eval_row.celery_task_id = None + previous_status = eval_row.status or "pending" + marked_failed = False + if ( + (source_row.diarised_transcript_status or "").lower() == "failed" + and eval_row.status == "pending" + ): + eval_row.status = "failed" + eval_row.error_message = ( + source_row.diarised_transcript_error or "Diarisation failed" + ) + eval_row.finished_at = _now() + marked_failed = True + + if marked_failed: + parent_db = catalog_db if catalog_db is not None else row_db + evaluation = ( + parent_db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == eval_row.evaluation_id) + .first() + ) + if evaluation is not None: + commit_terminal_row_and_rollup( + row_db, + evaluation, + eval_row, + previous_row_status=previous_status, + catalog_db=( + catalog_db + if catalog_db is not None and catalog_db is not row_db + else None + ), + ) + else: + row_db.commit() + else: + # Eval-chain transcribe stores the Celery id on the source row for + # ownership; clear it once this worker finishes so bulk/fair + # diarisation dispatch is not blocked (dispatch only considers rows + # with ``celery_task_id IS NULL``). + if (source_row.diarised_transcript_status or "").lower() == "completed": + source_row.celery_task_id = None + row_db.commit() + finally: + close_row_sessions(row_db, catalog_db) + + +def _was_cancelled_by_row_id(row_id: str | UUID) -> bool: + """Re-read the row in a short-lived session during slow I/O.""" + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, catalog_db, row, _ = locate_call_import_row(row_id) + except LookupError: + return False + try: + return _was_cancelled_externally(row_db, row) + finally: + close_row_sessions(row_db, catalog_db) + + +def _persist_diarization_failure( + row_id: str | UUID, + error_message: str, + *, + reason: str, + task_id: str | None = None, +) -> dict[str, Any]: + """Write a terminal diarisation failure without holding a long session.""" + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, catalog_db, row, _ = locate_call_import_row(row_id) + except LookupError: + return {"status": "skipped", "reason": "row_not_found"} + try: + if _was_cancelled_externally(row_db, row): + return {"status": "cancelled", "reason": "cancelled_by_user"} + values = { + "diarised_transcript_status": "failed", + "diarised_transcript_error": error_message, + } + if task_id is not None: + if not _diarisation_update_if_owned( + row_db, row_id, task_id, values, row=row + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "failed", "reason": reason} + row.diarised_transcript_status = "failed" + row.diarised_transcript_error = error_message + row_db.commit() + return {"status": "failed", "reason": reason} + finally: + close_row_sessions(row_db, catalog_db) + + +def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: + """STT / S3 / LLM diarisation without a long-lived DB session.""" + from app.models.enums import ModelProvider + from app.services.usage.call_import_context import call_import_row_usage_context + from app.services.usage.context import llm_usage_context + from app.workers.tasks.helpers.llm_diarisation import ( + LLMDiarisationError, + diarize_audio_with_llm, + diarize_transcript_with_llm, + ) + + row_id = ctx["row_id"] + normalised_mode = ctx["normalised_mode"] + recording_key = ctx["recording_key"] + organization_id = ctx["organization_id"] + llm_provider_value = ctx["llm_provider_value"] + llm_model_value = ctx["llm_model_value"] + llm_credential_uuid = ctx["llm_credential_uuid"] + effective_prompt = ctx["effective_prompt"] + + evaluation_id = ctx.get("evaluation_id") + call_import_id = ctx.get("call_import_id") + if not call_import_id: + raise ValueError("call_import_id missing from transcribe pipeline context") + + with llm_usage_context( + call_import_row_usage_context( + organization_id=organization_id, + workspace_id=ctx.get("workspace_id"), + call_import_id=call_import_id, + evaluation_id=evaluation_id, + ) + ): + return _run_diarization_pipeline_inner( + ctx, + row_id=row_id, + normalised_mode=normalised_mode, + recording_key=recording_key, + organization_id=organization_id, + llm_provider_value=llm_provider_value, + llm_model_value=llm_model_value, + llm_credential_uuid=llm_credential_uuid, + effective_prompt=effective_prompt, + ModelProvider=ModelProvider, + LLMDiarisationError=LLMDiarisationError, + diarize_audio_with_llm=diarize_audio_with_llm, + diarize_transcript_with_llm=diarize_transcript_with_llm, + ) + + +def _run_diarization_pipeline_inner( + ctx: dict[str, Any], + *, + row_id, + normalised_mode, + recording_key, + organization_id, + llm_provider_value, + llm_model_value, + llm_credential_uuid, + effective_prompt, + ModelProvider, + LLMDiarisationError, + diarize_audio_with_llm, + diarize_transcript_with_llm, +) -> dict[str, Any]: + plain_text: Optional[str] = None + raw_turns: Optional[List[Dict[str, Any]]] = None + + if normalised_mode == "stt_llm": + from app.services.ai.transcription_service import transcription_service + + provider_enum = ModelProvider(ctx["stt_provider"]) + stt_model = ctx["stt_model"] + credential_uuid = ctx["credential_uuid"] + language = ctx["language"] + + try: + stt_db = SessionLocal() + try: + result = transcription_service.transcribe( + audio_file_key=recording_key, + stt_provider=provider_enum, + stt_model=stt_model, + organization_id=organization_id, + db=stt_db, + language=language, + enable_speaker_diarization=False, + credential_id=credential_uuid, + ) + finally: + stt_db.close() + except Exception as exc: # noqa: BLE001 - want message on the row + logger.exception( + "transcribe_call_import_row failed for row {}", row_id + ) + if _was_cancelled_by_row_id(row_id): + logger.info( + "Row {} was cancelled by the user mid-flight; " + "preserving cancelled state instead of writing " + "STT failure.", + row_id, + ) + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + _summarize_exc(exc), + reason="transcription_error", + task_id=ctx.get("owner_task_id"), + ) + + plain_text = (result.get("transcript") or "").strip() or None + if not plain_text: + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + "Transcription returned an empty result.", + reason="empty_transcript", + task_id=ctx.get("owner_task_id"), + ) + + try: + llm_db = SessionLocal() + try: + raw_turns = diarize_transcript_with_llm( + plain_text, + llm_provider=llm_provider_value, + llm_model=llm_model_value, + organization_id=organization_id, + db=llm_db, + custom_prompt=ctx.get("operator_prompt"), + credential_id=llm_credential_uuid, + emit_native_text=bool(ctx.get("emit_native_text")), + ) + finally: + llm_db.close() + except LLMDiarisationError as exc: + logger.warning( + "LLM diarisation failed for row {}: {}", row_id, exc + ) + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + _compact_diarisation_error(str(exc)), + reason="llm_diarisation_error", + task_id=ctx.get("owner_task_id"), + ) + except Exception as exc: # noqa: BLE001 — same surfacing as STT + logger.exception( + "LLM diarisation crashed for row {}", row_id + ) + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + _summarize_exc(exc), + reason="llm_diarisation_error", + task_id=ctx.get("owner_task_id"), + ) + else: + from app.services.storage.s3_service import s3_service + + try: + audio_bytes = s3_service.download_file_by_key(recording_key) + except Exception as exc: # noqa: BLE001 — surfaced to the row + logger.exception( + "Failed to fetch recording for LLM-only diarise on row {}", + row_id, + ) + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + "Failed to download recording from storage: " + f"{_summarize_exc(exc)}", + reason="recording_download_error", + task_id=ctx.get("owner_task_id"), + ) + + try: + llm_db = SessionLocal() + try: + raw_turns = diarize_audio_with_llm( + audio_bytes, + llm_provider=llm_provider_value, + llm_model=llm_model_value, + organization_id=organization_id, + db=llm_db, + custom_prompt=ctx.get("operator_prompt"), + credential_id=llm_credential_uuid, + audio_file_key=recording_key, + emit_native_text=bool(ctx.get("emit_native_text")), + ) + finally: + llm_db.close() + except LLMDiarisationError as exc: + logger.warning( + "LLM-only diarisation failed for row {}: {}", row_id, exc + ) + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + _compact_diarisation_error(str(exc)), + reason="llm_only_diarisation_error", + task_id=ctx.get("owner_task_id"), + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "LLM-only diarisation crashed for row {}", row_id + ) + if _was_cancelled_by_row_id(row_id): + return {"status": "cancelled", "reason": "cancelled_by_user"} + return _persist_diarization_failure( + row_id, + _summarize_exc(exc), + reason="llm_only_diarisation_error", + task_id=ctx.get("owner_task_id"), + ) + + if not raw_turns: + logger.info( + "transcribe_call_import_row: row {} LLM-only diariser " + "returned no turns (no audible speech detected); " + "marking diarisation completed with empty transcript.", + row_id, + ) + if _was_cancelled_by_row_id(row_id): + return { + "status": "cancelled", + "reason": "cancelled_by_user", + } + return { + "status": "success", + "reason": "no_speech_detected", + "plain_text": None, + "raw_turns": [], + } + + return { + "status": "success", + "reason": "pipeline_complete", + "plain_text": plain_text, + "raw_turns": raw_turns, + } + + +def _mark_diarisation_pass(pipeline_ctx: dict[str, Any], row_id: str | UUID) -> None: + """Record that this row finished diarisation for the current eval pass.""" + evaluation_id = pipeline_ctx.get("evaluation_id") + if not evaluation_id: + return + try: + from app.workers.concurrency.fair_dispatch import mark_row_diarised_in_pass + + mark_row_diarised_in_pass(evaluation_id, row_id) + except Exception: # noqa: BLE001 — marker is advisory, never fail the row + logger.exception( + "Failed to mark row {} diarised in pass for evaluation {}", + row_id, + evaluation_id, + ) + + +def _finalize_diarization_row( + row_id: str | UUID, + *, + pipeline_ctx: dict[str, Any], + work_result: dict[str, Any], +) -> dict[str, Any]: + """Persist diarisation results after slow I/O completes.""" + from app.models.enums import ModelProvider + + normalised_mode = pipeline_ctx["normalised_mode"] + llm_provider_value = pipeline_ctx["llm_provider_value"] + llm_model_value = pipeline_ctx["llm_model_value"] + effective_prompt = pipeline_ctx["effective_prompt"] + stt_model = pipeline_ctx.get("stt_model") + owner_task_id = pipeline_ctx.get("owner_task_id") + provider_enum: Optional[ModelProvider] = None + if pipeline_ctx.get("stt_provider"): + provider_enum = ModelProvider(pipeline_ctx["stt_provider"]) + + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, catalog_db, row, _ = locate_call_import_row(row_id) + except LookupError: + return {"status": "skipped", "reason": "row_not_found"} + + def _complete_owned(values: dict[str, Any]) -> dict[str, Any] | None: + if not _diarisation_update_if_owned( + row_db, row_id, owner_task_id, values, row=row + ): + return {"status": "skipped", "reason": "superseded"} + _mark_diarisation_pass(pipeline_ctx, row_id) + return None + + try: + emit_native_text = bool(pipeline_ctx.get("emit_native_text")) + if work_result.get("reason") == "no_speech_detected": + if _was_cancelled_externally(row_db, row): + return {"status": "cancelled", "reason": "cancelled_by_user"} + superseded = _complete_owned( + { + "diarised_transcript": "", + "diarised_segments": [], + "diarised_native_transcript": None, + "diarised_speaker_swap": False, + "transcribe_mode": normalised_mode, + "diarised_transcript_status": "completed", + "diarised_transcript_error": ( + "No audible speech detected in the recording; " + "diariser returned no turns." + ), + "diarised_prompt": effective_prompt, + "diarised_at": _now(), + } + ) + if superseded is not None: + return superseded + return { + "status": "completed", + "row_id": str(row_id), + "mode": normalised_mode, + "reason": "no_speech_detected", + "turn_count": 0, + "characters": 0, + } + + plain_text = work_result.get("plain_text") + raw_turns = work_result.get("raw_turns") or [] + native_transcript = None + if emit_native_text and normalised_mode == "stt_llm": + native_transcript = (plain_text or "").strip() or None + turns = _segments_to_user_agent_turns( + raw_turns, + keep_native=emit_native_text, + ) + + distinct_roles = {turn.get("speaker") for turn in turns} + has_real_diarisation = len(distinct_roles - {None, ""}) >= 2 + rendered_turns = ( + _render_turns_as_text(turns, swap=False) + if turns and has_real_diarisation + else "" + ) + + transcript_to_store = ( + rendered_turns or plain_text or _plain_text_from_turns(turns) or "" + ).strip() + + if not transcript_to_store: + logger.info( + "transcribe_call_import_row: row {} diariser produced no " + "storable transcript; marking completed with empty transcript.", + row_id, + ) + if _was_cancelled_externally(row_db, row): + return { + "status": "cancelled", + "reason": "cancelled_by_user", + } + superseded = _complete_owned( + { + "diarised_transcript": "", + "diarised_segments": turns if has_real_diarisation else None, + "diarised_native_transcript": None, + "diarised_speaker_swap": False, + "transcribe_mode": normalised_mode, + "diarised_transcript_status": "completed", + "diarised_transcript_error": ( + "Diariser returned no storable transcript text." + ), + "diarised_prompt": effective_prompt, + "diarised_at": _now(), + } + ) + if superseded is not None: + return superseded + return { + "status": "completed", + "row_id": str(row_id), + "mode": normalised_mode, + "reason": "empty_transcript", + "turn_count": len(turns) if has_real_diarisation else 0, + "characters": 0, + } + + if _was_cancelled_externally(row_db, row): + logger.info( + "Row {} was cancelled by the user mid-flight; " + "skipping success write and preserving cancelled state.", + row_id, + ) + return {"status": "cancelled", "reason": "cancelled_by_user"} + + success_values: dict[str, Any] = { + "diarised_transcript": transcript_to_store, + "diarised_segments": turns if has_real_diarisation else None, + "diarised_native_transcript": native_transcript, + "diarised_speaker_swap": False, + "transcribe_mode": normalised_mode, + "diarised_transcript_status": "completed", + "diarised_transcript_error": None, + "diarised_prompt": effective_prompt, + "diarised_at": _now(), + } + if normalised_mode == "stt_llm" and provider_enum is not None: + success_values["diarised_transcript_provider"] = provider_enum.value + success_values["diarised_transcript_model"] = stt_model + superseded = _complete_owned(success_values) + if superseded is not None: + return superseded + + return { + "status": "completed", + "row_id": str(row_id), + "mode": normalised_mode, + "provider": provider_enum.value if provider_enum else "llm_only", + "model": stt_model if normalised_mode == "stt_llm" else llm_model_value, + "llm_provider": llm_provider_value, + "llm_model": llm_model_value, + "characters": len(transcript_to_store), + "turn_count": len(turns) if has_real_diarisation else 0, + } + finally: + close_row_sessions(row_db, catalog_db) + + +@celery_app.task( + name="transcribe_call_import_row", + bind=True, + max_retries=2, + time_limit=15 * 60, + soft_time_limit=12 * 60, +) +def transcribe_call_import_row_task( + self, + row_id: str, + stt_provider: Optional[str] = None, + stt_model: Optional[str] = None, + credential_id: Optional[str] = None, + language: Optional[str] = None, + overwrite_existing: bool = False, + run_eval_row_id: Optional[str] = None, + diarization_llm_provider: Optional[str] = None, + diarization_llm_model: Optional[str] = None, + diarization_llm_credential_id: Optional[str] = None, + diarization_prompt: Optional[str] = None, + mode: str = "stt_llm", + eval_restricted_metric_ids: Optional[List[str]] = None, + _eval_slot_task_id: Optional[str] = None, + emit_native_text: bool = False, +): + """Diarise a single row's recording. + + Skips rows with no S3 recording, or with an existing diarised + transcript when ``overwrite_existing`` is False. Errors are recorded + on the row (``diarised_transcript_status='failed'`` + + ``diarised_transcript_error``) so the UI can surface a per-row + diagnostic without polling Celery directly. The CSV-supplied + production ``transcript`` is never touched by this task. + + Two pipeline shapes: + + * ``mode="stt_llm"`` (default, backward-compat) — STT produces plain + text, then ``diarization_llm_*`` splits it into structured + ``agent`` / ``user`` turns. ``stt_provider`` / ``stt_model`` are + required in this mode. + * ``mode="llm_only"`` — STT is skipped; the audio bytes are sent + directly to a multimodal ``diarization_llm_*`` model along with + ``diarization_prompt`` for a single-pass transcribe + diarise. + The STT kwargs are ignored. + + The ``diarization_llm_*`` kwargs identify the chat model that + produces the structured turns. They are required in both modes: + the worker fails the row when either ``diarization_llm_provider`` + or ``diarization_llm_model`` is missing. + """ + + from app.models.enums import ModelProvider + from app.workers.tasks.helpers.llm_diarisation import ( + resolve_diarisation_system_prompt, + ) + + evaluation_id_for_dispatch: Optional[str] = None + # ``self.request.id`` is only present under a real Celery request. Direct + # ``.run()`` calls (tests, repair scripts) pass the row id as ``self``, so + # resolve the owning task id defensively. + request = getattr(self, "request", None) + slot_task_id: Optional[str] = _eval_slot_task_id or getattr(request, "id", None) + pipeline_ctx: dict[str, Any] | None = None + try: + from app.db_sharding.row_ops import ( + close_row_sessions, + locate_call_import_evaluation_row, + locate_call_import_row, + ) + + row_db = catalog_db = None + eval_row_for_chain = None + try: + if run_eval_row_id: + try: + _er_db, _cat, eval_row_for_chain, _, _ = ( + locate_call_import_evaluation_row(UUID(run_eval_row_id)) + ) + from app.workers.tasks.evaluate_call_import_row_core import ( + was_cancelled_externally, + ) + + if was_cancelled_externally(_er_db, eval_row_for_chain): + close_row_sessions(_er_db, _cat) + return { + "status": "skipped", + "reason": "eval_cancelled_by_user", + } + evaluation_id_for_dispatch = str( + eval_row_for_chain.evaluation_id + ) + close_row_sessions(_er_db, _cat) + eval_row_for_chain = None + except LookupError: + pass + + row_db, catalog_db, row, _shard_id = locate_call_import_row(row_id) + row_uuid = row.id + if row is None: + logger.warning( + "transcribe_call_import_row: row {} not found, skipping", + row_id, + ) + return {"status": "skipped", "reason": "row_not_found"} + + if ( + (row.diarised_transcript_status or "").lower() == "failed" + and (row.diarised_transcript_error or "") == _CANCELLED_BY_USER_ERROR + ): + logger.info( + "transcribe_call_import_row: row {} already cancelled by " + "user (likely a redelivery after SIGTERM revoke); skipping", + row_id, + ) + return {"status": "skipped", "reason": "cancelled_by_user"} + + existing_diarised = (row.diarised_transcript or "").strip() + if existing_diarised and not overwrite_existing: + if _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + {"diarised_transcript_status": "completed"}, + row=row, + ): + return { + "status": "skipped", + "reason": "transcript_present", + "row_id": row_id, + } + return {"status": "skipped", "reason": "superseded"} + + recording_key = (row.recording_s3_key or "").strip() + if not recording_key: + if not _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": ( + "No recording available for this row; cannot diarise." + ), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "skipped", "reason": "no_recording"} + + normalised_mode = (mode or "stt_llm").strip().lower() + if normalised_mode not in {"stt_llm", "llm_only"}: + if not _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": ( + f"Unknown diarisation mode '{mode}'. Expected " + "'stt_llm' or 'llm_only'." + ), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "failed", "reason": "unknown_mode"} + + provider_enum: Optional[ModelProvider] = None + if normalised_mode == "stt_llm": + if not stt_provider or not stt_model: + if not _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": ( + "STT provider/model not configured. Pick an STT " + "model in the Diarise modal." + ), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "failed", "reason": "missing_stt"} + try: + provider_enum = ModelProvider(stt_provider.lower()) + except ValueError: + if not _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": ( + f"Unknown STT provider '{stt_provider}'." + ), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "failed", "reason": "unknown_provider"} + + llm_provider_value = (diarization_llm_provider or "").strip() + llm_model_value = (diarization_llm_model or "").strip() + if not llm_provider_value or not llm_model_value: + if not _diarisation_update_if_owned( + row_db, + row_uuid, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": ( + "Diarisation LLM provider/model not configured. Pick " + "a chat model in the Diarise modal." + ), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + return {"status": "failed", "reason": "missing_llm_diariser"} + + try: + stt_credential_uuid = UUID(credential_id) if credential_id else None + except (TypeError, ValueError): + stt_credential_uuid = None + try: + llm_credential_uuid = ( + UUID(diarization_llm_credential_id) + if diarization_llm_credential_id + else None + ) + except (TypeError, ValueError): + llm_credential_uuid = None + + operator_prompt = (diarization_prompt or "").strip() or None + effective_prompt = resolve_diarisation_system_prompt( + operator_prompt, + emit_native_text=bool(emit_native_text), + ) + + claim_values: dict[str, Any] = { + "diarised_llm_provider": llm_provider_value, + "diarised_llm_model": llm_model_value, + "diarised_llm_credential_id": llm_credential_uuid, + "diarised_prompt": effective_prompt, + } + if normalised_mode == "stt_llm" and provider_enum is not None: + claim_values["diarised_transcript_provider"] = provider_enum.value + claim_values["diarised_transcript_model"] = stt_model + else: + claim_values["diarised_transcript_provider"] = "llm_only" + claim_values["diarised_transcript_model"] = llm_model_value + + if not _claim_diarisation_running( + row_db, row_uuid, slot_task_id, claim_values, row=row + ): + return {"status": "skipped", "reason": "superseded"} + + pipeline_ctx = { + "row_id": row_id, + "owner_task_id": slot_task_id, + "normalised_mode": normalised_mode, + "recording_key": recording_key, + "organization_id": row.organization_id, + "workspace_id": getattr(row, "workspace_id", None), + "call_import_id": row.call_import_id, + "evaluation_id": ( + UUID(evaluation_id_for_dispatch) + if evaluation_id_for_dispatch + else None + ), + "evaluation_row_id": ( + UUID(run_eval_row_id) if run_eval_row_id else None + ), + "stt_provider": provider_enum.value if provider_enum else None, + "stt_model": stt_model, + "credential_uuid": stt_credential_uuid, + "language": language, + "llm_provider_value": llm_provider_value, + "llm_model_value": llm_model_value, + "llm_credential_uuid": llm_credential_uuid, + "effective_prompt": effective_prompt, + "operator_prompt": operator_prompt, + "emit_native_text": bool(emit_native_text), + } + finally: + if row_db is not None: + close_row_sessions(row_db, catalog_db) + + if pipeline_ctx is None: + return {"status": "skipped", "reason": "setup_incomplete"} + + work_result = _run_diarization_pipeline(pipeline_ctx) + if work_result["status"] in {"failed", "cancelled"}: + return work_result + + return _finalize_diarization_row( + row_id, + pipeline_ctx=pipeline_ctx, + work_result=work_result, + ) + except Exception as exc: # noqa: BLE001 — terminal row state + no retry loop + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + logger.exception( + "transcribe_call_import_row crashed for row {}", row_id + ) + try: + row_db, catalog_db, row, _ = locate_call_import_row(row_id) + except LookupError: + return {"status": "failed", "reason": "unexpected_error"} + try: + if _was_cancelled_externally(row_db, row): + return {"status": "cancelled", "reason": "cancelled_by_user"} + if not _diarisation_update_if_owned( + row_db, + row_id, + slot_task_id, + { + "diarised_transcript_status": "failed", + "diarised_transcript_error": _summarize_exc(exc), + }, + row=row, + ): + return {"status": "skipped", "reason": "superseded"} + except Exception: + logger.exception( + "Failed to persist unexpected-error state for row {}", + row_id, + ) + try: + row_db.rollback() + except Exception: + pass + finally: + close_row_sessions(row_db, catalog_db) + return {"status": "failed", "reason": "unexpected_error"} + finally: + if run_eval_row_id: + try: + _apply_eval_chain_transcribe_cleanup(run_eval_row_id) + except Exception: + logger.exception( + "Failed to finalize eval-chain transcribe cleanup for row {}", + run_eval_row_id, + ) + if run_eval_row_id: + from app.workers.concurrency.fair_dispatch import ( + finish_eval_work_and_redispatch, + ) + + finish_eval_work_and_redispatch( + slot_task_id, + restricted_metric_ids=eval_restricted_metric_ids, + ) + else: + from app.workers.concurrency.limits import slot_registered_for_task + + if slot_registered_for_task(slot_task_id): + from app.workers.concurrency.fair_diarization_dispatch import ( + finish_diarization_work_and_redispatch, + ) + + finish_diarization_work_and_redispatch(slot_task_id) diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py index 2c8968ac..2f466c28 100644 --- a/app/workers/tasks/tts_comparison.py +++ b/app/workers/tasks/tts_comparison.py @@ -38,94 +38,10 @@ def _compute_wer_cer(ground_truth: str, predicted: str): Normalized scores reduce false penalties on numeric/currency phrasing differences (for example "$1,234.56" vs "one thousand two hundred..."). """ - try: - from jiwer import cer, wer - except ImportError: - logger.warning("[TTS Eval] jiwer not installed – skipping WER/CER") - return { - "raw_wer": None, - "raw_cer": None, - "normalized_wer": None, - "normalized_cer": None, - } - - number_words = { - "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", - "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", - "seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", "fifty", - "sixty", "seventy", "eighty", "ninety", "hundred", "thousand", "million", - "billion", "trillion", "point", "and", - } - currency_words = { - "dollar", "dollars", "usd", "cent", "cents", "rupee", "rupees", "inr", - "euro", "euros", "eur", "pound", "pounds", "gbp", - } - - def _is_numeric_token(token: str) -> bool: - return bool(re.fullmatch(r"\d+(?:\.\d+)?", token)) - - def _normalize_base(text: str) -> str: - punct_table = str.maketrans("", "", string.punctuation) - return text.lower().translate(punct_table).strip() - - def _normalize_entities(text: str) -> str: - tokens = _normalize_base(text).split() - normalized_tokens = [] - i = 0 - while i < len(tokens): - token = tokens[i] - is_entity_token = ( - _is_numeric_token(token) - or token in number_words - or token in currency_words - ) - if not is_entity_token: - normalized_tokens.append(token) - i += 1 - continue - - j = i - has_currency = token in currency_words - while j < len(tokens): - t = tokens[j] - if _is_numeric_token(t) or t in number_words or t in currency_words: - if t in currency_words: - has_currency = True - j += 1 - continue - break - - normalized_tokens.append("" if has_currency else "") - i = j - - return " ".join(normalized_tokens) - - ref = _normalize_base(ground_truth) - hyp = _normalize_base(predicted) - - if not ref: - return { - "raw_wer": None, - "raw_cer": None, - "normalized_wer": None, - "normalized_cer": None, - } + from app.services.evaluation.transcript_error_rates import compute_tts_wer_cer try: - raw_wer = round(wer(ref, hyp), 4) - raw_cer = round(cer(ref, hyp), 4) - - norm_ref = _normalize_entities(ground_truth) - norm_hyp = _normalize_entities(predicted) - normalized_wer = round(wer(norm_ref, norm_hyp), 4) if norm_ref else None - normalized_cer = round(cer(norm_ref, norm_hyp), 4) if norm_ref else None - - return { - "raw_wer": raw_wer, - "raw_cer": raw_cer, - "normalized_wer": normalized_wer, - "normalized_cer": normalized_cer, - } + return compute_tts_wer_cer(ground_truth, predicted) except Exception as e: logger.warning(f"[TTS Eval] WER/CER calculation error: {e}") return { diff --git a/config.docker.yml b/config.docker.yml index 1c6c3aa6..1757efd9 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -78,8 +78,8 @@ diarization: # CORS Settings cors: origins: - - "http://localhost:3000" - "http://localhost:8000" + - "https://sandbox.efficientai.cloud" # API Settings api: diff --git a/config.yml.example b/config.yml.example index e7845a3c..d455192e 100644 --- a/config.yml.example +++ b/config.yml.example @@ -145,8 +145,13 @@ s3: secret_access_key: "your-secret-access-key" endpoint_url: null # Optional: for S3-compatible services (e.g., MinIO, DigitalOcean Spaces) prefix: "audio/" # Prefix for audio files in bucket - -# GCS Configuration (alternative to S3 when storage.blob_provider is gcs) + # + # Browser call-import uploads (CSV/XLSX/audio) use presigned PUT directly to + # this bucket from the app origin. FastAPI CORS is not enough — the bucket + # must allow your frontend origins too. Example: + # ./scripts/apply_s3_call_import_cors.sh --config config.yml + # # or: aws s3api put-bucket-cors --bucket YOUR_BUCKET --cors-configuration file://scripts/s3-bucket-cors-call-imports.example.json + # Add production app URL to AllowedOrigins; use the same rules on MinIO/R2 CORS UI. gcs: enabled: false bucket_name: "your-gcs-bucket-name" @@ -177,8 +182,9 @@ diarization: # CORS Settings cors: origins: - - "http://localhost:3000" - "http://localhost:8000" + - "https://sandbox.efficientai.cloud" + # Add http://localhost:3000 only if you run `npm run dev` in frontend/ (Vite), not docker :8000 # API Settings api: diff --git a/docker-compose.yml b/docker-compose.yml index 03a6f35b..80e6606b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: db: - image: postgres:15 + image: pgvector/pgvector:pg15 container_name: efficientai_db environment: POSTGRES_USER: ${POSTGRES_USER:-efficientai} @@ -307,6 +307,39 @@ services: - ./config.docker.yml:/app/config.yml:ro command: eai worker --config /app/config.yml --loglevel info --queues traces --pool threads --concurrency 8 + # Harbor simulation runs (test agent ↔ production agent). Harbor needs Python 3.12, + # so this worker builds its own image. Each task runs one Harbor job subprocess that + # fans out trials itself (SimulationRun.n_concurrent), hence the low concurrency. + # Exported tasks + job dirs land in ./.data/harbor (shared with the api for trajectories). + worker-harbor: + image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest}-harbor + build: + context: . + dockerfile: docker/Dockerfile.worker + args: + PYTHON_VERSION: "3.12" + INSTALL_EXTRAS: "harbor" + TORCH_CPU_ONLY: "false" + container_name: efficientai_worker_harbor + env_file: + - .env + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} + REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ./uploads:/app/uploads + - ./.data:/app/.data + - ./config.docker.yml:/app/config.yml:ro + command: eai worker --config /app/config.yml --loglevel info --queues harbor-sim --concurrency 2 + volumes: postgres_data: clickhouse_data: diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 2db20550..ad4a80ee 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -1,4 +1,5 @@ -FROM python:3.11-slim +ARG PYTHON_VERSION=3.11 +FROM python:${PYTHON_VERSION}-slim # Build arguments for configurable installs ARG INSTALL_EXTRAS="qualitative-voice" diff --git a/docs-fumadocs/components/enterprise-comparison.tsx b/docs-fumadocs/components/enterprise-comparison.tsx index 5f3f8b1c..558909ae 100644 --- a/docs-fumadocs/components/enterprise-comparison.tsx +++ b/docs-fumadocs/components/enterprise-comparison.tsx @@ -74,6 +74,11 @@ const SECTIONS: ComparisonSection[] = [ oss: { type: 'no' }, enterprise: { type: 'yes' }, }, + { + feature: 'Scenario metrics', + oss: { type: 'no' }, + enterprise: { type: 'yes' }, + }, { feature: 'Alerts', oss: { type: 'no' }, diff --git a/docs-fumadocs/content/docs/products/call-imports.mdx b/docs-fumadocs/content/docs/products/call-imports.mdx index da4ef357..21da6468 100644 --- a/docs-fumadocs/content/docs/products/call-imports.mdx +++ b/docs-fumadocs/content/docs/products/call-imports.mdx @@ -41,6 +41,12 @@ Abort and force-fail run on the `eval-control` queue (before `evaluations` on `w When running locally via `eai start` or `eai worker`, the imports worker must consume **`imports,diarization,eval-control,evaluations`**. If `eval-control` is missing, Run Evaluation will enqueue materialize tasks that never run and recording imports will not start. +## Presigned uploads and S3 CORS + +CSV/XLSX staging and manual audio uploads PUT files **from the browser straight to object storage** (presigned URL). Your **S3 bucket** (or R2/MinIO equivalent) must allow CORS from the app origin (`http://localhost:8000`, production URL, etc.). API `cors.origins` alone does not fix this. + +Example AWS CLI: `scripts/s3-bucket-cors-call-imports.example.json` — see comments in `config.yml.example` under `s3:`. + ## Database sharding (enterprise scale) For large batches (10k+ rows), call-import row data can be spread across multiple PostgreSQL data shards with a catalog database for metadata and routing. See [Call Import Sharding](/docs/operations/call-import-sharding/) for the full system design, configuration, and scaling guide. diff --git a/docs/scale-audit/01-Capacity-1-million-a-day.docx b/docs/scale-audit/01-Capacity-1-million-a-day.docx new file mode 100644 index 00000000..2328486a Binary files /dev/null and b/docs/scale-audit/01-Capacity-1-million-a-day.docx differ diff --git a/docs/scale-audit/02-Code-shape.docx b/docs/scale-audit/02-Code-shape.docx new file mode 100644 index 00000000..18c3efe2 Binary files /dev/null and b/docs/scale-audit/02-Code-shape.docx differ diff --git a/docs/scale-audit/03-Design-patterns.docx b/docs/scale-audit/03-Design-patterns.docx new file mode 100644 index 00000000..cf7d1ab3 Binary files /dev/null and b/docs/scale-audit/03-Design-patterns.docx differ diff --git a/docs/scale-audit/EfficientAI-Scale-Audit.docx b/docs/scale-audit/EfficientAI-Scale-Audit.docx new file mode 100644 index 00000000..bd276dda Binary files /dev/null and b/docs/scale-audit/EfficientAI-Scale-Audit.docx differ diff --git a/docs/scale-audit/PLATFORM-SCALE-AUDIT.md b/docs/scale-audit/PLATFORM-SCALE-AUDIT.md new file mode 100644 index 00000000..09b5e42d --- /dev/null +++ b/docs/scale-audit/PLATFORM-SCALE-AUDIT.md @@ -0,0 +1,1482 @@ +# EfficientAI platform scale audit + +**Date:** 26 October 2026 +**Audience:** engineering, for a working discussion +**Target:** 1 million simulations a day, 1 million production calls a day, and 1 million live traces a day, at the same time, as a multi-tenant SaaS product +**How this was done:** a read of the backend (`app/`), the frontend (`frontend/src/`, about 146,000 lines), workers, sharding, and `docker-compose.yml`. This is not a load test. Every capacity number below is calculated from the caps and worker settings that ship in the repo. Measure them before treating them as a promise. + +The product is deployed on Kubernetes. This repository has no Kubernetes manifests, replica counts, CPU limits, or database settings for that cluster. The numbers here are **what one replica can do with the defaults in this repo**. Multiply by the real replica count, then subtract what those replicas share: one Postgres `max_connections`, one Redis, one set of provider rate limits. + +This document has four parts. **Part A** (sections 1–17) is capacity: what breaks when the numbers get large, including the infrastructure you can add later. **Part B** is the code shape: extra queries, extra downloads, and files that only grow by getting longer. **Part C** is the pattern scorecard. **Part D** is what those three parts still left out: data lifetime, lost updates, deploy safety, cost, tenant leaks, and how you know the system is sick. A new teammate has to live in that code tomorrow. + +## Start here + +Read this section in the meeting. Use the rest of the document as the evidence when someone asks "where?". + +The code cannot carry 1 million simulations, 1 million calls, and 1 million traces a day. Adding pods will multiply whatever each request already does. Fix the request first, then add machines. + +**The problems that block the target** + +1. **Live traces.** One API key is allowed about 2 exports a second. The API parses every batch, uploads it, and the worker parses it again. Every few seconds the worker rereads the whole trace. Call ids are 6 digits, so the id space runs out near 900,000. +2. **Simulations.** One worker slot holds the entire run for up to 6 hours. The default fleet has two slots. A dead worker leaves the run on `RUNNING` forever. Voice calls sit inside a worker process for the whole call. +3. **Imports and judging.** A full transcribe-and-judge day lands around 150,000–350,000 calls, not 1 million. Recordings are held in memory and downloaded more than once. The chart loads every score row. Each finished row waits on a billing HTTP call. +4. **The shared platform.** Each API pod is one process, and database calls block it. Connection pools can exceed what Postgres allows as soon as workers are busy. One Redis holds the queue, the locks, and the counters. Migrations run on every pod. Uploads still use local disk. +5. **The shape of the code.** The files people must edit are thousands of lines (`call_import_evaluations.py` is 9,937, the evaluation screen is 10,368, `api.ts` is 7,066). The next feature will be pasted into those files unless the review rule changes. Two people saving the same agent overwrite each other. Nothing deletes one organisation's data. + +**What we can do, in this order** + +1. **Stop doing the work twice.** Do not parse a trace on the API and again in the worker. Do not download a recording three times. Do not poll a full transcript to update a counter. Do not write the same JSON to the catalog and the shard. +2. **Make one request cheap and safe.** A list returns a summary. A poll returns a status. A create sends an idempotency key. A save sends the version it read and gets a conflict instead of a silent overwrite. If Redis is down, refuse the work. +3. **Put the next feature in a new file.** One use case, one module. A new provider or metric type is a new strategy, the same way voice providers and speech-to-text clients already work. A route does not gain SQL. A page that is already over 800 lines does not gain a fetch. +4. **Then add capacity.** Connection pooler, one worker pool per queue, trace derive that only reads new spans, import slots that match the target, ClickHouse retention, object storage instead of local disk. + +**What is already in good shape, and should not be rewritten** + +Fair scheduling for imports. Chunked inserts. Usage counters flushed on a timer. Telephony that places the call and returns, with audio on a separate process. Trace ingest that answers 202 and keeps the raw batch in object storage. Password login that keeps tokens in memory. A large backend test suite that already runs on pull requests. + +Section 16 and Part C both talk about patterns. In the meeting, use Part C9 (the ten patterns). Section 16 is the short version of the same point. Part B is the file-by-file evidence for item 3 above. Part D is the list of problems that have no sidebar item: lost updates, retention, deploys, spend, and tenant filters. + +--- + +## How to read this + +Sections follow the sidebar, top to bottom: Dashboard, Metrics, Simulations, Playground, Evaluations, Imports, Prompts, Observability, Usage, Alerting, Configurations, then login, IAM, and platform admin. Two sections come first, because every later page sits on them: + +1. The target, in plain numbers. +2. The shared platform (API, workers, Postgres, Redis, files). + +Each product section has the same shape: + +- **What it is today** — how a click becomes a request, a queue job, and a database write. +- **What it can carry today** — a ceiling, and what hits that ceiling first. +- **Backend** — logic that will not survive the target, and the change that fixes it. +- **Frontend** — each important control (button, list, search, poll) and whether it still works when the lists get large. +- **Keep** — behaviour that is already the right shape. Do not rewrite these while fixing the rest. + +Severity means: + +- **Critical** — the target is impossible, or data can be wrong or lost, until this changes. +- **High** — works for a pilot, breaks for a large tenant or for many tenants at once. +- **Medium** — pain grows with the team and with data size; not the first outage. +- **Low** — cleanup that matters once the above is done. + +--- + +## 1. The target, in plain numbers + +1 million events a day is about **12 per second** on average. A SaaS day is not flat. Plan for a **10× peak**, about **120 per second**, and assume the three streams run together. + +The streams are not the same kind of work: + +| Stream | What "1 million a day" actually means | What has to be in flight at peak | +| --- | --- | --- | +| Text simulations | A trial is a back-and-forth of LLM calls, often around 30 seconds, plus judge calls | About 350 trials running at once | +| Voice simulations or live voice calls | A call lasts minutes. At 5 minutes each, 1 million a day is not 12 short requests | About 3,500 calls connected at once | +| Live traces | One call exports spans many times. A 3 minute call flushing every 5 seconds is about 36 HTTP exports | About 2,000 calls on average, about 400 export requests per second average, about 4,000 per second at peak | +| Imported calls | Each call is a download, often a transcription, then one or more LLM judge calls | Slot time, not request count, is the limit | + +A page view is cheap next to any of those. The pages still matter: a handful of open browsers polling a heavy endpoint will take down the API that is supposed to accept the traces. + +--- + +## 2. Shared platform + +This is the box everything else runs in. If this section stays as it is, raising a cap in one product area only moves the failure to Postgres or Redis. + +### What it is today + +`docker-compose.yml` is a single machine: + +- **API** (`eai start`): one Uvicorn process, no worker processes. Kubernetes can run more pods, but each pod is still one process, and many routes are `async` while they run synchronous database calls on that one event loop. One slow list freezes every other request on that pod, including health. +- **Media** (`eai telephony-worker`): a second single process for live voice sockets. +- **Celery workers**, all talking to the same Redis database 0: + - `celery` + `audio-metrics`, prefork, concurrency 8 + - `platform-imports`, `import-control`, `imports`, `diarization`, `eval-control`, `evaluations` together, threads, concurrency 12 + - `usage`, threads, concurrency 4 + - `traces`, threads, concurrency 8 + - `harbor-sim`, prefork, concurrency 2 + - `platform` (alerts, cron tick, trace sweep), concurrency 2, started in the background next to Beat +- **Beat**: one process. Its schedule file lives in the container filesystem, not on a volume. A restart can double-fire jobs. If the platform worker thread dies, Beat keeps running and nothing consumes `platform`. +- **Postgres**: one server in compose. Sharding can be turned on (`database.sharding` in config). Rows are not split by organisation. Call-import rows are hashed by `(import id, row index / 500)`. Live payloads are hashed by `(workspace id, entity id)`. The catalog (users, orgs, API keys, import metadata, run rows) stays on one database. The example shards are three databases on that same Postgres, so they share connections, disk, and WAL. Sharding in that shape does not add write capacity. +- **Redis**: one node, no password, no volume, no `maxmemory` in compose. It is the broker, the result backend, the locks, the rate limits, the usage counters, and the slot leases. +- **ClickHouse**: one node, empty password, used for traces when `clickhouse.url` is set. The example config leaves that block commented out. +- **Files**: `./uploads` and `./.data` are host folders mounted into the API, the media process, and the workers. A second pod does not see another pod's disk. + +Database pools are `pool_size` 10 plus overflow 20, so **30 connections per engine per process**. Prefork children each get their own. With sharding, each child also opens a pool per shard. Postgres in the compose image allows about **100 connections**. One busy audio worker can ask for more than that by itself. There is no PgBouncer, no `pool_timeout`, and no `statement_timeout`. + +Every API pod runs migrations on startup (`create_all` plus the migration runner). The only lock is inside one process. Two pods starting together will apply the same schema change at once. A failure on one shard leaves shards on different schemas while the other pods keep serving. + +Authenticated requests hit Postgres even when the API-key cache hits: the cache still loads the key row and the organisation. Local login does several queries per request (user, membership, credential, org) with no cache. Mutating requests authenticate twice, because the read-only middleware resolves the caller and the route resolves them again. + +Uploads up to 500 MB are read fully into memory. Playback of recordings is streamed back out through the API process even when a presigned S3 URL exists. + +There is no circuit breaker around LLM, speech-to-text, speech-to-synthesis, or telephony providers. Slot leases **fail open**: if Redis errors, the code treats the slot as acquired. A Redis blip removes the only global cap and, on the usage path, falls back to a Postgres insert per event. + +HTTP rate limits are off unless `rate_limits.enforce` is set, and bulk import, eval, and trace routes are exempt. Trace ingest has its own limiter (section 11) and that one fails closed. + +The app defaults are `DEBUG=True` and a placeholder `SECRET_KEY`. Compose falls back to a known secret if the env var is missing. A missed env var in a new cluster boots with docs exposed and a known signing key. The local `config.yml` is gitignored and contains cloud credentials; if that file was ever copied, rotate those keys. Do not commit it. + +Liveness (`/health`) is cheap. Readiness runs migration checks and opens a session per database. Prometheus, when enabled, scrapes a single static API address. Workers, media, queue depth, and pool saturation are not in that scrape. + +### What it can carry today + +One API pod of this shape serves **tens of requests per second** of real database work, not a sustained 150. Redis itself can do 150 simple operations per second. The connection budget and the single event loop fail first. + +1 million Celery tasks a day also means 1 million result keys in the same Redis, kept until the default expiry (about a day), because `task_track_started` is on and `result_expires` is not tightened. + +### What has to change + +**Critical. Put a connection budget in front of Postgres.** Add PgBouncer (transaction mode) and size every pool so that `(pods × processes × engines × pool)` stays under `max_connections` with headroom. Set `pool_timeout` and `statement_timeout`. Do this before raising any product cap. More replicas without this will start failing checkouts at peak. + +**Critical. Stop blocking the API event loop.** Routes that call synchronous SQLAlchemy should be plain `def` endpoints, or the database work should go to a thread pool. Run more than one worker process per API pod only after pools are budgeted. Keep voice sockets on the media deployment, not on the API. + +**Critical. Split Redis by job, and make it durable.** One instance (or database) for the broker, one for ephemeral counters and locks, short result expiry, AOF or a managed primary with a replica. A restart today drops queues, slot leases, usage counters, and the rebalance lock. + +**Critical. Run migrations once, with a lock.** A Job that takes a Postgres advisory lock per database, then starts the API with migrations skipped. Two replicas must not both alter the schema. + +**Critical. Get files off the node disk.** Every upload and every simulation artifact goes to object storage. Presign downloads. The API should not sit in the middle of the bytes. Media call state that must survive a second replica already belongs in Redis; the in-memory fallback (section 7) has to go. + +**High. One worker deployment per queue.** The imports container drains six queues on 12 threads. A big import occupies every thread, and cancel/retry (`eval-control`) waits behind scoring. Prefetch 1 and late acks are already right; the missing piece is a bulkhead per queue. + +**High. Beat needs a durable scheduler.** RedBeat or a database scheduler, and the platform consumer as its own deployment with a health check. Do not scale Beat past one replica. + +**High. Authenticate once per request, from cache.** Trust the cached principal for its TTL. Store a hash of the API key, not the key itself (`api_keys` is looked up in plaintext). Sample `last_used` instead of writing it on the hot path. At 150 requests per second the catalog table of keys is the hottest table in the system. + +**High. Fail closed when Redis is down.** Slot acquire, usage flush, and rate limits should reject or queue locally. They should not grant unlimited work and they should not write the catalog synchronously from the hot path. + +**High. Per-provider circuit breaker.** A provider outage today retries (up to 40 times on some eval tasks) and fills the worker for the 30 minute task limit. Shed load before the retry budget is spent. + +**Medium. Readiness should be `SELECT 1` plus Redis ping.** Schema checks belong in the migration Job, not in the load balancer probe. + +**Medium. Instrument the things you will scale.** Queue age, pool checkout wait, media socket count, trace ingest lag. One scrape target named `api:8000` cannot see a second pod. + +### Keep + +Queue split between audio ML, imports, usage, traces, Harbor, and platform. Late acks, prefetch 1, visibility timeout above the short task limit, prefork child recycle for the ML worker. Redis Lua slot leases for import and eval fairness (the lease shape is right; the fail-open and the single integer counter are not — see Imports). Usage token counts buffered in Redis and flushed on a timer. Liveness with no I/O. API-key cache key is a hash of the secret even though the database still stores the secret. + +--- + +## 3. Scorecard + +| Section | Verdict | Ceiling with current defaults, one replica | First thing that breaks | +| --- | --- | --- | --- | +| Shared platform | Red | Tens of API requests/s; connection errors as soon as workers are busy | Postgres connections, one event loop, one Redis | +| Dashboard | Amber | The page is cheap. It also does not show the business | A failed summary looks like an empty account | +| Metrics | Amber | Fine until the metric library or the agent list is large | Whole catalog in the browser; a failed load can seed default metrics | +| Simulations | Red | Two runs at a time. Roughly 20k text trials/day at the default concurrency of 4, or about 2k voice calls/day if all 8 shared worker slots were voice | One worker slot owns the whole run; voice calls pin a worker for the whole call | +| Playground | Red for voice, amber for chat | A few dozen concurrent playground sessions before the API thread pool and the single media process fill up | Polling and audio run inside the API or one media process | +| Evaluations | Red | On the order of 150k–350k judged audio calls/day globally, not per tenant | 128 shared slots, and charts that load every row | +| Imports | Red | About 170k recording downloads/day at 16 slots and 8 seconds each. A CSV over 15 MB is rejected | Download cap, whole file in memory, billing HTTP call per row | +| Prompts | Green | Not on the million-call path | Optimization only sees 50 results | +| Observability and traces | Red | About 2 accepted trace exports per second per API key (about 170k exports/day) | The 120/minute ingest limit, then full-trace recompute | +| Usage | Amber | Counters in Redis are fine. The usage page shows the top 100 groups | A billing HTTP call on every completed import row | +| Alerting | Red once call volume is real | The 5 minute schedule is fine | Each alert loads every call JSON in its window | +| Configurations | Amber | Fine at tens of integrations. Breaks as catalogs grow | Full lists, 1,000-key S3 browse, 23 serial model-catalog calls | +| IAM, login, platform admin | Amber | Fine for hundreds of users. The platform org table stops at 200 | Long-lived tokens in `localStorage`; workspace cache mix-ups | + +--- + +## 4. Dashboard + +Route: `/`. Files: `frontend/src/pages/dashboard/Dashboard.tsx`, `app/api/v1/routes/dashboard.py`. + +### What it is today + +The home page is a quick-start (four setup cards) plus a summary of catalog counts: agents, personas, metrics, imports, and similar. It does not aggregate simulations, production calls, trace error rate, queue lag, or cost. The summary is cached in the browser for 60 seconds. The query key does not include the workspace id (see section 15 for why that matters on every page). + +A carousel of TTS models loads the full model catalog and rotates it every 6 seconds. + +### What it can carry today + +The endpoint itself will not fall over at 1 million calls a day, because it does not read those calls. It also cannot answer "are we healthy right now?". People will use Observability for that, and that page is one of the expensive ones. + +### Backend + +**High.** The summary counts catalog rows. Add a small rollup the page can read in one query: simulations in flight, imports in flight, calls in the last hour, trace ingest lag, error rate. Compute those in the workers as counters, not by scanning call tables on each page view. + +### Frontend controls + +- **Quick-start cards.** Always shown. There is no "setup already done" state, so returning users scroll past onboarding on every login. Collapse them once the summary says setup is complete. +- **Summary blocks.** If the request fails, the highlights render nothing. A 500 looks the same as a brand new account. Show an error with a retry. +- **Model carousel.** It is a catalog fetch on the hottest page in the product, and it is not the user's data. Move it off the home page or load it only when the card is opened. + +### Keep + +A 60 second stale time on the summary. That is the right instinct. Most other pages refetch on every visit (section 15). + +--- + +## 5. Metrics + +Routes: `/metrics-management`, Metrics Studio. Files: `frontend/src/pages/metrics/MetricsManagement.tsx` (4,518 lines), `MetricsStudio.tsx`, `app/api/v1/routes/metrics.py`, `metric_studio.py`, `app/workers/tasks/evaluate_studio_run_item.py`. + +### What it is today + +Metrics Management is the catalog of judge metrics (score metrics and classification metrics). The page calls `listMetrics` with no page size, then search, tag filter, and sort run in the browser on that full array. The search box filters on every keystroke. The component is one file: list, create, classification editor, categories, AI generate, and prompt-partial import. + +If the list comes back empty, the page seeds default metrics. The empty array is also the default while the request is in flight or after it fails, and the seed does not check `isError`. A 500 or a blip on a new workspace creates a second set of default metrics. + +Scope pickers load up to 500 agents and 500 scenarios into a multi-select. Anything past 500 cannot be selected, and the UI does not say the list was cut. + +Metrics Studio lets someone try a metric against calls, import rows, recordings, or evaluator results. Each source list is capped at 100 and the query keys do not include the workspace. While a studio run is active the page polls every 3 seconds. Studio items are one Celery task each, and they draw from the same global eval slot pool as production judging (128). That is fine for a handful of samples. It is not fine if someone points Studio at a large import. + +The studio run detail page sets that 3 second poll twice. + +### What it can carry today + +Metric count is a team-size problem, not a million-call problem. A library of a few hundred metrics will make this page janky because the whole library re-renders a 4,500 line component on each character. Studio competing with production judging becomes a problem only when Studio is used as a back door to evaluate a large corpus. + +### Backend + +**Medium.** Keep Studio on its own small queue and its own cap so a studio experiment cannot take the 128 production slots. + +**Low.** Classification metrics on the judge path are not all batched (section 8). The metric editor is where that fan-out is configured, so a metric with 10 classification questions can turn 1 million calls into 10 million LLM calls. Show that cost in the editor before save. + +### Frontend controls + +- **Search, tag filter, sort.** Client-side over the full list, no debounce. Move the filter to the server, debounce the input, and page the table. +- **Create / edit / AI-generate / import partial.** All live in the same component as the list. Split one component per dialog so a change to the editor does not risk the list. This is a team-growth issue: the file is already hard to review. +- **Agent and scenario scope multi-selects.** Server search with a small page, and a clear "showing the first N" state. Silent truncation means a metric is scoped to the wrong set. +- **Studio source pickers.** Same search. Include the workspace id in the query key. +- **Studio run polling.** One poll, only while the run is non-terminal, and only while the tab is visible. The run detail page should not poll the same query twice. + +### Keep + +Metrics Management is one of the few pages whose query key already includes the workspace id. Copy that pattern everywhere else. + +--- + +## 6. Simulations + +Sidebar: Agents, Personas, Scenarios, Tools, and the Simulations tab on an agent. This is the pre-production path: a persona talks to the customer's agent across scenarios, for several trials, and the platform scores the result. + +Files: `app/api/v1/routes/simulation_runs.py`, `app/workers/tasks/run_harbor_simulation.py`, `app/services/harbor/`, `frontend/src/pages/simulations/`, `frontend/src/pages/agents/`, `personas/`, `scenarios/`, `toolConnections/`. + +### What it is today + +Creating a run inserts one `SimulationRun` in `QUEUED` and enqueues **one** Celery task for the whole run. That task lands on `harbor-sim`. The compose worker has concurrency **2**. The task holds its slot for the entire run, up to **6 hours**. It starts a subprocess that runs up to `n_concurrent` trials at once (default 4, maximum 32) as threads. There is not one queue item per trial, so a second replica cannot steal the remaining trials of a big run. + +Each trial holds a database connection for the whole conversation. The pool is 30 connections. `n_concurrent` of 32, plus the scorer, can exhaust it. Every turn also writes the transcript. The parent task, every 5 seconds, reloads every trial row, including the heavy payload, just to update two counters (`completed` and `failed`). + +If the worker dies after the run has left `QUEUED`, the redelivered task sees `RUNNING` and returns. Nothing in the scheduler reconciles simulation runs. The run stays `RUNNING` forever. The broker visibility timeout is 2 hours, which is shorter than the 6 hour task limit, so a long run is also redelivered while it is still running. + +`run_id` is a random 6 digit number (`simulation_runs.py`, `randint(100000, 999999)`), with up to 100 existence checks. The space is 900,000 values, global, not per organisation. One million runs or trials does not fit. The same pattern is used for call short ids (section 11) and evaluator result ids. + +LLM calls on this path have no shared rate limit, no retry budget, and no cache. A 429 fails the trial. Harbor retries the trial once, which doubles spend. Metric judging runs inline on the same thread, after the conversation. One organisation's large run consumes the provider key for everyone else. + +The run's working directory is on local disk (`HARBOR_WORK_DIR`, `./.data` in compose). The tarball is built in memory and uploaded as one blob. The download endpoint loads that blob back into the API. Extra API pods without that disk cannot read trajectories. The work directory is not removed after upload. + +Sharding writes the heavy JSON (transcript, scores, call data) to the catalog **and** to a shard payload. The catalog is rewritten every turn. The shard key includes the result id, so a large org's payloads do spread out. The catalog hotspot does not. + +There is no per-org cap and no global cap on this queue. The fair-share leases that exist for imports are not used here. + +### What it can carry today + +At the default of 4 concurrent trials and about 30 seconds per trial, one Harbor replica does roughly **20,000 trials a day** if both slots stay full. At concurrency 32, if the database pool allowed it, roughly **180,000 a day**. 1 million trials a day needs on the order of **350 trials in flight**, which is several replicas at concurrency 32, plus a provider rate limit those replicas do not have. + +Two long runs occupy the entire simulation fleet of one replica. A 1,000 trial run at concurrency 4 sits on one slot for hours. + +Voice simulations are a different machine and are worse. See section 7. + +### Backend + +**Critical. One queue item per trial, with a lease.** The imports fair-dispatcher is the pattern already in the codebase. A run is a parent row; trials are leased work; any replica can take the next trial. Cap in-flight trials per workspace, per org, and globally. Reject or queue in Postgres when the cap is full, instead of `.delay()` with no limit. + +**Critical. A sweeper for stuck runs.** Heartbeat on the run. If the lease expires, requeue. Set the broker visibility timeout above the task limit. Reclaim `RUNNING` when the owner is gone. Today a deploy strands every in-flight run. + +**Critical. Stop holding a connection for the whole conversation.** Open a session per turn. Size the pool to the concurrency you actually run. Keep status and counters on the catalog row; write the transcript only to the shard (or to object storage). + +**Critical. Replace 6 digit public ids.** Use the UUID that already exists, or a per-org sequence. The random probe is a catalog read storm as the space fills up. + +**High. A Redis limiter per LLM credential, and a token ceiling per run.** When the ceiling is hit, fail the remaining trials cleanly. Do not retry a whole trial because one call returned 429. + +**High. Progress counters on the run row.** The 5 second reload of every transcript exists only to count finished trials. Increment the counter when a trial finishes. + +**High. Stream artifacts to object storage and delete the work dir.** The API reads trajectories from the object store, not from local disk. + +**Medium. Idempotency-Key on create.** A double click inserts two runs and two multi-hour jobs. Cancel should set the flag and revoke the task id. Avoid `terminate=True` on delete unless that child is known to be this task; killing a prefork process can take the next org's job with it. + +### Frontend controls + +- **Agent list** (`AgentsWorkspace`, and the list in the screenshot). `listAgents` defaults to **100**. Past that, agents exist on the server and are not in the sidebar list or the global agent switcher. Search and the All / Voice / Chat filter run on whatever page came back. There is no virtualization and no "load more". The header "N total" can disagree with what is on screen once an account has more than 100 agents. +- **Select all** on that list. It selects the loaded page, not the full server set. Say so, or make it a server-side selection. +- **Search by id or name.** Debounce it and send it to the server. Filtering 100 cards in the browser hides the rest. +- **Create Agent.** Fine as a single-entity write. The wizard should not prefetch every voice bundle, integration, and model catalog up front (VoiceBundle already does that waterfall; see section 14). +- **Overview tab.** Reads one agent. Name, language, call setup, live-session hangup. This tab is cheap. Keep it that way: do not hang live run stats on it by loading every trial. +- **Test Agent / Voice Agent tabs.** Configuration of one agent. Cheap if they load that agent and the selected bundle only. They become expensive if they pull every integration and every model to populate dropdowns. Load options for the provider the user actually picks. +- **Simulations tab and New simulation.** `NewSimulationRunModal` fetches up to 500 scenarios, 500 personas, the tool catalog, and every metric when it opens. Personas are filtered to voice vs text in the browser. Past 500 the run is silently incomplete. Use server search and show that the list is truncated. +- **Run detail.** While the run is active, the page refetches the run **and every trial** every **4 seconds** (`SimulationRunDetail.tsx`). The matrix draws one cell per scenario × persona × attempt, with no windowing. A few people watching a 1,000 trial run each pull a large JSON every 4 seconds, and the API recomputes the live summary by hydrating every transcript. + - Poll a small progress payload (status, completed, failed, eta). + - Page or virtualize the matrix. + - Load a transcript only when the trial drawer opens. +- **Trial drawer.** Refetches the growing transcript and the whole trajectory on a short interval, and the trajectory is a file on the API disk. Append-only events, or stop refetching once the trial leaves "in progress". +- **Personas, Scenarios, Tools pages.** Same shape: first page only (client default 100, or no page size for tools), client-side filter, no virtualization. A workspace with 120 scenarios looks like it has 100. Page on the server. + +### Keep + +The run status machine (`QUEUED` → `PREPARING` → `RUNNING` → `INGESTING` → terminal). The subprocess boundary, so cancel is a signal to that process and the Celery child keeps its broker connection. The `QUEUED`-only start, which is a real guard against starting the same run twice. List endpoints for runs do not recompute the live summary; only the detail view does. Payload routing hashes workspace and entity id, so simulation payloads are not all pinned to one shard. + +--- + +## 7. Playground and live voice + +Sidebar: Agent Playground, Voice Playground. Also the live call path used by voice simulations and telephony (Vobiz / Plivo / Twilio / Telnyx), because it is the same media stack. + +Files: `app/api/v1/routes/playground.py`, `voice_agent.py`, `voice_playground.py`, `app/workers/tasks/run_evaluator.py`, `initiate_vobiz_outbound.py`, `app/services/testing/test_agent_bridge_service.py`, `app/services/webrtc_bridge/`, `app/services/telephony/`, `frontend/src/pages/playground/`. + +### What it is today + +Two different shapes are mixed together. + +**Telephony outbound** is the shape to copy. The API stores a call session in Redis (1 hour TTL) and takes a pool slot (default **5 concurrent calls per organisation**). A short Celery task places the carrier call and returns. The audio socket lands on the media process. Any media replica can accept it, as long as Redis is up. On hangup, a worker merges the two WAV tracks and uploads them. + +**Voice playground and the voice evaluator bridge** do the opposite. A voice evaluation runs the WebRTC or WebSocket call **inside a Celery child** (`run_evaluator` on the default `celery` queue, shared with audio metrics, concurrency 8, 30 minute task limit). One child hosts one call. The bridge gives up at 300 seconds and may then poll the provider for up to about 10 minutes. The WebRTC path that actually completes is ElevenLabs; other platforms raise. A million 5 minute voice calls a day is about **3,500 concurrent calls**, which would be on the order of 400 of these 8-slot workers, and only if audio-metric jobs were not in the same pool. + +**Agent Playground web calls.** The browser owns the Vapi call. Retell and ElevenLabs are created on the server, then a FastAPI background task polls the provider for up to 5 minutes **inside the API process**, sleeping, holding a database session, with the decrypted integration key in the task arguments. The Starlette thread pool is on the order of 40 threads. Many concurrent playground users fill it. + +The media server is one Uvicorn process. Audio chunks are appended to an in-memory list for the life of the call. Merge reads both WAVs and then the merged file into memory for S3. Speech-to-text, speech-to-synthesis, and VAD for the in-browser voice agent run in that same process. + +If Redis errors, the telephony session falls back to a dict on that process, and the outbound pool falls back to an in-process counter. A second media pod will not see the session and will drop the call. The pool increment is get-then-set, not atomic. A missed release blocks a slot for up to 2 hours. The outbound initiate task, on error, marks the call failed and releases the slot, then retries, which places a second carrier call. + +`telephony_credential_rate_limit` covers recording fetches during import. It does not cover live Vapi, ElevenLabs, Retell, or Twilio concurrency. + +### What it can carry today + +Telephony: tens of calls per media pod before CPU (VAD, STT, TTS) saturates a core, and only 5 outbound calls per org. Playground: tens of concurrent polls before the API thread pool is busy doing `sleep`. Voice evaluator: about **2,000 five-minute calls a day per shared worker replica**, and fewer whenever an audio-metric task takes a slot. + +### Backend + +**Critical. The call does not belong inside a Celery child.** Initiate and finalize in workers. The media leg lives on the media deployment, which is how telephony already works. Put a Redis cap in front of each voice provider, the same shape as the import slot leases, and fail closed when Redis is down. + +**High. Atomic pool slots with a lease per call.** `INCR` (or a sorted set of call ids), release on hangup, and a sweeper for leaked slots. Raise the per-org cap as a product limit, not a hardcoded 5, if outbound is part of the million-call target. + +**High. Do not retry a carrier call that already failed the session.** Retry only before the carrier accepts. Send an idempotency key. + +**High. Stream recordings.** Multipart upload from a temp file. Cap the in-memory playout buffer. Run more than one media process behind the load balancer. Sticky sessions are only needed if state is still in the process; with Redis sessions they are not. + +**Medium. Playground provider polling belongs in a short task or a webhook.** Pass the credential id, not the decrypted key. + +### Frontend controls + +- **Agent Playground lists.** While anything is in progress, the page refetches the evaluator-result list and the call-recording list every **5 seconds**. Pagination slices an array it already downloaded, so the page size does not reduce the request. The recording list defaults to 100, so an in-progress row outside that page can stop the poll early, or an old non-terminal row can make it poll forever. Poll the specific non-terminal ids. +- **Voice Playground.** Polls at 3–4 seconds, and one panel polls external responses every 15 seconds even when idle. A WebSocket client also uses a 4 second `setInterval` that keeps firing in a background tab. Gate every interval on "visible and not finished". +- **Start call / hang up.** These are fine as single actions. The risk is the background poll and the media process, not the button. +- **Sample texts and provider pickers.** Load the catalog for the selected provider, not every provider, on open. + +### Keep + +Short initiate task, media process, Redis session, per-org pool. That is the right design, and the voice-eval path should be moved onto it. Provider metric polls inside the live bridge use a thread, so a 30 second HTTP call does not freeze the audio loop. LLM usage is attributed per leg (test vs production) and priced, which is the base for the budgets that are still missing. Stdio MCP tools are off unless the deployment opts in; leave them off for SaaS. + +--- + +## 8. Evaluations + +Sidebar: Evaluators, Evaluation Results, Judge Alignment. Imports have their own section; they share the judge workers with this one. + +Files: `app/services/evaluators/evaluator_run_service.py`, `app/workers/tasks/run_evaluator.py`, `process_evaluator_result.py`, `app/workers/tasks/helpers/llm_evaluation.py`, `app/services/evaluators/evaluator_results_aggregate.py`, `frontend/src/pages/evaluators/`, `judgeAlignment/`. + +### What it is today + +A suite run inserts one `EvaluatorResult` per combination and enqueues one `run_evaluator` task each, on the default queue, with no concurrency cap. The insert loop commits and probes a unique 6 digit id **per trial on the request thread**. A large suite holds the API request for the whole loop and then floods the queue. + +Chat and internal-LLM trials call the simulation function, then enqueue metric scoring. Voice trials take the bridge described in section 7. + +For imported calls, scoring is the fair queue: metrics that share one LLM provider, model, and credential are **one prompt and one completion**. Classification metrics on the Jev path are each their own call, so 10 classification metrics multiply the LLM bill by about 10. There is no application-level requests-per-minute limiter in front of the provider. A 429 fails the row and the task retries. + +Caps in `app/config.py`: + +| Pool | Default | +| --- | --- | +| Eval + transcription, global / org / workspace / one job | 128 / 128 / 100 / 75 | +| Recording fetch, global / org / workspace | 16 / 16 / 8 | + +Transcription and judging **share** the 128. A job that needs both occupies the pool twice. + +The results overview loads the whole filtered set into Python and builds histograms there. The results dashboard in the browser computes trends from the newest **500** rows and tells the user the rest were dropped, while still downloading full score JSON for those 500. + +Judge alignment runs one serial LLM call per sample, holding a database session across the run, and it does not use the eval slot pool. Alignment sets are small, so this does not move the million-call number. A large labeled set blocks one worker for the whole run. + +### What it can carry today + +One LLM call at 128 slots and about 15 seconds is roughly **740,000 judge calls a day** globally, and about **430,000** for a single job capped at 75. At 30 seconds (long transcript, or transcription then LLM on the same pool) that falls to about **370,000**. Combined with the download cap, a full transcribe-and-judge day lands around **150,000–350,000 calls**, not 1 million. One tenant cannot burst past the job cap of 75, which is correct for fairness and too low for a single large daily batch unless the slot time gets much shorter. + +### Backend + +**High. Separate transcription slots from judge slots.** Set the job cap from the number of workers you are paying for, not from a fixed 75, once the provider limiter exists. + +**High. A token bucket per provider credential.** Backoff that does not burn the row's retry budget. The slot lease is fairness between tenants. It is not a provider rate limit. 128 parallel calls to one key will trip the provider before the slot cap is the real limit. + +**High. SQL aggregates for the results overview.** The same rule as import charts (section 9): do not `.all()` score documents into the API process. + +**Medium. Bulk-insert a suite, then one dispatcher.** The request should not commit once per trial. + +**Medium. Batch classification questions** into the same completion that score metrics already share. + +**Low. Chunk judge-alignment samples** through the same fair dispatcher, so a large labeled set does not pin a worker outside the slot pool. + +### Frontend controls + +- **Run evaluation / run suite.** Needs a disabled state and a server-side cap error ("this org already has N runs in flight"). Today the click always enqueues. +- **Results list and dashboard.** Server-paged. The trend chart should call a bucketed endpoint (daily pass rate, latency) instead of downloading 500 full score blobs and dropping the rest in the browser. +- **Result detail.** Load one result. Do not prefetch every sibling in the run to render one page. (The simulation trial endpoint has this bug; check the evaluator detail the same way when you touch it.) +- **Filters.** If they filter a page that was already fetched, they hide rows that are on later pages. Send the filter to the server. +- **Judge Alignment screens.** Poll every 3 seconds on the evaluate screen and every 5 seconds on the dataset screen, including when idle on the dataset page. Poll only while a run is non-terminal. + +### Keep + +Shared-config metrics already go out as one prompt. That is the difference between 1 million and several million LLM calls, and it should stay the default for every metric type. Result list queries on the frontend are among the ones that already include the workspace id. + +--- + +## 9. Imports + +Sidebar: Imports (call imports, chat imports, schemas, tags). This is the path most likely to be asked to carry 1 million production calls a day, by CSV, by file, or by pulling history from Vapi, Retell, or ElevenLabs. + +Files: `app/api/v1/routes/call_imports.py`, `app/services/call_imports/`, `app/workers/concurrency/import_dispatch.py`, `fair_import_dispatch.py`, `limits.py`, `app/workers/tasks/process_call_import_row.py`, `transcribe_call_import_row.py`, `evaluate_call_import_row.py`, `frontend/src/pages/callImports/`. + +### What it is today + +The current UI upload is staged, which is the right idea, and then undone by how the bytes are handled: + +1. The browser posts the CSV or Excel file to the API. The client checks a **15 MB** cap and does not parse the file itself. +2. The API reads the entire upload into memory and rejects anything over 15 MB. +3. The staged path stores those bytes in S3. A worker downloads them back with a full `read()`. +4. CSV is decoded into one string buffer. Excel builds a full list of row dicts. Mapping holds every accepted row in a Python list. +5. Rows are inserted in chunks of 1,000, onto shards when sharding is on. +6. Dispatch does **not** enqueue one Celery task per row up front. A fair round-robin pulls the next pending row, capped by Redis inflight slots. This is the right design. +7. Each recording is downloaded with `httpx` and held as `response.content`, capped at 50 MB (300 MB for platform recordings). The bytes are uploaded to S3. Transcription downloads the object again into memory. The billing step may download it a third time to probe duration. +8. Evaluation is a second fair queue on the same 128 slot pool as transcription. +9. Progress is a Redis counter plus a debounced SQL count. The UI polls every 3–5 seconds and pages rows at 50 or 100. + +The slot counter is a single `INCR` with a 20 minute TTL refreshed on every acquire. A crashed worker never decrements it. New acquires keep the TTL alive, so leaked slots accumulate and the cap stays "full" forever. Redis errors return success, so the cap disappears during an outage. The platform importer already has a better lease (a sorted set in `platform_import_limits.py`). The main import path does not use it. + +A legacy `/upload` route still parses and inserts inside the request. + +Manual audio upload reads up to **100 files × 500 MB** into the API process before S3. + +Platform pulls are in better shape: time slices, cursor pages of 100, a per-task page budget, RPM tiers of 30 / 120 / 300, and Redis leases. Discovery can list about 30,000 calls a minute on the fast tier. Hydration is then capped at 64 in flight globally and 12 per import, and each recording is still fully buffered. Listing outruns storage, and the per-import lease serializes a large customer. + +Re-runs and "select all matching ids" load every id into a Python list. Search uses `ILIKE '%term%'`, which cannot use a normal index. Some detail handlers still accept a `row_limit` up to **5,000** and return full row bodies, including transcripts. The UI asks for 50–100, but the API allows the expensive call. + +Fair dispatch, on each tick, asks every shard for pending import ids and then runs one "next row" query per dispatched row. + +### Charts, insights, export + +`GET .../aggregate` loads every row's `metric_scores` from every shard into the API process and folds them in Python. The Visualizations tab polls that **every 5 seconds** while a run is in progress (`CallImportEvaluationDetail.tsx`, about 10,400 lines). + +Insights, metric clusters, and PDF generation load every eval/source pair, including transcripts, into memory, then cap the LLM calls (default 200) and shuffle. The worker runs out of memory before the LLM budget matters. + +CSV export can split parts, but the call site passes a part size of 2³¹, so it is one gzip. Excel is one temp file. Row reads are paged at 1,000, which is good. The write is one giant object. + +On each completed audio row, the worker makes a **synchronous HTTP call to Flexprice** before it is free to take the next row. Token usage itself is buffered in Redis. The per-row billing call is not. + +### What it can carry today + +| Step | Ceiling at the default caps | +| --- | --- | +| Recording download, 16 slots, ~8 s each | About 170,000 recordings/day | +| Recording download, 16 slots, ~3 s each | About 460,000/day | +| One LLM judge, 128 slots, ~15 s | About 740,000/day globally, ~430,000 for one job | +| Transcribe and judge on the shared pool | About 150,000–350,000 calls/day | +| One CSV | 15 MB. Sparse rows might be 50–80k. Rows with transcripts are a few thousand | + +To hold 1 million audio calls a day at 30 seconds of slot time you need on the order of **350 concurrent judge slots** and a download pool several times 16, plus streaming so those slots are not also holding 50 MB buffers. Celery enqueue of 1 million messages is not the first failure, because dispatch only fills the inflight window. That part is already right. + +### Backend + +**Critical. Raise the download pool and stream the bytes.** Stream to a temp file or to S3 multipart. Pass a file path to speech-to-text. Store duration once, at ingest, so billing does not download the file again. Scale download workers separately from LLM workers. + +**Critical. Replace the inflight integer with a per-task lease, and fail closed.** Copy `platform_import_limits.py`. A crash must release the slot without waiting 20 minutes, and a Redis error must not mean "unlimited". + +**Critical. Stop loading every score and every transcript for charts and insights.** Maintain per-metric counts as each row completes (Redis or a rollup table) and serve the chart from that. Sample rows for insights with a keyset scan. Do not join transcript text until a row is in the sample. + +**Critical. Move the Flexprice call off the worker hot path.** Push the billing event into the same Redis buffer as token usage, and flush in batches. A billing outage should not hold a judge slot for an HTTP timeout. One million extra RPCs a day, each holding a worker and a database session, will not work. + +**High. Presigned upload, stream-parse, drop the 15 MB cap for streamed CSV.** The API receives a small "file is staged" message. Parse from S3 in batches. Do not build the full row list. Point any leftover caller of `/upload` at the staged flow and remove the inline path. + +**High. Presign manual audio.** The API records the object key. It does not receive the bytes. + +**High. Decouple platform list rate from download concurrency.** Stream those recordings too. Size the hydrate pool for the target, with a per-org cap so one customer cannot take all 64. + +**High. Do not harvest every id for a retry.** Keyset-page ids into the bulk insert. Replace leading-wildcard search with trigram or prefix search. + +**High. Export in parts.** Keep the 1,000 row read. Emit many gzip parts. Do not offer Excel for a million-row run. + +**Medium. A Redis set of imports that still have pending rows.** Claim the next row with `UPDATE … RETURNING` on the partial index that already exists. The dispatcher should not scatter-gather every shard on every tick at 12 completions a second. + +**Medium. Cap list payloads.** Lower the 5,000 row max. Exclude transcript text from list responses. The UI never needed it. + +**Medium. A per-provider token bucket for speech-to-text**, same as for LLM. Sarvam's batch API for long audio is the right idea for the other providers. + +### Frontend controls + +- **Upload CSV / upload audio / upload chat.** The browser correctly does not parse the CSV. The bytes still go through FastAPI, and the 15 MB cap is enforced in two places that can drift. Switch the button to a presigned PUT, then a small JSON call. Show progress from the object store, not from a request body the API is buffering. +- **Import detail** (`CallImportDetail.tsx`, about 5,300 lines). Rows are paged at 100. Keep that. Do not add a client-side "load all". The 5 second poll should refetch the progress counters and the current page, not the aggregate. +- **Evaluation detail** (`CallImportEvaluationDetail.tsx`, about 10,400 lines). Several polls overlap (around 3–5 seconds). The visualizations poll is the dangerous one, because it triggers the full-table aggregate. Split the file by tab. Poll the chart only while that tab is visible, and only from the rollup, not from a full scan. Stop the poll when the run is terminal. +- **Select all / bulk retry / bulk re-eval.** These must mean "all rows matching the filter on the server", processed in pages. If they collect ids in the browser, they will truncate or freeze the tab. The API already has a path that loads every id; change both sides together. +- **Search.** Debounce, server-side, no leading wildcard. +- **Schemas and tags.** Small catalogs. A full list is fine until an org has hundreds. Page them before that, using the same table component as imports, so you do not grow a second unbounded list. +- **Export button.** Disable or switch to "we will email a link" once the run is large. A single gzip download of a million transcript rows will time out in the browser and fill a disk on the API. + +### Keep + +Fair round-robin across workspaces, orgs, and jobs. Chunked inserts of 1,000. Partial indexes for the pending-row claim. Redis progress counters with a debounced flush, instead of `COUNT(*)` on every completion. Platform discovery is cursor-paginated and RPM-tiered. Export already reads 1,000 rows at a time. S3 already has a chunk iterator the hot path can switch to. The UI tables are server-paged. Aggregates are not fetched until the visualizations tab is open — the bug is what that tab fetches, not the fact that it waits. + +--- + +## 10. Prompts + +Sidebar: Partials, Optimization. Files: `frontend/src/pages/promptPartials/PromptPartials.tsx` (2,813 lines), `promptOptimization/`, `app/workers/tasks/run_prompt_optimization.py`, `app/services/optimization/gepa_service.py`. + +### What it is today + +Partials are snippets injected into judge or agent prompts. They do not add an extra LLM call by themselves. The page is one large editor. Optimization trains on **50** evaluator results, with a default budget of about 20 metric calls, and truncates transcripts to 3,000 characters. The page polls every 5 seconds. + +### What it can carry today + +This will not melt the cluster at 1 million calls a day. It also cannot learn from a million-call corpus, because it never reads it. + +### Backend + +**Low.** If optimization is meant to learn from production, sample call-import rows with an explicit cap and a keyset scan. Do not `SELECT` the full table into the trainer. + +### Frontend controls + +- **Partial list and editor.** Same maintainability note as Metrics: split the editor from the list before more people are in this file. The list itself is a small catalog. +- **Run optimization.** Show the sample size and the metric-call budget on the button's confirmation, so a run cannot surprise the LLM bill. Poll only while status is running. +- **Save.** Single-row write. Fine. + +### Keep + +A hard budget on metric calls (`max_metric_calls`). Every other LLM path should grow a ceiling like this one. + +--- + +## 11. Observability, live traces, and production call webhooks + +Sidebar: Observability Overview and Calls. This is the live ingest path: customer agents export spans while calls are happening, and carrier webhooks deliver production calls. + +Files: `app/api/v1/routes/synthetic_traces.py`, `observability.py`, `telnyx_voice_webhooks.py`, `twilio_voice_webhooks.py`, `vobiz_telephony.py`, `app/services/observability/`, `app/services/synthetic_traces/` (ingest pipeline), `app/services/clickhouse/`, `app/workers/tasks/trace_tasks.py`, `frontend/src/pages/observability/`, `test-insights/`. + +### What it is today + +The intended ingest path is the right shape, and the default limit makes it unusable at the target: + +1. `POST /api/v1/observability/traces` authenticates, then applies a Redis sliding window of **120 requests per minute per API key** (`TRACES_RATE_LIMIT_PER_MINUTE`, enforced by default). Over the limit returns 429. Redis down returns 503 (fail closed, which is correct). +2. Body cap is 4 MB (413 over that). Good. +3. `defer_parse_to_worker` still **fully parses** the OTLP body on the API, to find the workspace, before the deferred path. The worker parses it again. +4. If ClickHouse is not configured, the deferred path returns **503**. The example config has ClickHouse commented out. A stock config cannot ingest traces. The old Postgres span tables are a legacy path; turning defer off writes one JSON batch and a commit per export, which will not reach 1 million traces a day. +5. On the hot path the API resolves or mints a trace id (Redis, else a ClickHouse `FINAL` lookup), increments a Redis sequence key that **never expires**, and does a **synchronous S3 PUT** of the raw body. Then it enqueues `process_s3_otlp_batch` and returns **202**. There is no check of queue depth. If S3 or Celery fails, the client gets 503, sometimes after the object was already written. +6. The worker inserts one ClickHouse row per span. Dedup is `ReplacingMergeTree` on span id, visible only after merge or on `FINAL` reads. A retry increments `span_count` again. Derive is scheduled with a 3 second debounce. Each derive **reloads every span for the trace**, recomputes turns, and rewrites the whole header, including the turns JSON. The ClickHouse path does not schedule another derive if newer spans arrive during the read. +7. Idle close: every 30 seconds, a query of `status = open` and `last_span_at` older than 120 seconds, **`LIMIT 200`**, over `FINAL`. Those rows stay `open` until a worker finishes, so the same 200 are queued again every 30 seconds. Closing a trace does not enqueue an evaluation. +8. Call short ids are 6 digits, 900,000 values (`app/utils/call_recordings.py`), and each attempt also probes Postgres and ClickHouse. One million retained calls do not fit. Near the cap, probes hammer both databases. Trace correlation is keyed by this id. + +The calls list, on each page, runs several Postgres `COUNT`s and several ClickHouse `count()` over `FINAL`, then loads `skip + limit` rows from both stores, sorts them in the API, and slices. Search filters that window in memory, so the total shrinks to the page you fetched. List queries include the turns JSON. `call_recordings` has no index on `(workspace, source, created_at)` or on agent + time. `created_at` is unindexed. + +Call detail returns the full `call_data` JSON. Audio is downloaded into a buffer in the API even though the client knows how to use a presigned URL; the panel also prefetches with `?proxy=true`. An SSE exists for live transcripts, and the panel **also** polls. + +Sharding dual-writes call JSON to a shard payload and still keeps the fat column on the catalog row. The catalog does not get smaller. + +ClickHouse tables have no TTL. Every hot read uses `FINAL`. Span attributes are JSON strings. At this rate a year is tens of billions of span rows if you keep them forever. + +Production webhooks: + +| Webhook | What the request does today | Scale problem | +| --- | --- | --- | +| Telnyx voice | Signature check only on `call.initiated`. Other events return 200 with no check. The request itself calls the provider to answer and start streaming. No event-id dedup | Answer latency, unsigned events, duplicate calls under retry | +| Twilio voice | Signature checked. TwiML built from the database in the request | Fine if the handler stays this small | +| Vobiz answer | Creates evaluator result, call session, and recording inside the webhook. A retry can insert another call | Duplicates at peak | +| Vobiz events | Writes status and the full payload. Last write wins, no event id | Lost updates, no dedup | +| Vobiz recording-ready | Downloads the audio and uploads to S3 **inside the webhook** | At ~100 call endings/s the API stalls. Retries re-download until the S3 key is set | +| Meta WhatsApp, SMS | Signature, then a Redis turn-complete. No LLM in the request | This is the right size | + +`finalize_telephony_recording` is the fast-ack pattern that already exists for the voice bundle. The carrier recording webhook does not use it. + +Alerts on this data are in section 13. They scan the same raw rows. + +### What it can carry today + +The 120/minute cap is **2 accepted exports per second per API key**, about 170,000 exports a day. That is roughly 17% of one request per trace at 1 million traces a day, and about 0.05% of a 5 second flush from 2,000 concurrent calls. A single tenant stays under the limit only while it has fewer than about **10 concurrent traced calls** flushing every 5 seconds. + +ClickHouse can take the row rate. The API cannot: it parses protobuf and waits on S3 on the request. Derive then multiplies the work. About 2,000 open traces / 3 seconds is on the order of **700 full-trace reads per second** average, and several thousand at peak. Idle close cannot drain a burst: 200 closes every 30 seconds, re-queued until they finish. A 20,000 call drop takes many minutes and does the work twice. + +The live-turn Redis cache lives 180 seconds. Idle close is 120 seconds, but the call-id cache is about 360 seconds. A long call (hold, support) falls back to a stale header, then a cache miss mints a **second trace id** for the same call. + +The orphan sweeper lists 2,000 S3 keys every 15 minutes. If workers stall past that, the rest of the batches are never replayed. + +### Backend + +**Critical. Raise the ingest limit to a byte or span budget per org, and keep a separate abuse limit.** 120 requests per minute per key cannot carry the product. Return 429 from **queue depth**, not from a cap that is already too small for ten concurrent calls. + +**Critical. Do not parse on the API.** Require the call id and workspace on the request (header or attribute the gateway can read without a full decode). Stream the bytes to the log. Parse only in the consumer. + +**Critical. Backpressure.** If the traces queue is deeper or older than a bound, return 429. Do not run derive inside the web process when the debounce lock fails (today a Redis error runs derive inline). + +**Critical. Incremental derive, and stop rewriting the whole header.** A dirty flag per trace, turn state updated from new spans, header row without the span blob. Put `workspace_id` in the lookup key so the read matches the sort order. The current lookup by trace id alone does not match `ORDER BY (workspace_id, started_at, trace_uuid)`. + +**Critical. Idle close as a queue, not a `FINAL` scan.** A Redis sorted set (or a state column in the primary key) of traces due to close. Mark `closing` when claimed so the same 200 are not re-queued. TTL on observations (days) and on headers (match the 90 day list window). Stop using `FINAL` on the hot path (`argMax` or a narrow header table). + +**Critical. Replace 6 digit call ids** with a time-ordered 64 bit id or a UUID. Keep a short code only as a display alias, if you still want one. + +**High. One cursor per stream on the calls list.** Counts from rollups. List columns that omit turns and failure blobs. Indexes on `(workspace_id, source, created_at)` and `(organization_id, agent_id, created_at)`. + +**High. Presign audio. Page transcripts.** Do not buffer the WAV in the API. Do not put the transcript in the call-detail poll. + +**High. Webhooks: verify every event, insert idempotently, return 200, do the rest in a worker.** Dedup on `(provider, call id, event id)`. Vobiz recording-ready should persist the URL and enqueue the finalize task that already exists. Telnyx must verify the signature on every event type, not only `call.initiated`. + +**Medium. Sequence keys need a TTL.** `INCR trace:batch_seq:{uuid}` is one Redis key per trace with no expiry: about 1 million new keys a day. Set `span_count` from a unique count, or drop it. Treat the S3 key plus a content hash as the idempotency token so retries do not double-count. + +**Medium. Hold the per-trace lock or retry.** Callers today proceed when the lock was not acquired. Concurrent batches lose `last_span_at`, and idle close uses a stale timestamp. + +**Medium. Catalog rows stay thin.** Ids, status, timestamps on the catalog. JSON only on the shard or in object storage. The dual-write does not reduce load while the fat column is still read. + +**Medium. Trace worker settings are the ML worker settings.** A 30 minute time limit and a child recycle every 20 tasks is wrong for short, numerous trace tasks. Give the traces deployment a short time limit and a high max-tasks. + +**Low. A minute rollup for the overview** (call count, error rate, latency percentiles). The dashboard route does not aggregate traces. The Observability overview downloads 500 evaluator results and charts them in the browser. + +### Frontend controls + +- **Overview charts.** They aggregate 500 rows in the browser. Point them at the rollup. Do not raise 500 to "load more". +- **Calls list.** Page size is small (about 15), which is good, and it refetches **every 3 seconds whenever any call is live**, with `staleTime: 0`. Each refetch reruns the hub counts. One live call makes every open Calls tab expensive. Poll a cheap generation counter, or use the SSE you already have, and refetch the list only when the generation changes. +- **Call detail.** Polls the call every 3 seconds and also opens an SSE. Pick one. Prefetch audio only as a presigned URL, not `?proxy=true`. +- **Trace panel.** Polls the trace and **all spans** every 3 seconds while the trace is open. A trace with thousands of spans is parsed in the browser on that interval. There is no virtualization. Poll spans only on the spans tab, and virtualize the list. +- **Search and filters on Calls.** They must be server filters. An in-memory filter on the fetched window reports the wrong total. +- **Evaluate on a call.** Single action, enqueue, return. Do not run the judge in the request. Confirm the button already does that; the ingest path does not evaluate on close, so this button is the manual path and should share the eval slot pool rather than bypassing it. +- **Date range.** Default the list to the 90 day window the API already has, and require an explicit action to go wider. + +### Keep + +202 plus an S3 write-ahead log plus a consumer is the right architecture. The 4 MB body cap is real. Trace rate limiting fails closed. ClickHouse is partitioned by month and observations dedupe by span id on merge. A cursor list API for traces already exists; the calls hub simply does not use it. Derive and close are already separate tasks on a `traces` queue. Live transcripts have Redis pub/sub SSE. SMS and WhatsApp webhooks only touch Redis. Shard payload reads for a page are grouped by shard. The API-key cache TTL for this path is configurable. + +The work is to stop doing extra work on the request, and to stop rereading the world on every derive, list, and sweep. + +--- + +## 12. Usage and billing + +Route: `/usage`. Files: `frontend/src/pages/usage/Usage.tsx` (about 1,470 lines), `app/api/v1/routes/org_usage.py`, `usage_pricing.py`, `app/services/usage/llm_usage.py`, `app/services/billing/flexprice_service.py`, `app/workers/tasks/flush_usage_counters.py`. + +### What it is today + +LLM, speech-to-text, and speech-to-synthesis usage is a Redis hash increment, flushed to Postgres about every 120 seconds by a beat task. Two flushers are guarded by a claim script so they do not double-apply. That design can take 1 million events a day. + +The exception is Flexprice: every completed import row with audio makes a synchronous HTTP ingest (section 9). On a Redis error, each usage event inserts into Postgres immediately, on the hot path. + +The Usage page sets `staleTime: 0`, refetches on every mount, and refetches on window focus, overriding the global default. Breakdown requests use `limit: 100`. The page does say when the result was truncated, so the missing groups are visible, and the numbers above the cut are not. Filter changes refetch the summary, the breakdown, and the whole option catalog. The client filters those options again and regroups the already-truncated rows. Pricing loads every AI provider and the model catalog up front. + +### What it can carry today + +The counter path is not the bottleneck. The page is wrong as a cost report once an org has more than 100 groups (agents, metrics, days). The Flexprice call is the bottleneck, and it sits on the eval worker, not on this page. + +### Backend + +**Critical.** Batch billing events. Covered in Imports; it is the same code. Do not add a second per-call HTTP client here. + +**High.** The breakdown API should group and page. The top 100 is a sample, and the UI should not have to regroup it. + +**Low.** Orphan recovery scans Redis with `SCAN` on the broker. Fine today. If flush claims leak, do that scan on the cache instance, not on the broker, once Redis is split (section 2). + +### Frontend controls + +- **Date range, group-by, filters.** Cache the option lists. Do not refetch the world on focus. One request per visible breakdown, server-grouped, with a real next page. +- **Searchable selects.** They filter the 100 returned options in the browser. If the agent you want is not in the top 100, you cannot pick them. Server search. +- **Pricing tab.** Load models for the provider being edited. +- **Export.** If you add one, generate it server-side. Do not download every usage row into the table first. + +### Keep + +`HINCRBY` plus a timer flush plus a claim script. This is the pattern Flexprice, aggregates, and alert counts should follow. + +--- + +## 13. Alerting + +Routes: `/alerts`, `/alerts/history`. Files: `frontend/src/pages/alerting/`, `app/workers/tasks/evaluate_alerts.py`, `evaluate_alerts_for_org.py`, `app/services/alerts/`, production metric queries. + +### What it is today + +Beat runs alert evaluation every 5 minutes and fans out one task per organisation that has an active alert. The schedule is fine. The query is not: production alerts load **every** `CallRecording` in the window into Python and compute duration and success from `call_data`. A 60 minute window at 1 million calls a day is about 40,000 JSON rows. A 24 hour window is about 1 million. Trace alerts run `count()` on `call_traces FINAL` per alert, so each extra alert repeats the scan. + +The alerts page calls `listAlerts()` with no limit, filters search and status in memory, and has no error state. A 500 looks like "no alerts configured". **Evaluate all** fires every rule from the browser. The agent multi-select uses the 100 agent cap. The query key has no workspace id. + +Alert history requests `skip 0, limit 100` and treats it as fresh forever (`staleTime: Infinity`). Search filters that page. The resolved and notify-failed counts are counts of those 100 rows. Open incidents correctly use a summary endpoint. Manual refresh has a 15 second cooldown, which is good. + +Alert detail loads the alert, up to 100 history rows, and all agents (capped at 100). + +Readers see create, delete, and evaluate-all. The API is expected to reject them. The UI does not check the role. + +### What it can carry today + +Hundreds of alert rules are fine. The evaluator falls over when the call window is large, which is exactly when you want alerts. + +### Backend + +**High. Aggregate in SQL or ClickHouse, one scan per org per tick.** Do not materialize `call_data` to compute a rate. Reuse the rollup from section 11 if you build it; do not scan raw calls a second way. + +**Medium. Evaluate-all should be one server call** for the current filter, queued, not N browser requests. + +### Frontend controls + +- **Search and status filter.** Server-side, paged. +- **Create, edit, delete, evaluate, evaluate all.** Hide them for readers. Show the API error instead of an empty list. +- **Agent scope.** Same typeahead as everywhere else, not the first 100 agents. +- **History table and the stat cards.** Cursor pagination. Stats from the summary endpoint, the way open incidents already work. Keep the 15 second refresh cooldown. +- **PagerDuty / email / Slack fields on the detail page.** Single-alert writes. Fine. Do not log routing keys to the console or to a toast. + +### Keep + +The 5 minute coordinator, the per-org fan-out, the open-incident summary endpoint, and the history refresh cooldown. Those are the right size. The metric query under them is what has to change. + +--- + +## 14. Configurations + +Sidebar: Cloud Storage, VoiceBundle, Integrations, Telephony Numbers, API Keys, Cron Jobs. Files: `frontend/src/pages/configurations/`, `app/api/v1/routes/data_sources.py`, `voicebundles.py`, `integrations.py`, `telephony.py`, `settings.py`, `cron_jobs.py`, `app/services/cron/job_dispatch.py`. + +These pages are not on the per-call hot path. They break when catalogs get large, and a few of them can pin an API pod by pulling a big object through it. Cron can dump a lot of work onto the eval pool at once. + +### Cloud Storage + +Browse calls `browseS3` with `maxKeys = 1000`, then sorts in the browser. There is no continuation token. A bucket of recordings for 1 million calls hides everything past 1,000 and, on failure, looks empty (errors are handled for upload and delete, not for browse). + +Upload and download of data-source objects read the whole body through the API (`await file.read()`, then the download returns the object body). + +**Change.** Page with the provider's cursor. Show a browse error. Presign upload and download. Playback stays on a presigned URL. + +**Controls.** Path breadcrumb, upload, delete, download, sort. Sort and filter belong on the page you fetched only if the page is complete. With a cursor, sort has to be server-side or clearly "this page only". + +### VoiceBundle + +Opening the page loops about **23 providers** and awaits a model-catalog call for each, cached 5 minutes, plus every bundle, AI provider, and integration. A provider outage fails that slot silently. Query keys are not scoped by workspace. The editor, the `?edit=` deep link, and three credential pickers live in one 1,750 line component. + +**Change.** One catalog endpoint, or fetch options for the provider the user picks. Split the form from the list. + +**Controls.** Create, edit, provider dropdowns, credential pickers. The dropdown is the expensive control. Everything else is a single-row write. + +### Integrations + +On mount: all integrations, all AI providers, telephony configs, and LLM gateway settings. Admin gating exists. Secrets are write-only password fields, and the copy says the browser does not display them. `getIntegrationApiKey` still returns a plaintext key and has no caller today. A later "reveal" button, or any XSS, can call it. VoiceBundle and Integrations use different query keys (`aiproviders` vs `ai-providers`) for the same list, so the cache does not dedupe. + +**Change.** Delete or tightly role-gate the plaintext key method. One query-key factory per resource. Split voice, LLM, and telephony forms (the file is about 2,050 lines). + +**Controls.** Connect, disconnect, save credentials. Single-row. The risk is the secret-read API, not the button. + +### Telephony numbers + +`listTelephonyNumbers()` has no paging. The page also loads every dial target and the outbound pool, then filters by provider in the browser. The derived "active numbers" array is rebuilt every render. + +**Change.** Page by provider. Memoize the derived lists. This inventory is also what the outbound pool in section 7 admits calls against; a truncated list and a pool of 5 are separate bugs that show up in the same screen. + +**Controls.** Add number, release, provider filter, outbound pool display. Filter server-side. The pool display should show the leased count from Redis, not a full refetch of every number. + +### API keys + +The product cap is 5 keys, so the list size is fine. The new secret is held in React state until the modal closes. There is no error state and no reader gate. A failed list looks empty. + +**Change.** Show the secret once, drop it from state on dismiss, show errors. The serious issue is the login key in `localStorage` (section 15), not this page's length. + +**Controls.** Create, revoke, copy. Fine at a cap of 5. If the cap is raised for automation, page the list then. + +### Cron jobs + +On mount the form downloads all cron jobs, all evaluators, all agents (100 cap), all personas (100 cap), and all scenarios (100 cap). None of the query keys include the workspace. Picking an evaluator past the cap is impossible, and a workspace switch can show another workspace's jobs until the slower request finishes. + +The dispatcher is a single flight (Redis lock), then enqueues **every** due job on that tick with no awareness of the 128 eval slots. Many orgs firing at the same minute land on the same pool as import judging. + +**Change.** Search evaluators by name. Include the workspace in the key. Enqueue cron runs through the fair dispatcher and cap how many jobs a tick may release. + +**Controls.** Create, enable/disable, schedule picker, "run now". Run-now must take a slot, not jump the queue. The agent/persona/scenario dropdowns are the silent-truncation bug again. + +### Knowledge bases (reached from Tools) + +Not in the top-level sidebar, but ingest is part of this configuration surface. Ingest loads the file, all chunks, embeds in batches of 32, and inserts vectors **one row at a time**. Retrieval is pgvector IVFFlat with `lists = 100`, which is the wrong index shape once chunks pass a few hundred thousand, and the filter is not applied before the vector scan. Upload is multipart through the API. + +**Change.** Presign the upload, multi-row insert, HNSW or a tuned index, pre-filter document ids. This matters because an agent tool call during a million-call eval will add this latency to every call that uses a knowledge base. + +### Keep + +Write-only secret fields. Admin gating on integrations. A product cap on API keys. Cron's single-flight lock (keep it, and add the slot cap in front of the enqueue). + +--- + +## 15. IAM, profile, login, and platform admin + +Files: `frontend/src/pages/iam/IAM.tsx`, `profile/Profile.tsx`, `auth/`, `platform/`, `frontend/src/lib/api.ts`, `frontend/src/store/authStore.ts`, `platformAdminStore.ts`, `frontend/src/components/Layout.tsx`, `WorkspaceSwitcher.tsx`. + +### Login and session + +Password login is in good shape. Access and refresh tokens stay in memory. Cookie sessions are on by default. CSRF is sent on writes. Refresh is single-flight. A disabled org is sent back to login with a message. Logout clears the session. OIDC keeps PKCE state in `sessionStorage`. + +The API key path is not. `authStore` reads and writes `apiKey` in `localStorage`, and the Axios interceptor sends it as `X-API-Key` when cookie sessions are off. Any script that runs on the origin can take a long-lived key. + +On refresh failure the client clears the session and hard-redirects to `/login`. It does not cancel in-flight queries, so they can 401 in a burst and stampede the refresh. Refresh failure also does not distinguish "this workspace is gone" from "this session is dead". + +Profile, on password change, clears the entire React Query cache, logs out, and redirects. That is correct, and it is why the cache must not hold secrets. + +### App shell (every page) + +`App.tsx` statically imports every page. There is no `React.lazy`. The production bundle is **one JavaScript file of about 5.5 MB** (built artifact `frontend/dist/assets/index-*.js`), plus about 376 KB of CSS. `dist` is about 17 MB because `public/` still contains persona photos of 1.3–1.6 MB and screenshots that nothing in the source references. Every login downloads LiveKit, Pipecat, ElevenLabs, Vapi, Retell, React Flow, and the chart library before Dashboard can paint. + +`frontend/src/lib/api.ts` is **7,066 lines** and `types/api.ts` is **2,795**. List methods return `Promise`. Nothing passes an `AbortSignal`. Navigating away does not cancel the request. There are about **746** uses of `any` under `frontend/src`. Lint does not ban it. There are 17 test files, almost all pure helpers. There is no test around the workspace switch or the auth interceptor. + +React Query defaults are `retry: 1` and no refetch on window focus, which is good, and `staleTime: 0`, which means every mount refetches. About **52** `refetchInterval` sites across 31 files. Six of them are fixed intervals even when the job is idle (called out in the sections above). + +**The workspace bug.** The active workspace id lives in Zustand and in `localStorage`. An interceptor stamps `X-Workspace-Id` at send time. Most query keys do **not** include the workspace id. Switching workspace calls `invalidateQueries()` with no filter. A slow response from workspace A can land in the cache after the user is looking at workspace B, because the key is just `['agents']` or `['alerts']`. Layout, Alerts, Cron, VoiceBundle, and Studio all share this shape. Metrics Management and the call-import queries already include the workspace. That is the pattern. + +Layout loads `listAgents` with the default **100** for the global agent dropdown and renders every row with no search. Past 100 agents the shell cannot select the rest, and every page pays for that list on mount. Agent-dropdown and sidebar state live in `Layout`, which also renders the page, so opening the menu re-renders whatever page is open, including the 4,500 line metrics screen. The mobile sidebar and the desktop sidebar are both mounted; each includes the workspace switcher, and the effect can invalidate the whole cache twice on first load. + +`OrgSwitcher` exists and is never mounted. Multi-org users change org from Profile, which lists every org with no search. + +Nav highlight is exact-match except for Tools and Metrics. `/alerts/:id` does not keep Alerts highlighted. Enterprise items stay in the sidebar even when the license will swap in an upgrade page after the click. + +There is no `dangerouslySetInnerHTML`. Markdown is rendered without raw HTML. Keep it that way. The only frontend env var is the API URL. + +### IAM + +The member table is the entire organisation, with loading and empty states and no error state. A 403 looks like an empty org. Password reset holds the new password in component state and can reveal it. That is acceptable for an admin reset if it is not logged, cached, or toasted. The file is about 1,030 lines; workspace members and roles are already separate components. + +### Platform admin + +The platform token is in `localStorage`, and `Authorization: Bearer` is read back from there. The org table requests `limit: 200` and ignores the API's `offset` and `search`. Members load the full user list. An admin can set a password in the modal. Orgs and signup codes use `useEffect` plus `useState`, not React Query, so they have no shared cancellation or retry. The page is outside the normal auth route and redirects in an effect. + +An XSS on the origin takes a cross-tenant admin bearer. A platform operator with more than 200 tenants cannot find an org. + +### What has to change + +**Critical. Put `workspaceId` (and org id) in every tenant query key, and pass `AbortSignal` into Axios.** Invalidate with a predicate, not a blanket wipe, except on logout and password change. This is the bug that makes every other page unsafe to use while switching workspaces under load. Do it once, in the query layer, not page by page after each incident. + +**High. Split the bundle.** `React.lazy` per route. Charts, the flow editor, and the voice SDKs in their own chunks. The login page should not download the voice stack. Delete the unreferenced public images. + +**High. Stop storing long-lived secrets in `localStorage`.** Cookie session for the product API key and for the platform admin, same as the password session. Until that ships, assume any XSS is a full compromise. + +**High. The global agent switcher is a typeahead**, against a paginated search. It is not a 100 row prefetch on every page. + +**High. Platform org list uses the API's search and offset.** Page members. Move those reads onto React Query so a 401 and a cancelled navigation behave like the rest of the app. + +**Medium. Show `isError` on every list** that today treats failure as empty: Dashboard, alerts, IAM, API keys, S3 browse, metrics seed. The metrics seed must run only after a successful empty response, once per workspace. + +**Medium. One sidebar instance.** Prefix-match nested routes. Hide or badge license-gated items. + +**Medium. Split `api.ts` by domain**, or generate the client from the API schema. New endpoints should not keep landing in a 7,000 line file with `any` returns. Ban new `any` on the client. Add tests for the query key and the 401 refresh interceptor. Those two are the tenancy and auth contracts. + +**Low. Cancel in-flight queries when refresh fails**, so a dead session does not fan out into a 401 storm. + +**Low. Mount the org switcher** in the header once orgs are numerous, with search. + +### Controls that show up on every page + +- **Workspace switcher.** Navigates off detail routes before invalidating. Keep that. Fix the cache key so the invalidation is not racing a stale response. +- **Org switch (profile).** Full cache clear is right because the session context changed. Show an error if the profile request failed; a blank form looks editable. +- **Global agent dropdown.** The single most repeated "loads too much, then stops" control in the product. Search, don't prefetch. +- **Sidebar links.** Cheap. Fix the active state so deep links still show where you are. +- **Logout.** Clears the session. Also clear the platform token and the API key. Confirm both stores are covered; the password path is, the platform path must be too. + +### Keep + +Cookie session, CSRF, in-memory refresh, single-flight refresh, no bearer token written for password login, no raw HTML injection, enterprise route gate, OSS quota checks on create, and the comment in `useRole` that the API is the real authority. The UI hiding a button is not the security boundary. Hiding it is still worth doing so the product matches the role. + +--- + +## 16. Patterns we are following, and the ones that are missing + +This is the same list a design review would use. It is here so a fix in one section does not invent a second pattern. + +**Already in the codebase, and worth keeping:** + +- Bulkheads by queue (partial: six import-related queues still share one worker). +- Fair-share leases for import and eval (workspace, org, global, per job). +- Delayed, aggregated usage flush. +- Slice sharding with a catalog registry and a Redis epoch so processes drop a stale map. +- A separate media process and a short "place the call, return" task for telephony. +- Idempotency-Key on platform-import create. +- Presigned URL support on the blob backends (the hot path often ignores it). +- Cursor listing for traces (the calls hub does not use it). +- 202 plus an object write-ahead log for trace ingest. +- Liveness separated from readiness. + +**Missing, and called out in the sections above:** + +| Pattern | Where it hurts | +| --- | --- | +| Backpressure at enqueue | Trace ingest returns 202 with no queue check. Simulation create always `.delay()`. HTTP rate limits skip the bulk APIs. Leases fail open | +| Leases and a reconciler | Simulations stick in `RUNNING`. Import slot integers leak. Trace idle-close re-queues the same 200 | +| Idempotency | Simulation create, carrier webhooks, Vobiz answer, outbound retry | +| Per-tenant and per-provider bulkheads | Harbor, voice eval, LLM, STT. Imports have the tenant half only | +| Circuit breakers | All external providers | +| Thin catalog, fat data elsewhere | Call JSON, trial transcripts, and scores are still written to the catalog | +| Keyset pagination and rollups | Calls list, import aggregate, alerts, results overview, usage breakdown | +| Outbox between catalog and shards | Rebalance copies, commits, then deletes. A failed epoch publish can delete the only copy. Not on the daily hot path, but it is how you will move data when a shard fills up | +| One migration runner | Every pod migrates itself | +| Connection budget | Pools are per process with no global ceiling | +| CQRS for read-heavy product surfaces | ClickHouse is only the trace store. Eval charts and alerts still scan the OLTP rows | + +Sharding does not fix the items in that table. It spreads stored bytes. It does not add fair scheduling, and the catalog remains the hotspot for auth, run rows, and the JSON we still dual-write. + +--- + +## 17. Where to start the discussion + +These are themes, not a sprint plan. The order is "what makes the other work possible". + +**First, so that more replicas are safe.** + +- Postgres connection budget and PgBouncer. +- Migrations as one locked Job. +- Redis split and durable. +- Object storage instead of `./uploads` and `./.data`. +- Workspace id on every query key, and no long-lived token in `localStorage`. +- Replace 6 digit ids (calls, simulation runs, evaluator results) before the space fills. This one gets worse by itself, even if traffic does not grow. + +**Second, so that 1 million a day is a capacity conversation instead of an architecture one.** + +- Trace ingest: higher org budget, no parse on the API, 429 when the queue is deep, incremental derive, idle-close that marks rows claimed, TTL, no `FINAL` on the hot path. +- Simulations: one leased trial per queue item, a stuck-run sweeper, no database connection held for the whole conversation, an LLM rate limit. +- Voice: calls on the media tier, not inside a Celery child, atomic per-org and per-provider caps. +- Imports: stream audio, lease slots that fail closed, a larger download pool split from the judge pool, rollups for charts, Flexprice off the hot path, presigned upload. +- Webhooks: verify, dedup, return 200, download in a worker. +- Alerts and the calls list read rollups, with the missing time indexes. + +**Third, so the team can keep shipping once the above is in place.** + +- Route-level code splitting and a smaller `api.ts`. +- Break up the largest screens: import evaluation detail (~10,400 lines), import detail (~5,300), metrics management (~4,500), prompt partials (~2,800), integrations (~2,050). +- Server search instead of silent caps at 100, 500, and 1,000. +- Poll only while work is in progress; prefer the SSE that traces already have. +- Error states that are not blank pages. +- Tests for tenancy and session refresh. + +A load test belongs next to the second group, not before the first. Useful scenarios: trace export at a few hundred requests per second, the calls list with one live call and many open browsers, a CSV import of a few hundred thousand rows, and a simulation run of a few thousand trials with two organisations at once. The numbers in this document are ceilings from the code. The test will show which ceiling you hit first on the real cluster. + +--- + +# Part B — Code shape and low-level design + +Part A says where the product runs out of room. This part says whether the code itself is doing unnecessary work, and whether the next feature has a place to go. Adding pods does not fix either. A request that does three database round trips still does three round trips on every pod. A 10,000 line screen still has to be understood by the next person who adds a button. + +## B1. The rule for this codebase + +**One user action, one purpose, one cheap round trip.** A list must not load a blob. A read must not write. A poll must not repeat a heavy query. A second request is allowed only when the screen shows a second thing the user asked for. + +**A file that grows past a few hundred lines is a failed split, not a badge.** Routes stay thin. A use-case function does one job. A query function returns one shape. A React page composes panels. A panel does not fetch the world. + +**New work extends a seam. It does not edit the giant function.** A new voice provider, a new metric type, a new import source, or a new chart is a new module that plugs into an existing interface. If the only way to add it is to scroll a 4,000 line file, the design is already wrong. + +The house style, in one picture: + +```text +Browser panel → feature hook (query key includes workspace) + → feature API module (one function, AbortSignal) + → route (auth, parse, status code) + → use-case service (the business rule) + → query module (one SQL or one ClickHouse statement) + → worker, only when the work is slow +``` + +Today the middle is often missing. The route holds the SQL, and the page holds the queries, the filters, and the markup. + +## B2. What is already modular + +This is not a junk repository with no structure. The folders are the right idea, and several seams are real. New code should extend these, not walk around them. + +- Backend is split into `app/api` (HTTP), `app/services` (use cases), `app/workers` (slow work), `app/db_sharding` (where a row lives), `app/core` (auth, limits, middleware), `app/models` (tables and DTOs). +- Speech-to-text clients live as one file per provider under `app/services/ai/stt_clients/`. That is a strategy: a new provider is a new file. +- Blob storage has more than one backend, and presign already exists as a method. +- Import and eval dispatch is its own package (`app/workers/concurrency/`) with caps per workspace, org, and job. That is the scheduling policy, and it is not buried in a route. +- Telephony already separates "place the call" from "hold the socket". +- The frontend is split by product area under `frontend/src/pages/`, and Imports already has a `components/` folder (upload modals, mapping, transcript, progress). Shared widgets live in `frontend/src/components/`. Pure helpers in `frontend/src/lib/` have the only real tests. +- Call-import and metrics screens already put the workspace id in the React Query key. That is the tenancy seam. Copy it. Do not invent a second one. + +The failure is that the **hot features outgrew the seam**. The next thousand lines went into the route and the page, because that was faster than opening a new file. That is the habit to stop. + +## B3. Cross-cutting code problems + +These show up in almost every section. They are listed once so the per-section notes can stay short. + +**Routes are doing the job of services.** `app/api/v1/routes/` is about 51,000 lines. `call_import_evaluations.py` is **9,937 lines, 44 endpoints, and about 80 direct `db.query` calls**. `call_imports.py` is 4,645 lines and about 31 queries. `agents.py` is 1,843 lines and about 40 queries. `playground.py` is 2,202 lines and about 43 queries. `voice_playground.py` is 2,549 lines and about 39 queries. A route function should be: authenticate, validate the body, call one service, map the result to a status code. SQL inside the route cannot be reused by a worker, cannot be tested without HTTP, and gets copied into the next endpoint. + +**Two god modules sit under every feature.** `app/models/database.py` is **3,737 lines** of every table. `app/models/schemas.py` is **6,372 lines** of every request and response. Adding a field to a call means opening a file that also owns personas, billing, and alerts. Split by domain: `models/calls.py`, `schemas/calls.py`, and the same for imports, sims, traces, identity. The tables can stay one database. The source files should not. + +**The judge is one 3,570 line module.** `app/workers/tasks/helpers/llm_evaluation.py` builds prompts, parses answers, and special-cases model families (Jev, Together) in one file. A new metric type means editing the middle of that file. The prompt builder, the parser, and each model dialect should be separate, with one interface: `score(transcript, metrics) -> scores`. + +**The frontend client is the same mistake.** `frontend/src/lib/api.ts` is **7,066 lines**. Every screen imports `apiClient` from it. There is no per-feature module, no generated types, and list methods return `any`. A change to the imports client is a change to the file that also does login. Split by domain, and pass `AbortSignal` so leaving a page cancels the request. + +**Pages fetch inside render trees that are too big to reason about.** The largest screens: + +| File | Lines | What it should become | +| --- | ---: | --- | +| `CallImportEvaluationDetail.tsx` | 10,368 | One shell, plus a tab component per view (rows, chart, insights, export) | +| `api.ts` | 7,066 | One module per product area | +| `CallImportDetail.tsx` | 5,331 | Shell, row table, upload, mapping | +| `MetricsManagement.tsx` | 4,518 | List page, and one component per dialog | +| `schemas.py` | 6,372 | One schema module per area | +| `call_import_evaluations.py` | 9,937 | Thin routes plus one service per use case | +| `database.py` | 3,737 | One model module per area | +| `llm_evaluation.py` | 3,570 | Prompt, parser, and one dialect per model family | +| `PromptPartials.tsx` | 2,813 | List and editor | +| `Integrations.tsx` | 2,055 | One form per provider family | + +A practical limit: **a new file past ~400 lines needs a reason, and a file past ~800 lines is a split before the next feature.** Do not boil the ocean in one change. The rule applies to the next edit: when you touch the file, extract the piece you are changing. + +**The same concept has two names.** Integrations uses the query key `['ai-providers']`. VoiceBundle uses `['aiproviders']` for the same list. Two caches, two requests, two places to forget the workspace id. One key factory per resource: `keys.integrations.list(workspaceId)`. + +**Reads that write, and lists that compute.** API-key auth updates `last_used` on the request path. Simulation progress reloads every transcript to increment a counter. The calls page runs several `COUNT`s and then loads the rows. The visualizations tab loads every score document to draw a chart. Each of these is an extra round trip that a counter or a rollup would remove. Command and query are mixed: the screen asks a question and the server changes state or scans a table to answer it. + +**Work is done twice because two layers do not trust each other.** Trace bodies are parsed on the API and again in the worker. Recordings are downloaded, stored, then downloaded again for transcription and sometimes a third time to measure duration. Trial JSON is written to the catalog and again to the shard. The UI polls a resource that an SSE stream is already pushing. Deduplicate at the boundary. One owner for each fact. + +**There is no repository, and that is fine if the query functions are real.** Do not introduce a generic repository class. Do extract `list_calls(workspace, cursor, filters)` into a module the route and the alert job both call. Today the alert job and the calls page each have their own scan. + +**Tests protect helpers, not behaviour.** Frontend tests are almost all pure functions (17 files). The workspace switch, the query keys, and the 401 refresh have no test. Backend use cases that live inside route functions are only testable through HTTP. Extract the use case, then test the use case. A 9,000 line route will not grow a useful test suite. + +**What "done" looks like for a new feature.** A short checklist, so reviewers can say no without a debate: + +1. The route file did not gain SQL or a new 200 line function. +2. The use case has a name and a test that does not boot the whole app. +3. The list endpoint returns ids and summaries, not transcripts or `call_data`. +4. The React Query key includes workspace id. The request takes an `AbortSignal`. +5. The screen does not poll unless something on it is in progress. +6. The new UI lives in a new component. The page file did not grow by hundreds of lines. +7. A new provider or metric type is a new class or file behind the existing interface. + +## B4. Per section + +Same sidebar order as Part A. Each note is the code, not the cluster. + +### Dashboard + +The page is small, which is the right size. The bug is what it asks for. `getDashboardSummary` is a catalog count, and the carousel then fetches the full TTS catalog on the home page, on every visit, for a widget the user did not request. That is a second request with no user action behind it. + +**Shape.** Keep `Dashboard.tsx` as composition: summary, quick-start, nothing else. The carousel is a separate lazy component, or it is deleted from this route. The summary endpoint stays one query against counters. It must not grow into "also load recent calls" inside the same handler. Recent calls are a different use case. + +**Next feature.** A new home-page card is a new endpoint that reads a counter, plus a new small component. It is not another query stuffed into `dashboard.py` and another fetch in the page. + +### Metrics + +`MetricsManagement.tsx` (4,518 lines) is the list, the create flow, the classification editor, categories, AI generate, and partial import. `metrics.py` is 2,555 lines and about 24 direct queries. Studio source pickers each fire their own list (`call imports`, `rows`, `recordings`, `calls`, `evaluator results`) capped at 100, so opening Studio is five requests that all return the wrong page of a large account. + +**Wasted work.** Search filters in the browser, so every keystroke re-renders the whole 4,500 line tree over the full metric array. The empty-list effect seeds default metrics, which is a write triggered by a read, including a failed read. + +**Shape.** One hook, `useMetrics(workspaceId, filter)`, that calls a paged endpoint. One component per dialog. The seed belongs in a migration or an explicit "load defaults" button, never in a `useEffect` on an empty array. Scope pickers are a shared `EntitySelect` with server search, used by metrics, cron, and alerts. Do not copy the 500-row fetch into a fourth screen. + +**Pattern.** Metric types (score, classification, and the next one you add) should be a strategy behind `build prompt` / `parse answer`, not more branches in `llm_evaluation.py`. The editor and the judge must share that strategy so the UI cannot configure a shape the judge cannot score in one call. + +### Simulations — Agents, Personas, Scenarios, Tools, runs + +Agent, persona, and scenario routes are each over 1,500 lines (`agents.py` 1,843 with about 40 queries, `personas.py` 1,525 with about 32). The pages call `listX()` with a default of 100 and filter on the client. The shell does it again for the global agent dropdown. Opening any page therefore pays for an agent list the page may not show, and the list is a different cache entry from the page's own list when the keys differ. + +**Wasted work on a run.** The worker reloads every trial payload every 5 seconds to update two integers. The open run page refetches the run and every trial every 4 seconds. One trial finishing causes a full reread on the server and a full reread in every open browser. The counter belongs on the run row. The page polls `{status, completed, failed}`. The matrix asks for one page of cells. The drawer asks for one transcript when it opens. + +**Shape.** `simulation_runs.py` should become: create, cancel, get progress, list trials, get trial. The Harbor subprocess stays the trial runner. Progress accounting is a few lines in the trial-finish path, not a loop in the parent. `NewSimulationRunModal` should not own four catalog fetches. It receives search callbacks. + +**Pattern.** A run is an aggregate: one parent, many trials. The code treats the parent task as the aggregate and keeps the trials inside it. Invert that. The parent is a row. The trial is the unit of work. That is both the scale fix from Part A and the design fix: you can test one trial without standing up a 6 hour job. + +**Next feature.** A new column on the run matrix is a field on the trial summary DTO. It is not a reason to load `call_data` into the list. + +### Playground and live voice + +`playground.py` (2,202 lines, ~43 queries) and `voice_playground.py` (2,549 lines, ~39 queries) are products living inside route files. Provider differences (Vapi in the browser, Retell and ElevenLabs created on the server, then a sleep-loop poll) are branches in the route. The poll holds a database session and the decrypted key. That is a use case (`watch_call`) pretending to be a request. + +**Wasted work.** While one call is in progress, the playground refetches the entire result list and the entire recording list. Pagination slices the array after the download. The voice page adds more intervals, including ones that run when nothing is in progress. + +**Shape.** One `PlaygroundCallService` with `start`, `status`, `stop`. Provider-specific start is a strategy object (`VapiBrowserCall`, `RetellServerCall`, `ElevenLabsServerCall`) so the next provider is a file, not a new branch in a 2,000 line route. Status comes from a webhook or a single-id poll. The screen subscribes to the ids it started, not to the whole history. + +**Pattern to copy.** Telephony's `initiate` task plus media process plus Redis session is already the strategy's skeleton. The playground and the voice evaluator bridge should call that skeleton. They should not grow a third copy of "how a call lives". + +### Evaluations + +Suite start does insert, commit, id probe, and enqueue **per trial inside the HTTP request**. That is N round trips for one click, and the id probe is the 6 digit collision check. Bulk insert, one dispatcher message. + +The results page downloads 500 full `metric_scores` documents and then drops most of the points on the floor to draw a trend. The chart's data shape is a bucket (`day, pass rate, count`). Returning the full document is the wrong DTO for that widget. List DTO and chart DTO are different types. They should not share a "give me the row" function. + +**Shape.** `evaluator_run_service.py` is the right place and should be the only place that creates runs. Routes in `evaluators.py` (~24 queries) and `evaluator_results.py` (~32 queries) shrink to call it. Judge alignment's serial loop becomes "enqueue N samples through the same dispatcher", not a private loop with its own session lifetime. + +### Imports + +This is the sharpest LLD problem in the repo, because the feature is also the largest. + +`call_import_evaluations.py` at 9,937 lines is not a module. It is forty-four handlers and the SQL for charts, exports, retries, and row detail. `call_imports.py` at 4,645 lines repeats the pattern for the import itself. `bulk_ops.py` (2,353) and `scatter_gather.py` (1,744) are real extractions, which proves the split works. The next extraction just did not happen. `call_import_evaluation_pdf_report.py` (2,945 lines) and `call_import_metric_clusters.py` (1,455) are further use cases that still reach around the route. + +On the frontend, `components/` already exists (upload, mapping, transcript, progress, side panel). The page files did not become shells. `CallImportEvaluationDetail.tsx` is still 10,368 lines and `CallImportDetail.tsx` is still 5,331. Extraction started and then stopped. Finish it: the page chooses the tab and passes ids. The tab owns its query. + +**Wasted work, per click or per poll.** + +- Upload reads the file into the API, writes S3, then a worker reads it all back to parse. +- One recording is downloaded up to three times (fetch, transcribe, duration probe). +- Visualizations polls a handler that loads every score row, every 5 seconds, while a run is open. +- "Select all" and retry build a Python list of every id. +- Opening a row can request up to 5,000 full bodies even though the table shows 100. +- Flexprice HTTP sits in the row-finished function, so billing latency is part of scoring. + +**Shape, use case by use case.** Each of these is its own module with its own input and output. None of them import the route file. + +| Use case | Owns | Must not do | +| --- | --- | --- | +| Stage upload | Accept an object key, parse in batches | Read the HTTP body into a `bytes` | +| Dispatch | Claim the next row id | Load the transcript | +| Fetch audio | Stream to object storage, store duration | Hold the file in a `bytes` and return it | +| Judge | Scores in, scores out | Call Flexprice or download audio | +| Chart | Read the rollup | `query.all()` of `metric_scores` | +| Insights | A sample of ids, then those transcripts | Load every pair, then sample | +| Export | Page through rows, write parts | One gzip of the whole run | + +**Pattern.** The pipeline is a state machine on the row (`pending → fetched → transcribed → scored`). Each state has one worker function. Adding a step (for example, redaction) is a new state and a new function, registered in the dispatcher. It is not a new block pasted into `evaluate_call_import_row_core.py` (already 1,107 lines) and a new tab pasted into the 10,000 line screen. + +**Frontend rule for this area.** A new button on the evaluation page is a new component file and a new hook. The pull request that adds it should show `CallImportEvaluationDetail.tsx` getting smaller or staying the same length. If the page file grows, the button was added in the wrong place. + +### Prompts + +`PromptPartials.tsx` is 2,813 lines for a catalog and an editor. `prompt_partials.py` is 981 lines and about 15 queries. The product is simple. The file size is the defect. Split list and editor. The optimization job is already a worker with a budget (`max_metric_calls`). Keep that function small. Do not let it grow a private copy of the judge. It should call the same `score(...)` interface as imports, on a sample. + +### Observability and live traces + +The pipeline modules exist (`ingest_pipeline`, `trace_service` at 1,988 lines, `otlp_mapper` at 1,496, ClickHouse store). That split is right. The waste is inside the steps: + +- The route parses the body the worker will parse again. The route's job is auth, size check, and handoff. +- Derive reloads every span to recompute turns. Turn state should be updated from the new batch only. +- The calls hub (`calls_hub.py`) merges two stores in the API process and counts them on every page view. Listing and counting are two queries with two DTOs. The list DTO does not include `turns`. +- Call detail and the audio proxy put the blob on the same endpoint as the metadata. Metadata, transcript page, and audio URL are three use cases. The panel currently polls the fat one and also opens an SSE. +- `observability.py` is 900 lines and about 17 queries. Webhook handlers still do provider calls and file downloads inline. A webhook function is: verify, dedup, enqueue, return. The download is the worker you already have (`finalize_telephony_recording`). + +**Pattern.** Ingest is a pipeline of pure steps: `accept → store raw → parse → append spans → fold header`. Each step has a function and a test with a fixture payload. `trace_service.py` at almost 2,000 lines is the next file to split along those step names, before the next trace feature lands in it. + +**Frontend.** `Observability.tsx` charting 500 evaluator results in the browser is a DTO error: the overview asked for rows when it wanted aggregates. `SyntheticCallTracePanel.tsx` (about 1,020 lines) polling every span is the same error. The spans tab requests a page of spans. The header tab requests the header. + +### Usage and billing + +`org_usage.py` is 1,717 lines for what should be a few read models. `llm_usage.py` (1,547) already has the right write path: increment a counter, flush later. `flexprice_service.py` (1,386) is called from the judge. That is a layering violation. The judge finishes a row. A billing subscriber reacts. The judge does not import the billing client. + +The Usage page refetches summary, breakdown, and the option catalog on every filter change and on window focus, then filters the options again in the client. Focus refetch is a request the user did not make. The breakdown DTO is already truncated at 100 and the client regroups it, which means the server returned the wrong grain. + +**Shape.** `UsageQuery` with `summary`, `breakdown(group, cursor)`, `suggest(field, text)`. The page holds filter state and renders the result. It does not re-aggregate. + +### Alerting + +`alerts.py` is 661 lines and about 21 queries. `alert_evaluation_service.py` is 1,117 lines and pulls every call JSON in the window into Python. The rule ("is the failure rate over the threshold") is a few lines. The scan is the rest. The rule should call the same rollup query the calls overview uses. If both screens have their own scan, they will drift, and you pay for both. + +The page loads every alert and filters in memory, and Evaluate-all is one request per rule from the browser. One request: `POST /alerts/evaluate?filter=...`. The server fans out. + +**Shape.** Alert rule, alert evaluation, and notification delivery are three modules. `send_alert_notifications` already exists as a task. Keep the rule evaluator from knowing about PagerDuty. It emits "rule X fired". The notifier decides the channel. + +### Configurations + +Each settings page loads full catalogs on mount, and several load the same catalog under different cache keys. VoiceBundle's loop of ~23 model-catalog requests is the clearest waste: one screen, 23 serial calls, most of them for providers the user will not open. One `GET /model-options?provider=` when the dropdown opens. + +`data_sources.py` streaming a whole object through the route is the route doing storage's job. The route returns a presigned URL. The browser talks to object storage. + +Cron's form fetches jobs, evaluators, agents, personas, and scenarios up front. Four of those are the silent 100-cap lists again. The form needs a typeahead, which is the shared `EntitySelect` from Metrics. Cron dispatch should call the eval dispatcher. It should not have its own "enqueue everything that is due" path that bypasses the slot cap. Two enqueue paths means two places that forget fairness. + +Knowledge-base ingest inserts vectors one row at a time after building the full chunk list. The use case is `ingest(document) -> chunks in batches`. The retrieval query is a separate function with the index hint. Do not hide both in the route (`knowledge_bases.py`, 653 lines, ~12 queries). + +### IAM, login, shell, platform admin + +**Shell.** `Layout.tsx` fetches agents, renders the sidebar twice, and owns UI state that re-renders the current page. The layout's job is chrome. Data for the agent switcher lives in a hook used only by that control. `WalkthroughProvider` selecting a fresh object from the license store re-renders the whole tree on a quota refresh. Select the fields the walkthrough reads. + +**API client and session.** `api.ts` handling login, refresh, workspace header, and every product call is why a billing change risks auth. Split `http.ts` (instance, CSRF, refresh, workspace header) from `features/*/api.ts`. The refresh interceptor is the one place that handles 401. Feature code does not. + +**Query keys.** A single `queryKeys` module. No string arrays written at the call site. A review rule: a `useQuery` whose key does not contain `workspaceId` is a bug, except for the platform-admin and the logged-in user. + +**IAM and platform admin.** `IAM.tsx` (~1,030 lines) mixes org members, invites, and password reset. The workspace sections are already separate components. Finish that for the org tab. Platform admin bypasses React Query with `useEffect`, so it does not get cancellation or the shared 401 behaviour. That is a second data layer. Use the same client. + +**Tokens.** `localStorage` for the API key and the platform token is a design bug, not only a security bug. It creates a second session mechanism next to the cookie session. One session mechanism. Feature code never reads a token. + +## B5. Design patterns — use these, skip the rest + +You do not need a pattern catalog. These six cover the ways this product actually grows. Each one is already started somewhere. The work is to make it the default. + +| Pattern | Use it for | Already exists | Stop doing this instead | +| --- | --- | --- | --- | +| Thin route + use case | Every new endpoint | `bulk_ops`, Harbor job runner, flexprice as a service (but called from the wrong place) | SQL and branching inside `routes/*.py` | +| Strategy | Voice providers, STT, TTS, metric dialects, import sources | `stt_clients/`, storage backends | `if provider ==` inside playground, judge, and webhooks | +| Pipeline / state machine | Import row, trace ingest, simulation trial | Row statuses, trace `open/closed`, run statuses | The next step pasted into a 1,000 line task | +| DTO per screen | Lists, charts, detail | List responses that omit `call_data` in some places | One "full row" response reused by a chart and a table | +| Feature module on the client | Each sidebar area | `pages//components` for imports | `api.ts` plus a page file of several thousand lines | +| Pub/sub for live UI | Traces, run progress, import progress | Live-transcript SSE, Redis progress counters | A 3 second poll of the heavy endpoint | + +Patterns to **not** add, because they would be ceremony here: a generic repository base class, a dependency-injection framework, an event bus for every write, CQRS as a project-wide architecture. ClickHouse for traces is already the one place where the read model is separate. Copy that idea for charts and alerts as a rollup table. Do not rebuild the whole app around it. + +SOLID, in the form that matters for this repo: + +- **One reason to change.** `llm_evaluation.py` changes when the prompt changes, when a parser changes, and when a model vendor changes. That is three reasons. Split them. +- **Open for a new provider, closed for the call loop.** The loop lives once. The provider is a new file. +- **A page depends on a hook, not on 40 query calls.** The hook depends on a small API module, not on `api.ts`. +- **The judge does not depend on Flexprice.** Billing depends on "a row finished". +- **Routes do not depend on SQLAlchemy query details.** They depend on a function. + +## B6. How to apply this without a rewrite + +A big-bang split of the 10,000 line files will stall the product. The rule is on the next change: + +1. Touch a giant file only to move the code you are modifying into a new module, then call it. +2. No new `db.query` in a route file. If you need a query, it goes in the use-case module you just extracted. +3. No new fetch in a page file over 800 lines. The fetch goes in a hook next to a new component. +4. No new poll of an endpoint that returns more than a status object. +5. A new provider or a new metric dialect is a new file that implements the existing function signature. The pull request shows that the old `if` chain was not extended. +6. Add a test on the extracted function. That is how the module stays a module. + +Do this first, because every later feature lands here anyway: + +- Extract chart, export, and retry out of `call_import_evaluations.py`, and split `CallImportEvaluationDetail.tsx` by tab. +- Split `llm_evaluation.py` into prompt, parse, and dialects before the next metric type. +- Split `api.ts` enough that imports, traces, and auth do not share a file. Add `queryKeys`. +- Make the judge emit a billing event instead of calling Flexprice. +- Make list and progress endpoints return summaries, and point the existing polls at them. + +Infra scaling (Part A, section 17) still matters. It will not make this code cheaper to change, and it will faithfully multiply whatever extra query you leave in the request. + +--- + +# Part C — Are we following the battle-tested patterns? + +A pattern is a named way to keep one kind of change from ripping through the codebase. We do not need all 23 Gang-of-Four patterns. We need the ones that match how this product grows: new providers, new metric types, new import sources, more tenants, and more people editing the same files. Where a pattern does not match the product, adding it would create the junk we are trying to avoid. Those are called out at the end, so nobody adds them "for completeness". + +Status words: + +- **Following** — the seam exists and new code can use it without editing a giant file. +- **Partial** — the pattern exists in one area and is bypassed in the area that is growing. +- **Missing** — the failure mode of this pattern is already in the code. This is what breaks later, not a style nit. + +## C1. Principles (the concepts). Patterns are how you enforce them. + +| Concept | The rule | Status | Where we stand | What breaks if we leave it | +| --- | --- | --- | --- | --- | +| Single responsibility | A module changes for one reason | **Missing** on the hot files | `llm_evaluation.py` changes for prompts, parsers, and model vendors. `call_import_evaluations.py` changes for charts, export, retry, and row detail. `api.ts` changes for every feature | Two people cannot edit the same feature. A fix in charts breaks export | +| Open/closed | Add a provider or metric type by adding a file, not by editing the loop | **Partial** | Voice providers, STT clients, auth providers, and historical import adapters are open/closed. The judge, playground call start, and webhook handlers are closed only by editing them | The next provider is a week inside a 2,000 line route, and it regresses the previous one | +| Liskov | A subclass must be usable wherever the base is used | **Partial** | `BaseVoiceProvider` promises `create_web_call` for every platform. The live bridge then special-cases ElevenLabs and raises for the others | Callers learn the subclass anyway, so the base type is a lie. Swapping a provider throws at runtime | +| Interface segregation | Do not force a client to depend on methods it does not use | **Missing** on the voice base | `BaseVoiceProvider` is one interface with many abstract methods (web call, agent CRUD, and more). A TTS-only provider still has to pretend to be a full telephony API | Stub methods that raise. The next small provider copies the stubs | +| Dependency inversion | High-level code depends on an interface. The concrete client is plugged in at the edge | **Partial** | FastAPI `Depends` for auth. Protocols for tools, historical imports, and auth providers. The judge imports Flexprice. Routes construct queries. Pages import the god `apiClient` | You cannot test the use case without the vendor, the database, and HTTP | +| Separation of concerns | HTTP, business rule, query, and UI are different modules | **Partial** | Folders exist (`api`, `services`, `workers`, `pages`). The hot path ignores them: SQL lives in routes, data fetching lives in page files | Every layer reimplements the same rule slightly differently | +| DRY | One fact, one place | **Missing** in a few expensive places | AI provider list has two React Query keys. Trace body parsed twice. Audio downloaded up to three times. Catalog JSON written twice. Suite-create and import-create each invent an id | The copies drift. One path gets the fix, the other keeps the bug, and you pay the cost twice per request | +| Command–query separation | A read does not write. A question does not scan a table the writer should have summarized | **Missing** on several screens | Auth updates `last_used` on read. Metrics seed writes because a read returned empty. Charts and alerts compute by loading rows | Reads get slower as data grows, and a failed read inserts data | +| Law of Demeter | A screen asks one neighbour for what it needs. It does not reach through four objects | **Missing** on large pages | A page calls `apiClient` for catalogs it will not show, then filters the result. Layout fetches agents for every page | One screen triggers a fan-out of requests. Changing the agent API breaks unrelated pages | +| Tell, don't ask | The object that owns the state performs the transition | **Partial** | Import rows and runs have status strings. Many places set the string directly instead of `row.mark_fetched()` | Illegal transitions (scored → pending, running forever) are easy to write and hard to find | +| Fail closed | If the protection mechanism is down, refuse the work | **Missing** on the slot lease | Trace rate limit fails closed. Import and eval slot acquire returns success when Redis errors | An outage removes the only cap. Tenants stampede the provider | +| Program to an interface | The caller names the operation, not the vendor | **Partial** | `get_voice_provider(platform)`, `HistoricalProviderAdapter`, `ToolProvider`, `AuthProvider` | The playground and the judge still say the vendor's name in the branch | +| Composition over inheritance | Build behaviour by combining small objects | **Partial** | Middleware is composed. Voice providers use a deep abstract base instead of small capabilities composed together | The base class becomes a junk drawer. Every provider inherits methods it does not want | +| High cohesion, low coupling | A package's files are about one thing, and other packages talk to it through one door | **Partial** | `db_sharding` and `workers/concurrency` are cohesive. `routes/call_import_evaluations.py` is not. Frontend features all couple to `api.ts` | A change in billing can break login, because they share a file | +| YAGNI / KISS | Do not add a pattern that has no change behind it | **Following** in spirit, then violated by file size | We did not build a generic repository framework. We did let files grow to 10,000 lines, which is the opposite of simple | The "simple" file is now the most expensive place to change | +| Encapsulation | Callers cannot touch the internals | **Partial** | Services exist, then routes reach into `db.query` and the UI reaches into full row JSON | Every caller depends on column layout. A column rename is a product-wide change | + +## C2. GRASP — who owns the work + +These are the responsibility rules behind the patterns in C3–C7. + +| Concept | Pattern that enforces it | Status | Evidence | +| --- | --- | --- | --- | +| Information expert | The module that holds the data does the calculation | **Missing** for counters | Trial progress is computed by reloading every transcript. Call success is computed by loading `call_data` in the alert worker. The row already has the facts. A counter on the parent should answer | +| Creator | The object that uses a thing is the one allowed to construct it | **Partial** | `get_voice_provider` creates providers. Routes and tasks also construct clients and sessions ad hoc | +| Controller | One use-case object receives the request | **Missing** on large routes | 44 endpoints in one file, each its own controller with its own SQL. There is no `EvaluateImport` or `GetCallChart` object | +| Polymorphism | A new type is a new class, dispatched by the interface | **Partial** | Voice, STT, auth, historical import. Not the judge, not call start, not webhooks | +| Pure fabrication | A service that is not a domain noun, invented to keep cohesion | **Following** where it exists | `fair_dispatch`, `bulk_ops`, `ingest_pipeline` are fabrications and they are the healthiest modules. The routes that did not get one are the problem | +| Indirection | A stable middle so the caller and the vendor can change separately | **Partial** | Provider registries. Missing between the judge and Flexprice, and between the page and `api.ts` | +| Protected variations | Wrap the thing that will change | **Partial** | Providers are wrapped. Metric dialects, chart queries, and list payloads are not. Those are exactly what the next quarter will change | + +## C3. Creational patterns + +| Pattern | What it is for | Status | Evidence | Breaks when | +| --- | --- | --- | --- | --- | +| Factory method / registry | Turn a string (`"vapi"`) into the right class without a chain of imports at the call site | **Following** | `VOICE_PROVIDERS` and `get_voice_provider` in `app/services/voice_providers/__init__.py`. `get_llm_provider_registry`. `ProviderRegistry` in `app/core/auth/providers.py`. `_SessionLocalFactory` | A new platform that bypasses the registry and is constructed at the call site. Review rule: if you `import VapiClient` outside the provider package, you skipped the factory | +| Abstract factory | A family of related objects created together (STT + TTS + LLM for one bundle) | **Partial** | A voice bundle is the family, but it is assembled with sequential lookups and per-provider catalog calls, not one factory that returns a consistent set | A bundle can mix incompatible pieces. The UI fires one request per family member | +| Builder | Build a complex object in steps, with a valid result at the end | **Missing** for prompts and runs | Evaluation prompts are concatenated in `llm_evaluation.py`. Simulation create validates inline | An invalid prompt or an invalid run is representable. The next metric option adds another string concat in the middle of the file. A `PromptBuilder` and a `RunSpec` that cannot be built half-finished fix this | +| Singleton | One shared instance per process | **Partial, and dangerous where accidental** | Settings are a process singleton, which is fine. In-memory telephony sessions and the outbound pool fall back to module-level dicts. Worker catalog caches are process-global dicts with no eviction | A second pod does not see the dict. The dict is a silent capacity leak. Real singletons in this system are Redis and Postgres, not Python globals | +| Prototype | Copy a fully built object instead of rebuilding it | **Not applicable** | We do not clone heavy graphs per request | Do not add it | +| Dependency injection | Pass collaborators in. Do not construct them inside the function | **Partial** | `Depends` for the current user and the DB session. Services then new up HTTP clients and open their own sessions | The function cannot be run in a test with a fake provider. Constructor injection on the use case, with FastAPI wiring it, is the version to standardise | + +## C4. Structural patterns + +| Pattern | What it is for | Status | Evidence | Breaks when | +| --- | --- | --- | --- | --- | +| Adapter | Make a third-party API look like ours | **Following** for historical import. **Missing** for live call start | `HistoricalProviderAdapter` plus `VapiHistoricalAdapter`, `RetellHistoricalAdapter`, `ElevenLabsHistoricalAdapter` | Live playground and webhooks speak each vendor's dialect in the route. A vendor change edits product code. New live providers should be adapters behind the same port the historical ones already use | +| Strategy | Swap an algorithm without editing the caller | **Following** for STT, storage, auth, historical pull. **Missing** for scoring dialects and call runtime | `stt_clients/` is one file per vendor. `TelephonyMergeStrategy` is an enum of strategies. Judge dialects are `if model` inside one module | Every new model family edits `llm_evaluation.py`. That file is already 3,570 lines. The next dialect will break an older one | +| Facade | One simple door in front of a subsystem | **Partial** | Blob storage is a door. `apiClient` is a facade that became the whole house (7,066 lines). Shard sessions are sometimes hidden, sometimes not | Callers depend on every subsystem. Split into one facade per area (`importsApi`, `tracesApi`), each small enough to read | +| Decorator | Add behaviour around an object without editing it | **Following** at the HTTP edge | Middleware stack in `app_factory.py`: CSRF, read-only role, usage context, security headers, trusted host. FastAPI dependencies decorate routes | Do not put business rules in middleware (the usage middleware and the double auth already lean that way). A decorator wraps a call. It does not replace the use case | +| Proxy | Stand in front of an expensive object and control access | **Partial, inverted** | Presigned URLs are the right proxy: the API authorises, object storage serves bytes. Several playback and download paths proxy the bytes through the API instead | The API becomes a file server. Memory and sockets scale with audio, not with decisions. The proxy should be the URL, not the process | +| Bridge | Separate the abstraction (a call, a score) from the implementation (which vendor) | **Partial** | `BaseVoiceProvider` is the abstraction. Implementations are the subclasses. The bridge collapses because the abstraction is too wide and some callers skip it | You cannot add a vendor without touching the call loop. Narrow the abstraction to the operations the loop actually needs (`start`, `stop`, `fetch_recording`) | +| Composite | Treat a group and a single item the same way | **Partial** | Metrics have parents and children, rendered ad hoc in the prompt builder. A simulation run is a group of trials, but the code treats the run as one task | A nested metric or a nested suite needs another special case. A composite `Metric` (`leaf` or `group`) with one `render` and one `parse` removes the special case | +| Flyweight | Share immutable data instead of copying it per row | **Missing** on hot payloads | Metric definitions, voice bundles, and provider config are re-read per row or per page instead of a short-lived cache keyed by id | At high row rates you repeat the same catalog read millions of times a day. Cache the immutable definition for the length of a job | +| Private class data / module boundary | Keep tables and DTOs from becoming a global namespace | **Missing** | `database.py` (3,737 lines) and `schemas.py` (6,372 lines) are one module for every area | Any model change conflicts with every other model change. Split by domain. This is the structural fix for coupling | + +## C5. Behavioural patterns + +| Pattern | What it is for | Status | Evidence | Breaks when | +| --- | --- | --- | --- | --- | +| Chain of responsibility | A request passes through handlers that each do one check | **Following** | The middleware chain. Also the import pipeline in spirit: fetch, then transcribe, then score | A new cross-cutting check (quota, audit) gets pasted into every route instead of one link in the chain. Add the link. Do not copy the check | +| State | An object behaves differently per status, and only legal moves exist | **Partial** | Runs (`QUEUED → PREPARING → RUNNING → …`), import rows, traces (`open` / `closed`) use status strings. Nothing owns the allowed edges. A dead worker leaves `RUNNING` because no state object times it out | Illegal writes, stuck states, and a sweeper that does not know the legal recovery move. One `transition(from, to)` per aggregate, with the timeout edge written down | +| Observer | When X changes, interested parties hear about it. The subject does not import them | **Partial** | `SutLatencyObserver`. Live transcript SSE is pub/sub. The judge calling Flexprice is the opposite of observer: the subject imports the listener | Billing, webhooks-out, and "refresh the chart" cannot be added without editing the judge. Emit "row finished". Subscribers do the rest | +| Pub/sub | Same idea, across processes | **Partial** | Redis for live turns and for progress counters. The UI still polls the heavy read | You pay for the stream and the poll. Subscribers should listen. The poll, if it stays, hits a generation counter | +| Strategy | Already in C4 | | | | +| Template method | The skeleton of an algorithm is fixed. Subclasses fill steps | **Following** on providers. **Missing** on jobs | `BaseVoiceProvider` defines the steps. Import processing is copy-and-edit across `process_call_import_row`, `transcribe_…`, `evaluate_…` rather than one skeleton with hooks | The next import-like pipeline (chat, platform, studio) copies the skeleton and forgets a step (lease release, billing, status). One `Pipeline.run()` with hooks | +| Command | Represent the work as an object: do, undo, log, retry | **Partial** | Celery tasks are commands. They are not idempotent, and cancel is not a first-class inverse. Create-run has no idempotency key | Retries double-charge or double-dial. A command carries a key and a `compensate()`. The task body checks the key before the side effect | +| Mediator | Widgets and services talk through one coordinator, not to each other | **Missing** on the big screens | `CallImportEvaluationDetail.tsx` knows about rows, charts, exports, insights, and polls. There is no page-level state machine those panels share | A new panel copies a poll and a fetch. A thin page mediator holds `{importId, tab, runStatus}` and panels subscribe | +| Iterator / cursor | Walk a large set one page at a time | **Partial** | S3 has a chunk iterator. Export reads 1,000 rows. Several callers then pass `part_rows=2**31` or `.all()` | Someone will call `.all()` on the next feature because the paged function is not the only door. The unpaged function should not exist for those tables | +| Memento, Visitor, Interpreter | Snapshot/restore, walk a tree with new operations, a little language | **Not applicable** | Metric JSON is not a language we should interpret with a new pattern. Prompt building is a builder, not an interpreter | Adding these now is junk. See C8 | + +## C6. Patterns that keep a SaaS from breaking under its own features + +These are the ones that fail in production even when the Gang-of-Four list is clean. They are still low-level design: they live in code, not in Kubernetes. + +| Pattern | The rule in this repo | Status | What is already there | What breaks | +| --- | --- | --- | --- | --- | +| Service layer | One use-case function. The route only maps HTTP | **Partial** | `bulk_ops`, Harbor runner, alert service, flexprice service | Routes of 2,000–10,000 lines. Workers cannot reuse the rule. Tests need HTTP | +| Repository / query object | One function per query shape, shared by the screen and the job | **Missing** as a rule | Queries are inlined. `scatter_gather` is the closest thing to a shared query layer | Alerts and the calls page drift into two scans. A new filter is implemented twice | +| DTO per use | The list, the chart, and the detail are different types | **Partial** | Some list paths omit `call_data`. Schemas exist | Charts receive full score documents. Detail and poll share one fat response. The client throws fields away after paying to load them | +| Unit of work | One transaction for one business action. Commit at the end | **Partial** | A SQLAlchemy session is a unit of work. Suite create commits once per trial. Shard check-in runs extra SQL on every connection return | A request holds a transaction open across provider HTTP calls, or commits a half-finished suite. Commit once, after the in-memory work, never across a vendor call | +| Idempotency | The same request twice is the same result once | **Partial** | Platform-import create accepts `Idempotency-Key`. Simulation create, webhooks, and outbound dial do not | Double click and carrier retry duplicate runs and phone calls | +| Retry with a budget | Retry a known-safe step. Do not retry a side effect that already happened | **Partial** | Celery `max_retries` on tasks. Outbound task retries after it has already failed the call and released the slot | Duplicate dials, duplicate LLM spend, slots held by poison messages. Retry only the step that did not complete, with a key | +| Circuit breaker | After N failures against a vendor, stop calling them for a short window | **Missing** | Retries are per task. Nothing is shared across tasks or orgs | One vendor outage fills the worker pool for the whole task time limit. Every tenant waits | +| Bulkhead | A failure in one workload cannot occupy the workers of another | **Partial** | Separate queues for traces, usage, Harbor, audio. Imports, diarization, and eval share 12 threads. Voice calls share the audio queue | A big import delays cancel and scoring. A voice call blocks audio metrics | +| Backpressure | When the queue is deep, the door refuses new work | **Missing** on create and on trace ingest | Import dispatch limits in-flight work, which is half of it. Create endpoints still enqueue without looking at depth. Slot acquire fails open | The broker grows without bound while the API returns 202. Users keep clicking | +| Lease / leader election | A worker owns a unit of work until a heartbeat dies, then someone else takes it | **Partial** | Platform import uses sorted-set leases. Eval slots are a single integer. Simulation runs have no lease. Beat is a single process with no durable lock around the schedule | Stuck `RUNNING`, leaked slots, double cron fires after a restart | +| Outbox | Write the business row and the "please do this next" event in the same transaction. A dispatcher publishes it | **Missing** | Code calls `task.delay()` after a commit, or before, depending on the path. Rebalance copies, commits, then deletes, and can delete if the publish step failed | A crash between commit and enqueue loses the job, or a crash after enqueue and before commit runs a job for a row that does not exist | +| Saga | A long flow has a forward step and a compensate step per stage | **Missing** for simulations and outbound | Status strings move forward. Compensate is "set FAILED" in some tasks and missing in others. Harbor has no resume | You cannot undo a half-placed call or resume a run after a deploy. Each stage needs `forward` and `compensate` | +| Quota / rate limit object | One implementation of "this tenant has spent its budget" | **Partial** | Trace limiter, telephony recording limiter, OSS quotas, eval slot caps. Each is its own mechanism, and some fail open | The next feature invents a fifth limiter and forgets fail-closed. One `Budget.try_acquire(tenant, kind)` | +| Cache-aside with a key that includes tenancy | Cache is keyed by tenant. Miss loads. Expiry is explicit | **Partial** | API-key cache and trace correlation cache exist. TTLs expire mid-call and mint a second trace. Frontend cache keys often omit workspace | Cross-tenant data on screen. Duplicate traces. Stale catalog. Key = tenant + id, TTL longer than the operation, invalidate on write | +| Anti-corruption layer | Our model stays stable when a vendor payload changes | **Partial** | OTLP mapper is a layer (`otlp_mapper.py`). Several webhooks store the raw vendor payload onto our call row and read fields from it later | A vendor field rename breaks alerts and the UI. Map at the edge into our types. Persist ours | + +## C7. Frontend patterns (same ideas, UI names) + +| Concept | Pattern | Status | What to standardise | +| --- | --- | --- | --- | +| One-way data flow | State lives in a store or a query cache. The screen renders it | **Partial** | React Query is the default. Platform admin uses `useEffect` and a second copy of loading/error. Ban new `useEffect` fetches | +| Container and presenter | The hook fetches. The component draws | **Missing** on the large pages | The page does both, so a visual change risks a request change. `useX(workspaceId)` returns data. The component takes props | +| Colocation / feature module | A feature's API, keys, hooks, and components sit together | **Partial** | `pages/callImports/components` exists. API calls still live in the global client and in the page | `features/imports/{api,keys,hooks,components}` | +| Query-key factory | Keys are a function, not a hand-written array | **Missing** | `['ai-providers']` and `['aiproviders']` | `keys.providers.list(workspaceId)` | +| Suspense / error/empty as states, not as blank | Loading, empty, and error are three renders | **Missing** on many lists | A failed load looks like "no data", and metrics then seed defaults | Every list component handles `isError` | +| Virtualization | Only draw the rows on screen | **Missing** | No virtualization library in the frontend | Required the day a table is not paged. Until then, server paging is the pattern. Do not render 500 cards | +| Code splitting | Load the code for the route the user opened | **Missing** | One 5.5 MB chunk. Static imports in `App.tsx` | `React.lazy` per route. Voice SDKs load with the playground only | +| Optimistic update vs refetch-the-world | After a write, update the one key or invalidate that key | **Partial** | Some mutations invalidate a key. Workspace switch invalidates everything | Invalidate the key you wrote. Full clear only on logout | +| Render nothing you did not ask for | A parent re-render should not redraw a heavy child | **Missing** | Layout state and the walkthrough context re-render the current page, including 4,500 line screens | Split chrome from the outlet. Pass stable props. Memoize the heavy child | + +## C8. Do not add these + +These are real patterns. They are the wrong tool here. Adding them to "follow LLD" would make the junk worse. + +| Pattern | Why it is the wrong tool | +| --- | --- | +| Generic repository base class | Hides the query you need to see. A named function per query is the repository | +| Visitor | We do not have a stable object tree that needs many new operations. Metric rendering is a builder | +| Interpreter | Prompts are not a language runtime. Do not build one | +| Prototype / clone | Nothing expensive is copied per request that way | +| Memento | We do not snapshot arbitrary object graphs. Status plus an event row is enough | +| Active record everywhere | The model that saves itself from a route is how SQL leaked into the routes. Keep persistence in the query module | +| A project-wide event bus | Observer for "row finished" and "call ended" is enough. A bus for every write becomes undebuggable | +| Full CQRS (a second write model for every table) | One rollup for charts and alerts, plus ClickHouse for spans. Not a second database per feature | +| Deep inheritance trees | `BaseVoiceProvider` is already too wide. Prefer a small protocol and composition | + +## C9. The short list that will actually break us + +If the team adopts nothing else, these are the patterns whose absence is already a defect, not a future risk. + +1. **Strategy + registry** for every vendor and every metric dialect. Copy `get_voice_provider` and `HistoricalProviderAdapter`. Stop adding `if provider` in routes and in `llm_evaluation.py`. +2. **State machine with explicit transitions** for runs, import rows, and traces, including the timeout edge. Status strings with no owner are why runs stick on `RUNNING`. +3. **Command with an idempotency key** on create-run, webhooks, and outbound dial. +4. **Unit of work**: one commit per action, no vendor HTTP inside the transaction, no per-row commit in a loop. +5. **DTO per screen**: list, chart, and detail are different shapes. Delete the path that returns the fat row to a list or a poll. +6. **Observer** from the judge and the call-ender. They emit an event. Billing, eval, and the UI subscribe. +7. **Circuit breaker + fail-closed lease** around every vendor and every slot. Retry without these fills the workers. +8. **Outbox** between "row committed" and "task enqueued". +9. **Query-key factory and feature hooks** on the frontend, so a new page cannot forget the workspace or fire a second catalog request. +10. **One reason to change per file.** The open/closed rule is unenforceable while the use case lives in a 10,000 line module. + +We already follow factory, adapter, strategy, template method, middleware chain, and pub/sub in the parts of the code that were given a package of their own. The breaks are in the parts that kept growing in the route and the page. New features go through the ten above. If a pull request cannot name which of the ten it used, it is extending the junk. + +--- + +# Part D — What the rest of the report does not cover + +Parts A–C follow the sidebar and the pattern list. A few risks do not belong to one screen, and they are easy to miss because the code looks fine until a customer asks a question the product cannot answer. This part is those gaps. It also corrects one understatement: the backend is not untested. + +## D1. The backend test suite is a real asset + +`tests/` holds on the order of **400 test modules**. Pull requests to `main` run them against Postgres (`.github/workflows/backend-tests-postgres.yml`). Coverage is uneven, and it is strongest where someone already extracted a function: judge parsing, fair dispatch, call-import routes, sharding, telephony URL checks, Flexprice, auth cookies. + +The frontend is the opposite. About **17** test files, almost all pure helpers. Nothing in CI fails when a React Query key drops the workspace id. Docs CI runs a frontend check (`.github/workflows/docs.yml`); it does not protect tenancy. + +**Do not** start a project to "add tests" in the abstract. When a use case is extracted (Part B), the test lands on that function in the same pull request. The first frontend tests worth writing are: query keys include workspace id, a 401 triggers one refresh, and a failed list does not render as empty. + +## D2. Two people editing the same thing + +There is no version column, no `ETag`, and no "updated at matches what I loaded" check on the aggregates people edit: agents, metrics, prompts, alert rules, integrations. The last save wins. The first person's change disappears with no error. + +At one editor this is rare. The day a second teammate joins an account, it becomes a weekly incident, and it looks like a bug in the feature rather than a missing lock. + +**Change.** On each editable aggregate, store `updated_at` or an integer version. The save sends the version it read. The update is `WHERE id = ? AND version = ?`. Zero rows means 409 conflict, and the UI reloads. Do this on the next entity you touch, then copy it. Do not boil every table in one migration. + +## D3. Data lifetime, offboarding, and PII + +The product stores call audio, transcripts, scores, and spans. The report's scale numbers assume that data stays. The product has almost no policy for letting it go. + +- ClickHouse trace tables have **no TTL** (Part A). A year of spans is tens of billions of rows if you keep them. +- Postgres call rows, import rows, and recordings have no retention job. Usage history has a pruner for the OSS rollup (`prune_oss_usage_history`). Customer call data does not. +- A search for deleting an organisation on the platform-admin route does not find a cascade. Offboarding a tenant means a person knows every table, every shard, and every S3 prefix. They will miss one. +- Transcripts and `call_data` are personal data. They are allowed in the database and in object storage. They are not allowed in application logs, error trackers, or analytics. There is no single helper that redacts them, so the next `logger.exception(row)` is an incident. + +**Change.** Write the policy in one place, then make the code match it: how long a trace span lives, how long a recording lives, how long a scored row lives, and the one job that deletes an organisation (catalog, every shard, every object prefix, ClickHouse). Until that job exists, "delete my account" is a manual project. Add a test that a log formatter cannot emit `call_data` or a transcript field. + +## D4. A click that succeeds before the data exists + +Several doors return before the work is visible: trace ingest returns 202 after an S3 put, simulation create returns after enqueue, imports return after staging. The UI then reads a store that may not have the row yet. Users refresh, double-submit, or think it failed. + +**Change.** Each of those responses includes a stable id and a status URL that is cheap (`queued`, not the full row). The button stays pending until that status exists. Create endpoints take an idempotency key (Part C) so the refresh is safe. This is the contract between the screen and the queue. Write it down per action. Do not leave it as "the poll will probably see it". + +## D5. Jobs that fail and then vanish + +Celery is configured with `acks_late` and `reject_on_worker_lost`. A crash redelivers the task. That is right. After `max_retries`, the message is gone. There is no dead-letter queue and no single screen of "poisoned" work. The row's status is the only record, and only if the task remembered to set it. Simulation runs do not (Part A): they stay `RUNNING`. + +**Change.** One outcome for every task: `succeeded`, `failed` with a reason on the row, or `retry`. A task that exits without writing one of those is a bug. A small admin query — rows stuck in a non-terminal status past their lease — is the operator view. A dead-letter queue is optional once the row itself holds the failure. + +## D6. Rolling deploys will fight the schema + +Part A says migrations must run once, under a lock. That is not enough. During a Kubernetes rollout, old pods and new pods run together. A migration that adds a `NOT NULL` column, renames a column, or changes a row's meaning will crash the pods that have not restarted. + +**Change.** Every schema change is expand, then code, then contract. First add the new column as nullable and write both shapes. Next release reads the new shape. A later release drops the old one. Never rename a column the previous release still reads. The migration Job runs before the new pods take traffic, and the old pods must still boot against the expanded schema. Put this in the pull-request checklist next to the ten patterns. One bad migration takes the API down in a way that more replicas make worse, because they migrate and serve at the same time today. + +## D7. Three configs, three products + +The same behaviour is set in `config.yml`, in environment variables, and in Python defaults, and they disagree. `DEBUG` defaults true in code and false in the Docker config. Sharding is on in a local config and off in `config.docker.yml`. ClickHouse is commented out in the example, and trace ingest then returns 503. A new pod that misses one variable is a different product. + +**Change.** One source wins: environment in the cluster, file only for local. Boot must fail when `DEBUG` is true, the secret is the placeholder, or trace ingest is enabled without ClickHouse. Print the effective values that matter (pool sizes, slot caps, sharding on or off, ClickHouse on or off) in the startup log. If two pods print different lines, you are not running one system. + +## D8. A tenant can spend without a ceiling + +Slot caps limit concurrency. They do not limit money. A simulation run, an import, and a cron job all call the LLM until the queue is empty. Optimization has `max_metric_calls`. Nothing else has a budget. One mis-clicked "run" on a large import is a bill, and a shared provider key means it is also everyone else's latency. + +**Change.** A budget object per organisation: tokens or judge-calls per day, checked when a trial or a row is claimed, not after the invoice. When it is spent, the rest of the queue fails with a clear status. The Usage page reads that budget. Flexprice stays a subscriber (Part B), not the thing that stops the worker. + +## D9. Tenant isolation is a query rule, not a middleware hope + +The workspace header is stamped on the client. The server must still ignore it when the caller is not a member, and every query must include the organisation or the workspace. Middleware that sets "current workspace" does not protect a query someone wrote without the filter. Sharding makes this worse: a bug can read another tenant's slice if the id is guessed and the query is not scoped. Public ids are 6 digits today, which makes guessing cheap (Part A). + +There is a security test file and enterprise-gating tests. There is not a habit that every new list query arrives with a test that a second organisation sees zero rows. + +**Change.** The query functions from Part B take `organization_id` as a required argument, not an optional filter. A test template: create two orgs, call the function as org A, assert org B's row is absent. Apply it to calls, imports, traces, runs, and API keys first. Those are the tables that hold customer data. + +Recording download already has the right instinct: `assert_recording_url_safe` rejects IP hosts, credentials in the URL, non-allowlisted hostnames, and private addresses after DNS resolution (`app/services/telephony/recording_download.py`). Keep every new outbound fetch on that helper. A CSV full of URLs is otherwise a way to make the worker call the internal network. + +## D10. You cannot see which tenant is hurting you + +Part A notes that Prometheus scrapes one API address and does not see workers. The missing product view is per organisation, not per pod: queue wait, LLM calls, error rate, ingest lag, slots in use. Without it, a noisy tenant and a broken deploy look the same, and the only tool is reading logs. + +**Change.** Four numbers, emitted by the workers you already have, labeled by organisation id: + +- Age of the oldest queued trial, import row, and trace batch. +- Slots in use against the cap. +- Judge calls and tokens in the last hour (the Redis counters already exist). +- API error rate. + +Page on queue age and on error rate. The dashboard in the product can show the same counters later. Do not build the chart before the numbers exist. + +Define the targets in one sentence each, so "scale" is a yes or no: API p99 under a second for anything that is not a file, trace ingest lag under a minute, no run left non-terminal more than ten minutes after its lease died, a list endpoint never reads a transcript. The load test in Part A measures these. It does not invent new ones. + +## D11. The frontend and the backend do not share a contract + +FastAPI can emit an OpenAPI document. The client does not generate from it. `api.ts` and `types/api.ts` are hand-written, full of `any`, and they drift. A renamed field becomes a silent `undefined` on a screen. That is how extra requests and blank states sneak in: the UI asks again, or it asks for the wrong shape and then fetches a second endpoint to fill the hole. + +**Change.** Generate the client types from the OpenAPI schema in CI, and fail the build when the committed types are stale. Start with the hot resources: calls, imports, simulation runs, traces. Hand-written wrappers can stay for auth. New endpoints do not get a new `any` method. + +## D12. What this report is not + +- It is not a load test. Every ceiling is arithmetic from caps in the repo. +- It is not a penetration test. Tenant-filter tests and the recording URL allowlist are the security items tied to scale. Auth cookies, CSRF, and security headers already have tests. Someone should still try cross-tenant ids on purpose before a public launch. +- It is not a line-by-line inventory of every button. The controls that fan out, poll, or silently truncate are covered. A button that saves one row is fine unless it is missing a version check (D2) or an error state (Part B). +- It does not know your replica counts, backup frequency, or restore drill. Those are not in the repository. The code gap that remains even with perfect backups is D3: you cannot restore or delete **one** organisation. A cluster backup is not a tenant export. + +Revisit this document when one of the ten patterns in Part C becomes the default for new code, or when a load test replaces a ceiling with a measurement. Until then, treat a contradiction between this file and the code as a bug in one of them, and fix the one that is wrong. diff --git a/docs/scale-audit/_md_to_docx.py b/docs/scale-audit/_md_to_docx.py new file mode 100644 index 00000000..c1fb643c --- /dev/null +++ b/docs/scale-audit/_md_to_docx.py @@ -0,0 +1,344 @@ +"""Render three standalone audit documents from PLATFORM-SCALE-AUDIT.md.""" + +import re +from pathlib import Path + +from docx import Document +from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.oxml import OxmlElement +from docx.oxml.ns import qn +from docx.shared import Cm, Pt, RGBColor + +ROOT = Path(__file__).parent +SRC = (ROOT / "PLATFORM-SCALE-AUDIT.md").read_text(encoding="utf-8").splitlines() + +NAVY = RGBColor(0x1F, 0x4E, 0x79) +TEAL = RGBColor(0x0F, 0x6E, 0x6E) +BODY = RGBColor(0x1F, 0x29, 0x37) +MUTED = RGBColor(0x4B, 0x55, 0x63) +HEADER_BG = "1F4E79" +ZEBRA = "F4F7FB" + + +def set_run_font(run, name="Calibri", size=11, bold=False, color=BODY, italic=False): + run.font.name = name + run._element.rPr.rFonts.set(qn("w:eastAsia"), name) + run.font.size = Pt(size) + run.bold = bold + run.italic = italic + run.font.color.rgb = color + + +def shade_cell(cell, hex_color): + tc_pr = cell._tc.get_or_add_tcPr() + shd = OxmlElement("w:shd") + shd.set(qn("w:fill"), hex_color) + shd.set(qn("w:val"), "clear") + tc_pr.append(shd) + + +def set_cell_border(cell): + tc_pr = cell._tc.get_or_add_tcPr() + borders = OxmlElement("w:tcBorders") + for edge in ("top", "left", "bottom", "right"): + el = OxmlElement(f"w:{edge}") + el.set(qn("w:val"), "single") + el.set(qn("w:sz"), "4") + el.set(qn("w:color"), "E5E7EB") + borders.append(el) + tc_pr.append(borders) + + +def add_inline(paragraph, text, size=11, color=BODY, base_bold=False): + pattern = re.compile(r"(\*\*[^*]+\*\*|`[^`]+`)") + pos = 0 + for match in pattern.finditer(text): + if match.start() > pos: + run = paragraph.add_run(text[pos:match.start()]) + set_run_font(run, size=size, bold=base_bold, color=color) + token = match.group(0) + if token.startswith("**"): + run = paragraph.add_run(token[2:-2]) + set_run_font(run, size=size, bold=True, color=color) + else: + run = paragraph.add_run(token[1:-1]) + set_run_font(run, name="Consolas", size=max(size - 1, 9), color=NAVY) + pos = match.end() + if pos < len(text): + run = paragraph.add_run(text[pos:]) + set_run_font(run, size=size, bold=base_bold, color=color) + + +def add_page_number(paragraph): + run = paragraph.add_run() + set_run_font(run, size=9, color=MUTED) + begin = OxmlElement("w:fldChar") + begin.set(qn("w:fldCharType"), "begin") + instr = OxmlElement("w:instrText") + instr.set(qn("xml:space"), "preserve") + instr.text = " PAGE " + end = OxmlElement("w:fldChar") + end.set(qn("w:fldCharType"), "end") + run._r.append(begin) + run._r.append(instr) + run._r.append(end) + + +def style_heading(paragraph, level): + sizes = {1: 20, 2: 16, 3: 13} + colors = {1: NAVY, 2: TEAL, 3: NAVY} + before = {1: 18, 2: 14, 3: 10} + for run in paragraph.runs: + set_run_font(run, size=sizes[level], bold=True, color=colors[level]) + pf = paragraph.paragraph_format + pf.space_before = Pt(before[level]) + pf.space_after = Pt(6) + pf.line_spacing = 1.08 + pf.keep_with_next = True + + +def body_paragraph(doc, text): + p = doc.add_paragraph() + p.paragraph_format.space_after = Pt(8) + p.paragraph_format.space_before = Pt(0) + p.paragraph_format.line_spacing = 1.12 + add_inline(p, text) + return p + + +def add_table(doc, rows): + cols = max(len(r) for r in rows) + table = doc.add_table(rows=len(rows), cols=cols) + table.autofit = True + tbl_pr = table._tbl.tblPr + tbl_w = OxmlElement("w:tblW") + tbl_w.set(qn("w:w"), "5000") + tbl_w.set(qn("w:type"), "pct") + tbl_pr.append(tbl_w) + for i, row_data in enumerate(rows): + row = table.rows[i] + tr_pr = row._tr.get_or_add_trPr() + tr_pr.append(OxmlElement("w:cantSplit")) + if i == 0: + tr_pr.append(OxmlElement("w:tblHeader")) + for j in range(cols): + cell = row.cells[j] + cell.text = "" + p = cell.paragraphs[0] + p.paragraph_format.space_after = Pt(2) + p.paragraph_format.space_before = Pt(2) + value = row_data[j] if j < len(row_data) else "" + white = RGBColor(0xFF, 0xFF, 0xFF) + add_inline( + p, + value.strip(), + size=9, + color=white if i == 0 else BODY, + base_bold=(i == 0), + ) + shade_cell(cell, HEADER_BG if i == 0 else (ZEBRA if i % 2 == 0 else "FFFFFF")) + set_cell_border(cell) + spacer = doc.add_paragraph() + spacer.paragraph_format.space_after = Pt(4) + + +def is_separator(line): + stripped = line.replace("|", "").replace(":", "").replace("-", "").strip() + return stripped == "" and "-" in line + + +def parse_table_row(line): + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def render(lines, out_path, eyebrow, title, subtitle, footer_label): + doc = Document() + section = doc.sections[0] + section.page_width = Cm(21.0) + section.page_height = Cm(29.7) + section.left_margin = Cm(1.7) + section.right_margin = Cm(1.7) + section.top_margin = Cm(1.8) + section.bottom_margin = Cm(1.8) + + header = section.header.paragraphs[0] + header.alignment = WD_ALIGN_PARAGRAPH.RIGHT + run = header.add_run(footer_label) + set_run_font(run, size=9, bold=True, color=NAVY) + + footer = section.footer.paragraphs[0] + footer.alignment = WD_ALIGN_PARAGRAPH.RIGHT + label = footer.add_run("1 million / day · page ") + set_run_font(label, size=9, color=MUTED) + add_page_number(footer) + + normal = doc.styles["Normal"] + normal.font.name = "Calibri" + normal.font.size = Pt(11) + normal.font.color.rgb = BODY + + cover = doc.add_paragraph() + cover.paragraph_format.space_before = Pt(72) + r = cover.add_run(eyebrow) + set_run_font(r, size=12, bold=True, color=TEAL) + + h = doc.add_paragraph() + h.paragraph_format.space_before = Pt(8) + h.paragraph_format.space_after = Pt(8) + r = h.add_run(title) + set_run_font(r, size=32, bold=True, color=NAVY) + + sub = doc.add_paragraph() + r = sub.add_run(subtitle) + set_run_font(r, size=14, color=MUTED) + + meta = doc.add_paragraph() + meta.paragraph_format.space_before = Pt(28) + r = meta.add_run( + "Scale target: 1 million simulations a day, 1 million calls a day,\n" + "and 1 million live traces a day, at the same time.\n\n" + "Engineering discussion · 26 October 2026\n" + "Read from the code. Not a load test. Numbers are per replica\n" + "unless the section says otherwise." + ) + set_run_font(r, size=12, color=BODY) + doc.add_page_break() + + in_code = False + code_buf = [] + para_buf = [] + i = 0 + + def flush_para(): + nonlocal para_buf + text = " ".join(s.strip() for s in para_buf).strip() + para_buf = [] + if text: + body_paragraph(doc, text) + + while i < len(lines): + line = lines[i] + if in_code: + if line.strip().startswith("```"): + p = doc.add_paragraph() + p.paragraph_format.left_indent = Cm(0.3) + p.paragraph_format.space_after = Pt(8) + run = p.add_run("\n".join(code_buf)) + set_run_font(run, name="Consolas", size=9, color=NAVY) + code_buf = [] + in_code = False + else: + code_buf.append(line) + i += 1 + continue + if line.strip().startswith("```"): + flush_para() + in_code = True + i += 1 + continue + if line.strip() == "---": + flush_para() + i += 1 + continue + if line.startswith("|"): + flush_para() + rows = [] + while i < len(lines) and lines[i].strip().startswith("|"): + if not is_separator(lines[i]): + rows.append(parse_table_row(lines[i])) + i += 1 + if rows: + add_table(doc, rows) + continue + if line.startswith("#"): + flush_para() + level = min(len(line) - len(line.lstrip("#")), 3) + text = line[level:].strip() if False else line.lstrip("#").strip() + p = doc.add_heading(text, level=level) + style_heading(p, level) + i += 1 + continue + bullet = re.match(r"^(\s*)[-*] (.*)$", line) + if bullet: + flush_para() + p = doc.add_paragraph() + p.paragraph_format.left_indent = Cm(0.75) + p.paragraph_format.first_line_indent = Cm(-0.35) + p.paragraph_format.space_after = Pt(3) + p.paragraph_format.line_spacing = 1.08 + add_inline(p, "• " + bullet.group(2).strip()) + i += 1 + continue + numbered = re.match(r"^(\s*)(\d+)\. (.*)$", line) + if numbered: + flush_para() + p = doc.add_paragraph() + p.paragraph_format.left_indent = Cm(0.75) + p.paragraph_format.first_line_indent = Cm(-0.4) + p.paragraph_format.space_after = Pt(3) + p.paragraph_format.line_spacing = 1.08 + add_inline(p, f"{numbered.group(2)}. {numbered.group(3).strip()}") + i += 1 + continue + if not line.strip(): + flush_para() + i += 1 + continue + para_buf.append(line) + i += 1 + flush_para() + doc.save(out_path) + print(out_path.name, "paragraphs", len(doc.paragraphs), "tables", len(doc.tables)) + + +def slice_lines(start, end): + """1-based inclusive line numbers.""" + return SRC[start - 1 : end] + + +part_a = slice_lines(1, 948) +part_a[0] = "# Part A — What breaks at 1 million a day" +part_a[9] = ( + "This is document 1 of 3. It walks the product from the top of the sidebar to the bottom and says what each area can carry today. " + "**Document 2** is the code shape. **Document 3** is the design-pattern scorecard. " + "A later note in the full markdown (Part D) covers retention, lost updates, deploys, and spend. It is not in these three files." +) + +part_b = [ + "This is document 2 of 3. The scale target is still **1 million simulations, 1 million calls, and 1 million traces a day**. " + "Document 1 says where the product runs out of room. This document says whether the code is doing extra work on every request, and whether the next feature has a file to go in. " + "Adding servers does not fix a request that already does the same work twice.", + "", +] + slice_lines(952, 1223) + +part_c = [ + "This is document 3 of 3. The scale target is still **1 million simulations, 1 million calls, and 1 million traces a day**. " + "A pattern is worth following when a new provider, a new metric, or a new teammate would otherwise edit a giant file and break the last feature. " + "Document 1 is capacity. Document 2 is the file-by-file code shape. This document is the scorecard.", + "", +] + slice_lines(1227, 1379) + +render( + part_a, + ROOT / "01-Capacity-1-million-a-day.docx", + "DOCUMENT 1 OF 3 · PART A", + "What breaks at\n1 million a day", + "Sidebar, top to bottom.\nWhat each area can carry today, what is wrong,\nand what to change before adding more servers.", + "PART A · CAPACITY", +) +render( + part_b, + ROOT / "02-Code-shape.docx", + "DOCUMENT 2 OF 3 · PART B", + "Code shape for\n1 million a day", + "Extra queries, extra downloads, and files\nthat are thousands of lines long.\nWhere the next feature should go.", + "PART B · CODE SHAPE", +) +render( + part_c, + ROOT / "03-Design-patterns.docx", + "DOCUMENT 3 OF 3 · PART C", + "Design patterns\nwe do and don't follow", + "Low-level design, scored against this repo.\nWhat is already a seam, what is only half done,\nand the ten patterns a change should use.", + "PART C · PATTERNS", +) diff --git a/env.example b/env.example index 27067d58..6e1da88f 100644 --- a/env.example +++ b/env.example @@ -88,7 +88,7 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0 # CLICKHOUSE_PASSWORD= # CORS -CORS_ORIGINS=["http://localhost:3000", "http://localhost:8000"] +CORS_ORIGINS=["http://localhost:8000", "https://sandbox.efficientai.cloud"] # API Settings API_KEY_HEADER=X-API-Key diff --git a/frontend/README.md b/frontend/README.md index 754a6e47..950446e9 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -181,9 +181,23 @@ The production build will be in the `dist/` directory. ### CORS Issues -If you see CORS errors, ensure: -1. The backend API allows requests from `http://localhost:3000` -2. The API is running and accessible +**API CORS (FastAPI):** If API calls fail from the browser, ensure `cors.origins` in `config.yml` includes your UI origin (typically `http://localhost:8000` when the API serves the SPA, plus `https://sandbox.efficientai.cloud` for hosted sandbox). + +**S3 CORS (call-import uploads):** Staged CSV/XLSX and presigned audio uploads use `fetch(..., PUT)` **directly to S3**. That is a cross-origin request to `*.s3.amazonaws.com`, not to the API. The bucket must expose CORS allowing your app origin and `PUT`/`Content-Type`, or you will see: + +`blocked by CORS policy ... No 'Access-Control-Allow-Origin' header` + +Fix: apply bucket CORS once per environment (see `scripts/s3-bucket-cors-call-imports.example.json`): + +```bash +export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_DEFAULT_REGION=us-east-1 +./scripts/apply_s3_call_import_cors.sh --config config.yml +# or: aws s3api put-bucket-cors --bucket voiceai-evals-test --cors-configuration file://scripts/s3-bucket-cors-call-imports.example.json +``` + +Edit `AllowedOrigins` in that file first (localhost + production URL). + +**401 on `/api/v1/...`:** Session or API key expired — sign in again. A failed S3 PUT often appears as a separate CORS error, not 401. ### API Key Issues diff --git a/frontend/public/tool-logos/googledrive.svg b/frontend/public/tool-logos/googledrive.svg new file mode 100644 index 00000000..1d393a0a --- /dev/null +++ b/frontend/public/tool-logos/googledrive.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/tool-logos/modelcontextprotocol.svg b/frontend/public/tool-logos/modelcontextprotocol.svg new file mode 100644 index 00000000..4722f94f --- /dev/null +++ b/frontend/public/tool-logos/modelcontextprotocol.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/tool-logos/notion.svg b/frontend/public/tool-logos/notion.svg new file mode 100644 index 00000000..fdcb83c3 --- /dev/null +++ b/frontend/public/tool-logos/notion.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/tool-logos/postgresql.svg b/frontend/public/tool-logos/postgresql.svg new file mode 100644 index 00000000..defffc26 --- /dev/null +++ b/frontend/public/tool-logos/postgresql.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/tool-logos/slack.svg b/frontend/public/tool-logos/slack.svg new file mode 100644 index 00000000..a57ff169 --- /dev/null +++ b/frontend/public/tool-logos/slack.svg @@ -0,0 +1 @@ + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 5b1875f4..0c79fa32 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { lazy, Suspense, useEffect, useState } from 'react' import { BrowserRouter, Routes, Route, Navigate, useParams, useLocation } from 'react-router-dom' import { useAuthStore } from './store/authStore' import { useLicenseStore } from './store/licenseStore' @@ -30,6 +30,10 @@ import Personas from './pages/personas/Personas' // Scenarios import Scenarios from './pages/scenarios/Scenarios' +// Knowledge bases +import { KnowledgeBaseDetail } from './pages/knowledgeBases' +import { ToolsHub } from './pages/toolConnections' + // Metrics import MetricsLayout from './pages/metrics/MetricsLayout' import MetricsManagement from './pages/metrics/MetricsManagement' @@ -59,6 +63,7 @@ import ResultsUnassigned from './pages/evaluators/results/ResultsUnassigned' import EvaluatorResultDetail from './pages/evaluators/results/EvaluatorResultDetail' import EvaluationDetail from './pages/evaluators/results/EvaluationDetail' import EvaluationsList from './pages/evaluators/results/EvaluationsList' +import SimulationRunDetail from './pages/simulations/SimulationRunDetail' // Observability import Observability from './pages/observability/Observability' @@ -102,11 +107,22 @@ import BlindTestForm from './pages/public/BlindTestForm' // Call Imports import CallImports from './pages/callImports/CallImports' import ChatImports from './pages/callImports/ChatImports' -import CallImportDetail from './pages/callImports/CallImportDetail' -import CallImportEvaluationDetail from './pages/callImports/CallImportEvaluationDetail' import CallImportTagsPage from './pages/callImports/Tags' import CallImportSchemasPage from './pages/callImports/Schemas' +const CallImportDetail = lazy(() => import('./pages/callImports/CallImportDetail')) +const CallImportEvaluationDetail = lazy( + () => import('./pages/callImports/CallImportEvaluationDetail'), +) + +function RouteSuspenseFallback() { + return ( +
+
+
+ ) +} + function PreserveSearchRedirect({ to }: { to: string }) { const location = useLocation() @@ -194,6 +210,9 @@ function App() { } /> } /> } /> + } /> + } /> + } /> } /> } /> } /> @@ -218,6 +237,7 @@ function App() { element={} /> } /> + } /> } /> } /> } /> @@ -255,10 +275,25 @@ function App() { } /> } /> } /> - } /> + + }> + + + + } + /> } + element={ + + }> + + + + } /> } /> } /> diff --git a/frontend/src/components/AIProviderModelPicker.tsx b/frontend/src/components/AIProviderModelPicker.tsx index 672188de..1c2ea416 100644 --- a/frontend/src/components/AIProviderModelPicker.tsx +++ b/frontend/src/components/AIProviderModelPicker.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react' +import { useEffect, useMemo } from 'react' import { useQuery } from '@tanstack/react-query' import { apiClient } from '../lib/api' import AnchoredSelect from './shared/AnchoredSelect' @@ -179,6 +179,11 @@ export default function AIProviderModelPicker({ const gatewayDirectModel = modelResolution.mode === 'gateway_direct' ? modelResolution.model : null + useEffect(() => { + if (!gatewayDirectModel || model === gatewayDirectModel) return + onModelChange(gatewayDirectModel) + }, [gatewayDirectModel, model, onModelChange]) + const catalogModels = modelResolution.mode === 'catalog' ? modelResolution.models : [] const llmModels = modelFilter ? catalogModels.filter(modelFilter) : catalogModels diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index e77cb564..21aae3ff 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -36,6 +36,7 @@ import { Github, Sparkles, Upload, + Wrench, } from 'lucide-react' import { useState, useEffect } from 'react' import { Info, X } from 'lucide-react' @@ -65,6 +66,7 @@ const navigationSections: NavSection[] = [ { name: 'Agents', href: '/agents', icon: Bot }, { name: 'Personas', href: '/personas', icon: Users }, { name: 'Scenarios', href: '/scenarios', icon: FileText }, + { name: 'Tools', href: '/tools', icon: Wrench }, ], }, { @@ -156,6 +158,10 @@ function getFlattenedNavItems(): NavItem[] { } function isNavItemActive(href: string, pathname: string): boolean { + if (href === '/tools') { + // Knowledge bases live under Tools; their detail pages keep the Tools item lit. + return pathname === href || pathname.startsWith('/knowledge-bases') + } if (href === '/metrics-management') { return ( pathname === href || pathname.startsWith('/metrics-management/') @@ -242,9 +248,8 @@ export default function Layout() { {/* Desktop sidebar */}
{/* Top bar */}
) @@ -635,6 +682,11 @@ export default function VoiceAgent({ ? 'Connected' : status}

+ {kbToolCallCount > 0 && ( +

+ Knowledge base searches this call: {kbToolCallCount} +

+ )} {!isConnected ? ( -
-
- {evaluatorScope ? ( -
+ return ( + <> + {createPortal( +
+
+
-

- Cluster scope -

-

- Choose which agent, scenarios, and simulation dates to include. +

+ Generate clusters +

+

+ Configure failure values per metric, choose calls, then generate + clusters from LLM rationales.

- - {draftAgentId ? ( -
-

- Scenarios (optional — leave all unchecked for every scenario) -

- {scenarioOptions.length ? ( -
- {scenarioOptions.map((scenario) => { - const checked = draftScenarioIds.includes(scenario.id) - return ( - - ) - })} -
- ) : ( -

- No scenarios found for this agent yet. + +

+
+ {evaluatorScope ? ( +
+
+

+ Cluster scope +

+

+ Choose which agent, scenarios, and simulation dates to include.

- )} +
+ + {draftAgentId ? ( +
+

+ Scenarios (optional — leave all unchecked for every scenario) +

+ {scenarioOptions.length ? ( +
+ {scenarioOptions.map((scenario) => { + const checked = draftScenarioIds.includes(scenario.id) + return ( + + ) + })} +
+ ) : ( +

+ No scenarios found for this agent yet. +

+ )} +
+ ) : null} + { + setDraftStartDate(start) + setDraftEndDate(end) + setPoliciesTouched(false) + }} + />
) : null} - { - setDraftStartDate(start) - setDraftEndDate(end) - setPoliciesTouched(false) - }} - /> +
+ {failurePoliciesQuery.isLoading ? ( +

Loading failure policies…

+ ) : failurePoliciesQuery.data ? ( + { + setPoliciesTouched(true) + setPolicies(next) + }} + disabled={generating} + /> + ) : null} +
+
+ {eligibleRowsQuery.isLoading ? ( +

Loading eligible calls…

+ ) : ( + + )} +
+
+

+ LLM for clustering +

+

+ Select a provider and model. Cluster generation stays off until + both are chosen. +

+ +
+ {error ?

{error}

: null} +
+
+ +
- ) : null} -
- {failurePoliciesQuery.isLoading ? ( -

Loading failure policies…

- ) : failurePoliciesQuery.data ? ( - { - setPoliciesTouched(true) - setPolicies(next) - }} - disabled={generating} - /> - ) : null} -
-
- {eligibleRowsQuery.isLoading ? ( -

Loading eligible calls…

- ) : ( - - )} -
-
-

- LLM for clustering -

-

- Provider and model used for failure signatures and cluster - synthesis. Defaults to this evaluation's scoring LLM when - unset. -

- { - setLlmPickerTouched(true) - setPickerProvider(next) - }} - onModelChange={(next) => { - setLlmPickerTouched(true) - setPickerModel(next) - }} - disabled={generating} - size="sm" - />
- {error ?

{error}

: null} -
-
- - -
-
-
, - document.body, +
, + document.body, + )} + setReplaceConfirmOpen(false)} + onConfirm={() => { + void startGeneration() + }} + /> + ) } export function MetricClustersPanel({ client, - defaultProvider = '', - defaultModel = '', state, isLoading, onGenerated, @@ -761,7 +758,6 @@ export function MetricClustersPanel({ const [error, setError] = useState(null) const [pickerProvider, setPickerProvider] = useState('') const [pickerModel, setPickerModel] = useState('') - const [llmPickerTouched, setLlmPickerTouched] = useState(false) const [clusterActionModalOpen, setClusterActionModalOpen] = useState(false) const setModalOpen = (open: boolean) => { @@ -773,23 +769,6 @@ export function MetricClustersPanel({ registerOpenGenerateModal?.(() => setModalOpen(true)) }, [registerOpenGenerateModal]) - useEffect(() => { - if (state?.provider) { - setPickerProvider(state.provider) - if (state.model) setPickerModel(state.model) - return - } - if (llmPickerTouched) return - if (defaultProvider) setPickerProvider(defaultProvider) - if (defaultModel) setPickerModel(defaultModel) - }, [ - state?.provider, - state?.model, - defaultProvider, - defaultModel, - llmPickerTouched, - ]) - const llmPickerDisabled = cancelling || state?.status === 'running' const llmPickerBlock = ( @@ -798,20 +777,13 @@ export function MetricClustersPanel({ LLM for clustering

- Provider and model used for failure signatures and cluster synthesis. - Defaults to this evaluation's scoring LLM when unset. + Select a provider and model before generating clusters.

{ - setLlmPickerTouched(true) - setPickerProvider(next) - }} - onModelChange={(next) => { - setLlmPickerTouched(true) - setPickerModel(next) - }} + onProviderChange={setPickerProvider} + onModelChange={setPickerModel} disabled={llmPickerDisabled} size="sm" /> @@ -826,8 +798,6 @@ export function MetricClustersPanel({ setError(null) }} client={client} - defaultProvider={defaultProvider} - defaultModel={defaultModel} state={state} onGenerated={onGenerated} onError={setError} @@ -889,8 +859,8 @@ export function MetricClustersPanel({ : 'flagged calls' const metricStage = progress?.current_metric_name && - progress?.total_metrics && - progress.total_metrics > 0 + progress?.total_metrics && + progress.total_metrics > 0 ? `Metric ${progress.current_metric_index ?? 0} of ${progress.total_metrics}: ${progress.current_metric_name}` : null return ( @@ -968,22 +938,22 @@ export function MetricClustersPanel({ return (
-

- Failure diagnostics stopped -

-

- {state.error_message || - 'Cluster generation was cancelled. Partial results were not saved.'} -

- {state.progress ? ( -

- Stopped at {state.progress.completed_llm_calls} /{' '} - {state.progress.total_llm_calls} LLM calls +

+ Failure diagnostics stopped +

+

+ {state.error_message || + 'Cluster generation was cancelled. Partial results were not saved.'} +

+ {state.progress ? ( +

+ Stopped at {state.progress.completed_llm_calls} /{' '} + {state.progress.total_llm_calls} LLM calls +

+ ) : null} +

+ Use New report above to start again.

- ) : null} -

- Use New report above to start again. -

{clusterGenerationModal}
@@ -994,15 +964,15 @@ export function MetricClustersPanel({ return (
-

- Failure diagnostics failed -

-

- {state.error_message || 'Cluster generation failed.'} -

-

- Use New report above to try again with a different scope or configuration. -

+

+ Failure diagnostics failed +

+

+ {state.error_message || 'Cluster generation failed.'} +

+

+ Use New report above to try again with a different scope or configuration. +

{clusterGenerationModal}
@@ -1013,15 +983,15 @@ export function MetricClustersPanel({ return (
-
-

- No cluster results yet -

-

- Select a cluster report above or use New report to generate clusters - for an agent, scenario set, and date range. -

-
+
+

+ No cluster results yet +

+

+ Select a cluster report above or use New report to generate clusters + for an agent, scenario set, and date range. +

+
{clusterGenerationModal}
@@ -1046,10 +1016,9 @@ export function MetricClustersPanel({ } const subtabClass = (view: ClusterReportView) => - `px-3 py-1.5 text-xs font-medium rounded transition ${ - activeView === view - ? 'bg-white text-primary-700 shadow-sm' - : 'text-gray-600 hover:text-gray-900' + `px-3 py-1.5 text-xs font-medium rounded transition ${activeView === view + ? 'bg-white text-primary-700 shadow-sm' + : 'text-gray-600 hover:text-gray-900' }` return ( diff --git a/frontend/src/components/providers/ProviderModelPicker.tsx b/frontend/src/components/providers/ProviderModelPicker.tsx index b1262efd..6049d6f3 100644 --- a/frontend/src/components/providers/ProviderModelPicker.tsx +++ b/frontend/src/components/providers/ProviderModelPicker.tsx @@ -35,8 +35,6 @@ import { resolveActiveAIProvider } from '../../lib/gatewayRouting' import { buildLLMProviderKeys, formatGatewayCredentialLabel, - isAudioCapableModel, - isGatewayAudioCapableCredential, resolveLLMModelsForCredential, shouldAlwaysShowCredentialPicker, } from '../../lib/llmModelOptions' @@ -98,33 +96,32 @@ export interface ProviderModelPickerProps { /** When true, show a "credential" row that lets the user pin a specific AIProvider row. */ allowCredentialPick?: boolean disabled?: boolean - /** - * When true, restrict the LLM picker to providers + models that accept - * audio input via Chat Completions. Set this from "LLM-only" diarisation - * modals so the operator can't pick e.g. ``gpt-4.1`` / ``gpt-5`` (which - * have no ``input_audio`` content part on Chat Completions) and hit a - * cryptic provider 400 at runtime. - * - * Implementation: providers are clamped to OpenAI + Google (the only - * two that have audio-capable chat models in our LiteLLM wiring today), - * and the model dropdown is filtered to substring matches in - * :data:`AUDIO_CAPABLE_MODEL_MATCHERS`. Both rules are intentionally - * lenient (substrings, not exact strings) so newer dated snapshots - * (e.g. ``gpt-4o-audio-preview-2025-06-03``) flow through automatically - * without a code change. - */ - audioCapableOnly?: boolean /** Restrict selectable LLM models (e.g. System 1-only for classification metrics). */ modelFilter?: (model: string) => boolean + /** LLM-only diarisation: keep providers and models that can accept audio. */ + audioCapableOnly?: boolean /** Shown when ``modelFilter`` leaves the selected provider/credential with nothing to pick. */ incompatibleHint?: string /** When false, hide the LLM advanced options panel (LLM kind only). */ showAdvancedOptions?: boolean } -// Voice-platform integrations that also expose LLM models. Credentials -// for these providers live in the Integration table (Configurations → -// Integrations → Voice Platform), not AIProvider. +const AUDIO_LLM_PROVIDERS = new Set(['openai', 'google', 'openrouter', 'custom']) + +/** Models the LLM-only diariser can send audio to. */ +export function isAudioCapableModel(provider: string, model: string): boolean { + const providerKey = provider.trim().toLowerCase() + const modelKey = model.trim().toLowerCase() + if (!modelKey) return false + if (providerKey === 'custom') return true + if (providerKey === 'openai') return modelKey.includes('audio') + if (providerKey === 'google') return modelKey.includes('gemini') + if (providerKey === 'openrouter') { + return modelKey.includes('gemini') || modelKey.includes('audio') + } + return false +} + import { INTEGRATION_LLM_PLATFORMS } from '../../lib/integrationLlmPlatforms' export default function ProviderModelPicker({ @@ -136,9 +133,9 @@ export default function ProviderModelPicker({ className, allowCredentialPick = false, disabled = false, - audioCapableOnly = false, showAdvancedOptions = true, modelFilter, + audioCapableOnly = false, incompatibleHint = 'No compatible models for this provider.', }: ProviderModelPickerProps) { const { data: aiProviders = [] } = useQuery({ @@ -202,7 +199,6 @@ export default function ProviderModelPicker({ kind === 'llm' ? buildLLMProviderKeys(aiProviders, integrationLlmKeys, { providerAllowList, - audioCapableOnly, }) : [] @@ -218,6 +214,11 @@ export default function ProviderModelPicker({ // provider; credential pinning happens in the secondary select. const providerKeys = Array.from( new Set(eligibleProviders.map((p) => p.provider)), + ).filter( + (key) => + !audioCapableOnly || + kind !== 'llm' || + AUDIO_LLM_PROVIDERS.has(key), ) const activeCredential = useMemo(() => { @@ -249,11 +250,11 @@ export default function ProviderModelPicker({ const unfilteredModels = catalogModels.mode === 'gateway_direct' ? [] - : audioCapableOnly && kind === 'llm' && value.provider - ? catalogModels.models.filter((m) => - isAudioCapableModel(value.provider as string, m), - ) - : catalogModels.models + : catalogModels.models.filter((model) => + audioCapableOnly && kind === 'llm' && value.provider + ? isAudioCapableModel(value.provider, model) + : true, + ) const models = modelFilter && kind === 'llm' ? unfilteredModels.filter(modelFilter) : unfilteredModels const hasIncompatibleSelection = Boolean( @@ -265,6 +266,18 @@ export default function ProviderModelPicker({ : unfilteredModels.length > 0 && models.length === 0), ) + useEffect(() => { + if (!audioCapableOnly || kind !== 'llm' || !value.provider) return + if (AUDIO_LLM_PROVIDERS.has(value.provider.toLowerCase())) return + onChange({ + provider: null, + model: null, + credential_id: null, + llm_config: null, + }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioCapableOnly, kind, value.provider]) + // Auto-pick the first model when the provider changes and the // currently-selected model isn't valid for it. Avoids a confusing // empty-model state after the user picks a provider. @@ -280,30 +293,6 @@ export default function ProviderModelPicker({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [value.provider, models, resolvedGatewayDirectModel]) - // When the audio-capable toggle flips ON after the user has already - // picked a non-audio provider/model (e.g. Anthropic + Claude), clear - // the stale selection so the parent form doesn't submit something we - // just hid from the dropdown. - useEffect(() => { - if (!audioCapableOnly) return - if (!value.provider) return - const providerAllowed = llmProviderKeys.includes(value.provider) - const credentialAllowed = - activeCredential && isGatewayAudioCapableCredential(activeCredential) - if (!providerAllowed && !credentialAllowed) { - onChange({ provider: null, model: null, credential_id: null }) - return - } - if ( - value.model && - !isAudioCapableModel(value.provider, value.model) && - !credentialAllowed - ) { - onChange({ ...value, model: null }) - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [audioCapableOnly, value.provider, value.model, llmProviderKeys]) - const selectedProviderKey = value.provider?.toLowerCase() ?? '' const credentialRows = selectedProviderKey @@ -394,13 +383,6 @@ export default function ProviderModelPicker({

Model is fixed on the integration — Bifrost gateway routing applies.

- {audioCapableOnly && - activeCredential && - activeCredential.provider?.toLowerCase() === 'custom' && ( -

- Audio support depends on the underlying Bifrost model. -

- )} ) : ( <> @@ -418,11 +400,7 @@ export default function ProviderModelPicker({ {!value.provider ? ( ) : models.length === 0 ? ( - + ) : ( models.map((m) => (
diff --git a/frontend/src/components/shared/StatusBadge.tsx b/frontend/src/components/shared/StatusBadge.tsx index 59c52e06..54986624 100644 --- a/frontend/src/components/shared/StatusBadge.tsx +++ b/frontend/src/components/shared/StatusBadge.tsx @@ -20,6 +20,7 @@ const STATUS_STYLES: Record = { evaluating: 'bg-blue-100 text-blue-800', running: 'bg-blue-100 text-blue-800', cancelled: 'bg-gray-100 text-gray-600', + auto_aborted: 'bg-gray-100 text-gray-600', processing: 'bg-blue-100 text-blue-800', completed: 'bg-green-100 text-green-800', partial: 'bg-amber-100 text-amber-800', @@ -49,7 +50,7 @@ export default function StatusBadge({ status, size = 'md' }: StatusBadgeProps) { if (normalizedStatus === 'completed' || normalizedStatus === 'call_ended') { return } - if (normalizedStatus === 'failed') { + if (normalizedStatus === 'failed' || normalizedStatus === 'auto_aborted') { return } if (normalizedStatus === 'partial') { @@ -67,7 +68,7 @@ export default function StatusBadge({ status, size = 'md' }: StatusBadgeProps) { return ( {renderIcon()} - {status} + {normalizedStatus === 'auto_aborted' ? 'Auto-aborted' : status} ) } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 6db124f0..0c984139 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,6764 +1,6625 @@ -import axios, { AxiosInstance } from 'axios' -import { - clearAuthSession, - csrfHeaders, - getApiErrorDetail, - hasRevocableUserCredentials, - isOrganizationAccessDenied, - organizationAccessDeniedMessage, - redirectToLoginWithMessage, - type UserSessionCredentials, -} from './authSession' -import type { - GenerateScenariosFromPromptParams, - GenerateTestPromptParams, - GenerateTestSetupParams, -} from '../types/agentTestSetupGeneration' -import type { - AudioFile, - Evaluation, - EvaluationCreate, - EvaluationResult, - DashboardSummary, - ModelConfigEntry, - APIKey, - MessageResponse, - EvaluationStatus, - OrganizationMember, - Invitation, - InvitationAcceptResponse, - InvitationCreate, - InvitationPreview, - Profile, - UserUpdate, - UserPreferences, - UserPreferencesUpdate, - Role, - Integration, - IntegrationCreate, - ListIntegrationVoiceAgentsResponse, - S3ConnectionTestResponse, - S3ListFilesResponse, - S3BrowseResponse, - S3Status, - CallImport, - CallImportDetail, - CallImportListResponse, - CallImportRow, - ObservabilityCall, - CallImportSchema, - CallImportSchemaCreate, - CallImportSchemaListResponse, - CallImportSchemaUpdate, - CallImportStatus, - CallImportTag, - CallImportUploadResponse, - CallImportPreviewResponse, - CallImportEvaluation, - CallImportEvaluationBaselineCandidatesResponse, - CallImportEvaluationExportJob, - CallImportEvaluationPdfReport, - CallImportEvaluationPdfReportListResponse, - CallImportEvaluationLLMOverride, - CallImportEvaluationListResponse, - CallImportEvaluationRow, - CallImportEvaluationRowListResponse, - CallImportEvaluationRetryResponse, - CallImportEvaluationBulkActionResponse, - CallImportEvaluationAggregateResponse, - CallImportInsightsResponse, - EvaluatorResultListResponse, - EvaluatorResultsOverviewResponse, - EvaluatorResultsAggregateResponse, - ListEvaluatorResultsParams, - CallImportTranscribeRequest, - CallImportTranscribeResponse, - CallImportRetryFailedRowsResponse, - PlatformImportProgress, - Workspace, - LLMGenerationConfig, - TestAgent, -} from '../types/api' - -export interface EnterpriseFeatureMeta { - title: string - description?: string - category?: string -} - -export type EnterpriseFeatureCatalog = Record - -export type VoicePlaygroundSourceType = 'tts' | 'recording' | 'upload' - -export interface VoicePlaygroundSideConfig { - source_type: VoicePlaygroundSourceType - provider?: string - model?: string - voices?: Array<{ id: string; name: string; sample_rate_hz?: number }> - call_import_row_ids?: string[] - upload_s3_keys?: string[] -} - -export type VoicePlaygroundBlindTestRefType = 'recording' | 'upload' | 'tts_sample' - -export interface VoicePlaygroundBlindTestAudioRef { - type: VoicePlaygroundBlindTestRefType - call_import_row_id?: string - upload_s3_key?: string - tts_sample_id?: string - label?: string -} - -export interface VoicePlaygroundBlindTestPair { - text?: string - x: VoicePlaygroundBlindTestAudioRef - y: VoicePlaygroundBlindTestAudioRef -} - -export interface UsagePolicy { - extended_history: boolean - max_history_days: number | null -} - -export interface OssQuotas { - max_user_metrics: number | null - max_agents: number | null - max_org_members: number | null - max_workspaces: number | null -} - -export interface OssQuotaUsage { - user_metrics: number - agents: number - org_members: number - workspaces: number -} - -export interface LicenseInfoResponse { - is_enterprise: boolean - gateway_routing_allowed?: boolean - enabled_features: string[] - all_enterprise_features: string[] - feature_catalog?: EnterpriseFeatureCatalog - organization?: string - usage_policy?: UsagePolicy - quotas?: OssQuotas - quota_usage?: OssQuotaUsage -} - -export interface ReportBranding { - heading?: string | null - has_logo: boolean - images: Array<{ - id: string - filename: string - content_type: string - size_bytes: number - role: 'internal' | 'external' | 'generic' - updated_at?: string | null - data_uri?: string | null - }> -} - -export interface AuthProviderConfig { - name: 'api_key' | 'local_password' | 'external_oidc' - enabled: boolean - display_name: string - description?: string - supports_password?: boolean - supports_signup?: boolean - oidc_issuer?: string | null - oidc_client_id?: string | null - oidc_authorize_url?: string | null -} - -export interface AuthConfigResponse { - providers: AuthProviderConfig[] - tier: 'oss' | 'enterprise' - gated_signup?: boolean - cookie_session_enabled?: boolean -} - -export interface AuthUserSummary { - id: string - email: string - name?: string | null - first_name?: string | null - last_name?: string | null - organization_id: string - role?: string | null - has_password?: boolean - email_is_placeholder?: boolean -} - -export interface TokenResponse { - access_token: string - refresh_token?: string - token_type: string - expires_in: number - user: AuthUserSummary - join_notice?: string | null -} - -export interface LoginOrgOption { - id: string - name: string - role: string -} - -export interface LoginOrgSelectionResponse { - requires_org_selection: true - organizations: LoginOrgOption[] -} - -export interface PlatformAdminSummary { - id: string - email: string -} - -export interface PlatformTokenResponse { - access_token: string - token_type: string - expires_in: number - admin: PlatformAdminSummary -} - -export interface PlatformOrganizationItem { - id: string - name: string - is_active: boolean - member_count: number - created_at?: string | null - disabled_at?: string | null -} - -export interface PlatformOrganizationListResponse { - items: PlatformOrganizationItem[] - total: number - offset: number - limit: number -} - -export interface PlatformOrganizationStats { - total: number - active: number - disabled: number -} - -export interface PlatformOrgUser { - id: string - email: string - role: string - is_active: boolean -} - -export interface PlatformSignupCode { - id: string - label?: string | null - max_uses?: number | null - use_count: number - expires_at?: string | null - is_active: boolean - created_at?: string | null - code?: string | null -} - -export type LoginResponse = TokenResponse | LoginOrgSelectionResponse - -export function isLoginOrgSelectionResponse( - response: LoginResponse -): response is LoginOrgSelectionResponse { - return 'requires_org_selection' in response && response.requires_org_selection === true -} - -export interface OrganizationSummary { - id: string - name: string -} - -export type LLMGatewayMode = 'inherit' | 'enabled' | 'disabled' -export type LLMGatewayType = 'inherit' | 'bifrost' | 'litellm_proxy' -export type LLMGatewayInterface = 'inherit' | 'litellm_shim' | 'native_openai' -export type LLMGatewayRouting = 'direct' | 'bifrost' | 'litellm_proxy' -export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' - -export interface LLMGatewaySettings { - mode: LLMGatewayMode - gateway_type: LLMGatewayType - gateway_interface?: LLMGatewayInterface - base_url?: string | null - has_virtual_key: boolean - has_master_key: boolean - platform_enabled: boolean - platform_gateway_type: 'bifrost' | 'litellm_proxy' - platform_gateway_interface?: 'litellm_shim' | 'native_openai' - platform_base_url?: string | null - effective_routing: LLMGatewayRouting - effective_gateway_type?: 'bifrost' | 'litellm_proxy' | null - effective_gateway_interface?: 'litellm_shim' | 'native_openai' | null - effective_base_url?: string | null - effective_has_virtual_key: boolean - effective_has_master_key: boolean - gateway_managed_credentials: boolean -} - -export interface LLMGatewaySettingsUpdate { - mode: LLMGatewayMode - gateway_type?: LLMGatewayType - gateway_interface?: LLMGatewayInterface - base_url?: string | null - virtual_key?: string | null - master_key?: string | null - clear_virtual_key?: boolean - clear_master_key?: boolean -} - -export interface AlertingSyncSettings { - sync_notification_lifecycle: boolean - pagerduty_webhook_url: string | null - has_pagerduty_webhook_signing_secret: boolean - pagerduty_inbound_ready: boolean - confirmation_phrase_required: string -} - -export interface AlertingSyncSettingsUpdate { - sync_notification_lifecycle: boolean - confirm_enable?: boolean - confirmation_phrase?: string - pagerduty_webhook_signing_secret?: string - clear_pagerduty_webhook_signing_secret?: boolean -} - -export interface TelephonyIntegrationResponse { - id: string - organization_id: string - provider: string - name?: string | null - verify_app_uuid?: string | null - voice_app_id?: string | null - sip_domain?: string | null - masking_config?: Record | null - is_active: boolean - /** True if this row is the default credential for (org, provider). */ - is_default?: boolean - last_tested_at?: string | null - created_at: string - updated_at: string -} - -export interface TelephonyIntegrationCreatePayload { - provider?: string - name?: string - auth_id: string - auth_token: string - verify_app_uuid?: string - voice_app_id?: string - sip_domain?: string - masking_config?: Record - /** Mark the new credential as the default for (org, provider). */ - is_default?: boolean -} - -export interface TelephonyIntegrationUpdatePayload { - /** When set, update this specific credential row instead of the legacy single-row flow. */ - id?: string - provider?: string - name?: string - auth_id?: string - auth_token?: string - verify_app_uuid?: string - voice_app_id?: string - sip_domain?: string - masking_config?: Record - is_active?: boolean -} - -export interface TelephonyPhoneNumberResponse { - id: string - phone_number: string - country_iso2?: string | null - region?: string | null - number_type?: string | null - capabilities?: Record | null - is_masking_pool: boolean - inbound_enabled?: boolean - outbound_enabled?: boolean - source?: string - agent_id?: string | null - linked_agent_name?: string | null - provider?: string | null - is_active: boolean - created_at: string -} - -export interface TelephonyAvailableNumber { - e164: string - provider_number_id?: string | null - country?: string | null - region?: string | null - capabilities?: Record | null - status?: string | null - application_id?: string | null - already_imported: boolean - imported_number_id?: string | null -} - -export interface TelephonyImportNumberResult { - number: string - success: boolean - message: string - answer_url: string - webhook_configured?: boolean | null - imported_number_id?: string | null - application_id?: string | null -} - -export interface TelephonyImportNumbersResponse { - provider: string - results: TelephonyImportNumberResult[] - answer_url: string -} - -/** @deprecated Use TelephonyAvailableNumber */ -export type VobizAvailableNumber = TelephonyAvailableNumber - -/** @deprecated Use TelephonyImportNumbersResponse */ -export type VobizImportNumbersResponse = TelephonyImportNumbersResponse - -/** @deprecated Use TelephonyImportNumberResult */ -export type VobizImportNumberResult = TelephonyImportNumberResult - -export interface PlatformOutboundPoolNumber { - phone_number: string - provider: string -} - -export interface PlatformOutboundPoolResponse { - numbers: PlatformOutboundPoolNumber[] - max_concurrent_per_org: number - shared_across_orgs: boolean -} - -/** @deprecated Use PlatformOutboundPoolResponse */ -export type VobizOutboundPoolResponse = PlatformOutboundPoolResponse - -export interface VobizOutboundCallRequest { - to_number: string - agent_id: string - from_number?: string - persona_id?: string - scenario_id?: string - evaluator_id?: string -} - -export interface TelephonyDialTargetResponse { - id: string - phone_number: string - label?: string | null - created_at: string - updated_at: string -} - -export interface VobizOutboundCallResponse { - provider_request_uuid: string - call_status: string - from_number: string - to_number: string - call_ref: string - call_short_id?: string - evaluator_result_id?: string - result_id?: string - message: string -} - -export interface EvaluatorSuiteCombination { - id: string - evaluator_id: string - persona_id?: string - persona_name?: string | null - scenario_id: string - scenario_name?: string | null - scenario_description?: string | null - scenario_required_info?: Record | null -} - -export interface EvaluatorSuitePersonaSummary { - id: string - name?: string | null - tts_provider?: string | null -} - -export interface EvaluatorSuite { - id: string - organization_id: string - name?: string | null - agent_id: string - persona_id: string - persona_ids?: string[] - personas?: EvaluatorSuitePersonaSummary[] - agent_name?: string | null - persona_name?: string | null - agent_call_type?: string | null - agent_call_medium?: string | null - voice_bundle_tts_provider?: string | null - metric_ids?: string[] | null - llm_provider?: string | null - llm_model?: string | null - tags?: string[] | null - default_runs_per_combination: number - round_robin_index: number - is_active: boolean - agent_suite_count: number - combination_count: number - combinations: EvaluatorSuiteCombination[] - created_at: string - updated_at: string -} - -export interface RunEvaluatorSuiteResponse { - total_runs: number - task_ids: string[] - evaluator_results: any[] - phone_call_refs: string[] -} - -export interface RunNextCombinationResponse { - evaluator_id: string - persona_id?: string | null - persona_name?: string | null - scenario_id: string - scenario_name: string - combination_index: number - next_index: number - evaluator_result_id?: string | null - result_id?: string | null - task_id?: string | null - phone_call_ref?: string | null - call_short_id?: string | null -} - -export interface ChooseNextCombinationResponse { - evaluator_id: string - persona_id?: string | null - persona_name?: string | null - scenario_id: string - scenario_name: string - combination_index: number - next_index: number -} - -type TTSReportOptionsPayload = { - show_runs?: boolean - min_runs_to_show?: number - include_latency?: boolean - include_ttfb?: boolean - include_endpoint?: boolean - include_naturalness?: boolean - include_hallucination?: boolean - include_prosody?: boolean - include_arousal?: boolean - include_valence?: boolean - include_cer?: boolean - include_wer?: boolean - include_hallucination_examples?: boolean - hallucination_examples_limit?: number - include_disclaimer_sections?: boolean - include_methodology_sections?: boolean - zone_threshold_overrides?: Record -} - -// When running in production (served from same origin), use relative path -// Otherwise use environment variable or default -export const apiBaseUrl = - import.meta.env.VITE_API_URL || - (import.meta.env.PROD ? '' : 'http://localhost:8000') - -const API_BASE_URL = apiBaseUrl - -let refreshPromise: Promise | null = null - -class ApiClient { - private client: AxiosInstance - private inMemoryAccessToken: string | null = null - private inMemoryRefreshToken: string | null = null - private cookieSessionEnabled = true - - constructor() { - this.client = axios.create({ - baseURL: API_BASE_URL, - withCredentials: true, - headers: { - 'Content-Type': 'application/json', - }, - }) - - this.client.interceptors.request.use((config) => { - const apiKey = this.cookieSessionEnabled ? null : localStorage.getItem('apiKey') - const workspaceId = localStorage.getItem('activeWorkspaceId') - if (this.inMemoryAccessToken) { - config.headers.Authorization = `Bearer ${this.inMemoryAccessToken}` - } else if (config.headers.Authorization) { - delete config.headers.Authorization - } - if (apiKey) { - config.headers['X-API-Key'] = apiKey - } else if (config.headers['X-API-Key']) { - delete config.headers['X-API-Key'] - } - if (workspaceId) { - config.headers['X-Workspace-Id'] = workspaceId - } else if (config.headers['X-Workspace-Id']) { - delete config.headers['X-Workspace-Id'] - } - const method = (config.method || 'get').toLowerCase() - if (method !== 'get' && method !== 'head' && method !== 'options') { - Object.assign(config.headers, csrfHeaders()) - } - return config - }) - - // Add response interceptor for error handling and silent token refresh - this.client.interceptors.response.use( - (response) => response, - async (error) => { - const response = error.response - if (response?.data instanceof Blob) { - try { - const text = await response.data.text() - const trimmed = text.trim() - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - response.data = JSON.parse(text) - } - } catch { - // Keep the original blob when it isn't JSON. - } - } - - const originalRequest = error.config - const requestUrl = String(originalRequest?.url || '') - const isAuthEndpoint = - requestUrl.includes('/auth/login') || - requestUrl.includes('/auth/signup') || - requestUrl.includes('/auth/refresh') || - requestUrl.includes('/auth/logout') || - requestUrl.includes('/auth/config') || - requestUrl.includes('/auth/password') - - const detail = getApiErrorDetail(error) - if ( - response?.status === 403 && - isOrganizationAccessDenied(detail) && - !isAuthEndpoint - ) { - redirectToLoginWithMessage(organizationAccessDeniedMessage(detail)) - return Promise.reject(error) - } - - if ( - response?.status === 401 && - originalRequest && - !originalRequest._retry && - !isAuthEndpoint - ) { - originalRequest._retry = true - const refreshed = await this.tryRefreshAccessToken() - if (refreshed) { - if (this.cookieSessionEnabled) { - delete originalRequest.headers.Authorization - } else if (this.inMemoryAccessToken) { - originalRequest.headers.Authorization = `Bearer ${this.inMemoryAccessToken}` - } - return this.client(originalRequest) - } - - clearAuthSession() - window.location.href = '/login' - } - return Promise.reject(error) - }, - ) - } - - private async tryRefreshAccessToken(): Promise { - if (!this.cookieSessionEnabled && !this.inMemoryRefreshToken) { - return false - } - - if (!refreshPromise) { - refreshPromise = axios - .post( - `${API_BASE_URL}/api/v1/auth/refresh`, - this.cookieSessionEnabled - ? {} - : { refresh_token: this.inMemoryRefreshToken }, - { - withCredentials: true, - headers: { 'Content-Type': 'application/json', ...csrfHeaders() }, - }, - ) - .then((res) => { - const { access_token, refresh_token: rotatedRefresh } = res.data as TokenResponse - if (access_token) { - this.setAccessToken(access_token) - } - if (rotatedRefresh) { - this.setRefreshToken(rotatedRefresh) - } - // Cookie sessions return empty tokens in JSON; new credentials are Set-Cookie only. - return Boolean( - this.cookieSessionEnabled || access_token || this.inMemoryAccessToken, - ) - }) - .catch((err) => { - const refreshDetail = getApiErrorDetail(err) - if ( - err?.response?.status === 403 && - isOrganizationAccessDenied(refreshDetail) - ) { - redirectToLoginWithMessage( - organizationAccessDeniedMessage(refreshDetail), - ) - } - return false - }) - .finally(() => { - refreshPromise = null - }) - } - - return refreshPromise - } - - setApiKey(apiKey: string) { - localStorage.setItem('apiKey', apiKey) - } - - clearApiKey() { - localStorage.removeItem('apiKey') - } - - setCookieSessionEnabled(enabled: boolean) { - this.cookieSessionEnabled = enabled - } - - isCookieSessionEnabled(): boolean { - return this.cookieSessionEnabled - } - - setAccessToken(accessToken: string) { - this.inMemoryAccessToken = accessToken - } - - clearAccessToken() { - this.inMemoryAccessToken = null - } - - setRefreshToken(refreshToken: string) { - this.inMemoryRefreshToken = refreshToken - } - - clearRefreshToken() { - this.inMemoryRefreshToken = null - } - - clearInMemoryTokens() { - this.inMemoryAccessToken = null - this.inMemoryRefreshToken = null - } - - getAccessToken(): string | null { - return this.inMemoryAccessToken - } - - // Workspace endpoints (in-org isolation boundary for call imports + metrics). - async listWorkspaces(): Promise { - const response = await this.client.get('/api/v1/workspaces') - return response.data - } - - async createWorkspace(payload: { - name: string - slug?: string - }): Promise { - const response = await this.client.post('/api/v1/workspaces', payload) - return response.data - } - - async updateWorkspace( - workspaceId: string, - payload: { name?: string; is_active?: boolean }, - ): Promise { - const response = await this.client.patch( - `/api/v1/workspaces/${workspaceId}`, - payload, - ) - return response.data - } - - async deleteWorkspace(workspaceId: string): Promise { - await this.client.delete(`/api/v1/workspaces/${workspaceId}`) - } - - async listCapabilities(): Promise { - const response = await this.client.get('/api/v1/capabilities') - return response.data - } - - async listWorkspaceRoles(): Promise { - const response = await this.client.get('/api/v1/workspace-roles') - return response.data - } - - async createWorkspaceRole( - payload: import('../types/api').WorkspaceRoleCreate, - ): Promise { - const response = await this.client.post('/api/v1/workspace-roles', payload) - return response.data - } - - async updateWorkspaceRole( - roleId: string, - payload: import('../types/api').WorkspaceRoleUpdate, - ): Promise { - const response = await this.client.patch( - `/api/v1/workspace-roles/${roleId}`, - payload, - ) - return response.data - } - - async deleteWorkspaceRole(roleId: string): Promise { - await this.client.delete(`/api/v1/workspace-roles/${roleId}`) - } - - async listWorkspaceMembers( - workspaceId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/workspaces/${workspaceId}/members`, - ) - return response.data - } - - async addWorkspaceMember( - workspaceId: string, - payload: { user_id: string; role_id: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/workspaces/${workspaceId}/members`, - payload, - ) - return response.data - } - - async updateWorkspaceMember( - workspaceId: string, - userId: string, - payload: { role_id: string }, - ): Promise { - const response = await this.client.patch( - `/api/v1/workspaces/${workspaceId}/members/${userId}`, - payload, - ) - return response.data - } - - async removeWorkspaceMember( - workspaceId: string, - userId: string, - ): Promise { - await this.client.delete( - `/api/v1/workspaces/${workspaceId}/members/${userId}`, - ) - } - - // Auth endpoints - async getAuthConfig(): Promise { - const response = await this.client.get('/api/v1/auth/config') - this.setCookieSessionEnabled(Boolean(response.data.cookie_session_enabled)) - return response.data - } - - async signup(data: { - email: string - password: string - organization_name?: string - first_name?: string - last_name?: string - reference_code?: string - invite_token?: string - }): Promise { - const response = await this.client.post('/api/v1/auth/signup', data) - return response.data - } - - async previewInvitation(token: string): Promise { - const response = await axios.get(`${API_BASE_URL}/api/v1/auth/invitations/preview/${encodeURIComponent(token)}`) - return response.data - } - - async acceptInvitationByToken(token: string): Promise { - const response = await this.client.post('/api/v1/auth/invitations/accept-by-token', { token }) - return response.data - } - - private platformHeaders(): Record { - const headers: Record = {} - const token = localStorage.getItem('platformAccessToken') - if (token) { - headers.Authorization = `Bearer ${token}` - } - return headers - } - - private platformRequestConfig( - method: string, - extraHeaders?: Record, - ): { headers: Record; withCredentials: boolean } { - const headers: Record = { - ...this.platformHeaders(), - ...extraHeaders, - } - const methodLower = method.toLowerCase() - if (methodLower !== 'get' && methodLower !== 'head' && methodLower !== 'options') { - Object.assign(headers, csrfHeaders()) - } - return { headers, withCredentials: true } - } - - async platformLogin(email: string, password: string): Promise { - const response = await axios.post( - `${API_BASE_URL}/api/v1/platform/auth/login`, - { email, password }, - { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, - }, - ) - return response.data - } - - async platformLogout(accessToken?: string | null): Promise<{ success: boolean; admin_id: string }> { - const token = accessToken ?? localStorage.getItem('platformAccessToken') - const headers: Record = { - 'Content-Type': 'application/json', - ...csrfHeaders(), - } - if (token) { - headers.Authorization = `Bearer ${token}` - } - const response = await axios.post( - `${API_BASE_URL}/api/v1/platform/auth/logout`, - {}, - { headers, withCredentials: true }, - ) - return response.data - } - - /** - * Voluntary logout: revoke JWT blacklist + refresh token on the server. - * Uses raw axios so credentials survive after local session is cleared. - */ - revokeUserSessionBestEffort(credentials: UserSessionCredentials): void { - if (!hasRevocableUserCredentials(credentials)) { - return - } - const auth = { accessToken: credentials.accessToken, apiKey: credentials.apiKey } - void this.revokeUserSession(credentials.refreshToken, auth) - .catch(() => this.revokeUserSession(credentials.refreshToken, auth)) - .catch(() => {}) - } - - revokePlatformSessionBestEffort(accessToken?: string | null): void { - void this.platformLogout(accessToken) - .catch(() => this.platformLogout(accessToken)) - .catch(() => {}) - } - - async getPlatformOrganizationStats(): Promise { - const response = await axios.get( - `${API_BASE_URL}/api/v1/platform/organizations/stats`, - this.platformRequestConfig('get'), - ) - return response.data - } - - async listPlatformOrganizations(params?: { - offset?: number - limit?: number - search?: string - is_active?: boolean - }): Promise { - const response = await axios.get(`${API_BASE_URL}/api/v1/platform/organizations`, { - ...this.platformRequestConfig('get'), - params, - }) - return response.data - } - - async updatePlatformOrganization( - orgId: string, - data: { is_active: boolean }, - ): Promise { - const response = await axios.patch( - `${API_BASE_URL}/api/v1/platform/organizations/${orgId}`, - data, - this.platformRequestConfig('patch', { 'Content-Type': 'application/json' }), - ) - return response.data - } - - async listPlatformOrganizationUsers( - orgId: string, - params?: { role?: string }, - ): Promise { - const response = await axios.get( - `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users`, - { ...this.platformRequestConfig('get'), params }, - ) - return response.data - } - - async platformResetUserPassword( - orgId: string, - userId: string, - newPassword: string, - ): Promise<{ user_id: string; email: string; message: string }> { - const response = await axios.post( - `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users/${userId}/reset-password`, - { new_password: newPassword }, - this.platformRequestConfig('post', { 'Content-Type': 'application/json' }), - ) - return response.data - } - - async listPlatformSignupCodes(): Promise { - const response = await axios.get( - `${API_BASE_URL}/api/v1/platform/signup-codes`, - this.platformRequestConfig('get'), - ) - return response.data - } - - async createPlatformSignupCode(data: { - code: string - label?: string - max_uses?: number - expires_at?: string - }): Promise { - const response = await axios.post( - `${API_BASE_URL}/api/v1/platform/signup-codes`, - data, - this.platformRequestConfig('post', { 'Content-Type': 'application/json' }), - ) - return response.data - } - - async updatePlatformSignupCode( - codeId: string, - data: { is_active?: boolean; max_uses?: number; label?: string }, - ): Promise { - const response = await axios.patch( - `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, - data, - this.platformRequestConfig('patch', { 'Content-Type': 'application/json' }), - ) - return response.data - } - - async deactivatePlatformSignupCode(codeId: string): Promise { - const response = await axios.delete( - `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, - this.platformRequestConfig('delete'), - ) - return response.data - } - - async establishOidcSession(oidcAccessToken: string): Promise { - const response = await axios.post( - `${API_BASE_URL}/api/v1/auth/oidc/session`, - {}, - { - withCredentials: true, - headers: { - Authorization: `Bearer ${oidcAccessToken}`, - 'Content-Type': 'application/json', - ...csrfHeaders(), - }, - }, - ) - const data = response.data as TokenResponse - if (data.access_token) { - this.setAccessToken(data.access_token) - } - if (data.refresh_token) { - this.setRefreshToken(data.refresh_token) - } - return data - } - - async loginWithPassword( - email: string, - password: string, - organizationId?: string, - inviteToken?: string, - ): Promise { - const response = await this.client.post('/api/v1/auth/login', { - email, - password, - ...(organizationId ? { organization_id: organizationId } : {}), - ...(inviteToken ? { invite_token: inviteToken } : {}), - }) - return response.data - } - - async logout( - refreshToken?: string | null, - credentials?: { accessToken?: string | null; apiKey?: string | null }, - ): Promise<{ success: boolean; auth_method: string }> { - return this.revokeUserSession(refreshToken, credentials) - } - - private async revokeUserSession( - refreshToken?: string | null, - credentials?: { accessToken?: string | null; apiKey?: string | null }, - ): Promise<{ success: boolean; auth_method: string }> { - const headers: Record = { - 'Content-Type': 'application/json', - ...csrfHeaders(), - } - const accessToken = credentials?.accessToken ?? this.inMemoryAccessToken - const apiKey = this.cookieSessionEnabled - ? credentials?.apiKey ?? null - : credentials?.apiKey ?? localStorage.getItem('apiKey') - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}` - } else if (apiKey) { - headers['X-API-Key'] = apiKey - } - const response = await axios.post( - `${API_BASE_URL}/api/v1/auth/logout`, - this.cookieSessionEnabled - ? {} - : { - refresh_token: - refreshToken ?? this.inMemoryRefreshToken ?? undefined, - }, - { headers, withCredentials: true }, - ) - return response.data - } - - async refreshSession(refreshToken?: string): Promise { - const response = await axios.post( - `${API_BASE_URL}/api/v1/auth/refresh`, - this.cookieSessionEnabled ? {} : { refresh_token: refreshToken }, - { - withCredentials: true, - headers: { 'Content-Type': 'application/json', ...csrfHeaders() }, - }, - ) - return response.data - } - - async getMe(): Promise { - const response = await this.client.get('/api/v1/auth/me') - return response.data - } - - async switchOrganization(organizationId: string): Promise { - const response = await this.client.post('/api/v1/auth/switch-org', { - organization_id: organizationId, - ...(this.cookieSessionEnabled - ? {} - : { refresh_token: this.inMemoryRefreshToken || undefined }), - }) - return response.data - } - - async setPassword(data: { - new_password: string - current_password?: string - email?: string - }): Promise { - const response = await this.client.post('/api/v1/auth/password', data) - return response.data - } - - async generateApiKey(name?: string): Promise { - const response = await this.client.post('/api/v1/auth/generate-key', { name }) - return response.data - } - - async validateApiKey(): Promise<{ valid: boolean; message: string }> { - const response = await this.client.post('/api/v1/auth/validate') - return response.data - } - - // Settings / API Key Management endpoints - async listApiKeys(): Promise { - const response = await this.client.get('/api/v1/settings/api-keys') - return response.data - } - - async createApiKey(name?: string): Promise { - const response = await this.client.post('/api/v1/settings/api-keys', { name }) - return response.data - } - - async deleteApiKey(keyId: string): Promise { - await this.client.delete(`/api/v1/settings/api-keys/${keyId}`) - } - - async regenerateApiKey(keyId: string): Promise { - const response = await this.client.post(`/api/v1/settings/api-keys/${keyId}/regenerate`) - return response.data - } - - // Audio endpoints - async uploadAudio(file: File): Promise { - const formData = new FormData() - formData.append('file', file) - const response = await this.client.post('/api/v1/audio/upload', formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }) - return response.data - } - - async getAudio(audioId: string): Promise { - const response = await this.client.get(`/api/v1/audio/${audioId}`) - return response.data - } - - async listAudio(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/audio', { - params: { skip, limit }, - }) - return response.data - } - - async deleteAudio(audioId: string): Promise { - const response = await this.client.delete(`/api/v1/audio/${audioId}`) - return response.data - } - - async downloadAudio(audioId: string): Promise { - const response = await this.client.get(`/api/v1/audio/${audioId}/download`, { - responseType: 'blob', - }) - return response.data - } - - // Evaluation endpoints - async createEvaluation(data: EvaluationCreate): Promise { - const response = await this.client.post('/api/v1/evaluations/create', data) - return response.data - } - - async getEvaluation(evaluationId: string): Promise { - const response = await this.client.get(`/api/v1/evaluations/${evaluationId}`) - return response.data - } - - async listEvaluations( - skip = 0, - limit = 100, - status?: EvaluationStatus - ): Promise { - const response = await this.client.get('/api/v1/evaluations', { - params: { skip, limit, status }, - }) - return response.data - } - - async cancelEvaluation(evaluationId: string): Promise { - const response = await this.client.post(`/api/v1/evaluations/${evaluationId}/cancel`) - return response.data - } - - async deleteEvaluation(evaluationId: string): Promise { - const response = await this.client.delete(`/api/v1/evaluations/${evaluationId}`) - return response.data - } - - // Results endpoints - async getEvaluationResult(evaluationId: string): Promise { - const response = await this.client.get(`/api/v1/results/${evaluationId}`) - return response.data - } - - async getMetrics(evaluationId: string): Promise<{ - evaluation_id: string - metrics: Record - processing_time?: number | null - }> { - const response = await this.client.get(`/api/v1/results/${evaluationId}/metrics`) - return response.data - } - - async getTranscript(evaluationId: string): Promise<{ - evaluation_id: string - transcript: string - }> { - const response = await this.client.get(`/api/v1/results/${evaluationId}/transcript`) - return response.data - } - - async compareEvaluations(evaluationIds: string[]): Promise<{ - evaluations: EvaluationResult[] - comparison_metrics: Record - }> { - const response = await this.client.post('/api/v1/results/compare', { - evaluation_ids: evaluationIds, - }) - return response.data - } - - // Agents endpoints - async getMessagingPublicBaseUrl(): Promise<{ - public_base_url: string - twilio_public_base_url?: string - telnyx_public_base_url?: string - meta_whatsapp_inbound_webhook_url?: string - }> { - const response = await this.client.get('/api/v1/chat/messaging/public-base-url') - return response.data - } - - async simulateAgentMetaWhatsappInbound( - agentId: string, - payload: { messaging_recipient: string; body?: string }, - ): Promise<{ ok: boolean }> { - const response = await this.client.post( - `/api/v1/agents/${agentId}/chat-messaging/simulate-meta-whatsapp-inbound`, - payload, - ) - return response.data - } - - async testAgentTwilioSms( - agentId: string, - overrides?: { - messaging_recipient?: string - twilio_from?: string - twilio_sms_trial_body_template?: string - }, - ): Promise<{ - ok: boolean - message_sid: string - to: string - from: string - body_sent: string - }> { - const response = await this.client.post( - `/api/v1/agents/${agentId}/chat-messaging/test-twilio-sms`, - overrides ?? {}, - ) - return response.data - } - - async testAgentMetaWhatsapp( - agentId: string, - overrides?: { messaging_recipient?: string }, - ): Promise<{ - ok: boolean - message_id: string - to: string - wa_id: string - template_sent: string - }> { - const response = await this.client.post( - `/api/v1/agents/${agentId}/chat-messaging/test-meta-whatsapp`, - overrides ?? {}, - ) - return response.data - } - - async testAgentTelnyxSms( - agentId: string, - overrides?: { - messaging_recipient?: string - telnyx_from?: string - }, - ): Promise<{ - ok: boolean - message_id: string - to: string - from: string - body_sent: string - }> { - const response = await this.client.post( - `/api/v1/agents/${agentId}/chat-messaging/test-telnyx-sms`, - overrides ?? {}, - ) - return response.data - } - - async createAgent(data: { - name: string - phone_number?: string - telephony_phone_number_id?: string - language: string - description?: string | null - call_type: string - call_medium: string - voice_bundle_id?: string - ai_provider_id?: string - voice_ai_integration_id?: string - voice_ai_agent_id?: string - provider_prompt?: string | null - silence_hangup_secs?: number - }): Promise { - const response = await this.client.post('/api/v1/agents', data) - return response.data - } - - async listAgents(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/agents', { - params: { skip, limit }, - }) - return response.data - } - - async getAgent(agentId: string): Promise { - const response = await this.client.get(`/api/v1/agents/${agentId}`) - return response.data - } - - async checkAgentPhoneAssignment(params: { - phoneNumber?: string - telephonyPhoneNumberId?: string - excludeAgentId?: string - }): Promise { - const response = await this.client.get('/api/v1/agents/check-phone-assignment', { - params: { - phone_number: params.phoneNumber, - telephony_phone_number_id: params.telephonyPhoneNumberId, - exclude_agent_id: params.excludeAgentId, - }, - }) - return response.data - } - - async updateAgent(agentId: string, data: { - name?: string - phone_number?: string | null - telephony_phone_number_id?: string | null - language?: string - description?: string | null - call_type?: string - call_medium?: string - silence_hangup_secs?: number - voice_bundle_id?: string | null - ai_provider_id?: string - voice_ai_integration_id?: string | null - voice_ai_agent_id?: string | null - provider_prompt?: string | null - prompt_variables?: Record - }): Promise { - const response = await this.client.put(`/api/v1/agents/${agentId}`, data) - return response.data - } - - async deleteAgent(agentId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/agents/${agentId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - async getAgentDeleteImpact(agentId: string): Promise<{ - agent_id: string - agent_name: string - dependencies: Record - can_delete_without_force: boolean - }> { - const response = await this.client.get(`/api/v1/agents/${agentId}/delete-impact`) - return response.data - } - - async generateAgentDescription(data: { - description: string - tone?: string - format_style?: string - provider?: string - model?: string - agent_id?: string - include_linked_scenarios?: boolean - append_scenarios_to_output?: boolean - }): Promise<{ content: string; provider: string; model: string }> { - const response = await this.client.post('/api/v1/agents/generate-description', data) - return response.data - } - - async generateTestPromptFromProduction( - data: GenerateTestPromptParams, - ): Promise<{ - sections: Array<{ key: string; title: string; content: string }> - test_agent_prompt: string - first_message: { - production_mode: string - production_message?: string | null - caller_mode: string - caller_message?: string | null - } - test_agent_template: { - sections: Array<{ key: string; title: string; content: string }> - first_message: { - production_mode: string - production_message?: string | null - caller_mode: string - caller_message?: string | null - } - } - provider: string - model: string - }> { - const response = await this.client.post('/api/v1/agents/generate-test-prompt', data) - return response.data - } - - async generateScenariosFromPrompt( - data: GenerateScenariosFromPromptParams, - ): Promise<{ - scenarios: Array<{ name: string; description: string; goal?: string | null }> - provider: string - model: string - }> { - const response = await this.client.post('/api/v1/agents/generate-scenarios-from-prompt', data) - return response.data - } - - async generateTestSetupFromProduction( - data: GenerateTestSetupParams, - ): Promise<{ - sections: Array<{ key: string; title: string; content: string }> - test_agent_prompt: string - scenarios: Array<{ name: string; description: string; goal?: string | null }> - provider: string - model: string - }> { - const response = await this.client.post('/api/v1/agents/generate-test-setup', data) - return response.data - } - - // Personas endpoints - async listPersonas(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/personas', { - params: { skip, limit }, - }) - return response.data - } - - async getPersona(personaId: string): Promise { - const response = await this.client.get(`/api/v1/personas/${personaId}`) - return response.data - } - - async createPersona(data: { - name: string - gender: string - tts_provider?: string - tts_voice_id?: string - tts_voice_name?: string - is_custom?: boolean - description?: string - tts_config?: Record - llm_temperature?: number - llm_max_tokens?: number - response_delay_ms?: number - max_turns?: number - allow_interruptions?: boolean - }): Promise { - const response = await this.client.post('/api/v1/personas', data) - return response.data - } - - async updatePersona(personaId: string, data: any): Promise { - const response = await this.client.put(`/api/v1/personas/${personaId}`, data) - return response.data - } - - async deletePersona(personaId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/personas/${personaId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - async clonePersona(personaId: string, name?: string): Promise { - const response = await this.client.post(`/api/v1/personas/${personaId}/clone`, { name }) - return response.data - } - - async seedDemoData(): Promise { - const response = await this.client.post('/api/v1/personas/seed-data') - return response.data - } - - async getPersonaAgentPromptSources(agentId: string): Promise<{ - agent_id: string - agent_name: string - test_agent_prompt: string - agent_prompt: string - }> { - const response = await this.client.get(`/api/v1/personas/agent-prompt-sources/${agentId}`) - return response.data - } - - async generatePersonaPrompt(data: { - agent_id: string - source?: 'test_agent' | 'agent' | 'auto' - persona_name?: string - persona_gender?: string - additional_context?: string - provider?: string - model?: string - credential_id?: string - llm_config?: LLMGenerationConfig | null - }): Promise<{ - persona_prompt: string - source_used: string - provider: string - model: string - }> { - const response = await this.client.post('/api/v1/personas/generate-prompt', data) - return response.data - } - - // Persona voice options (built-in + custom voices, ungated) - async getPersonaVoiceOptions(provider?: string): Promise<{ - providers: Array<{ - id: string - name: string - voices: Array<{ - id: string - name: string - gender: string - is_custom: boolean - custom_voice_id?: string - description?: string | null - }> - }> - }> { - const response = await this.client.get('/api/v1/personas/voice-options', { - params: provider ? { provider } : undefined, - }) - return response.data - } - - // Custom voice CRUD (persona-scoped, ungated) - async listPersonaCustomVoices(provider?: string): Promise { - const response = await this.client.get('/api/v1/personas/custom-voices', { - params: provider ? { provider } : undefined, - }) - return response.data - } - - async createPersonaCustomVoice(data: { - provider: string - voice_id: string - name: string - gender?: string - description?: string - }): Promise { - const response = await this.client.post('/api/v1/personas/custom-voices', data) - return response.data - } - - async updatePersonaCustomVoice(customVoiceId: string, data: { - voice_id?: string - name?: string - gender?: string - description?: string - }): Promise { - const response = await this.client.put(`/api/v1/personas/custom-voices/${customVoiceId}`, data) - return response.data - } - - async deletePersonaCustomVoice(customVoiceId: string): Promise { - const response = await this.client.delete(`/api/v1/personas/custom-voices/${customVoiceId}`) - return response.data - } - - async listAmbientPresets(): Promise<{ presets: { id: string; label: string }[] }> { - const response = await this.client.get('/api/v1/personas/ambient-presets') - return response.data - } - - async listAmbientLibrary(): Promise< - Array<{ - id: string - name: string - original_filename?: string | null - created_at?: string - }> - > { - const response = await this.client.get('/api/v1/personas/ambient-library') - return response.data - } - - async uploadAmbientLibraryAsset(file: File, name?: string): Promise { - const formData = new FormData() - formData.append('file', file) - if (name) { - formData.append('name', name) - } - const response = await this.client.post('/api/v1/personas/ambient-library', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async updateAmbientLibraryAsset(assetId: string, data: { name: string }): Promise { - const response = await this.client.patch(`/api/v1/personas/ambient-library/${assetId}`, data) - return response.data - } - - async deleteAmbientLibraryAsset(assetId: string): Promise { - await this.client.delete(`/api/v1/personas/ambient-library/${assetId}`) - } - - async previewAmbientLibraryAsset(assetId: string): Promise { - const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview`, { - responseType: 'blob', - }) - return response.data - } - - async getAmbientLibraryPreviewUrl( - assetId: string, - expiration: number = 3600, - ): Promise<{ url: string; expires_in: number }> { - const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview-url`, { - params: { expiration }, - }) - return response.data - } - - async previewAmbientPreset(presetId: string): Promise { - const response = await this.client.get(`/api/v1/personas/ambient-presets/${presetId}/preview`, { - responseType: 'blob', - }) - return response.data - } - - async uploadPersonaAmbientAudio(personaId: string, file: File): Promise { - const formData = new FormData() - formData.append('file', file) - const response = await this.client.post(`/api/v1/personas/${personaId}/ambient-audio`, formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async deletePersonaAmbientAudio(personaId: string): Promise { - const response = await this.client.delete(`/api/v1/personas/${personaId}/ambient-audio`) - return response.data - } - - // Scenarios endpoints - async listScenarios(skip = 0, limit = 100, agentId?: string): Promise { - const response = await this.client.get('/api/v1/scenarios', { - params: { skip, limit, ...(agentId ? { agent_id: agentId } : {}) }, - }) - return response.data - } - - async getScenario(scenarioId: string): Promise { - const response = await this.client.get(`/api/v1/scenarios/${scenarioId}`) - return response.data - } - - async createScenario(data: { - name: string - agent_id?: string | null - description?: string | null - required_info: Record - }): Promise { - const response = await this.client.post('/api/v1/scenarios', data) - return response.data - } - - async updateScenario(scenarioId: string, data: { - name?: string - agent_id?: string | null - description?: string | null - required_info?: Record - }): Promise { - const response = await this.client.put(`/api/v1/scenarios/${scenarioId}`, data) - return response.data - } - - async deleteScenario(scenarioId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/scenarios/${scenarioId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - // Chat/Inference endpoints - async chatCompletion(data: { - messages: Array<{ role: string; content: string }> - provider: string - model: string - temperature?: number - max_tokens?: number - llm_config?: LLMGenerationConfig | null - }): Promise<{ - text: string - model: string - usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } - processing_time?: number - }> { - const response = await this.client.post('/api/v1/chat/completion', data) - return response.data - } - - // VoiceBundle endpoints - async listVoiceBundles(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/voicebundles', { - params: { skip, limit }, - }) - return response.data - } - - async getVoiceBundle(voicebundleId: string): Promise { - const response = await this.client.get(`/api/v1/voicebundles/${voicebundleId}`) - return response.data - } - - async createVoiceBundle(data: any): Promise { - const response = await this.client.post('/api/v1/voicebundles', data) - return response.data - } - - async updateVoiceBundle(voicebundleId: string, data: any): Promise { - const response = await this.client.put(`/api/v1/voicebundles/${voicebundleId}`, data) - return response.data - } - - async deleteVoiceBundle(voicebundleId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/voicebundles/${voicebundleId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - // AI Provider endpoints - async listAIProviders(): Promise { - const response = await this.client.get('/api/v1/aiproviders') - return response.data - } - - async getAIProvider(aiproviderId: string): Promise { - const response = await this.client.get(`/api/v1/aiproviders/${aiproviderId}`) - return response.data - } - - async createAIProvider(data: any): Promise { - const response = await this.client.post('/api/v1/aiproviders', data) - return response.data - } - - async updateAIProvider(aiproviderId: string, data: any): Promise { - const response = await this.client.put(`/api/v1/aiproviders/${aiproviderId}`, data) - return response.data - } - - async deleteAIProvider(aiproviderId: string): Promise { - await this.client.delete(`/api/v1/aiproviders/${aiproviderId}`) - } - - async setDefaultAIProvider(aiproviderId: string): Promise { - const response = await this.client.post( - `/api/v1/aiproviders/${aiproviderId}/set-default`, - ) - return response.data - } - - async testAIProvider(aiproviderId: string): Promise { - const response = await this.client.post(`/api/v1/aiproviders/${aiproviderId}/test`) - return response.data - } - - async getLLMGatewaySettings(): Promise { - const response = await this.client.get('/api/v1/organizations/llm-gateway') - return response.data - } - - async updateLLMGatewaySettings(data: LLMGatewaySettingsUpdate): Promise { - const response = await this.client.put('/api/v1/organizations/llm-gateway', data) - return response.data - } - - async getAlertingSyncSettings(): Promise { - const response = await this.client.get('/api/v1/organizations/alerting-sync') - return response.data - } - - async updateAlertingSyncSettings(data: AlertingSyncSettingsUpdate): Promise { - const response = await this.client.put('/api/v1/organizations/alerting-sync', data) - return response.data - } - - // IAM endpoints - async getOrganization(): Promise { - const response = await this.client.get('/api/v1/iam/organization') - return response.data - } - - async updateOrganization(data: { name: string }): Promise { - const response = await this.client.patch('/api/v1/iam/organization', data) - return response.data - } - - async listOrganizationUsers(): Promise { - const response = await this.client.get('/api/v1/iam/users') - return response.data - } - - async inviteUser(data: InvitationCreate): Promise { - const response = await this.client.post('/api/v1/iam/invitations', data) - return response.data - } - - async listInvitations(): Promise { - const response = await this.client.get('/api/v1/iam/invitations') - return response.data - } - - async updateUserRole(userId: string, role: Role): Promise { - const response = await this.client.put(`/api/v1/iam/users/${userId}/role`, { role }) - return response.data - } - - async removeUser(userId: string): Promise { - await this.client.delete(`/api/v1/iam/users/${userId}`) - } - - async cancelInvitation(invitationId: string): Promise { - await this.client.delete(`/api/v1/iam/invitations/${invitationId}`) - } - - /** - * Admin-initiated password reset for another organization member. - * - * The caller must be an ADMIN of the same organization as `userId`. The - * backend ignores any current-password concept here (that's for the - * self-service `/auth/password` endpoint). Communicate the new password - * to the user out-of-band. - */ - async adminResetUserPassword( - userId: string, - newPassword: string, - ): Promise<{ user_id: string; email: string; message: string }> { - const response = await this.client.post( - `/api/v1/iam/users/${userId}/reset-password`, - { new_password: newPassword }, - ) - return response.data - } - - // Profile endpoints - async getProfile(): Promise { - const response = await this.client.get('/api/v1/profile') - return response.data - } - - async updateProfile(data: UserUpdate): Promise { - const response = await this.client.put('/api/v1/profile', data) - return response.data - } - - async getMyInvitations(): Promise { - const response = await this.client.get('/api/v1/profile/invitations') - return response.data - } - - async acceptInvitation(invitationId: string): Promise { - const response = await this.client.post(`/api/v1/profile/invitations/${invitationId}/accept`) - return response.data - } - - async declineInvitation(invitationId: string): Promise { - const response = await this.client.post(`/api/v1/profile/invitations/${invitationId}/decline`) - return response.data - } - - // User Preferences endpoints - async getUserPreferences(): Promise { - const response = await this.client.get('/api/v1/profile/preferences') - return response.data - } - - async updateUserPreferences(data: UserPreferencesUpdate): Promise { - const response = await this.client.put('/api/v1/profile/preferences', data) - return response.data - } - - // Integration endpoints - async listIntegrations(): Promise { - const response = await this.client.get('/api/v1/integrations') - return response.data - } - - async getIntegration(integrationId: string): Promise { - const response = await this.client.get(`/api/v1/integrations/${integrationId}`) - return response.data - } - - async previewIntegrationAgentPrompt( - integrationId: string, - voiceAiAgentId: string, - options?: { agentChannel?: 'voice' | 'chat' }, - ): Promise<{ provider_prompt: string }> { - const response = await this.client.post( - `/api/v1/integrations/${integrationId}/preview-agent-prompt`, - { - voice_ai_agent_id: voiceAiAgentId, - ...(options?.agentChannel ? { agent_channel: options.agentChannel } : {}), - }, - ) - return response.data - } - - async listIntegrationVoiceAgents( - integrationId: string, - options?: { refresh?: boolean; search?: string; agentKind?: 'voice' | 'chat' }, - ): Promise { - const response = await this.client.get( - `/api/v1/integrations/${integrationId}/voice-agents`, - { - params: { - ...(options?.refresh ? { refresh: true } : {}), - ...(options?.search ? { search: options.search } : {}), - ...(options?.agentKind ? { agent_kind: options.agentKind } : {}), - }, - }, - ) - return response.data - } - - async createIntegration(data: IntegrationCreate): Promise { - const response = await this.client.post('/api/v1/integrations', data) - return response.data - } - - async updateIntegration(integrationId: string, data: Partial): Promise { - const response = await this.client.put(`/api/v1/integrations/${integrationId}`, data) - return response.data - } - - async deleteIntegration(integrationId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/integrations/${integrationId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - async setDefaultIntegration(integrationId: string): Promise { - const response = await this.client.post( - `/api/v1/integrations/${integrationId}/set-default`, - ) - return response.data - } - - async getIntegrationApiKey(integrationId: string): Promise<{ api_key: string; public_key?: string | null }> { - const response = await this.client.get(`/api/v1/integrations/${integrationId}/api-key`) - return response.data - } - - // Telephony endpoints (provider-agnostic) - async createTelephonyConfig(data: TelephonyIntegrationCreatePayload): Promise { - const response = await this.client.post('/api/v1/telephony/config', data) - return response.data - } - - async getTelephonyConfig(provider: string = 'plivo'): Promise { - const response = await this.client.get('/api/v1/telephony/config', { params: { provider } }) - return response.data - } - - async listTelephonyConfigs(provider?: string): Promise { - const response = await this.client.get('/api/v1/telephony/configs', { - params: provider ? { provider } : undefined, - }) - return response.data - } - - async updateTelephonyConfig(data: TelephonyIntegrationUpdatePayload): Promise { - const response = await this.client.put('/api/v1/telephony/config', data) - return response.data - } - - async setDefaultTelephonyConfig(integrationId: string): Promise { - const response = await this.client.post( - `/api/v1/telephony/config/${integrationId}/set-default`, - ) - return response.data - } - - async deleteTelephonyConfig(integrationId: string): Promise { - await this.client.delete(`/api/v1/telephony/config/${integrationId}`) - } - - async listTelephonyNumbers(provider?: string): Promise { - const response = await this.client.get('/api/v1/telephony/numbers', { - params: provider ? { provider } : undefined, - }) - return response.data - } - - async listAvailableTelephonyNumbers( - provider: string, - credentialId?: string, - ): Promise { - const response = await this.client.get('/api/v1/telephony/numbers/available', { - params: { - provider, - ...(credentialId ? { credential_id: credentialId } : {}), - }, - }) - return response.data - } - - async importTelephonyNumbers( - provider: string, - numbers: string[], - agentId?: string, - credentialId?: string, - ): Promise { - const response = await this.client.post('/api/v1/telephony/numbers/import', { - provider, - numbers, - agent_id: agentId || undefined, - credential_id: credentialId || undefined, - }) - return response.data - } - - async listVobizAvailableNumbers(): Promise { - return this.listAvailableTelephonyNumbers('vobiz') - } - - async importVobizNumbers(numbers: string[], agentId?: string): Promise { - return this.importTelephonyNumbers('vobiz', numbers, agentId) - } - - async deleteTelephonyNumber(numberId: string): Promise { - await this.client.delete(`/api/v1/telephony/numbers/${numberId}`) - } - - async deleteImportedVobizNumber(numberId: string): Promise { - await this.deleteTelephonyNumber(numberId) - } - - async listTelephonyOutboundPool(): Promise { - const response = await this.client.get('/api/v1/telephony/outbound-pool') - return response.data - } - - async listVobizOutboundPool(): Promise { - return this.listTelephonyOutboundPool() - } - - async listTelephonyDialTargets(): Promise { - const response = await this.client.get('/api/v1/telephony/dial-targets') - return response.data - } - - async createVobizOutboundCall(data: VobizOutboundCallRequest): Promise { - const response = await this.client.post('/api/v1/telephony/vobiz/calls/outbound', data) - return response.data - } - - async listDialTargets(): Promise { - const response = await this.client.get('/api/v1/telephony/dial-targets') - return response.data - } - - async createDialTarget(data: { phone_number: string; label?: string }): Promise { - const response = await this.client.post('/api/v1/telephony/dial-targets', data) - return response.data - } - - async updateDialTarget( - targetId: string, - data: { phone_number?: string; label?: string }, - ): Promise { - const response = await this.client.put(`/api/v1/telephony/dial-targets/${targetId}`, data) - return response.data - } - - async deleteDialTarget(targetId: string): Promise { - await this.client.delete(`/api/v1/telephony/dial-targets/${targetId}`) - } - - // Data Sources endpoints - async testS3Connection(): Promise { - const response = await this.client.post('/api/v1/data-sources/s3/test') - return response.data - } - - async listS3Files(prefix?: string, maxKeys = 1000): Promise { - const response = await this.client.get('/api/v1/data-sources/s3/files', { - params: { prefix, max_keys: maxKeys }, - }) - return response.data - } - - async browseS3(path = '', maxKeys = 1000): Promise { - const response = await this.client.get('/api/v1/data-sources/s3/browse', { - params: { path, max_keys: maxKeys }, - }) - return response.data - } - - async uploadToS3(file: File, customFilename?: string): Promise { - const formData = new FormData() - formData.append('file', file) - if (customFilename) { - formData.append('filename', customFilename) - } - const response = await this.client.post('/api/v1/data-sources/s3/upload', formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }) - return response.data - } - - async getS3Status(): Promise { - const response = await this.client.get('/api/v1/data-sources/s3/status') - return response.data - } - - async downloadFromS3(fileKey: string): Promise { - const response = await this.client.get(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}/download`, { - responseType: 'blob', - }) - // Create a blob URL and trigger download - const url = window.URL.createObjectURL(new Blob([response.data])) - const link = document.createElement('a') - link.href = url - link.setAttribute('download', fileKey.split('/').pop() || 'file') - document.body.appendChild(link) - link.click() - link.remove() - window.URL.revokeObjectURL(url) - } - - async getS3PresignedUrl(fileKey: string, expiration: number = 3600): Promise<{ url: string; expires_in: number }> { - const response = await this.client.get(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}/presigned-url`, { - params: { expiration }, - }) - return response.data - } - - async deleteFromS3(fileKey: string): Promise { - const response = await this.client.delete(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}`) - return response.data - } - - // Call Imports endpoints - - /** - * Inspect an uploaded CSV / Excel file and get its sheets + headers. - * - * Drives the column-mapping UI without forcing the frontend to parse - * CSV / xlsx itself. CSV files come back as a single synthetic sheet; - * Excel workbooks come back with one entry per worksheet. - */ - async previewCallImportFile(file: File): Promise { - const formData = new FormData() - formData.append('file', file) - const response = await this.client.post( - '/api/v1/call-imports/preview', - formData, - { headers: { 'Content-Type': 'multipart/form-data' } }, - ) - return response.data - } - - async uploadCallImport( - file: File, - options: { - provider: string - telephonyIntegrationId: string - /** Reusable Input Parameter schema this upload is mapped against. */ - schemaId: string - /** - * Schema-driven mapping: ``{parameter_name: csv_header}``. Must - * include every required parameter on the schema. Pass an empty - * string / omit a parameter to leave it unmapped (only allowed - * for optional parameters). - */ - parameterMapping: Record - /** - * CSV/Excel headers the user has explicitly skipped. Every source - * column must either appear in ``parameterMapping`` or here, or - * the backend rejects the upload with 400. - */ - skippedColumns?: string[] - dataset?: string | null - tagIds?: string[] - /** - * Worksheet name to import when the file is an Excel workbook. - * Required for .xlsx / .xlsm uploads; must be left undefined / null - * for CSV uploads (the backend rejects a non-empty value for CSV). - */ - sheetName?: string | null - } - ): Promise { - const formData = new FormData() - formData.append('file', file) - formData.append('provider', options.provider) - formData.append('telephony_integration_id', options.telephonyIntegrationId) - formData.append('schema_id', options.schemaId) - formData.append('parameter_mapping', JSON.stringify(options.parameterMapping)) - formData.append('skipped_columns', JSON.stringify(options.skippedColumns || [])) - if (options.dataset !== undefined && options.dataset !== null) { - formData.append('dataset', options.dataset) - } - if (options.tagIds && options.tagIds.length > 0) { - for (const tagId of options.tagIds) { - formData.append('tag_ids', tagId) - } - } - if (options.sheetName) { - formData.append('sheet_name', options.sheetName) - } - const response = await this.client.post('/api/v1/call-imports/upload', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - /** - * UPLOAD stage of the staged call-import flow. - * - * Persists the CSV / Excel file to S3, captures a sheets snapshot, - * and creates a ``CallImport`` row with ``status='uploaded'``. The - * returned record has no mapping / provider / rows yet — the caller - * follows up with {@link updateCallImportMapping} and - * {@link startCallImport} to advance the batch. - */ - async createCallImport( - file: File, - options: { - dataset: string - tagIds?: string[] - /** - * Optional schema pre-pick. The user can still change it during - * the MAP stage; provided here only so the detail page can pre- - * select the schema dropdown. - */ - schemaId?: string | null - /** Set to `chat` for post-prod transcript-only imports. */ - contentModality?: 'chat' | 'voice' - }, - ): Promise { - const formData = new FormData() - formData.append('file', file) - formData.append('dataset', options.dataset) - if (options.tagIds && options.tagIds.length > 0) { - for (const tagId of options.tagIds) { - formData.append('tag_ids', tagId) - } - } - if (options.schemaId) { - formData.append('schema_id', options.schemaId) - } - if (options.contentModality) { - formData.append('content_modality', options.contentModality) - } - const response = await this.client.post('/api/v1/call-imports', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async uploadCallImportAudio( - files: File[], - options: { - dataset: string - tagIds?: string[] - batchName?: string - }, - ): Promise { - const formData = new FormData() - for (const file of files) { - formData.append('files', file) - } - formData.append('dataset', options.dataset) - if (options.batchName?.trim()) { - formData.append('batch_name', options.batchName.trim()) - } - if (options.tagIds && options.tagIds.length > 0) { - for (const tagId of options.tagIds) { - formData.append('tag_ids', tagId) - } - } - const response = await this.client.post('/api/v1/call-imports/audio-upload', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async appendCallImportAudio( - importId: string, - files: File[], - ): Promise { - const formData = new FormData() - for (const file of files) { - formData.append('files', file) - } - const response = await this.client.post( - `/api/v1/call-imports/${importId}/audio-append`, - formData, - { - headers: { 'Content-Type': 'multipart/form-data' }, - }, - ) - return response.data - } - - async uploadCallImportAudioChunked( - files: File[], - options: { - dataset: string - tagIds?: string[] - batchName?: string - onProgress?: (progress: CallImportAudioUploadProgress) => void - }, - ): Promise { - if (files.length === 0) { - throw new Error('At least one audio file is required.') - } - - const chunks: File[][] = [] - for (let i = 0; i < files.length; i += AUDIO_UPLOAD_CHUNK_SIZE) { - chunks.push(files.slice(i, i + AUDIO_UPLOAD_CHUNK_SIZE)) - } - - let result: CallImportUploadResponse | null = null - for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) { - const chunk = chunks[chunkIndex] - options.onProgress?.({ - chunkIndex: chunkIndex + 1, - totalChunks: chunks.length, - uploadedFiles: chunkIndex * AUDIO_UPLOAD_CHUNK_SIZE, - totalFiles: files.length, - }) - - if (chunkIndex === 0) { - result = await this.uploadCallImportAudio(chunk, { - dataset: options.dataset, - tagIds: options.tagIds, - batchName: options.batchName, - }) - } else { - result = await this.appendCallImportAudio(result!.id, chunk) - } - - options.onProgress?.({ - chunkIndex: chunkIndex + 1, - totalChunks: chunks.length, - uploadedFiles: Math.min( - (chunkIndex + 1) * AUDIO_UPLOAD_CHUNK_SIZE, - files.length, - ), - totalFiles: files.length, - }) - } - - return result! - } - - async uploadCallImportAudioAppendChunked( - importId: string, - files: File[], - options?: { - onProgress?: (progress: CallImportAudioUploadProgress) => void - }, - ): Promise { - if (files.length === 0) { - throw new Error('At least one audio file is required.') - } - - const chunks: File[][] = [] - for (let i = 0; i < files.length; i += AUDIO_UPLOAD_CHUNK_SIZE) { - chunks.push(files.slice(i, i + AUDIO_UPLOAD_CHUNK_SIZE)) - } - - let result: CallImportUploadResponse | null = null - for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) { - const chunk = chunks[chunkIndex] - options?.onProgress?.({ - chunkIndex: chunkIndex + 1, - totalChunks: chunks.length, - uploadedFiles: chunkIndex * AUDIO_UPLOAD_CHUNK_SIZE, - totalFiles: files.length, - }) - - result = await this.appendCallImportAudio(importId, chunk) - - options?.onProgress?.({ - chunkIndex: chunkIndex + 1, - totalChunks: chunks.length, - uploadedFiles: Math.min( - (chunkIndex + 1) * AUDIO_UPLOAD_CHUNK_SIZE, - files.length, - ), - totalFiles: files.length, - }) - } - - return result! - } - - /** - * MAP stage of the staged call-import flow. - * - * Persists the schema + sheet + parameter mapping on the batch. The - * backend validates against the sheet snapshot it cached at upload - * time, so this is purely a metadata operation (no S3 fetch). - * Idempotent: safe to call repeatedly while the batch is in - * ``uploaded`` or ``mapped`` state. - */ - async updateCallImportMapping( - id: string, - options: { - schemaId: string - sheetName?: string | null - parameterMapping: Record - skippedColumns?: string[] - }, - ): Promise { - const response = await this.client.patch( - `/api/v1/call-imports/${id}/mapping`, - { - schema_id: options.schemaId, - sheet_name: options.sheetName ?? null, - parameter_mapping: options.parameterMapping, - skipped_columns: options.skippedColumns ?? [], - }, - ) - return response.data - } - - /** - * IMPORT stage of the staged call-import flow. - * - * Materialises rows from the staged source file using the persisted - * mapping and fans them out to the ``imports`` Celery queue. Returns - * the same shape as the legacy one-shot ``uploadCallImport``. - */ - async startCallImport( - id: string, - options: { - provider: string | null - telephonyIntegrationId: string | null - }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${id}/import`, - { - provider: options.provider, - telephony_integration_id: options.telephonyIntegrationId, - }, - ) - return response.data - } - - // ------------------------------------------------------------------- - // Call Import Schemas (reusable Input Parameter schemas) - // ------------------------------------------------------------------- - - async listCallImportSchemas(): Promise { - const response = await this.client.get('/api/v1/call-import-schemas') - return response.data - } - - async getCallImportSchema(id: string): Promise { - const response = await this.client.get(`/api/v1/call-import-schemas/${id}`) - return response.data - } - - async createCallImportSchema( - payload: CallImportSchemaCreate, - ): Promise { - const response = await this.client.post('/api/v1/call-import-schemas', payload) - return response.data - } - - async updateCallImportSchema( - id: string, - payload: CallImportSchemaUpdate, - ): Promise { - const response = await this.client.patch( - `/api/v1/call-import-schemas/${id}`, - payload, - ) - return response.data - } - - async deleteCallImportSchema( - id: string, - options?: { force?: boolean }, - ): Promise { - await this.client.delete(`/api/v1/call-import-schemas/${id}`, { - params: options?.force ? { force: true } : undefined, - }) - } - - async previewPlatformCallImport(payload: { - voice_integration_id: string - window_start: string - window_end: string - window_timezone?: string - dataset?: string - filters?: { - agent_ids?: string[] - min_duration_seconds?: number - speed?: string - import_provider_transcript?: boolean - medium?: 'voice' | 'chat' - agents?: Array<{ id: string; name: string; medium?: string | null }> - } - }): Promise<{ - recordings_in_window: number - recordings_count_complete: boolean - count_pages_scanned?: number - discovery_slices?: number - sample_count: number - samples: Array<{ - conversation_id: string - agent_id?: string | null - started_at?: string | null - status?: string | null - }> - }> { - const response = await this.client.post('/api/v1/call-imports/platform/preview', payload) - return response.data - } - - async createPlatformCallImport(payload: { - voice_integration_id: string - window_start: string - window_end: string - window_timezone?: string - name?: string - dataset?: string - max_calls?: number - filters?: { - agent_ids?: string[] - min_duration_seconds?: number - speed?: string - import_provider_transcript?: boolean - medium?: 'voice' | 'chat' - agents?: Array<{ id: string; name: string; medium?: string | null }> - } - }, idempotencyKey?: string): Promise<{ id: string; status: string; discovery_status?: string }> { - const response = await this.client.post('/api/v1/call-imports/platform', payload, { - headers: idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : undefined, - }) - return response.data - } - - async pausePlatformImport(importId: string): Promise<{ status: string }> { - const response = await this.client.post(`/api/v1/call-imports/platform/${importId}/pause`) - return response.data - } - - async resumePlatformImport(importId: string): Promise<{ status: string }> { - const response = await this.client.post(`/api/v1/call-imports/platform/${importId}/resume`) - return response.data - } - - async retryPlatformDiscovery(importId: string): Promise<{ status: string }> { - const response = await this.client.post( - `/api/v1/call-imports/platform/${importId}/retry-discovery`, - ) - return response.data - } - - async listPlatformImportAgents( - integrationId: string, - ): Promise> { - const response = await this.client.get('/api/v1/call-imports/platform/agents', { - params: { integration_id: integrationId }, - }) - return response.data - } - - async getPlatformImportProgress(importId: string): Promise { - const response = await this.client.get( - `/api/v1/call-imports/platform/${importId}/progress`, - ) - return response.data - } - - async listCallImports( - params: { - page?: number - page_size?: number - status?: CallImportStatus - dataset?: string - tag_id?: string[] - source_format?: string - } = {} - ): Promise { - // Send tag_id repeated rather than as a JSON array. - const search = new URLSearchParams() - if (params.page !== undefined) search.set('page', String(params.page)) - if (params.page_size !== undefined) search.set('page_size', String(params.page_size)) - if (params.status) search.set('status', params.status) - if (params.dataset !== undefined) search.set('dataset', params.dataset) - if (params.source_format) search.set('source_format', params.source_format) - for (const tag of params.tag_id || []) search.append('tag_id', tag) - const response = await this.client.get('/api/v1/call-imports', { params: search }) - return response.data - } - - async listCallImportDatasets(): Promise { - const response = await this.client.get('/api/v1/call-imports/datasets') - return response.data - } - - async getCallImport( - id: string, - params: { - row_limit?: number - row_offset?: number - q?: string - /** - * Optional filter on ``CallImportRow.diarised_transcript_status``. - * When set, ``filtered_total_rows`` on the response reflects the - * post-filter row count (combined with ``q`` when both are - * supplied) so the UI can paginate against the same slice. - */ - diarised_status?: 'pending' | 'running' | 'completed' | 'failed' - } = {} - ): Promise { - const response = await this.client.get(`/api/v1/call-imports/${id}`, { params }) - return response.data - } - - async getCallImportRow(id: string, rowId: string): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${id}/rows/${rowId}`, - ) - return response.data - } - - async listCallImportRowIds( - id: string, - params: { - q?: string - diarised_status?: 'pending' | 'running' | 'completed' | 'failed' - } = {} - ): Promise<{ ids: string[]; total: number }> { - const response = await this.client.get( - `/api/v1/call-imports/${id}/row-ids`, - { params }, - ) - return response.data - } - - async updateCallImport( - id: string, - payload: { - original_filename?: string | null - dataset?: string | null - tag_ids?: string[] - /** - * Reassign the Input Parameter schema. Only honoured while the - * batch is in ``uploaded`` / ``mapped`` state. Switching schemas - * resets the mapping and rewinds the batch to ``uploaded``. - */ - schema_id?: string | null - }, - ): Promise { - const response = await this.client.patch(`/api/v1/call-imports/${id}`, payload) - return response.data - } - - async deleteCallImport( - id: string, - ): Promise<{ id: string; status: 'accepted' | 'completed' }> { - const response = await this.client.delete(`/api/v1/call-imports/${id}`) - return response.data - } - - async deleteCallImportRow(id: string, rowId: string): Promise { - await this.client.delete(`/api/v1/call-imports/${id}/rows/${rowId}`) - } - - async bulkDeleteCallImportRows( - id: string, - rowIds: string[], - ): Promise<{ deleted: number; status?: 'completed' | 'accepted' }> { - const response = await this.client.post( - `/api/v1/call-imports/${id}/rows/bulk-delete`, - { row_ids: rowIds }, - ) - return response.data - } - - async createCallImportEvaluation( - callImportId: string, - payload: { - metric_ids: string[] - name?: string | null - /** - * Which transcript to score against. ``'diarised'`` (default) - * auto-diarizes then scores; ``'production'`` scores the CSV - * transcript directly. - */ - transcript_sources?: Array<'production' | 'diarised'> - /** Run-level LLM provider key. Leave undefined for legacy default. */ - llm_provider?: string | null - llm_model?: string | null - llm_credential_id?: string | null - llm_config?: LLMGenerationConfig | null - /** Per-metric LLM override map keyed by metric UUID. */ - metric_llm_overrides?: Record | null - /** When true, diarize rows missing transcripts before evaluation. */ - auto_transcribe?: boolean - transcribe_overwrite?: boolean - /** - * Diarisation pipeline shape for the auto-transcribe step. - * 'stt_llm' (default) runs STT then an LLM diariser over the - * resulting text. 'llm_only' skips STT and feeds the audio - * directly to the multimodal ``diarization_llm_*`` model — STT - * fields must be omitted in that case. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - stt_provider?: string | null - stt_model?: string | null - stt_credential_id?: string | null - stt_language?: string | null - /** - * LLM diariser config: post-STT, an LLM splits the plain - * transcript into agent/user turns using ``diarization_prompt`` - * (or the canonical default when empty). Required by the server - * when ``auto_transcribe`` is set. Also used in 'llm_only' mode - * where this same LLM receives the audio directly. - */ - diarization_llm_provider?: string | null - diarization_llm_model?: string | null - diarization_llm_credential_id?: string | null - diarization_prompt?: string | null - /** - * Opt into LLM-driven discovery of brand-new top-level metrics - * for this run. Surfaces candidates in the Discovered metrics - * panel on the evaluation detail Flow tab. - */ - discover_new_metrics?: boolean - /** Telephony credentials when starting from a mapped batch. */ - provider?: string | null - telephony_integration_id?: string | null - }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations`, - payload, - ) - return response.data - } - - /** Aggregated metric distributions for a single evaluation run. */ - async getCallImportEvaluationAggregate( - callImportId: string, - evaluationId: string, - baselineEvaluationId?: string | null, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/aggregate`, - { - params: baselineEvaluationId - ? { baseline_evaluation_id: baselineEvaluationId } - : undefined, - }, - ) - return response.data - } - - /** - * Read the cached LLM TLDR for an evaluation. Returns ``null`` when - * the user has never generated one (the empty-state CTA renders - * this case). - */ - async getCallImportEvaluationInsights( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/insights`, - ) - return response.data ?? null - } - - /** - * Generate (or return cached) the LLM TLDR for an evaluation run. - * - * - Pass ``regenerate: true`` to force a fresh LLM call even when a - * cached summary exists at the current ``completed_rows`` watermark. - * - Pass ``provider`` + ``model`` to pin a specific LLM. Omit both to - * let the backend auto-detect the org's first active OpenAI / - * Anthropic / Google credential (mirroring Prompt Partials). - */ - async generateCallImportEvaluationInsights( - callImportId: string, - evaluationId: string, - options?: { - regenerate?: boolean - provider?: string | null - model?: string | null - credential_id?: string | null - max_llm_calls?: number | null - }, - ): Promise { - const body: Record = { - regenerate: Boolean(options?.regenerate), - } - if (options?.provider) body.provider = options.provider - if (options?.model) body.model = options.model - if (options?.credential_id) body.credential_id = options.credential_id - if (options?.max_llm_calls != null) body.max_llm_calls = options.max_llm_calls - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/insights`, - body, - ) - return response.data - } - - async getCallImportEvaluationUserInsights( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/user-insights`, - ) - return response.data - } - - async getCallImportEvaluationMetricClusters( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters`, - ) - return response.data - } - - async getCallImportEvaluationPromptImprovements( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/prompt-improvements`, - ) - return response.data - } - - async generateCallImportEvaluationPromptImprovements( - callImportId: string, - evaluationId: string, - data: { - imported_agent_id?: string - regenerate?: boolean - force?: boolean - provider?: string - model?: string - credential_id?: string - }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/prompt-improvements`, - data, - ) - return response.data - } - - async reviewCallImportEvaluationPromptImprovement( - callImportId: string, - evaluationId: string, - data: { - suggestion_id: string - review_status: 'pending' | 'accepted' | 'rejected' - edited_text?: string | null - }, - ): Promise { - const response = await this.client.patch( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/prompt-improvements/review`, - data, - ) - return response.data - } - - async previewCallImportEvaluationPromptPush( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/prompt-improvements/push-preview`, - ) - return response.data - } - - async pushCallImportEvaluationPrompt( - callImportId: string, - evaluationId: string, - confirm: string, - options?: { pushToLive?: boolean }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/prompt-improvements/push`, - { confirm, push_to_live: options?.pushToLive ?? true }, - ) - return response.data - } - - async listCallImportEvaluationMetricClusterEligibleRows( - callImportId: string, - evaluationId: string, - options?: { - limit?: number - count_only?: boolean - }, - ): Promise { - const params = new URLSearchParams() - if (options?.limit != null) params.set('limit', String(options.limit)) - if (options?.count_only) params.set('count_only', 'true') - const query = params.toString() - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters/eligible-rows${query ? `?${query}` : ''}`, - ) - return response.data - } - - async getCallImportEvaluationMetricClusterFailurePolicies( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters/failure-policies`, - ) - return response.data - } - - async saveCallImportEvaluationMetricClusterFailurePolicies( - callImportId: string, - evaluationId: string, - policies: Record, - ): Promise { - const response = await this.client.put( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters/failure-policies`, - { policies, source: 'user' }, - ) - return response.data - } - - async generateCallImportEvaluationMetricClusters( - callImportId: string, - evaluationId: string, - options?: { - regenerate?: boolean - force?: boolean - provider?: string | null - model?: string | null - credential_id?: string | null - max_llm_calls?: number | null - evaluation_row_ids?: string[] | null - row_limit?: number | null - failure_policies?: Record< - string, - import('../types/api').MetricFailurePolicy - > | null - }, - ): Promise { - const body: Record = { - regenerate: Boolean(options?.regenerate), - force: Boolean(options?.force), - } - if (options?.provider) body.provider = options.provider - if (options?.model) body.model = options.model - if (options?.credential_id) body.credential_id = options.credential_id - if (options?.max_llm_calls != null) body.max_llm_calls = options.max_llm_calls - if (options?.evaluation_row_ids?.length) { - body.evaluation_row_ids = options.evaluation_row_ids - } - if (options?.row_limit != null) body.row_limit = options.row_limit - if (options?.failure_policies) { - body.failure_policies = options.failure_policies - } - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters`, - body, - ) - return response.data - } - - async cancelCallImportEvaluationMetricClusters( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/metric-clusters/cancel`, - ) - return response.data - } - - async generateCallImportEvaluationUserInsights( - callImportId: string, - evaluationId: string, - options?: { - regenerate?: boolean - force?: boolean - provider?: string | null - model?: string | null - credential_id?: string | null - max_llm_calls?: number | null - }, - ): Promise { - const body: Record = { - regenerate: Boolean(options?.regenerate), - force: Boolean(options?.force), - } - if (options?.provider) body.provider = options.provider - if (options?.model) body.model = options.model - if (options?.credential_id) body.credential_id = options.credential_id - if (options?.max_llm_calls != null) body.max_llm_calls = options.max_llm_calls - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/user-insights`, - body, - ) - return response.data - } - - /** - * Aggregate flow chart for a parent metric: returns nodes/edges built - * from per-row LLM-inferred ``sequence`` arrays. Used by the React - * Flow visualisation on the evaluation overview. - */ - async getCallImportEvaluationFlow( - callImportId: string, - evaluationId: string, - parentMetricId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/flow`, - { params: { parent_metric_id: parentMetricId } }, - ) - return response.data - } - - /** - * List LLM-discovered candidate sub-labels for a parent metric in an - * evaluation. Returned items power the Discovered Labels panel where - * users decide which candidates to promote or merge. - */ - async getCallImportEvaluationDiscoveredLabels( - callImportId: string, - evaluationId: string, - parentMetricId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-labels`, - { params: { parent_metric_id: parentMetricId } }, - ) - return response.data - } - - /** - * Merge a discovered slug into another within an evaluation. Rewrites - * every row's discovered_labels and sequence array so the panel + - * flow chart converge on the surviving slug. - */ - async mergeCallImportEvaluationDiscoveredLabels( - callImportId: string, - evaluationId: string, - body: { parent_metric_id: string; from_key: string; to_key: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-labels/merge`, - body, - ) - return response.data - } - - /** - * Delete (tombstone) an LLM-discovered candidate. Strips the slug - * from every row's discovered_labels + sequence and records a - * deletion alias on the evaluation so workers finishing later don't - * resurrect it. Use for gibberish candidates the LLM proposed. - */ - async deleteCallImportEvaluationDiscoveredLabel( - callImportId: string, - evaluationId: string, - body: { parent_metric_id: string; key: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-labels/delete`, - body, - ) - return response.data - } - - /** - * Promote an LLM-discovered candidate into a real child Metric under - * the given parent. ``key`` must equal ``slugify(name)`` so existing - * rows' sequence arrays auto-resolve against the new child. - * - * ``capture_rationale`` defaults to true on the backend (matches the - * Discovered Labels UX: candidates are LLM-suggested with rationales, - * so the user almost always wants future rows that hit them to keep - * capturing rationales). - */ - async promoteDiscoveredChild( - parentMetricId: string, - body: { - key: string - name: string - description?: string | null - capture_rationale?: boolean - }, - ): Promise { - const response = await this.client.post( - `/api/v1/metrics/${parentMetricId}/children/from-discovered`, - body, - ) - return response.data - } - - /** - * List LLM-discovered candidate TOP-LEVEL metrics for an evaluation. - * Mirrors :func:`getCallImportEvaluationDiscoveredLabels` but is - * scoped to the evaluation as a whole (no ``parent_metric_id``). - * Returns an empty ``items`` list when the evaluation didn't opt - * into top-level metric discovery, so callers can fetch - * unconditionally. - */ - async getCallImportEvaluationDiscoveredMetrics( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-metrics`, - ) - return response.data - } - - /** - * Merge one discovered top-level metric slug into another. Rewrites - * every row's ``__discovered_metrics__`` list and records the - * alias on the evaluation so workers finishing later converge on - * the surviving slug. - */ - async mergeCallImportEvaluationDiscoveredMetrics( - callImportId: string, - evaluationId: string, - body: { from_key: string; to_key: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-metrics/merge`, - body, - ) - return response.data - } - - /** - * Tombstone an LLM-discovered top-level metric candidate. Strips - * the slug from every row's ``__discovered_metrics__`` list and - * records a deletion alias on the evaluation so workers finishing - * later can't resurrect it. - */ - async deleteCallImportEvaluationDiscoveredMetric( - callImportId: string, - evaluationId: string, - body: { key: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/discovered-metrics/delete`, - body, - ) - return response.data - } - - /** - * Promote an LLM-discovered top-level metric candidate into a real - * standalone :class:`Metric` row. ``key`` must equal - * ``slugify(name)`` so already-scored rows that referenced the - * candidate keep resolving against the promoted metric. - * - * ``metric_type`` selects how the new metric will be scored on - * future runs (``boolean`` / ``rating`` / ``category``). ``category`` - * creates a ``multi_label`` parent with no children — the user adds - * children via the existing Metrics page. - */ - async promoteDiscoveredMetric(body: { - key: string - name: string - description?: string | null - metric_type?: 'boolean' | 'rating' | 'category' - capture_rationale?: boolean - custom_config?: Record | null - }): Promise { - const response = await this.client.post( - `/api/v1/metrics/from-discovered`, - body, - ) - return response.data - } - - /** Cross-run insights for the call import detail page. */ - async getCallImportInsights( - callImportId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/insights`, - ) - return response.data - } - - /** - * Fetch the canonical LLM diariser prompt. Used by the Transcribe / - * Run Evaluation modals to pre-fill the prompt textarea so the - * operator sees the actual default they'd otherwise get. - */ - async getCallImportDiarisationPromptDefault(): Promise { - const response = await this.client.get( - `/api/v1/call-imports/diarisation-prompt-default`, - ) - return (response.data?.prompt ?? '') as string - } - - /** Fan out diarization tasks for a batch of rows. */ - async transcribeCallImport( - callImportId: string, - payload: CallImportTranscribeRequest, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/transcribe`, - payload, - ) - return response.data - } - - /** Diarize a single row's recording. */ - async transcribeCallImportRow( - callImportId: string, - rowId: string, - payload: CallImportTranscribeRequest, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/rows/${rowId}/transcribe`, - payload, - ) - return response.data - } - - /** - * Re-enqueue every failed import row in this batch. - * - * Rows are reset to `pending` first so the UI can immediately show that - * the retry sweep started; rows that fail to enqueue again are returned - * in `enqueue_failed` and stay `failed`. - */ - async retryFailedCallImportRows( - callImportId: string, - options?: { - provider: string | null - telephonyIntegrationId: string | null - }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/retry-failed`, - options - ? { - provider: options.provider, - telephony_integration_id: options.telephonyIntegrationId, - } - : undefined, - ) - return response.data - } - - /** - * Abort an in-flight (or queued) diarisation for a single row. - * - * Idempotent — calling on a row whose diarisation is already in a - * terminal state (``completed`` / ``failed`` / ``idle``) returns the - * row unchanged so the UI can wire this to a "Stop" button without - * pre-checking state. The backend flips the row's - * ``diarised_transcript_status`` to ``failed`` and stamps - * ``"Diarisation cancelled by user"`` as the error so the existing - * diarisation pill in the row header renders the right state on the - * next poll. - */ - async cancelCallImportRowDiarisation( - callImportId: string, - rowId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/rows/${rowId}/cancel-diarisation`, - ) - return response.data - } - - /** - * Abort in-flight diarisation for many rows in one call. - * - * Pass ``rowIds`` to cancel a specific subset. Omit it (or pass - * ``null``) to cancel every row in the import whose - * ``diarised_transcript_status`` is currently ``pending`` or - * ``running`` — the "stop everything" affordance. - */ - async cancelCallImportDiarisation( - callImportId: string, - rowIds?: string[] | null, - ): Promise<{ cancelled: number; skipped: number }> { - const body = - rowIds === undefined || rowIds === null - ? {} - : { row_ids: rowIds } - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/cancel-diarisation`, - body, - ) - return response.data - } - - /** - * Flip ``diarised_speaker_swap`` on a row and re-render - * ``diarised_transcript`` from ``diarised_segments``. Returns the - * updated row so the caller can swap it into local state without an - * extra refetch. - * - * The backend rejects this with 409 when the row has no structured - * segments (e.g. legacy diarisations done before turns were - * persisted) — callers should surface that as "re-diarise to enable - * speaker swapping". - */ - async toggleCallImportRowSpeakerSwap( - callImportId: string, - rowId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/rows/${rowId}/diarised-speaker-swap`, - ) - return response.data - } - - async updateCallImportEvaluation( - callImportId: string, - evaluationId: string, - payload: { name?: string | null }, - ): Promise { - const response = await this.client.patch( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}`, - payload, - ) - return response.data - } - - async bulkDeleteCallImportEvaluations( - callImportId: string, - evaluationIds: string[], - ): Promise<{ deleted: number }> { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/bulk-delete`, - { evaluation_ids: evaluationIds }, - ) - return response.data - } - - async deleteCallImportEvaluationRow( - callImportId: string, - evaluationId: string, - evalRowId: string, - ): Promise { - await this.client.delete( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/rows/${evalRowId}`, - ) - } - - /** - * Re-enqueue failed evaluation rows. With no body, every row in the - * run whose status is ``failed`` is re-run with the run's saved - * config. Pass ``evalRowIds`` to scope the retry to a subset (used - * by the per-row "Retry" button), and/or the ``llm_*`` / ``stt_*`` - * fields to swap out the LLM / STT provider before re-enqueueing - * (the backend validates + persists them onto the run). - * - * Rows still in progress / already completed are silently skipped - * on the server and reported in the response's ``skipped`` list — - * the mutation never fails just because the caller's selection is - * stale. - */ - async retryCallImportEvaluation( - callImportId: string, - evaluationId: string, - options?: { - evalRowIds?: string[] - llmProvider?: string | null - llmModel?: string | null - llmCredentialId?: string | null - llmConfig?: LLMGenerationConfig | null - sttProvider?: string | null - sttModel?: string | null - sttCredentialId?: string | null - transcribeOverwrite?: boolean - transcribeMode?: 'stt_llm' | 'llm_only' - diarizationLlmProvider?: string | null - diarizationLlmModel?: string | null - diarizationLlmCredentialId?: string | null - diarizationPrompt?: string | null - /** - * Override telephony credentials on the batch when failed rows must - * re-fetch recordings. Send both fields together; omit to keep the - * batch's existing pinned credentials. - */ - provider?: string | null - telephonyIntegrationId?: string | null - /** - * Metric-subset retry: when set, only these metrics are - * recomputed and merged into the row's existing metric_scores - * (other metrics' previous values are preserved). The backend - * auto-flips ``include_completed`` to true in this case so - * already-successful rows are eligible. - */ - metricIds?: string[] - /** - * When true, rows currently ``completed`` are eligible for - * retry too (otherwise only ``failed`` rows are picked up). - * Required-and-implied when ``metricIds`` is set. - */ - includeCompleted?: boolean - /** - * Per-metric LLM overrides, keyed by leaf metric id. Replaces the - * run's stored overrides, so callers should merge with the existing - * map. Used to route classification metrics to a System 1 model. - */ - metricLlmOverrides?: Record - }, - ): Promise { - const body: Record = {} - if (options?.evalRowIds && options.evalRowIds.length > 0) { - body.eval_row_ids = options.evalRowIds - } - if (options?.llmProvider) body.llm_provider = options.llmProvider - if (options?.llmModel) body.llm_model = options.llmModel - if (options?.llmCredentialId !== undefined) { - body.llm_credential_id = options.llmCredentialId - } - if (options?.llmConfig !== undefined) { - body.llm_config = options.llmConfig - } - if (options?.metricLlmOverrides !== undefined) { - body.metric_llm_overrides = options.metricLlmOverrides - } - if (options?.sttProvider) body.stt_provider = options.sttProvider - if (options?.sttModel) body.stt_model = options.sttModel - if (options?.sttCredentialId !== undefined) { - body.stt_credential_id = options.sttCredentialId - } - if (options?.transcribeOverwrite) { - body.transcribe_overwrite = true - } - if (options?.transcribeMode) { - body.transcribe_mode = options.transcribeMode - } - if (options?.diarizationLlmProvider) { - body.diarization_llm_provider = options.diarizationLlmProvider - } - if (options?.diarizationLlmModel) { - body.diarization_llm_model = options.diarizationLlmModel - } - if (options?.diarizationLlmCredentialId !== undefined) { - body.diarization_llm_credential_id = options.diarizationLlmCredentialId - } - if (options?.diarizationPrompt !== undefined) { - body.diarization_prompt = options.diarizationPrompt - } - if ( - options?.provider !== undefined || - options?.telephonyIntegrationId !== undefined - ) { - body.provider = options.provider ?? null - body.telephony_integration_id = options.telephonyIntegrationId ?? null - } - if (options?.metricIds && options.metricIds.length > 0) { - body.metric_ids = options.metricIds - } - if (options?.includeCompleted) { - body.include_completed = true - } - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/retry`, - body, - ) - return response.data - } - - /** - * Re-enqueue a single failed evaluation row. Returns the refreshed - * row so the UI can flip the badge to ``pending`` immediately - * without waiting for the next polling tick. Server returns 409 if - * the row is still in progress or already completed. - */ - async retryCallImportEvaluationRow( - callImportId: string, - evaluationId: string, - evalRowId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/rows/${evalRowId}/retry`, - ) - return response.data - } - - /** - * Abort an entire in-flight (or queued) evaluation run. - * - * Idempotent — calling on a run whose rows are already terminal - * returns the run unchanged so the UI can wire this to an "Abort - * run" button without pre-checking state. The backend flips every - * cancellable row's ``status`` to ``failed`` and stamps - * ``"Evaluation cancelled by user"`` as the error so the polling UI - * surfaces the cancel on the next tick. Pairs with the worker's - * cancellation guard which prevents a late-finishing task from - * overwriting the cancelled state. - */ - async cancelCallImportEvaluation( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/cancel`, - ) - return response.data - } - - /** - * Force-fail only rows currently in ``pending`` for an evaluation run. - * - * Unlike ``cancelCallImportEvaluation``, this does NOT touch rows that - * are already ``running``. Useful when queued rows are stuck indefinitely - * but active workers should keep progressing. - */ - async forceFailCallImportEvaluationPending( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/force-fail-pending`, - ) - return response.data - } - - /** - * Abort an in-flight (or queued) evaluation for a single row. - * - * Mirrors :func:`cancelCallImportEvaluation` but scoped to one row, - * intended for the per-row Stop button on the evaluation detail - * rows table. The parent run's counters are rolled up server-side - * so the run-level pill updates on the same response. - */ - async cancelCallImportEvaluationRow( - callImportId: string, - evaluationId: string, - evalRowId: string, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/rows/${evalRowId}/cancel`, - ) - return response.data - } - - async listCallImportEvaluations(callImportId: string): Promise { - const response = await this.client.get(`/api/v1/call-imports/${callImportId}/evaluations`) - return response.data - } - - async getCallImportEvaluation( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}`, - ) - return response.data - } - - async listCallImportEvaluationRows( - callImportId: string, - evaluationId: string, - params: { - page?: number - page_size?: number - q?: string - metric_id?: string - metric_value?: string - classification_yes_no?: string - classification_choice?: string - classification_level?: string - status?: string - // Flow-chart drilldown: filter to rows whose sequence under the - // given parent contains ``flow_node`` (and optionally is - // immediately followed by ``flow_edge_target`` for edge clicks). - // Accepts a child UUID, a ``disc:`` discovered id, or a - // raw slug. - flow_parent_id?: string - flow_node?: string - flow_edge_target?: string - // Discovered-label filters: either pin to a specific discovered - // slug or surface every row that produced any candidate under - // ``discovered_parent_id``. - discovered_parent_id?: string - discovered_label_key?: string - has_discovered?: boolean - // Column-click sorting from the UI. ``sort_by`` is either a - // built-in column key (``row_index`` / ``conversation_id`` / - // ``status``) or ``metric:`` for per-metric value - // sorting; ``sort_dir`` is ``asc`` (default) or ``desc``. - sort_by?: string - sort_dir?: 'asc' | 'desc' - } = {}, - ): Promise { - const cleaned: Record = {} - for (const [k, v] of Object.entries(params)) { - if (v !== undefined && v !== null && v !== '') cleaned[k] = v - } - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/rows`, - { params: cleaned }, - ) - return response.data - } - - async getCallImportEvaluationRow( - callImportId: string, - evaluationId: string, - rowId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/rows/${rowId}`, - ) - return response.data - } - - async startCallImportEvaluationExport( - callImportId: string, - evaluationId: string, - format: 'csv' | 'xlsx' = 'csv', - includeTranscripts = true, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/export`, - { format, include_transcripts: includeTranscripts }, - ) - return response.data - } - - async getCallImportEvaluationExport( - callImportId: string, - evaluationId: string, - jobId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/export-jobs/${jobId}`, - ) - return response.data - } - - async generateCallImportEvaluationPdfReport( - callImportId: string, - evaluationId: string, - vendorName: string, - reportType: 'external' | 'internal', - includeWeeklyDelta = false, - options?: { - internalBrandImageId?: string | null - externalBrandImageId?: string | null - useCase?: string | null - baselineEvaluationId?: string | null - reportConfig?: Record - platformBaseUrl?: string | null - }, - ): Promise { - const response = await this.client.post( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-report`, - { - vendor_name: vendorName, - report_type: reportType, - include_weekly_delta: includeWeeklyDelta, - include_period_delta: includeWeeklyDelta, - baseline_evaluation_id: options?.baselineEvaluationId || null, - use_case: options?.useCase || null, - internal_brand_image_id: options?.internalBrandImageId || null, - external_brand_image_id: options?.externalBrandImageId || null, - report_config: options?.reportConfig || {}, - platform_base_url: options?.platformBaseUrl || null, - }, - { responseType: 'arraybuffer' }, - ) - const contentType = String(response.headers['content-type'] || '') - if (contentType.includes('application/pdf')) { - return new Blob([response.data], { type: 'application/pdf' }) - } - const text = new TextDecoder().decode(response.data) - return JSON.parse(text) as CallImportEvaluationPdfReport - } - - async listCallImportEvaluationPdfReports( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports`, - ) - return response.data - } - - async getCallImportEvaluationPdfReport( - callImportId: string, - evaluationId: string, - reportId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports/${reportId}`, - ) - return response.data - } - - async downloadCallImportEvaluationPdfReport( - callImportId: string, - evaluationId: string, - reportId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports/${reportId}/download`, - { responseType: 'blob' }, - ) - return response.data - } - - async listCallImportEvaluationBaselineCandidates( - callImportId: string, - evaluationId: string, - ): Promise { - const response = await this.client.get( - `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/baseline-candidates`, - ) - return response.data - } - - async deleteCallImportEvaluation(callImportId: string, evaluationId: string): Promise { - await this.client.delete(`/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}`) - } - - // Call Import Tags endpoints - async listCallImportTags(): Promise { - const response = await this.client.get('/api/v1/call-import-tags') - return response.data - } - - async createCallImportTag(payload: { - name: string - color?: string | null - }): Promise { - const response = await this.client.post('/api/v1/call-import-tags', payload) - return response.data - } - - async updateCallImportTag( - tagId: string, - payload: { name?: string; color?: string | null }, - ): Promise { - const response = await this.client.patch( - `/api/v1/call-import-tags/${tagId}`, - payload, - ) - return response.data - } - - async deleteCallImportTag(tagId: string): Promise { - await this.client.delete(`/api/v1/call-import-tags/${tagId}`) - } - - // Model Config endpoints - async getDashboardSummary(): Promise { - const response = await this.client.get('/api/v1/dashboard/summary') - return response.data - } - - async getOrgUsageFxRate(): Promise<{ - base: string - quote: string - rate: number - as_of: string - source: string - }> { - const response = await this.client.get('/api/v1/organizations/usage/fx-rate') - return response.data - } - - async getOrgUsageSummary(params: { - start?: string - end?: string - tz?: string - workspace_id?: string - product_section?: string - model?: string - resource_id?: string - usage_kind?: string - call_import_id?: string - evaluation_id?: string - dataset?: string - tag_id?: string - }): Promise<{ - start: string - end: string - totals: { - prompt_tokens: number - completion_tokens: number - total_tokens: number - cache_read_tokens: number - cache_creation_tokens: number - reasoning_tokens: number - audio_seconds: number - tts_characters: number - call_count: number - input_cost_micro_usd?: number - output_cost_micro_usd?: number - cache_read_cost_micro_usd?: number - cache_creation_cost_micro_usd?: number - reasoning_cost_micro_usd?: number - audio_cost_micro_usd?: number - tts_cost_micro_usd?: number - total_cost_micro_usd?: number - costs?: { - input_cost_usd: number - output_cost_usd: number - cache_read_cost_usd: number - cache_write_cost_usd: number - reasoning_cost_usd: number - audio_cost_usd: number - tts_cost_usd: number - total_cost_usd: number - currency: string - has_unpriced_usage: boolean - } - } - last_updated_at?: string | null - }> { - const response = await this.client.get('/api/v1/organizations/usage/summary', { - params, - }) - return response.data - } - - async getOrgUsageBreakdown(params: { - start?: string - end?: string - tz?: string - group_by?: 'workspace' | 'product_section' | 'model' | 'resource' | 'usage_kind' | 'call_import' - workspace_id?: string - product_section?: string - model?: string - resource_id?: string - usage_kind?: string - call_import_id?: string - evaluation_id?: string - dataset?: string - tag_id?: string - limit?: number - offset?: number - }): Promise<{ - start: string - end: string - group_by: string - rows: Array<{ - workspace_id?: string | null - workspace_name?: string | null - product_section?: string | null - product_section_label?: string | null - model?: string | null - resource_id?: string | null - resource_type?: string | null - resource_label?: string | null - call_import_id?: string | null - call_import_label?: string | null - usage_kind?: string | null - prompt_tokens: number - completion_tokens: number - total_tokens: number - cache_read_tokens: number - cache_creation_tokens: number - reasoning_tokens: number - audio_seconds: number - tts_characters: number - call_count: number - input_cost_micro_usd?: number - output_cost_micro_usd?: number - cache_read_cost_micro_usd?: number - cache_creation_cost_micro_usd?: number - reasoning_cost_micro_usd?: number - audio_cost_micro_usd?: number - tts_cost_micro_usd?: number - total_cost_micro_usd?: number - costs?: { - input_cost_usd: number - output_cost_usd: number - cache_read_cost_usd: number - cache_write_cost_usd: number - reasoning_cost_usd: number - audio_cost_usd: number - tts_cost_usd: number - total_cost_usd: number - currency: string - has_unpriced_usage: boolean - } - }> - total_count: number - truncated_at_limit?: boolean - last_updated_at?: string | null - }> { - const response = await this.client.get('/api/v1/organizations/usage/breakdown', { - params, - }) - return response.data - } - - async listUsagePricingAvailableModels(): Promise<{ models: string[] }> { - const response = await this.client.get( - '/api/v1/organizations/usage/pricing/available-models', - ) - return response.data - } - - async getUsagePricingEffective( - model: string, - params?: { usage_kind?: string; as_of?: string }, - ): Promise<{ - model: string - usage_kind: string - as_of: string - catalog_rates?: Record | null - effective_rates?: Record | null - effective_source?: string | null - has_override: boolean - override?: { - effective_from: string - effective_to?: string | null - rates: Record - } | null - }> { - const response = await this.client.get( - `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, - { params }, - ) - return response.data - } - - async listUsagePricingOverrides(params?: { - model?: string - usage_kind?: string - }): Promise< - Array<{ - id: string - organization_id: string - model: string - usage_kind: string - effective_from: string - effective_to?: string | null - rates: { - input_per_1m?: number | null - output_per_1m?: number | null - cache_read_per_1m?: number | null - cache_write_per_1m?: number | null - reasoning_per_1m?: number | null - audio_per_minute?: number | null - tts_per_1m_characters?: number | null - } - recompute_enqueued?: boolean - recompute_job_id?: string | null - }> - > { - const response = await this.client.get('/api/v1/organizations/usage/pricing/overrides', { - params, - }) - return response.data - } - - async upsertUsagePricingOverride( - model: string, - body: { - usage_kind?: string - effective_from: string - effective_to?: string - rates: { - input_per_1m?: number - output_per_1m?: number - cache_read_per_1m?: number - cache_write_per_1m?: number - reasoning_per_1m?: number - audio_per_minute?: number - tts_per_1m_characters?: number - } - recompute?: boolean - } - ): Promise<{ - id: string - model: string - usage_kind: string - effective_from: string - effective_to?: string | null - rates: Record - recompute_enqueued?: boolean - recompute_job_id?: string | null - }> { - const response = await this.client.put( - `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, - body - ) - return response.data - } - - async deleteUsagePricingOverride( - model: string, - params?: { usage_kind?: string; effective_from?: string; recompute?: boolean } - ): Promise<{ - deleted: boolean - model: string - usage_kind: string - recompute_enqueued?: boolean - recompute_job_id?: string | null - }> { - const response = await this.client.delete( - `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, - { params } - ) - return response.data - } - - async triggerUsageCostRecompute(body?: { - start_date?: string - end_date?: string - model?: string - usage_kind?: string - }): Promise<{ - id: string - organization_id: string - status: string - model?: string | null - usage_kind?: string | null - start_date?: string | null - end_date?: string | null - updated_rows: number - error_message?: string | null - celery_task_id?: string | null - created_at?: string | null - updated_at?: string | null - completed_at?: string | null - }> { - const response = await this.client.post( - '/api/v1/organizations/usage/pricing/recompute', - body ?? {} - ) - return response.data - } - - async getUsageCostRecomputeJob(jobId: string): Promise<{ - id: string - organization_id: string - status: string - model?: string | null - usage_kind?: string | null - start_date?: string | null - end_date?: string | null - updated_rows: number - error_message?: string | null - celery_task_id?: string | null - created_at?: string | null - updated_at?: string | null - completed_at?: string | null - }> { - const response = await this.client.get( - `/api/v1/organizations/usage/pricing/recompute/${jobId}` - ) - return response.data - } - - async getOrgUsageFilters(params?: { - start?: string - end?: string - tz?: string - workspace_id?: string - product_section?: string - model?: string - resource_id?: string - evaluation_id?: string - usage_kind?: string - call_import_id?: string - q?: string - dataset?: string - tag_id?: string - }): Promise<{ - workspaces: Array<{ id: string; name: string }> - product_sections: Array<{ id: string; label: string }> - call_imports: Array<{ id: string; label: string }> - evaluations: Array<{ id: string; label: string }> - models: string[] - resources: Array<{ id: string; type?: string; label: string; product_section?: string }> - usage_kinds: Array<{ id: string; label: string }> - datasets: string[] - tags: Array<{ id: string; label: string }> - }> { - const response = await this.client.get('/api/v1/organizations/usage/filters', { - params, - }) - return response.data - } - - async getAllModels(): Promise> { - const response = await this.client.get('/api/v1/model-config/models') - return response.data - } - - async getModelConfig(modelName: string): Promise { - const response = await this.client.get(`/api/v1/model-config/models/${modelName}`) - return response.data - } - - async getModelsByProvider(provider: string): Promise { - const response = await this.client.get(`/api/v1/model-config/providers/${provider}/models`) - return response.data - } - - async getModelOptions(provider: string): Promise<{ stt: string[]; llm: string[]; tts: string[]; s2s: string[]; tts_voices: Record; tts_sample_rates?: number[] }> { - const response = await this.client.get(`/api/v1/model-config/providers/${provider}/options`) - const data = response.data - // Ensure s2s and tts_voices are always present (for backward compatibility) - return { - ...data, - s2s: data.s2s || [], - tts_voices: data.tts_voices || {}, - } - } - - async getModelsByType(provider: string, modelType: 'stt' | 'llm' | 'tts'): Promise { - const response = await this.client.get(`/api/v1/model-config/providers/${provider}/types/${modelType}/models`) - return response.data - } - - // Manual Evaluations endpoints - async listManualEvaluationAudioFiles(prefix?: string, maxKeys = 1000): Promise { - const response = await this.client.get('/api/v1/manual-evaluations/audio-files', { - params: { prefix, max_keys: maxKeys }, - }) - return response.data - } - - async getAudioPresignedUrl(fileKey: string, expiration = 3600): Promise<{ url: string; expires_in: number }> { - const response = await this.client.get( - `/api/v1/manual-evaluations/audio-files/${encodeURIComponent(fileKey)}/presigned-url`, - { - params: { expiration }, - } - ) - return response.data - } - - async transcribeAudio(data: { - audio_file_key: string - stt_provider: string - stt_model: string - name?: string - language?: string - enable_speaker_diarization?: boolean - }): Promise { - const response = await this.client.post('/api/v1/manual-evaluations/transcribe', data) - return response.data - } - - async listManualTranscriptions(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/manual-evaluations/transcriptions', { - params: { skip, limit }, - }) - return response.data - } - - async getManualTranscription(transcriptionId: string): Promise { - const response = await this.client.get(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`) - return response.data - } - - async updateManualTranscription(transcriptionId: string, name: string): Promise { - const response = await this.client.patch(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`, { name }) - return response.data - } - - async deleteManualTranscription(transcriptionId: string): Promise { - const response = await this.client.delete(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`) - return response.data - } - - // Test Agent endpoints - async createTestAgentConversation(data: { - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - conversation_metadata?: Record - }): Promise { - const response = await this.client.post('/api/v1/test-agents/conversations', data) - return response.data - } - - async getTestAgentConversation(conversationId: string): Promise { - const response = await this.client.get(`/api/v1/test-agents/conversations/${conversationId}`) - return response.data - } - - async listTestAgentConversations(): Promise { - const response = await this.client.get('/api/v1/test-agents/conversations') - return response.data - } - - async startTestAgentConversation(conversationId: string): Promise { - const response = await this.client.post(`/api/v1/test-agents/conversations/${conversationId}/start`) - return response.data - } - - async processTestAgentAudio(conversationId: string, audioFile: File, chunkTimestamp?: number): Promise { - const formData = new FormData() - formData.append('audio_file', audioFile) - if (chunkTimestamp !== undefined) { - formData.append('chunk_timestamp', chunkTimestamp.toString()) - } - const response = await this.client.post( - `/api/v1/test-agents/conversations/${conversationId}/process-audio`, - formData, - { - headers: { - 'Content-Type': 'multipart/form-data', - }, - } - ) - return response.data - } - - async getTestAgentResponseAudio(conversationId: string): Promise { - const response = await this.client.get( - `/api/v1/test-agents/conversations/${conversationId}/response-audio`, - { - responseType: 'blob', - } - ) - return response.data - } - - async endTestAgentConversation(conversationId: string, finalAudioKey?: string): Promise { - const response = await this.client.post(`/api/v1/test-agents/conversations/${conversationId}/end`, { - final_audio_key: finalAudioKey, - }) - return response.data - } - - async deleteTestAgentConversation(conversationId: string): Promise { - await this.client.delete(`/api/v1/test-agents/conversations/${conversationId}`) - } - - // Voice Agent endpoints - async getVoiceAgentConnection( - query?: Record, - ): Promise<{ ws_url: string; endpoint?: string }> { - const search = new URLSearchParams() - if (query) { - for (const [key, value] of Object.entries(query)) { - if (value !== undefined && value !== null && value !== '') { - search.set(key, String(value)) - } - } - } - const qs = search.toString() - const path = `/api/v1/voice-agent/connect${qs ? `?${qs}` : ''}` - const response = await this.client.post(path) - return response.data - } - - // Playground endpoints - async createWebCall(data: { - agent_id: string - metadata?: Record - retell_llm_dynamic_variables?: Record - custom_sip_headers?: Record - ui_surface?: 'agents_talk' | 'agent_playground' - }): Promise<{ - call_type: string - access_token?: string - call_id?: string - agent_id: string - agent_version?: number - call_status?: string - agent_name?: string - metadata?: Record - retell_llm_dynamic_variables?: Record - sample_rate?: number - call_short_id?: string - signed_url?: string - host?: string - room_name?: string - conversation_id?: string - }> { - const response = await this.client.post('/api/v1/playground/web-call', data) - return response.data - } - - async listCallRecordings(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/playground/call-recordings', { - params: { skip, limit }, - }) - return response.data - } - - async getCallRecording(callShortId: string): Promise { - const response = await this.client.get(`/api/v1/playground/call-recordings/${callShortId}`) - return response.data - } - - getCallRecordingAudioStreamUrl(callShortId: string, options?: { stereo?: boolean }): string { - const params = new URLSearchParams({ proxy: 'true' }) - if (options?.stereo) params.set('stereo', 'true') - return this.buildAuthenticatedApiUrl( - `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, - ) - } - - async getCallRecordingLogs(callShortId: string): Promise<{ - platform: string - entries: Array<{ - time?: string | null - level?: string | null - category?: string | null - summary?: string | null - raw?: Record - }> - count: number - }> { - const response = await this.client.get(`/api/v1/playground/call-recordings/${callShortId}/logs`) - return response.data - } - - async refreshCallRecording( - callShortId: string, - providerCallId?: string | null, - ): Promise<{ message: string }> { - const response = await this.client.post( - `/api/v1/playground/call-recordings/${callShortId}/refresh`, - providerCallId ? { provider_call_id: providerCallId } : undefined, - ) - return response.data - } - - async finalizePlaygroundCallRecording( - callShortId: string, - providerCallId?: string | null, - ): Promise { - const maxAttempts = 6 - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { - try { - await this.refreshCallRecording(callShortId, providerCallId) - return - } catch (error: unknown) { - const axiosError = error as { - response?: { status?: number; data?: { detail?: string } } - } - const status = axiosError.response?.status - const detail = String(axiosError.response?.data?.detail || '') - const missingProvider = - status === 400 && detail.toLowerCase().includes('provider') - if (!missingProvider || attempt === maxAttempts - 1) { - throw error - } - await new Promise((resolve) => setTimeout(resolve, 400 * (attempt + 1))) - } - } - } - - async updateCallRecording(callShortId: string, providerCallId: string): Promise<{ message: string; provider_call_id: string }> { - const response = await this.client.put(`/api/v1/playground/call-recordings/${callShortId}`, { - provider_call_id: providerCallId - }) - return response.data - } - - async deleteCallRecording(callShortId: string): Promise<{ message: string }> { - const response = await this.client.delete(`/api/v1/playground/call-recordings/${callShortId}`) - return response.data - } - - async reEvaluateCallRecording(callShortId: string): Promise<{ - message: string - evaluator_result_id: string - result_id: string - audio_s3_key: string - task_id: string - }> { - const response = await this.client.post(`/api/v1/playground/call-recordings/${callShortId}/re-evaluate`) - return response.data - } - - async getCallRecordingAudioBlob( - callShortId: string, - options?: { stereo?: boolean }, - ): Promise { - const params = new URLSearchParams({ proxy: 'true' }) - if (options?.stereo) params.set('stereo', 'true') - const response = await this.client.get( - `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, - { responseType: 'blob' }, - ) - return response.data as Blob - } - - async getCallRecordingAudioUrl(callShortId: string, options?: { stereo?: boolean }): Promise { - const blob = await this.getCallRecordingAudioBlob(callShortId, options) - return URL.createObjectURL(blob) - } - - async getCallRecordingAudioBuffer( - callShortId: string, - options?: { stereo?: boolean }, - ): Promise { - const params = new URLSearchParams({ proxy: 'true' }) - if (options?.stereo) params.set('stereo', 'true') - const response = await this.client.get( - `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, - { responseType: 'arraybuffer' }, - ) - return response.data as ArrayBuffer - } - - async createCustomWebsocketSession(data: { - agent_id: string - websocket_url: string - transcript_entries: Array<{ role: 'user' | 'agent'; content: string; timestamp: string }> - started_at?: string - ended_at?: string - call_short_id?: string - audio_file?: File - }): Promise<{ - message: string - call_short_id: string - audio_s3_key?: string | null - evaluator_result_id?: string | null - }> { - const formData = new FormData() - formData.append('agent_id', data.agent_id) - formData.append('websocket_url', data.websocket_url) - formData.append('transcript_entries', JSON.stringify(data.transcript_entries)) - if (data.started_at) { - formData.append('started_at', data.started_at) - } - if (data.ended_at) { - formData.append('ended_at', data.ended_at) - } - if (data.call_short_id) { - formData.append('call_short_id', data.call_short_id) - } - if (data.audio_file) { - formData.append('audio_file', data.audio_file) - } - - const response = await this.client.post('/api/v1/playground/custom-websocket-sessions', formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }) - return response.data - } - - async evaluateCustomWebsocketSession(callShortId: string): Promise<{ - message: string - evaluator_result_id: string - result_id: string - task_id: string - }> { - const response = await this.client.post(`/api/v1/playground/custom-websocket-sessions/${callShortId}/evaluate`) - return response.data - } - - async getAgentSttConfig(agentId: string): Promise<{ - available: boolean - provider?: string - model?: string - reason?: string - }> { - const response = await this.client.get(`/api/v1/playground/agents/${agentId}/stt-config`) - return response.data - } - - async transcribeTurn( - agentId: string, - channel: 'user' | 'agent', - audioBlob: Blob, - ): Promise<{ transcript: string; channel: string }> { - const formData = new FormData() - formData.append('agent_id', agentId) - formData.append('channel', channel) - formData.append('audio_file', audioBlob, `turn_${channel}_${Date.now()}.wav`) - const response = await this.client.post('/api/v1/playground/transcribe-turn', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async summarizeTranscript(params: { - transcript?: string - entries?: Array<{ role: string; content: string; timestamp?: string }> - callShortId?: string - agentId?: string - force?: boolean - }): Promise<{ - summary: string - provider: string - model: string - source?: 'voice_bundle' | 'org_fallback' - cached?: boolean - generated_at?: string - usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } - }> { - const body: Record = {} - if (params.transcript) body.transcript = params.transcript - if (params.entries && params.entries.length > 0) body.entries = params.entries - if (params.callShortId) body.call_short_id = params.callShortId - if (params.agentId) body.agent_id = params.agentId - if (params.force) body.force = true - const response = await this.client.post('/api/v1/playground/summarize-transcript', body) - return response.data - } - - // Observability endpoints - async listObservabilityCalls(skip = 0, limit = 100): Promise { - const response = await this.client.get('/api/v1/observability/calls', { - params: { skip, limit }, - }) - return response.data - } - - async listCallsHub(params?: { - skip?: number - limit?: number - status?: string - search?: string - event?: string - }): Promise<{ - items: Array<{ kind: 'obs' | 'trace'; sort_at?: string; obs?: ObservabilityCall; trace?: Record }> - total: number - page: number - page_size: number - page_count: number - summary: { - total: number - obs_total: number - obs_live: number - obs_ended: number - obs_started: number - obs_other: number - traces_open: number - traces_closed: number - } - }> { - const response = await this.client.get('/api/v1/observability/calls-hub', { params }) - return response.data - } - - private buildAuthenticatedApiUrl(path: string): string { - const normalizedPath = path.startsWith('/') ? path : `/${path}` - const configuredBase = (this.client.defaults.baseURL || '').replace(/\/$/, '') - const url = configuredBase - ? new URL(`${configuredBase}${normalizedPath}`) - : new URL(normalizedPath, window.location.origin) - - const workspaceId = localStorage.getItem('activeWorkspaceId') - if (!this.cookieSessionEnabled) { - const accessToken = this.inMemoryAccessToken - const apiKey = localStorage.getItem('apiKey') - if (accessToken) { - url.searchParams.set('token', accessToken) - } else if (apiKey) { - url.searchParams.set('api_key', apiKey) - } - } - if (workspaceId) { - url.searchParams.set('workspace_id', workspaceId) - } - return url.toString() - } - - /** SSE / EventSource cannot use axios; must send session cookies explicitly. */ - openAuthenticatedEventSource(path: string): EventSource { - const url = this.buildAuthenticatedApiUrl(path) - return new EventSource(url, { withCredentials: true }) - } - - async getObservabilityCall(callShortId: string): Promise { - const response = await this.client.get(`/api/v1/observability/calls/${callShortId}`) - return response.data - } - - getObservabilityCallLiveEventsUrl(callShortId: string): string { - return this.buildAuthenticatedApiUrl( - `/api/v1/observability/calls/${callShortId}/live-events`, - ) - } - - getObservabilityCallAudioStreamUrl(callShortId: string): string { - return this.buildAuthenticatedApiUrl( - `/api/v1/observability/calls/${callShortId}/audio?proxy=true`, - ) - } - - async getObservabilityCallAudioBuffer(callShortId: string): Promise { - const response = await this.client.get( - `/api/v1/observability/calls/${callShortId}/audio?proxy=true`, - { responseType: 'arraybuffer' }, - ) - return response.data as ArrayBuffer - } - - async getObservabilityCallAudioBlob(callShortId: string): Promise { - const response = await this.client.get( - `/api/v1/observability/calls/${callShortId}/audio`, - { responseType: 'blob' }, - ) - return response.data as Blob - } - - async getObservabilityCallAudioUrl(callShortId: string): Promise { - const blob = await this.getObservabilityCallAudioBlob(callShortId) - return URL.createObjectURL(blob) - } - - async deleteObservabilityCall(callShortId: string): Promise<{ message: string }> { - const response = await this.client.delete(`/api/v1/observability/calls/${callShortId}`) - return response.data - } - - async evaluateObservabilityCall(callShortId: string, evaluatorId: string): Promise { - const response = await this.client.post(`/api/v1/observability/calls/${callShortId}/evaluate`, { - evaluator_id: evaluatorId, - }) - return response.data - } - - // Evaluator endpoints - async createEvaluator(data: { - name?: string - agent_id?: string - persona_id?: string - scenario_id?: string - custom_prompt?: string - metric_ids?: string[] - llm_provider?: string - llm_model?: string - tags?: string[] - }): Promise { - const response = await this.client.post('/api/v1/evaluators', data) - return response.data - } - - async formatCustomPrompt(prompt: string): Promise<{ formatted_prompt: string }> { - const response = await this.client.post('/api/v1/evaluators/format-prompt', { prompt }) - return response.data - } - - async createEvaluatorsBulk(data: { - name?: string - agent_id: string - scenario_id: string - persona_ids: string[] - tags?: string[] - }): Promise { - const response = await this.client.post('/api/v1/evaluators/bulk', data) - return response.data - } - - async listEvaluators(): Promise { - const response = await this.client.get('/api/v1/evaluators') - return response.data - } - - async getEvaluator(evaluatorId: string): Promise { - const response = await this.client.get(`/api/v1/evaluators/${evaluatorId}`) - return response.data - } - - async updateEvaluator(evaluatorId: string, data: { - agent_id?: string - persona_id?: string - scenario_id?: string - name?: string - custom_prompt?: string - metric_ids?: string[] - llm_provider?: string - llm_model?: string - tags?: string[] - }): Promise { - const response = await this.client.put(`/api/v1/evaluators/${evaluatorId}`, data) - return response.data - } - - async deleteEvaluator(evaluatorId: string, force?: boolean): Promise { - const response = await this.client.delete(`/api/v1/evaluators/${evaluatorId}`, { - params: force ? { force: true } : undefined, - }) - return response.data - } - - async runEvaluators(evaluatorIds: string[]): Promise<{ task_ids: string[]; evaluator_results: any[] }> { - const response = await this.client.post('/api/v1/evaluators/run', { evaluator_ids: evaluatorIds }) - return response.data - } - - async createEvaluatorSuite(data: { - name?: string - agent_id: string - persona_id?: string - persona_ids?: string[] - scenario_ids: string[] - metric_ids?: string[] - llm_provider?: string - llm_model?: string - tags?: string[] - default_runs_per_combination?: number - }): Promise { - const response = await this.client.post('/api/v1/evaluator-suites', data) - return response.data - } - - async listEvaluatorSuites(): Promise { - const response = await this.client.get('/api/v1/evaluator-suites') - return response.data - } - - async getEvaluatorSuite(suiteId: string): Promise { - const response = await this.client.get(`/api/v1/evaluator-suites/${suiteId}`) - return response.data - } - - async updateEvaluatorSuite( - suiteId: string, - data: { - name?: string - metric_ids?: string[] | null - llm_provider?: string | null - llm_model?: string | null - tags?: string[] - default_runs_per_combination?: number - }, - ): Promise { - const response = await this.client.put(`/api/v1/evaluator-suites/${suiteId}`, data) - return response.data - } - - async addEvaluatorSuiteScenarios(suiteId: string, scenarioIds: string[]): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/scenarios`, { - scenario_ids: scenarioIds, - }) - return response.data - } - - async removeEvaluatorSuiteScenario(suiteId: string, scenarioId: string): Promise { - const response = await this.client.delete( - `/api/v1/evaluator-suites/${suiteId}/scenarios/${scenarioId}`, - ) - return response.data - } - - async addEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/personas`, { - persona_ids: personaIds, - }) - return response.data - } - - async replaceEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { - const response = await this.client.put(`/api/v1/evaluator-suites/${suiteId}/personas`, { - persona_ids: personaIds, - }) - return response.data - } - - async removeEvaluatorSuitePersona(suiteId: string, personaId: string): Promise { - const response = await this.client.delete( - `/api/v1/evaluator-suites/${suiteId}/personas/${personaId}`, - ) - return response.data - } - - async deleteEvaluatorSuite(suiteId: string): Promise { - await this.client.delete(`/api/v1/evaluator-suites/${suiteId}`) - } - - async runEvaluatorSuite( - suiteId: string, - data: { runs_per_combination: number; to_number?: string; from_number?: string }, - ): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/run`, data) - return response.data - } - - async runNextCombination( - suiteId: string, - data?: { from_number?: string }, - ): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/run-next`, data || {}) - return response.data - } - - async chooseNextCombination(suiteId: string): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/choose-next`) - return response.data - } - - async activateEvaluatorSuite(suiteId: string): Promise { - const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/activate`) - return response.data - } - - // Metric endpoints - async createMetric(data: { - name: string - description?: string - /** - * Optional illustrative example surfaced alongside the description - * in the LLM judge's rubric. Mainly used by categorization label - * children but accepted on every metric. - */ - example?: string | null - metric_type: 'number' | 'boolean' | 'rating' | 'text' - trigger?: 'always' - enabled?: boolean - metric_origin?: 'default' | 'custom' - supported_surfaces?: string[] - enabled_surfaces?: string[] - custom_data_type?: 'boolean' | 'enum' | 'number_range' - custom_config?: Record - tags?: string[] - capture_rationale?: boolean - parent_metric_id?: string | null - selection_mode?: 'single_choice' | 'multi_label' | null - /** - * Scope of the new metric: - * - "workspace" (default): stamped with the active X-Workspace-Id - * header so it only appears inside that workspace. - * - "organization": stored with workspace_id=NULL so it surfaces - * in every workspace of the caller's org. - * Ignored when parent_metric_id is set — children always inherit - * their parent's scope. - */ - scope?: 'workspace' | 'organization' - }): Promise { - const response = await this.client.post('/api/v1/metrics', data) - return response.data - } - - /** - * Atomically create a parent (category) metric + N children, used by - * the "Create category" flow on the Metrics page and by the - * /parse-bulk → save path when a hierarchy was requested. - */ - async createMetricWithChildren(data: { - name: string - description?: string | null - selection_mode: 'single_choice' | 'multi_label' - enabled?: boolean - supported_surfaces?: string[] - enabled_surfaces?: string[] - tags?: string[] | null - allow_discovery?: boolean - /** - * Parent-level "Enable LLM Rationale" toggle. When true the LLM - * judge emits one rationale at the category level (children never - * carry rationales in hierarchical mode). - */ - capture_rationale?: boolean - children: Array<{ - name: string - description?: string | null - /** - * Optional illustrative example for this label, surfaced - * alongside the description in the LLM judge's rubric. - */ - example?: string | null - enabled?: boolean - capture_rationale?: boolean | null - tags?: string[] | null - }> - /** - * Same semantics as {@link createMetric}'s ``scope``: when - * ``"organization"`` the parent + every child are stored with - * ``workspace_id=NULL`` so the whole category subtree is visible - * in every workspace of the org. - */ - scope?: 'workspace' | 'organization' - }): Promise { - const response = await this.client.post( - '/api/v1/metrics/with-children', - data, - ) - return response.data - } - - async addMetricChild(parentMetricId: string, data: { - name: string - description?: string | null - /** - * Optional illustrative example for this label, surfaced alongside - * the description in the LLM judge's rubric. - */ - example?: string | null - enabled?: boolean - capture_rationale?: boolean | null - tags?: string[] | null - }): Promise { - const response = await this.client.post( - `/api/v1/metrics/${parentMetricId}/children`, - data, - ) - return response.data - } - - async listMetrics( - surface?: string, - includeChildren: boolean = true, - options?: { includeDrafts?: boolean; draftsOnly?: boolean; enabledOnly?: boolean }, - ): Promise { - const response = await this.client.get('/api/v1/metrics', { - params: { - ...(surface ? { surface } : {}), - include_children: includeChildren, - ...(options?.includeDrafts ? { include_drafts: true } : {}), - ...(options?.draftsOnly ? { drafts_only: true } : {}), - ...(options?.enabledOnly ? { enabled_only: true } : {}), - }, - }) - return response.data - } - - async createMetricDraft(data: { - name: string - description?: string - metric_type: string - trigger: string - metric_origin?: string - studio_notes?: string - parent_metric_id?: string - selection_mode?: string - custom_data_type?: string - custom_config?: Record - capture_rationale?: boolean - compare_transcripts?: boolean - scope?: 'workspace' | 'organization' - supported_surfaces?: string[] - enabled_surfaces?: string[] - tags?: string[] - enabled?: boolean - }): Promise { - const response = await this.client.post('/api/v1/metrics/drafts', data) - return response.data - } - - async createMetricDraftWithChildren(data: { - name: string - description?: string | null - selection_mode: 'single_choice' | 'multi_label' - allow_discovery?: boolean - capture_rationale?: boolean - supported_surfaces: string[] - enabled_surfaces: string[] - children: Array<{ - name: string - description?: string | null - example?: string | null - capture_rationale?: boolean - enabled?: boolean - }> - scope?: 'workspace' | 'organization' - studio_notes?: string - }): Promise { - const response = await this.client.post('/api/v1/metrics/drafts/with-children', data) - return response.data - } - - async promoteMetric(metricId: string): Promise<{ metric: any; promoted_at: string }> { - const response = await this.client.post(`/api/v1/metrics/${metricId}/promote`) - return response.data - } - - async createMetricStudioRun(data: { - name?: string - metric_ids: string[] - sources: Array<{ - source_kind: 'call_import_row' | 'call_recording' | 'evaluator_result' - source_ref: string - display_label?: string - }> - transcript_source?: 'production' | 'diarised' - llm_provider?: string - llm_model?: string - llm_credential_id?: string - llm_config?: LLMGenerationConfig | null - metric_llm_overrides?: Record - }): Promise { - const response = await this.client.post('/api/v1/metric-studio/runs', data) - return response.data - } - - async listMetricStudioRuns(skip = 0, limit = 50): Promise<{ items: any[]; total: number }> { - const response = await this.client.get('/api/v1/metric-studio/runs', { - params: { skip, limit }, - }) - return response.data - } - - async getMetricStudioRun(runId: string): Promise { - const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}`) - return response.data - } - - async listMetricStudioRunResults( - runId: string, - skip = 0, - limit = 100, - ): Promise<{ items: any[]; total: number }> { - const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}/results`, { - params: { skip, limit }, - }) - return response.data - } - - async retryMetricStudioRun( - runId: string, - resultIds?: string[], - ): Promise { - const response = await this.client.post(`/api/v1/metric-studio/runs/${runId}/retry`, { - result_ids: resultIds, - }) - return response.data - } - - async deleteMetricStudioRun(runId: string): Promise { - await this.client.delete(`/api/v1/metric-studio/runs/${runId}`) - } - - // Evaluator Results endpoints - async listEvaluatorResults( - paramsOrEvaluatorId?: ListEvaluatorResultsParams | string, - playground?: boolean, - testAgentsOnly?: boolean, - agentId?: string - ): Promise { - let params: Record = {} - if (typeof paramsOrEvaluatorId === 'string' || paramsOrEvaluatorId === undefined) { - if (paramsOrEvaluatorId) { - params.evaluator_id = paramsOrEvaluatorId - } - if (agentId) { - params.agent_id = agentId - } - if (playground !== undefined) { - params.playground = playground - } - if (testAgentsOnly !== undefined) { - params.test_agents_only = testAgentsOnly - } - } else { - const p = paramsOrEvaluatorId - if (p.skip !== undefined) params.skip = p.skip - if (p.limit !== undefined) params.limit = p.limit - if (p.evaluatorId) params.evaluator_id = p.evaluatorId - if (p.agentId) params.agent_id = p.agentId - if (p.suiteId) params.suite_id = p.suiteId - if (p.scenarioId) params.scenario_id = p.scenarioId - if (p.status) params.status = p.status - if (p.since) params.since = p.since - if (p.until) params.until = p.until - if (p.unassignedOnly) params.unassigned_only = true - if (p.playground !== undefined) params.playground = p.playground - if (p.testAgentsOnly !== undefined) params.test_agents_only = p.testAgentsOnly - } - const response = await this.client.get('/api/v1/evaluator-results', { params }) - return response.data - } - - async getEvaluatorResultsOverview(params?: { - agentId?: string - suiteId?: string - since?: string - until?: string - }): Promise { - const query: Record = {} - if (params?.agentId) query.agent_id = params.agentId - if (params?.suiteId) query.suite_id = params.suiteId - if (params?.since) query.since = params.since - if (params?.until) query.until = params.until - const response = await this.client.get('/api/v1/evaluator-results/overview', { params: query }) - return response.data - } - - async getEvaluatorResultsAggregate(params: { - suiteId?: string - agentId?: string - scenarioId?: string - since?: string - until?: string - }): Promise { - const query: Record = {} - if (params.suiteId) query.suite_id = params.suiteId - if (params.agentId) query.agent_id = params.agentId - if (params.scenarioId) query.scenario_id = params.scenarioId - if (params.since) query.since = params.since - if (params.until) query.until = params.until - const response = await this.client.get('/api/v1/evaluator-results/aggregate', { params: query }) - return response.data - } - - private _evaluatorResultClusterQuery(scope: { - agentId: string - scenarioIds?: string[] - since?: string - until?: string - suiteId?: string - scenarioId?: string - scopeKey?: string - jobId?: string - }): Record { - const query: Record = { - agent_id: scope.agentId, - } - if (scope.jobId) query.job_id = scope.jobId - else if (scope.scopeKey) query.scope_key = scope.scopeKey - if (scope.scenarioIds?.length) { - query.scenario_ids = scope.scenarioIds - } - if (scope.since) query.since = scope.since - if (scope.until) query.until = scope.until - if (scope.suiteId) query.suite_id = scope.suiteId - if (scope.scenarioId) query.scenario_id = scope.scenarioId - return query - } - - async getEvaluatorResultMetricClusterFailurePolicies(scope: { - agentId: string - scenarioIds?: string[] - since?: string - until?: string - }): Promise { - const response = await this.client.get( - '/api/v1/evaluator-results/metric-clusters/failure-policies', - { params: this._evaluatorResultClusterQuery(scope) }, - ) - return response.data - } - - async saveEvaluatorResultMetricClusterFailurePolicies( - scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, - policies: Record, - ): Promise { - const response = await this.client.put( - '/api/v1/evaluator-results/metric-clusters/failure-policies', - { policies }, - { params: this._evaluatorResultClusterQuery(scope) }, - ) - return response.data - } - - async listEvaluatorResultMetricClusterEligibleRows( - scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, - options?: { limit?: number; count_only?: boolean }, - ): Promise { - const response = await this.client.get( - '/api/v1/evaluator-results/metric-clusters/eligible-rows', - { params: { ...this._evaluatorResultClusterQuery(scope), ...options } }, - ) - return response.data - } - - async listEvaluatorResultMetricClusterScopes(): Promise< - import('../types/api').EvaluatorResultClusterScopeListResponse - > { - const response = await this.client.get( - '/api/v1/evaluator-results/metric-clusters/scopes', - ) - return response.data - } - - async getEvaluatorResultMetricClusters(scope: { - agentId: string - scenarioIds?: string[] - since?: string - until?: string - jobId?: string - scopeKey?: string - }): Promise { - const response = await this.client.get('/api/v1/evaluator-results/metric-clusters', { - params: this._evaluatorResultClusterQuery(scope), - }) - return response.data - } - - async generateEvaluatorResultMetricClusters( - scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, - body: { - force?: boolean - regenerate?: boolean - provider?: string - model?: string - row_limit?: number - failure_policies?: Record - }, - ): Promise { - const response = await this.client.post('/api/v1/evaluator-results/metric-clusters', body, { - params: this._evaluatorResultClusterQuery(scope), - }) - return response.data - } - - async cancelEvaluatorResultMetricClusters(scope: { - agentId: string - scenarioIds?: string[] - since?: string - until?: string - scopeKey?: string - }): Promise { - const response = await this.client.post( - '/api/v1/evaluator-results/metric-clusters/cancel', - {}, - { params: this._evaluatorResultClusterQuery(scope) }, - ) - return response.data - } - - async deleteEvaluatorResultMetricClusters(scope: { - agentId: string - scenarioIds?: string[] - since?: string - until?: string - jobId?: string - scopeKey?: string - }): Promise { - await this.client.delete('/api/v1/evaluator-results/metric-clusters', { - params: this._evaluatorResultClusterQuery(scope), - }) - } - - async getEvaluatorResult(id: string, includeRelations: boolean = true): Promise { - const params = includeRelations ? { include_relations: 'true' } : {} - const response = await this.client.get(`/api/v1/evaluator-results/${id}`, { params }) - return response.data - } - - getEvaluatorResultLiveEventsUrl(resultId: string): string { - return this.buildAuthenticatedApiUrl( - `/api/v1/evaluator-results/${resultId}/live-events`, - ) - } - - async listConversationEvaluations(agentId: string): Promise { - const response = await this.client.get('/api/v1/conversation-evaluations', { - params: { agent_id: agentId }, - }) - return response.data - } - - async createConversationEvaluation(data: { - transcription_id: string - agent_id: string - }): Promise { - const response = await this.client.post('/api/v1/conversation-evaluations', data) - return response.data - } - - async deleteConversationEvaluation(evaluationId: string): Promise { - await this.client.delete(`/api/v1/conversation-evaluations/${evaluationId}`) - } - - async updateConversationEvaluation( - evaluationId: string, - data: Record, - ): Promise { - const response = await this.client.patch( - `/api/v1/conversation-evaluations/${evaluationId}`, - data, - ) - return response.data - } - - async getEvaluatorResultMetrics(id: string): Promise { - const response = await this.client.get(`/api/v1/evaluator-results/${id}/metrics`) - return response.data - } - - async getSyntheticCallTraceForResult( - evaluatorResultId: string, - includeSpans = true, - ): Promise { - const response = await this.client.get( - `/api/v1/observability/traces/results/${evaluatorResultId}`, - { params: { include_spans: includeSpans } }, - ) - return response.data - } - - async listSyntheticCallTraces(params?: { - skip?: number - limit?: number - status?: string - cursor?: string - }): Promise<{ items: any[]; total: number; next_cursor?: string; has_more?: boolean }> { - const response = await this.client.get('/api/v1/observability/traces', { params }) - return response.data - } - - async deleteSyntheticCallTrace(traceId: string): Promise<{ message: string }> { - const response = await this.client.delete(`/api/v1/observability/traces/${traceId}`) - return response.data - } - - async getSyntheticCallTraceSpans(traceId: string): Promise { - const response = await this.client.get(`/api/v1/observability/traces/${traceId}/spans`) - return response.data - } - - async getSyntheticCallTrace(traceId: string, includeSpans = true): Promise { - const response = await this.client.get(`/api/v1/observability/traces/${traceId}`, { - params: { include_spans: includeSpans }, - }) - return response.data - } - - async getSyntheticCallTraceByCallShortId(callShortId: string, includeSpans = true): Promise { - const response = await this.client.get( - `/api/v1/observability/traces/by-call-short-id/${callShortId}`, - { params: { include_spans: includeSpans } }, - ) - return response.data - } - - async getSyntheticCallTraceSetup(): Promise { - const response = await this.client.get('/api/v1/observability/traces/setup') - return response.data - } - - async createSyntheticTraceSession(data: { - transport?: 'webrtc' | 'websocket' | 'phone' | 'custom' - evaluator_result_id?: string - agent_id?: string - }): Promise<{ - trace_id: string - call_short_id: string - workspace_id: string - transport: string - status: string - otel_correlation: Record - }> { - const response = await this.client.post('/api/v1/observability/traces/sessions', data) - return response.data - } - - async closeSyntheticTraceSession(callShortId: string): Promise<{ - trace_id: string - call_short_id: string - status: string - }> { - const response = await this.client.post( - `/api/v1/observability/traces/sessions/${callShortId}/close`, - ) - return response.data - } - - async ingestSyntheticTraceJson(data: { - call_short_id: string - spans: Array<{ - name: string - turn_number: number - ttfb_ms?: number - attributes?: Record - }> - }): Promise<{ - accepted_spans: number - synthetic_call_trace_id?: string - correlated: boolean - }> { - const response = await this.client.post('/api/v1/observability/traces/ingest', data) - return response.data - } - - async getEvaluatorResultOtelCorrelation(id: string): Promise { - const response = await this.client.get(`/api/v1/evaluator-results/${id}/otel-correlation`) - return response.data - } - - getEvaluatorResultAudioStreamUrl(resultId: string): string { - return this.buildAuthenticatedApiUrl( - `/api/v1/evaluator-results/${resultId}/audio`, - ) - } - - async getEvaluatorResultAudioBuffer(resultId: string): Promise { - const response = await this.client.get( - `/api/v1/evaluator-results/${resultId}/audio`, - { responseType: 'arraybuffer' }, - ) - return response.data as ArrayBuffer - } - - async getEvaluatorResultAudioBlob(resultId: string): Promise { - const response = await this.client.get( - `/api/v1/evaluator-results/${resultId}/audio`, - { responseType: 'blob' }, - ) - return response.data as Blob - } - - async getEvaluatorResultAudioUrl(resultId: string): Promise { - const blob = await this.getEvaluatorResultAudioBlob(resultId) - return URL.createObjectURL(blob) - } - - async createEvaluatorResultManual(data: { - evaluator_id: string - audio_s3_key: string - duration_seconds?: number - }): Promise { - const response = await this.client.post('/api/v1/evaluator-results', data) - return response.data - } - - async reEvaluateResult(id: string): Promise { - const response = await this.client.post(`/api/v1/evaluator-results/${id}/re-evaluate`) - return response.data - } - - async deleteEvaluatorResult(id: string): Promise { - await this.client.delete(`/api/v1/evaluator-results/${id}`) - } - - async deleteEvaluatorResultsBulk(ids: string[]): Promise { - // FastAPI expects multiple query parameters with the same name - // Build query string manually: result_ids=id1&result_ids=id2 - const params = new URLSearchParams() - ids.forEach(id => params.append('result_ids', id)) - await this.client.delete(`/api/v1/evaluator-results?${params.toString()}`) - } - - async getMetric(metricId: string): Promise { - const response = await this.client.get(`/api/v1/metrics/${metricId}`) - return response.data - } - - async updateMetric(metricId: string, data: { - name?: string - description?: string - /** - * Pass ``""`` (empty string) to clear the stored example; omit to - * leave the persisted value untouched. - */ - example?: string | null - metric_type?: 'number' | 'boolean' | 'rating' | 'text' - trigger?: 'always' - enabled?: boolean - metric_origin?: 'default' | 'custom' - supported_surfaces?: string[] - enabled_surfaces?: string[] - custom_data_type?: 'boolean' | 'enum' | 'number_range' - custom_config?: Record - tags?: string[] - capture_rationale?: boolean - selection_mode?: 'single_choice' | 'multi_label' | null - /** Only honored on multi_label parents; backend 400s otherwise. */ - allow_discovery?: boolean - }): Promise { - const response = await this.client.put(`/api/v1/metrics/${metricId}`, data) - return response.data - } - - async deleteMetric(metricId: string): Promise { - await this.client.delete(`/api/v1/metrics/${metricId}`) - } - - async seedDefaultMetrics(): Promise { - const response = await this.client.post('/api/v1/metrics/seed-defaults') - return response.data - } - - async generateMetric(data: { - mode: 'description' | 'examples' - surface: 'agent' | 'chat_agent' | 'voice_playground' - description?: string - examples?: Array<{ transcript: string; rating: any; notes?: string }> - provider?: string - model?: string - credential_id?: string - llm_config?: Record - }): Promise<{ - name: string - description: string - metric_type: 'rating' | 'boolean' | 'number' | 'text' - custom_data_type: 'boolean' | 'enum' | 'number_range' | null - custom_config: Record - supported_surfaces: string[] - enabled_surfaces: string[] - suggested_tags: string[] - provider?: string - model?: string - }> { - const response = await this.client.post('/api/v1/metrics/generate', data) - return response.data - } - - async parseBulkMetric(data: { - prompt: string - surface: 'agent' | 'chat_agent' | 'voice_playground' - /** When set, the response includes a ``parent`` block + all labels are children. */ - parent_name?: string - parent_description?: string - selection_mode?: 'single_choice' | 'multi_label' - }): Promise<{ - metrics: Array<{ - name: string - description: string - metric_type: 'rating' | 'boolean' | 'number' | 'text' - custom_data_type: 'boolean' | 'enum' | 'number_range' | null - custom_config: Record - supported_surfaces: string[] - enabled_surfaces: string[] - capture_rationale: boolean - suggested_tags: string[] - source_label: { label_name: string; definition: string; examples: string } - }> - parent?: { - name: string - description: string | null - selection_mode: 'single_choice' | 'multi_label' - supported_surfaces: string[] - enabled_surfaces: string[] - } | null - }> { - const response = await this.client.post('/api/v1/metrics/parse-bulk', data) - return response.data - } - - // Alert endpoints - async listAlerts(status?: string): Promise { - const params: any = {} - if (status) { - params.status_filter = status - } - const response = await this.client.get('/api/v1/alerts', { params }) - return response.data - } - - async getAlert(alertId: string): Promise { - const response = await this.client.get(`/api/v1/alerts/${alertId}`) - return response.data - } - - async createAlert(data: { - name: string - description?: string | null - data_source?: string - metric_type: string - aggregation: string - operator: string - threshold_value: number - time_window_minutes: number - agent_ids?: string[] | null - notify_frequency: string - notify_emails?: string[] - notify_webhooks?: string[] - notify_pagerduty_routing_keys?: string[] - }): Promise { - const response = await this.client.post('/api/v1/alerts', data) - return response.data - } - - async updateAlert(alertId: string, data: { - name?: string - description?: string | null - data_source?: string - metric_type?: string - aggregation?: string - operator?: string - threshold_value?: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: string - notify_emails?: string[] - notify_webhooks?: string[] - notify_pagerduty_routing_keys?: string[] - status?: string - }): Promise { - const response = await this.client.put(`/api/v1/alerts/${alertId}`, data) - return response.data - } - - async deleteAlert(alertId: string): Promise { - await this.client.delete(`/api/v1/alerts/${alertId}`) - } - - async toggleAlertStatus(alertId: string): Promise { - const response = await this.client.post(`/api/v1/alerts/${alertId}/toggle`) - return response.data - } - - // Alert History endpoints - async getOpenIncidentSummary(): Promise<{ total: number; by_alert_id: Record }> { - const response = await this.client.get('/api/v1/alerts/history/open-summary') - return response.data - } - - async listAlertHistory( - status?: string, - alertId?: string, - skip = 0, - limit = 100, - openOnly = false - ): Promise { - const params: any = { skip, limit } - if (openOnly) { - params.open_only = true - } else if (status) { - params.status_filter = status - } - if (alertId) { - params.alert_id = alertId - } - const response = await this.client.get('/api/v1/alerts/history/all', { params }) - return response.data - } - - async getAlertHistoryItem(historyId: string): Promise { - const response = await this.client.get(`/api/v1/alerts/history/${historyId}`) - return response.data - } - - async acknowledgeAlertHistory(historyId: string): Promise { - const response = await this.client.put(`/api/v1/alerts/history/${historyId}`, { - status: 'acknowledged' - }) - return response.data - } - - async resolveAlertHistory(historyId: string, resolutionNotes?: string): Promise { - const response = await this.client.put(`/api/v1/alerts/history/${historyId}`, { - status: 'resolved', - resolution_notes: resolutionNotes - }) - return response.data - } - - // Alert Evaluation & Notification endpoints - async triggerAlert(alertId: string): Promise { - const response = await this.client.post(`/api/v1/alerts/${alertId}/trigger`) - return response.data - } - - async evaluateAllAlerts(): Promise { - const response = await this.client.post('/api/v1/alerts/evaluate/all') - return response.data - } - - async testAlertNotification(alertId: string, data: { - webhook_url?: string - email?: string - pagerduty_routing_key?: string - }): Promise { - const response = await this.client.post(`/api/v1/alerts/${alertId}/test-notification`, data) - return response.data - } - - // Cron Job endpoints - async listCronJobs(): Promise { - const response = await this.client.get('/api/v1/cron-jobs') - return response.data - } - - async getCronJob(cronJobId: string): Promise { - const response = await this.client.get(`/api/v1/cron-jobs/${cronJobId}`) - return response.data - } - - async createCronJob(data: { - name: string - cron_expression: string - timezone: string - max_runs: number - interval_days?: number | null - evaluator_ids: string[] - }): Promise { - const response = await this.client.post('/api/v1/cron-jobs', data) - return response.data - } - - async updateCronJob(cronJobId: string, data: { - name?: string - cron_expression?: string - timezone?: string - max_runs?: number - interval_days?: number | null - evaluator_ids?: string[] - status?: string - }): Promise { - const response = await this.client.put(`/api/v1/cron-jobs/${cronJobId}`, data) - return response.data - } - - async deleteCronJob(cronJobId: string): Promise { - await this.client.delete(`/api/v1/cron-jobs/${cronJobId}`) - } - - async toggleCronJobStatus(cronJobId: string): Promise { - const response = await this.client.post(`/api/v1/cron-jobs/${cronJobId}/toggle`) - return response.data - } - - async runCronJobNow(cronJobId: string): Promise { - const response = await this.client.post(`/api/v1/cron-jobs/${cronJobId}/run`) - return response.data - } - - // Voice Playground (TTS Comparison) endpoints - async listTTSProviders(): Promise { - const response = await this.client.get('/api/v1/voice-playground/tts-providers') - return response.data - } - - async listCustomTTSVoices(provider?: string): Promise> { - const response = await this.client.get('/api/v1/voice-playground/custom-voices', { - params: provider ? { provider } : undefined, - }) - return response.data - } - - async createCustomTTSVoice(data: { - provider: string - voice_id: string - name: string - gender?: string - accent?: string - description?: string - }): Promise { - const response = await this.client.post('/api/v1/voice-playground/custom-voices', data) - return response.data - } - - async updateCustomTTSVoice(customVoiceId: string, data: { - voice_id?: string - name?: string - gender?: string - accent?: string - description?: string - }): Promise { - const response = await this.client.put(`/api/v1/voice-playground/custom-voices/${customVoiceId}`, data) - return response.data - } - - async deleteCustomTTSVoice(customVoiceId: string): Promise { - const response = await this.client.delete(`/api/v1/voice-playground/custom-voices/${customVoiceId}`) - return response.data - } - - async createTTSComparison(data: { - name?: string - mode?: 'benchmark' | 'blind_test_only' - // Legacy benchmark fields - provider_a?: string - model_a?: string - voices_a?: Array<{ id: string; name: string; sample_rate_hz?: number }> - provider_b?: string - model_b?: string - voices_b?: Array<{ id: string; name: string; sample_rate_hz?: number }> - // New per-side benchmark config - side_a?: VoicePlaygroundSideConfig - side_b?: VoicePlaygroundSideConfig - // Common - sample_texts?: string[] - num_runs?: number - eval_stt_provider?: string - eval_stt_model?: string - // blind_test_only - pairs?: Array - }): Promise { - const response = await this.client.post('/api/v1/voice-playground/comparisons', data) - return response.data - } - - async listVoicePlaygroundCallImportRows(params: { - call_import_id?: string - with_recording?: boolean - skip?: number - limit?: number - } = {}): Promise<{ - items: Array<{ - id: string - call_import_id: string - call_import_filename: string | null - conversation_id: string - transcript: string | null - recording_s3_key: string | null - has_recording: boolean - status: string - created_at: string | null - }> - total: number - skip: number - limit: number - }> { - const response = await this.client.get('/api/v1/voice-playground/call-import-rows', { params }) - return response.data - } - - async uploadVoicePlaygroundAudio(file: File): Promise<{ - s3_key: string - presigned_url: string | null - filename: string - size_bytes: number - content_type: string - }> { - const form = new FormData() - form.append('file', file) - const response = await this.client.post('/api/v1/voice-playground/uploads', form, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async listTTSComparisons(skip = 0, limit = 50): Promise { - const response = await this.client.get('/api/v1/voice-playground/comparisons', { - params: { skip, limit }, - }) - return response.data - } - - async getTTSComparison(comparisonId: string): Promise { - const response = await this.client.get(`/api/v1/voice-playground/comparisons/${comparisonId}`) - return response.data - } - - async generateTTSComparison(comparisonId: string): Promise<{ message: string; task_id: string }> { - const response = await this.client.post(`/api/v1/voice-playground/comparisons/${comparisonId}/generate`) - return response.data - } - - async submitBlindTest(comparisonId: string, results: Array<{ - sample_index: number - preferred: 'A' | 'B' - voice_a_id: string - voice_b_id: string - }>): Promise { - const response = await this.client.post( - `/api/v1/voice-playground/comparisons/${comparisonId}/blind-test`, - { results } - ) - return response.data - } - - async deleteTTSComparison(comparisonId: string): Promise { - await this.client.delete(`/api/v1/voice-playground/comparisons/${comparisonId}`) - } - - // Blind Test Sharing (owner side) - async createBlindTestShare(comparisonId: string, payload: { - title: string - description?: string - creator_notes?: string - custom_metrics: Array<{ key: string; label: string; type: 'rating' | 'comment'; scale?: number }> - }): Promise { - const response = await this.client.post( - `/api/v1/voice-playground/comparisons/${comparisonId}/share`, - payload - ) - return response.data - } - - async getBlindTestShare(comparisonId: string): Promise { - const response = await this.client.get( - `/api/v1/voice-playground/comparisons/${comparisonId}/share` - ) - return response.data - } - - async updateBlindTestShare(shareId: string, payload: { - title?: string - description?: string - creator_notes?: string - custom_metrics?: Array<{ key: string; label: string; type: 'rating' | 'comment'; scale?: number }> - status?: 'open' | 'closed' - }): Promise { - const response = await this.client.patch( - `/api/v1/voice-playground/shares/${shareId}`, - payload - ) - return response.data - } - - async deleteBlindTestShare(shareId: string): Promise { - await this.client.delete(`/api/v1/voice-playground/shares/${shareId}`) - } - - async listBlindTestResponses(shareId: string, skip = 0, limit = 100): Promise<{ - items: any[] - total: number - }> { - const response = await this.client.get( - `/api/v1/voice-playground/shares/${shareId}/responses`, - { params: { skip, limit } } - ) - return response.data - } - - async generateSampleTexts(params: { - voice_bundle_id?: string - provider?: string - model?: string - scenario?: string - count?: number - length?: string - temperature?: number - llm_config?: LLMGenerationConfig | null - }): Promise<{ samples: string[]; provider: string; model: string }> { - const response = await this.client.post('/api/v1/voice-playground/generate-samples', params) - return response.data - } - - async getTTSAnalytics(): Promise> { - const response = await this.client.get('/api/v1/voice-playground/analytics') - return response.data - } - - async downloadTTSComparisonReport( - comparisonId: string, - includeUnfinishedSamples = false, - reportOptions?: TTSReportOptionsPayload - ): Promise { - const response = await this.client.get( - `/api/v1/voice-playground/comparisons/${comparisonId}/report.pdf`, - { - params: { - include_unfinished_samples: includeUnfinishedSamples, - ...(reportOptions ? { report_options: JSON.stringify(reportOptions) } : {}), - }, - responseType: 'blob', - } - ) - return response.data - } - - async createTTSComparisonReportJob( - comparisonId: string, - reportOptions?: TTSReportOptionsPayload - ): Promise<{ - id: string - comparison_id: string - status: string - format: string - task_id?: string - report_options?: TTSReportOptionsPayload - created_at?: string | null - }> { - const response = await this.client.post( - `/api/v1/voice-playground/comparisons/${comparisonId}/reports`, - reportOptions ? { report_options: reportOptions } : {} - ) - return response.data - } - - async getVoicePlaygroundReportThresholdDefaults(): Promise<{ - zone_threshold_overrides: NonNullable - is_custom: boolean - }> { - const response = await this.client.get('/api/v1/voice-playground/report-threshold-defaults') - return response.data - } - - async updateVoicePlaygroundReportThresholdDefaults(data: { - zone_threshold_overrides?: NonNullable - reset_to_system_defaults?: boolean - }): Promise<{ - zone_threshold_overrides: NonNullable - is_custom: boolean - message: string - }> { - const response = await this.client.put('/api/v1/voice-playground/report-threshold-defaults', data) - return response.data - } - - async getTTSComparisonReportJob(reportJobId: string): Promise<{ - id: string - comparison_id: string - status: string - format: string - filename?: string | null - error_message?: string | null - task_id?: string | null - download_url?: string | null - report_options?: TTSReportOptionsPayload - created_at?: string | null - updated_at?: string | null - }> { - const response = await this.client.get(`/api/v1/voice-playground/reports/${reportJobId}`) - return response.data - } - - // Prompt Partials - async listPromptPartials( - skip = 0, - limit = 100, - search?: string, - kind: 'all' | 'partial' | 'imported_agent' | 'metric' = 'all', - ): Promise { - const response = await this.client.get('/api/v1/prompt-partials', { - params: { - skip, - limit, - kind, - ...(search ? { search } : {}), - }, - }) - return response.data - } - - async listImportedAgents(skip = 0, limit = 100, search?: string): Promise { - return this.listPromptPartials(skip, limit, search, 'imported_agent') - } - - async listMetricPartials(skip = 0, limit = 100, search?: string): Promise { - return this.listPromptPartials(skip, limit, search, 'metric') - } - - async createImportedAgent(data: { - name: string - description?: string | null - content: string - }): Promise { - const response = await this.client.post('/api/v1/prompt-partials', { - ...data, - tags: ['__imported_agent__'], - }) - return response.data - } - - async createMetricPartial(data: { - name: string - description?: string | null - content: string - }): Promise { - const response = await this.client.post('/api/v1/prompt-partials', { - ...data, - tags: ['__metric_partial__'], - }) - return response.data - } - - async updateMetricPartial( - partialId: string, - data: { - name?: string - description?: string | null - content?: string - change_summary?: string - }, - ): Promise { - const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, { - ...data, - tags: ['__metric_partial__'], - }) - return response.data - } - - async generateAgentFlowchart( - partialId: string, - options?: { - provider?: string - model?: string - credential_id?: string - regenerate?: boolean - }, - ): Promise { - const response = await this.client.post( - `/api/v1/prompt-partials/${partialId}/flowchart`, - { - provider: options?.provider, - model: options?.model, - credential_id: options?.credential_id, - regenerate: options?.regenerate ?? false, - }, - ) - return response.data - } - - async saveAgentFlowchartLayout( - partialId: string, - nodes: Array<{ id: string; position_x: number; position_y: number }>, - ): Promise { - const response = await this.client.put( - `/api/v1/prompt-partials/${partialId}/flowchart/layout`, - { nodes }, - ) - return response.data - } - - async mapAgentFlowchartPromptSections( - partialId: string, - options?: { provider?: string; model?: string; credential_id?: string }, - ): Promise { - const response = await this.client.post( - `/api/v1/prompt-partials/${partialId}/flowchart/prompt-map`, - { - provider: options?.provider, - model: options?.model, - credential_id: options?.credential_id, - }, - ) - return response.data - } - - async updateImportedAgent( - partialId: string, - data: { - name?: string - description?: string | null - content?: string - change_summary?: string - tags?: string[] - }, - ): Promise { - const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, data) - return response.data - } - - async getPromptPartial(partialId: string): Promise { - const response = await this.client.get(`/api/v1/prompt-partials/${partialId}`) - return response.data - } - - async createPromptPartial(data: { - name: string - description?: string - content: string - tags?: string[] - }): Promise { - const response = await this.client.post('/api/v1/prompt-partials', data) - return response.data - } - - async updatePromptPartial(partialId: string, data: { - name?: string - description?: string | null - content?: string - tags?: string[] - change_summary?: string - }): Promise { - const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, data) - return response.data - } - - async deletePromptPartial(partialId: string): Promise { - await this.client.delete(`/api/v1/prompt-partials/${partialId}`) - } - - async listPromptPartialVersions(partialId: string): Promise { - const response = await this.client.get(`/api/v1/prompt-partials/${partialId}/versions`) - return response.data - } - - async getPromptPartialVersion(partialId: string, versionNumber: number): Promise { - const response = await this.client.get(`/api/v1/prompt-partials/${partialId}/versions/${versionNumber}`) - return response.data - } - - async revertPromptPartial(partialId: string, versionNumber: number): Promise { - const response = await this.client.post(`/api/v1/prompt-partials/${partialId}/revert/${versionNumber}`) - return response.data - } - - async previewPromptPartialVersionPush( - partialId: string, - versionNumber: number, - ): Promise { - const response = await this.client.get( - `/api/v1/prompt-partials/${partialId}/versions/${versionNumber}/push-preview`, - ) - return response.data - } - - async pushPromptPartialVersion( - partialId: string, - versionNumber: number, - confirm: string, - ): Promise { - const response = await this.client.post( - `/api/v1/prompt-partials/${partialId}/versions/${versionNumber}/push`, - { confirm }, - ) - return response.data - } - - async clonePromptPartial(partialId: string): Promise { - const response = await this.client.post(`/api/v1/prompt-partials/${partialId}/clone`) - return response.data - } - - async generatePromptWithAI(data: { - description: string - tone?: string - format_style?: string - provider?: string - model?: string - credential_id?: string - llm_config?: LLMGenerationConfig | null - }): Promise<{ content: string; provider: string; model: string }> { - const response = await this.client.post('/api/v1/prompt-partials/generate', data) - return response.data - } - - async improvePromptWithAI(data: { - content: string - instructions?: string - provider?: string - model?: string - credential_id?: string - llm_config?: LLMGenerationConfig | null - }): Promise<{ content: string; provider: string; model: string }> { - const response = await this.client.post('/api/v1/prompt-partials/improve', data) - return response.data - } - - // License / Enterprise - async getLicenseInfo(): Promise { - const response = await this.client.get('/api/v1/settings/license-info') - return response.data - } - - async getReportBranding(): Promise { - const response = await this.client.get('/api/v1/settings/report-branding') - return response.data - } - - async updateReportBranding(data: { heading?: string | null }): Promise { - const response = await this.client.patch('/api/v1/settings/report-branding', data) - return response.data - } - - async uploadReportBrandingImages( - files: File[], - role: 'internal' | 'external' | 'generic' = 'generic', - ): Promise { - const formData = new FormData() - for (const file of files) { - formData.append('files', file) - } - formData.append('role', role) - const response = await this.client.post('/api/v1/settings/report-branding/images', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }) - return response.data - } - - async deleteReportBrandingImage(imageId: string): Promise { - const response = await this.client.delete(`/api/v1/settings/report-branding/images/${imageId}`) - return response.data - } - - // GEPA Prompt Optimization (Enterprise) - async createOptimizationRun(data: { - agent_id: string - evaluator_id?: string - voice_bundle_id?: string - config?: Record - }): Promise { - const response = await this.client.post('/api/v1/prompt-optimization/runs', data) - return response.data - } - - async deleteOptimizationRun(runId: string): Promise { - await this.client.delete(`/api/v1/prompt-optimization/runs/${runId}`) - } - - async listOptimizationRuns(agentId?: string): Promise { - const params = agentId ? { agent_id: agentId } : {} - const response = await this.client.get('/api/v1/prompt-optimization/runs', { params }) - return response.data - } - - async getOptimizationRun(runId: string): Promise { - const response = await this.client.get(`/api/v1/prompt-optimization/runs/${runId}`) - return response.data - } - - async listOptimizationCandidates(runId: string): Promise { - const response = await this.client.get(`/api/v1/prompt-optimization/runs/${runId}/candidates`) - return response.data - } - - async acceptCandidate(runId: string, candidateId: string): Promise { - const response = await this.client.post( - `/api/v1/prompt-optimization/runs/${runId}/candidates/${candidateId}/accept` - ) - return response.data - } - - async pushCandidateToProvider(runId: string, candidateId: string): Promise { - const response = await this.client.post( - `/api/v1/prompt-optimization/runs/${runId}/candidates/${candidateId}/push` - ) - return response.data - } - - async syncProviderPrompt(agentId: string): Promise<{ - synced: boolean - provider_prompt: string | null - provider_prompt_synced_at: string | null - }> { - const response = await this.client.post(`/api/v1/agents/${agentId}/sync-provider-prompt`) - return response.data - } - - // ------------------------------------------------------------------- - // Judge Alignment (AlignEval-style hybrid integration) - // - // Per-org thresholds and judge model selection are sourced from the - // backend (no hardcoded defaults in the frontend). - // ------------------------------------------------------------------- - - async getJudgeAlignmentSettings(): Promise { - const response = await this.client.get('/api/v1/judge-alignment/settings') - return response.data - } - - async updateJudgeAlignmentSettings( - data: { min_labels_to_evaluate: number; min_labels_to_optimize: number } - ): Promise { - const response = await this.client.patch('/api/v1/judge-alignment/settings', data) - return response.data - } - - async listJudgeCapableModels(): Promise { - const response = await this.client.get('/api/v1/judge-alignment/available-models') - return response.data - } - - async listJudgeDatasets(): Promise { - const response = await this.client.get('/api/v1/judge-alignment/datasets') - return response.data - } - - async createJudgeDataset(data: { - name: string - description?: string - source_type: 'transcript' | 'metric_output' | 'csv' - source_config?: Record - input_field?: string - output_field?: string - }): Promise { - const response = await this.client.post('/api/v1/judge-alignment/datasets', data) - return response.data - } - - async uploadJudgeDatasetCsv( - file: File, - name: string, - description?: string - ): Promise { - const formData = new FormData() - formData.append('file', file) - formData.append('name', name) - if (description) formData.append('description', description) - const response = await this.client.post( - '/api/v1/judge-alignment/datasets/upload-csv', - formData, - { headers: { 'Content-Type': 'multipart/form-data' } } - ) - return response.data - } - - async getJudgeDataset(datasetId: string): Promise { - const response = await this.client.get(`/api/v1/judge-alignment/datasets/${datasetId}`) - return response.data - } - - async deleteJudgeDataset(datasetId: string): Promise { - await this.client.delete(`/api/v1/judge-alignment/datasets/${datasetId}`) - } - - async listJudgeSamples( - datasetId: string, - options: { only_labeled?: boolean; skip?: number; limit?: number } = {} - ): Promise { - const response = await this.client.get( - `/api/v1/judge-alignment/datasets/${datasetId}/samples`, - { params: options } - ) - return response.data - } - - async labelJudgeSample( - sampleId: string, - label: 'pass' | 'fail' | null - ): Promise { - const response = await this.client.patch( - `/api/v1/judge-alignment/samples/${sampleId}`, - { label } - ) - return response.data - } - - async bulkLabelJudgeSamples( - datasetId: string, - items: Array<{ sample_id: string; label: 'pass' | 'fail' | null }> - ): Promise<{ updated: number; requested: number }> { - const response = await this.client.post( - `/api/v1/judge-alignment/datasets/${datasetId}/samples/bulk-label`, - { items } - ) - return response.data - } - - async listJudgeRuns(datasetId: string): Promise { - const response = await this.client.get( - `/api/v1/judge-alignment/datasets/${datasetId}/runs` - ) - return response.data - } - - async triggerJudgeRun( - datasetId: string, - data: { evaluator_id: string; split?: 'all' | 'dev' | 'test'; sample_ids?: string[] } - ): Promise { - const response = await this.client.post( - `/api/v1/judge-alignment/datasets/${datasetId}/run`, - data - ) - return response.data - } - - async getJudgeRun(runId: string): Promise { - const response = await this.client.get(`/api/v1/judge-alignment/runs/${runId}`) - return response.data - } - - async recomputeJudgeRunMetrics(runId: string): Promise { - const response = await this.client.post( - `/api/v1/judge-alignment/runs/${runId}/recompute-metrics` - ) - return response.data - } - - async deleteJudgeRun(runId: string): Promise { - await this.client.delete(`/api/v1/judge-alignment/runs/${runId}`) - } - - async optimizeJudge( - datasetId: string, - data: { - evaluator_id: string - dev_ratio?: number - seed?: number - max_metric_calls?: number - minibatch_size?: number - agent_id?: string - } - ): Promise { - const response = await this.client.post( - `/api/v1/judge-alignment/datasets/${datasetId}/optimize`, - data - ) - return response.data - } -} - -// ------------------------------------------------------------------- -// Judge Alignment shared types -// ------------------------------------------------------------------- - -export interface JudgeAlignmentSettings { - min_labels_to_evaluate: number - min_labels_to_optimize: number - defaults: { min_labels_to_evaluate: number; min_labels_to_optimize: number } -} - -export interface JudgeModelCatalogEntry { - provider: string - provider_label: string - model: string - label: string -} - -export interface JudgeDataset { - id: string - name: string - description?: string | null - source_type: 'transcript' | 'metric_output' | 'csv' - source_config: Record - input_field: string - output_field: string - total_samples: number - labeled_samples: number - unlabeled_samples: number - created_at?: string - updated_at?: string -} - -export interface JudgeSample { - id: string - dataset_id: string - external_id?: string | null - input_text: string - output_text: string - label?: 'pass' | 'fail' | null - labeled_by?: string | null - labeled_at?: string | null - extra?: Record | null - created_at?: string -} - -export interface JudgeRunMetrics { - precision: number - recall: number - f1: number - kappa: number - tp: number - fp: number - tn: number - fn: number - n: number -} - -export interface JudgeRun { - id: string - dataset_id: string - evaluator_id?: string | null - split: 'all' | 'dev' | 'test' - llm_provider?: string | null - llm_model?: string | null - status: 'pending' | 'queued' | 'running' | 'completed' | 'failed' - metrics?: JudgeRunMetrics | null - predictions?: Record | null - error_message?: string | null - celery_task_id?: string | null - gepa_optimization_id?: string | null - created_at?: string - updated_at?: string -} - -export interface JudgeOptimizeResponse { - optimization_run_id: string - dev_sample_count: number - test_sample_count: number -} - -export const AUDIO_UPLOAD_CHUNK_SIZE = 25 - -export interface CallImportAudioUploadProgress { - chunkIndex: number - totalChunks: number - uploadedFiles: number - totalFiles: number -} - -// Factory function to create ApiClient instance -// This ensures TypeScript correctly infers the type as ApiClient, not AxiosInstance -function createApiClient(): ApiClient { - return new ApiClient() -} - -// Create instance -const apiClientInstance = createApiClient() - -// Export with explicit type annotation - CRITICAL for TypeScript to recognize as ApiClient -// Without this explicit type, TypeScript may incorrectly infer AxiosInstance -export const apiClient: ApiClient = apiClientInstance - -// Re-export the type for explicit typing if needed -export type { ApiClient } - - -// ---------------------------------------------------------------------- -// Public (unauthenticated) blind-test API. -// Uses a bare axios instance so no Authorization / X-API-Key headers are -// attached. Anyone with the share_token in the URL can call these. -// ---------------------------------------------------------------------- - -const publicClient = axios.create({ - baseURL: API_BASE_URL, - headers: { 'Content-Type': 'application/json' }, -}) - -export interface PublicBlindTestEntrySubmit { - sample_index: number - preferred: 'X' | 'Y' - ratings_x?: Record - ratings_y?: Record - comment?: string -} - -export const publicBlindTestApi = { - async getForm(shareToken: string): Promise { - const res = await publicClient.get(`/api/v1/public/blind-tests/${shareToken}`) - return res.data - }, - async submit(shareToken: string, payload: { - rater_name: string - rater_email: string - client_token: string - responses: PublicBlindTestEntrySubmit[] - }): Promise { - const res = await publicClient.post( - `/api/v1/public/blind-tests/${shareToken}/responses`, - payload - ) - return res.data - }, -} +import axios, { AxiosInstance } from 'axios' +export { uploadFileToPresignedPutUrl } from './presignedUpload' +import { CallImportsApi, type CallImportAudioUploadProgress } from '../pages/callImports/api/callImportsApi' +export type { CallImportAudioUploadProgress } from '../pages/callImports/api/callImportsApi' +export { AUDIO_UPLOAD_CHUNK_SIZE } from '../pages/callImports/api/callImportsApi' +import { + clearAuthSession, + csrfHeaders, + getApiErrorDetail, + hasRevocableUserCredentials, + isOrganizationAccessDenied, + organizationAccessDeniedMessage, + redirectToLoginWithMessage, + type UserSessionCredentials, +} from './authSession' +import type { + GenerateScenariosFromPromptParams, + GenerateTestPromptParams, + GenerateTestSetupParams, +} from '../types/agentTestSetupGeneration' +import type { + AgentToolCatalogEntry, + AgentToolConnectionLink, + ExpectedToolCall, + ToolConnection, + ToolConnectionCreate, + ToolConnectionSecrets, + ToolSimulationMode, + AudioFile, + Evaluation, + EvaluationCreate, + EvaluationResult, + DashboardSummary, + ModelConfigEntry, + APIKey, + MessageResponse, + EvaluationStatus, + OrganizationMember, + Invitation, + InvitationAcceptResponse, + InvitationCreate, + InvitationPreview, + Profile, + UserUpdate, + UserPreferences, + UserPreferencesUpdate, + Role, + Integration, + IntegrationCreate, + ListIntegrationVoiceAgentsResponse, + S3ConnectionTestResponse, + S3ListFilesResponse, + S3BrowseResponse, + S3Status, + CallImport, + CallImportDetail, + CallImportListResponse, + CallImportRow, + ObservabilityCall, + CallImportSchema, + CallImportSchemaCreate, + CallImportSchemaListResponse, + CallImportSchemaUpdate, + CallImportStatus, + CallImportTag, + CallImportUploadResponse, + CallImportPreviewResponse, + CallImportEvaluation, + CallImportEvaluationBaselineCandidatesResponse, + CallImportEvaluationExportJob, + CallImportEvaluationPdfReport, + CallImportEvaluationPdfReportListResponse, + CallImportEvaluationLLMOverride, + CallImportEvaluationListResponse, + CallImportEvaluationRow, + CallImportEvaluationRowListResponse, + CallImportEvaluationRetryResponse, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationAggregateResponse, + CallImportInsightsResponse, + EvaluatorResultListResponse, + EvaluatorResultsOverviewResponse, + EvaluatorResultsAggregateResponse, + ListEvaluatorResultsParams, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportRowSelectionFilter, + CallImportRetryFailedRowsResponse, + PlatformImportProgress, + Workspace, + LLMGenerationConfig, + TestAgent, + SimulationRun, + SimulationRunCreate, + SimulationTrial, + SimulationTrialDetail, + SimulationFailureAnalysis, + SimulationFailureAnalysisRequest, + SimulationComposedPrompt, + SimulationRunPdfReportSections, +} from '../types/api' + +export interface EnterpriseFeatureMeta { + title: string + description?: string + category?: string +} + +export type EnterpriseFeatureCatalog = Record + +export type VoicePlaygroundSourceType = 'tts' | 'recording' | 'upload' + +export interface VoicePlaygroundSideConfig { + source_type: VoicePlaygroundSourceType + provider?: string + model?: string + voices?: Array<{ id: string; name: string; sample_rate_hz?: number }> + call_import_row_ids?: string[] + upload_s3_keys?: string[] +} + +export type VoicePlaygroundBlindTestRefType = 'recording' | 'upload' | 'tts_sample' + +export interface VoicePlaygroundBlindTestAudioRef { + type: VoicePlaygroundBlindTestRefType + call_import_row_id?: string + upload_s3_key?: string + tts_sample_id?: string + label?: string +} + +export interface VoicePlaygroundBlindTestPair { + text?: string + x: VoicePlaygroundBlindTestAudioRef + y: VoicePlaygroundBlindTestAudioRef +} + +export interface UsagePolicy { + extended_history: boolean + max_history_days: number | null +} + +export interface OssQuotas { + max_user_metrics: number | null + max_agents: number | null + max_org_members: number | null + max_workspaces: number | null +} + +export interface OssQuotaUsage { + user_metrics: number + agents: number + org_members: number + workspaces: number +} + +export interface LicenseInfoResponse { + is_enterprise: boolean + gateway_routing_allowed?: boolean + enabled_features: string[] + all_enterprise_features: string[] + feature_catalog?: EnterpriseFeatureCatalog + organization?: string + usage_policy?: UsagePolicy + quotas?: OssQuotas + quota_usage?: OssQuotaUsage +} + +export interface ReportBranding { + heading?: string | null + has_logo: boolean + images: Array<{ + id: string + filename: string + content_type: string + size_bytes: number + role: 'internal' | 'external' | 'generic' + updated_at?: string | null + data_uri?: string | null + }> +} + +export interface AuthProviderConfig { + name: 'api_key' | 'local_password' | 'external_oidc' + enabled: boolean + display_name: string + description?: string + supports_password?: boolean + supports_signup?: boolean + oidc_issuer?: string | null + oidc_client_id?: string | null + oidc_authorize_url?: string | null +} + +export interface AuthConfigResponse { + providers: AuthProviderConfig[] + tier: 'oss' | 'enterprise' + gated_signup?: boolean + cookie_session_enabled?: boolean +} + +export interface AuthUserSummary { + id: string + email: string + name?: string | null + first_name?: string | null + last_name?: string | null + organization_id: string + role?: string | null + has_password?: boolean + email_is_placeholder?: boolean +} + +export interface TokenResponse { + access_token: string + refresh_token?: string + token_type: string + expires_in: number + user: AuthUserSummary + join_notice?: string | null +} + +export interface LoginOrgOption { + id: string + name: string + role: string +} + +export interface LoginOrgSelectionResponse { + requires_org_selection: true + organizations: LoginOrgOption[] +} + +export interface PlatformAdminSummary { + id: string + email: string +} + +export interface PlatformTokenResponse { + access_token: string + token_type: string + expires_in: number + admin: PlatformAdminSummary +} + +export interface PlatformOrganizationItem { + id: string + name: string + is_active: boolean + member_count: number + created_at?: string | null + disabled_at?: string | null +} + +export interface PlatformOrganizationListResponse { + items: PlatformOrganizationItem[] + total: number + offset: number + limit: number +} + +export interface PlatformOrganizationStats { + total: number + active: number + disabled: number +} + +export interface PlatformOrgUser { + id: string + email: string + role: string + is_active: boolean +} + +export interface PlatformSignupCode { + id: string + label?: string | null + max_uses?: number | null + use_count: number + expires_at?: string | null + is_active: boolean + created_at?: string | null + code?: string | null +} + +export type LoginResponse = TokenResponse | LoginOrgSelectionResponse + +export function isLoginOrgSelectionResponse( + response: LoginResponse +): response is LoginOrgSelectionResponse { + return 'requires_org_selection' in response && response.requires_org_selection === true +} + +export interface OrganizationSummary { + id: string + name: string +} + +export type LLMGatewayMode = 'inherit' | 'enabled' | 'disabled' +export type LLMGatewayType = 'inherit' | 'bifrost' | 'litellm_proxy' +export type LLMGatewayInterface = 'inherit' | 'litellm_shim' | 'native_openai' +export type LLMGatewayRouting = 'direct' | 'bifrost' | 'litellm_proxy' +export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' + +export interface LLMGatewaySettings { + mode: LLMGatewayMode + gateway_type: LLMGatewayType + gateway_interface?: LLMGatewayInterface + base_url?: string | null + has_virtual_key: boolean + has_master_key: boolean + platform_enabled: boolean + platform_gateway_type: 'bifrost' | 'litellm_proxy' + platform_gateway_interface?: 'litellm_shim' | 'native_openai' + platform_base_url?: string | null + effective_routing: LLMGatewayRouting + effective_gateway_type?: 'bifrost' | 'litellm_proxy' | null + effective_gateway_interface?: 'litellm_shim' | 'native_openai' | null + effective_base_url?: string | null + effective_has_virtual_key: boolean + effective_has_master_key: boolean + gateway_managed_credentials: boolean +} + +export interface LLMGatewaySettingsUpdate { + mode: LLMGatewayMode + gateway_type?: LLMGatewayType + gateway_interface?: LLMGatewayInterface + base_url?: string | null + virtual_key?: string | null + master_key?: string | null + clear_virtual_key?: boolean + clear_master_key?: boolean +} + +export interface AlertingSyncSettings { + sync_notification_lifecycle: boolean + pagerduty_webhook_url: string | null + has_pagerduty_webhook_signing_secret: boolean + pagerduty_inbound_ready: boolean + confirmation_phrase_required: string +} + +export interface AlertingSyncSettingsUpdate { + sync_notification_lifecycle: boolean + confirm_enable?: boolean + confirmation_phrase?: string + pagerduty_webhook_signing_secret?: string + clear_pagerduty_webhook_signing_secret?: boolean +} + +export interface TelephonyIntegrationResponse { + id: string + organization_id: string + provider: string + name?: string | null + verify_app_uuid?: string | null + voice_app_id?: string | null + sip_domain?: string | null + masking_config?: Record | null + is_active: boolean + /** True if this row is the default credential for (org, provider). */ + is_default?: boolean + last_tested_at?: string | null + created_at: string + updated_at: string +} + +export interface TelephonyIntegrationCreatePayload { + provider?: string + name?: string + auth_id: string + auth_token: string + verify_app_uuid?: string + voice_app_id?: string + sip_domain?: string + masking_config?: Record + /** Mark the new credential as the default for (org, provider). */ + is_default?: boolean +} + +export interface TelephonyIntegrationUpdatePayload { + /** When set, update this specific credential row instead of the legacy single-row flow. */ + id?: string + provider?: string + name?: string + auth_id?: string + auth_token?: string + verify_app_uuid?: string + voice_app_id?: string + sip_domain?: string + masking_config?: Record + is_active?: boolean +} + +export interface TelephonyPhoneNumberResponse { + id: string + phone_number: string + country_iso2?: string | null + region?: string | null + number_type?: string | null + capabilities?: Record | null + is_masking_pool: boolean + inbound_enabled?: boolean + outbound_enabled?: boolean + source?: string + agent_id?: string | null + linked_agent_name?: string | null + provider?: string | null + is_active: boolean + created_at: string +} + +export interface TelephonyAvailableNumber { + e164: string + provider_number_id?: string | null + country?: string | null + region?: string | null + capabilities?: Record | null + status?: string | null + application_id?: string | null + already_imported: boolean + imported_number_id?: string | null +} + +export interface TelephonyImportNumberResult { + number: string + success: boolean + message: string + answer_url: string + webhook_configured?: boolean | null + imported_number_id?: string | null + application_id?: string | null +} + +export interface TelephonyImportNumbersResponse { + provider: string + results: TelephonyImportNumberResult[] + answer_url: string +} + +/** @deprecated Use TelephonyAvailableNumber */ +export type VobizAvailableNumber = TelephonyAvailableNumber + +/** @deprecated Use TelephonyImportNumbersResponse */ +export type VobizImportNumbersResponse = TelephonyImportNumbersResponse + +/** @deprecated Use TelephonyImportNumberResult */ +export type VobizImportNumberResult = TelephonyImportNumberResult + +export interface PlatformOutboundPoolNumber { + phone_number: string + provider: string +} + +export interface PlatformOutboundPoolResponse { + numbers: PlatformOutboundPoolNumber[] + max_concurrent_per_org: number + shared_across_orgs: boolean +} + +/** @deprecated Use PlatformOutboundPoolResponse */ +export type VobizOutboundPoolResponse = PlatformOutboundPoolResponse + +export interface VobizOutboundCallRequest { + to_number: string + agent_id: string + from_number?: string + persona_id?: string + scenario_id?: string + evaluator_id?: string +} + +export interface TelephonyDialTargetResponse { + id: string + phone_number: string + label?: string | null + created_at: string + updated_at: string +} + +export interface VobizOutboundCallResponse { + provider_request_uuid: string + call_status: string + from_number: string + to_number: string + call_ref: string + call_short_id?: string + evaluator_result_id?: string + result_id?: string + message: string +} + +export interface EvaluatorSuiteCombination { + id: string + evaluator_id: string + persona_id?: string + persona_name?: string | null + scenario_id: string + scenario_name?: string | null + scenario_description?: string | null + scenario_required_info?: Record | null +} + +export interface EvaluatorSuitePersonaSummary { + id: string + name?: string | null + tts_provider?: string | null +} + +export interface EvaluatorSuite { + id: string + organization_id: string + name?: string | null + agent_id: string + persona_id: string + persona_ids?: string[] + personas?: EvaluatorSuitePersonaSummary[] + agent_name?: string | null + persona_name?: string | null + agent_call_type?: string | null + agent_call_medium?: string | null + voice_bundle_tts_provider?: string | null + metric_ids?: string[] | null + llm_provider?: string | null + llm_model?: string | null + tags?: string[] | null + default_runs_per_combination: number + round_robin_index: number + is_active: boolean + agent_suite_count: number + combination_count: number + combinations: EvaluatorSuiteCombination[] + created_at: string + updated_at: string +} + +export interface RunEvaluatorSuiteResponse { + total_runs: number + task_ids: string[] + evaluator_results: any[] + phone_call_refs: string[] +} + +export interface RunNextCombinationResponse { + evaluator_id: string + persona_id?: string | null + persona_name?: string | null + scenario_id: string + scenario_name: string + combination_index: number + next_index: number + evaluator_result_id?: string | null + result_id?: string | null + task_id?: string | null + phone_call_ref?: string | null + call_short_id?: string | null +} + +export interface ChooseNextCombinationResponse { + evaluator_id: string + persona_id?: string | null + persona_name?: string | null + scenario_id: string + scenario_name: string + combination_index: number + next_index: number +} + +type TTSReportOptionsPayload = { + show_runs?: boolean + min_runs_to_show?: number + include_latency?: boolean + include_ttfb?: boolean + include_endpoint?: boolean + include_naturalness?: boolean + include_hallucination?: boolean + include_prosody?: boolean + include_arousal?: boolean + include_valence?: boolean + include_cer?: boolean + include_wer?: boolean + include_hallucination_examples?: boolean + hallucination_examples_limit?: number + include_disclaimer_sections?: boolean + include_methodology_sections?: boolean + zone_threshold_overrides?: Record +} + +// When running in production (served from same origin), use relative path +// Otherwise use environment variable or default +export const apiBaseUrl = + import.meta.env.VITE_API_URL || + (import.meta.env.PROD ? '' : 'http://localhost:8000') + +const API_BASE_URL = apiBaseUrl + + +let refreshPromise: Promise | null = null + +class ApiClient { + private client: AxiosInstance + readonly callImports: CallImportsApi + private inMemoryAccessToken: string | null = null + private inMemoryRefreshToken: string | null = null + private cookieSessionEnabled = true + + constructor() { + this.client = axios.create({ + baseURL: API_BASE_URL, + withCredentials: true, + headers: { + 'Content-Type': 'application/json', + }, + }) + + this.callImports = new CallImportsApi(this.client) + + this.client.interceptors.request.use((config) => { + const apiKey = this.cookieSessionEnabled ? null : localStorage.getItem('apiKey') + const workspaceId = localStorage.getItem('activeWorkspaceId') + if (this.inMemoryAccessToken) { + config.headers.Authorization = `Bearer ${this.inMemoryAccessToken}` + } else if (config.headers.Authorization) { + delete config.headers.Authorization + } + if (apiKey) { + config.headers['X-API-Key'] = apiKey + } else if (config.headers['X-API-Key']) { + delete config.headers['X-API-Key'] + } + if (workspaceId) { + config.headers['X-Workspace-Id'] = workspaceId + } else if (config.headers['X-Workspace-Id']) { + delete config.headers['X-Workspace-Id'] + } + const method = (config.method || 'get').toLowerCase() + if (method !== 'get' && method !== 'head' && method !== 'options') { + Object.assign(config.headers, csrfHeaders()) + } + return config + }) + + // Add response interceptor for error handling and silent token refresh + this.client.interceptors.response.use( + (response) => response, + async (error) => { + const response = error.response + if (response?.data instanceof Blob) { + try { + const text = await response.data.text() + const trimmed = text.trim() + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + response.data = JSON.parse(text) + } + } catch { + // Keep the original blob when it isn't JSON. + } + } + + const originalRequest = error.config + const requestUrl = String(originalRequest?.url || '') + const isAuthEndpoint = + requestUrl.includes('/auth/login') || + requestUrl.includes('/auth/signup') || + requestUrl.includes('/auth/refresh') || + requestUrl.includes('/auth/logout') || + requestUrl.includes('/auth/config') || + requestUrl.includes('/auth/password') + + const detail = getApiErrorDetail(error) + if ( + response?.status === 403 && + isOrganizationAccessDenied(detail) && + !isAuthEndpoint + ) { + redirectToLoginWithMessage(organizationAccessDeniedMessage(detail)) + return Promise.reject(error) + } + + if ( + response?.status === 401 && + originalRequest && + !originalRequest._retry && + !isAuthEndpoint + ) { + originalRequest._retry = true + const refreshed = await this.tryRefreshAccessToken() + if (refreshed) { + if (this.cookieSessionEnabled) { + delete originalRequest.headers.Authorization + } else if (this.inMemoryAccessToken) { + originalRequest.headers.Authorization = `Bearer ${this.inMemoryAccessToken}` + } + return this.client(originalRequest) + } + + clearAuthSession() + window.location.href = '/login' + } + return Promise.reject(error) + }, + ) + } + + private async tryRefreshAccessToken(): Promise { + if (!this.cookieSessionEnabled && !this.inMemoryRefreshToken) { + return false + } + + if (!refreshPromise) { + refreshPromise = axios + .post( + `${API_BASE_URL}/api/v1/auth/refresh`, + this.cookieSessionEnabled + ? {} + : { refresh_token: this.inMemoryRefreshToken }, + { + withCredentials: true, + headers: { 'Content-Type': 'application/json', ...csrfHeaders() }, + }, + ) + .then((res) => { + const { access_token, refresh_token: rotatedRefresh } = res.data as TokenResponse + if (access_token) { + this.setAccessToken(access_token) + } + if (rotatedRefresh) { + this.setRefreshToken(rotatedRefresh) + } + // Cookie sessions return empty tokens in JSON; new credentials are Set-Cookie only. + return Boolean( + this.cookieSessionEnabled || access_token || this.inMemoryAccessToken, + ) + }) + .catch((err) => { + const refreshDetail = getApiErrorDetail(err) + if ( + err?.response?.status === 403 && + isOrganizationAccessDenied(refreshDetail) + ) { + redirectToLoginWithMessage( + organizationAccessDeniedMessage(refreshDetail), + ) + } + return false + }) + .finally(() => { + refreshPromise = null + }) + } + + return refreshPromise + } + + setApiKey(apiKey: string) { + localStorage.setItem('apiKey', apiKey) + } + + clearApiKey() { + localStorage.removeItem('apiKey') + } + + setCookieSessionEnabled(enabled: boolean) { + this.cookieSessionEnabled = enabled + } + + isCookieSessionEnabled(): boolean { + return this.cookieSessionEnabled + } + + setAccessToken(accessToken: string) { + this.inMemoryAccessToken = accessToken + } + + clearAccessToken() { + this.inMemoryAccessToken = null + } + + setRefreshToken(refreshToken: string) { + this.inMemoryRefreshToken = refreshToken + } + + clearRefreshToken() { + this.inMemoryRefreshToken = null + } + + clearInMemoryTokens() { + this.inMemoryAccessToken = null + this.inMemoryRefreshToken = null + } + + getAccessToken(): string | null { + return this.inMemoryAccessToken + } + + // Workspace endpoints (in-org isolation boundary for call imports + metrics). + async listWorkspaces(): Promise { + const response = await this.client.get('/api/v1/workspaces') + return response.data + } + + async createWorkspace(payload: { + name: string + slug?: string + }): Promise { + const response = await this.client.post('/api/v1/workspaces', payload) + return response.data + } + + async updateWorkspace( + workspaceId: string, + payload: { name?: string; is_active?: boolean }, + ): Promise { + const response = await this.client.patch( + `/api/v1/workspaces/${workspaceId}`, + payload, + ) + return response.data + } + + async deleteWorkspace(workspaceId: string): Promise { + await this.client.delete(`/api/v1/workspaces/${workspaceId}`) + } + + async listCapabilities(): Promise { + const response = await this.client.get('/api/v1/capabilities') + return response.data + } + + async listWorkspaceRoles(): Promise { + const response = await this.client.get('/api/v1/workspace-roles') + return response.data + } + + async createWorkspaceRole( + payload: import('../types/api').WorkspaceRoleCreate, + ): Promise { + const response = await this.client.post('/api/v1/workspace-roles', payload) + return response.data + } + + async updateWorkspaceRole( + roleId: string, + payload: import('../types/api').WorkspaceRoleUpdate, + ): Promise { + const response = await this.client.patch( + `/api/v1/workspace-roles/${roleId}`, + payload, + ) + return response.data + } + + async deleteWorkspaceRole(roleId: string): Promise { + await this.client.delete(`/api/v1/workspace-roles/${roleId}`) + } + + async listWorkspaceMembers( + workspaceId: string, + ): Promise { + const response = await this.client.get( + `/api/v1/workspaces/${workspaceId}/members`, + ) + return response.data + } + + async addWorkspaceMember( + workspaceId: string, + payload: { user_id: string; role_id: string }, + ): Promise { + const response = await this.client.post( + `/api/v1/workspaces/${workspaceId}/members`, + payload, + ) + return response.data + } + + async updateWorkspaceMember( + workspaceId: string, + userId: string, + payload: { role_id: string }, + ): Promise { + const response = await this.client.patch( + `/api/v1/workspaces/${workspaceId}/members/${userId}`, + payload, + ) + return response.data + } + + async removeWorkspaceMember( + workspaceId: string, + userId: string, + ): Promise { + await this.client.delete( + `/api/v1/workspaces/${workspaceId}/members/${userId}`, + ) + } + + // Auth endpoints + async getAuthConfig(): Promise { + const response = await this.client.get('/api/v1/auth/config') + this.setCookieSessionEnabled(Boolean(response.data.cookie_session_enabled)) + return response.data + } + + async signup(data: { + email: string + password: string + organization_name?: string + first_name?: string + last_name?: string + reference_code?: string + invite_token?: string + }): Promise { + const response = await this.client.post('/api/v1/auth/signup', data) + return response.data + } + + async previewInvitation(token: string): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/auth/invitations/preview/${encodeURIComponent(token)}`) + return response.data + } + + async acceptInvitationByToken(token: string): Promise { + const response = await this.client.post('/api/v1/auth/invitations/accept-by-token', { token }) + return response.data + } + + private platformHeaders(): Record { + const headers: Record = {} + const token = localStorage.getItem('platformAccessToken') + if (token) { + headers.Authorization = `Bearer ${token}` + } + return headers + } + + private platformRequestConfig( + method: string, + extraHeaders?: Record, + ): { headers: Record; withCredentials: boolean } { + const headers: Record = { + ...this.platformHeaders(), + ...extraHeaders, + } + const methodLower = method.toLowerCase() + if (methodLower !== 'get' && methodLower !== 'head' && methodLower !== 'options') { + Object.assign(headers, csrfHeaders()) + } + return { headers, withCredentials: true } + } + + async platformLogin(email: string, password: string): Promise { + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/auth/login`, + { email, password }, + { + headers: { 'Content-Type': 'application/json' }, + withCredentials: true, + }, + ) + return response.data + } + + async platformLogout(accessToken?: string | null): Promise<{ success: boolean; admin_id: string }> { + const token = accessToken ?? localStorage.getItem('platformAccessToken') + const headers: Record = { + 'Content-Type': 'application/json', + ...csrfHeaders(), + } + if (token) { + headers.Authorization = `Bearer ${token}` + } + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/auth/logout`, + {}, + { headers, withCredentials: true }, + ) + return response.data + } + + /** + * Voluntary logout: revoke JWT blacklist + refresh token on the server. + * Uses raw axios so credentials survive after local session is cleared. + */ + revokeUserSessionBestEffort(credentials: UserSessionCredentials): void { + if (!hasRevocableUserCredentials(credentials)) { + return + } + const auth = { accessToken: credentials.accessToken, apiKey: credentials.apiKey } + void this.revokeUserSession(credentials.refreshToken, auth) + .catch(() => this.revokeUserSession(credentials.refreshToken, auth)) + .catch(() => {}) + } + + revokePlatformSessionBestEffort(accessToken?: string | null): void { + void this.platformLogout(accessToken) + .catch(() => this.platformLogout(accessToken)) + .catch(() => {}) + } + + async getPlatformOrganizationStats(): Promise { + const response = await axios.get( + `${API_BASE_URL}/api/v1/platform/organizations/stats`, + this.platformRequestConfig('get'), + ) + return response.data + } + + async listPlatformOrganizations(params?: { + offset?: number + limit?: number + search?: string + is_active?: boolean + }): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/platform/organizations`, { + ...this.platformRequestConfig('get'), + params, + }) + return response.data + } + + async updatePlatformOrganization( + orgId: string, + data: { is_active: boolean }, + ): Promise { + const response = await axios.patch( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}`, + data, + this.platformRequestConfig('patch', { 'Content-Type': 'application/json' }), + ) + return response.data + } + + async listPlatformOrganizationUsers( + orgId: string, + params?: { role?: string }, + ): Promise { + const response = await axios.get( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users`, + { ...this.platformRequestConfig('get'), params }, + ) + return response.data + } + + async platformResetUserPassword( + orgId: string, + userId: string, + newPassword: string, + ): Promise<{ user_id: string; email: string; message: string }> { + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users/${userId}/reset-password`, + { new_password: newPassword }, + this.platformRequestConfig('post', { 'Content-Type': 'application/json' }), + ) + return response.data + } + + async listPlatformSignupCodes(): Promise { + const response = await axios.get( + `${API_BASE_URL}/api/v1/platform/signup-codes`, + this.platformRequestConfig('get'), + ) + return response.data + } + + async createPlatformSignupCode(data: { + code: string + label?: string + max_uses?: number + expires_at?: string + }): Promise { + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/signup-codes`, + data, + this.platformRequestConfig('post', { 'Content-Type': 'application/json' }), + ) + return response.data + } + + async updatePlatformSignupCode( + codeId: string, + data: { is_active?: boolean; max_uses?: number; label?: string }, + ): Promise { + const response = await axios.patch( + `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, + data, + this.platformRequestConfig('patch', { 'Content-Type': 'application/json' }), + ) + return response.data + } + + async deactivatePlatformSignupCode(codeId: string): Promise { + const response = await axios.delete( + `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, + this.platformRequestConfig('delete'), + ) + return response.data + } + + async establishOidcSession(oidcAccessToken: string): Promise { + const response = await axios.post( + `${API_BASE_URL}/api/v1/auth/oidc/session`, + {}, + { + withCredentials: true, + headers: { + Authorization: `Bearer ${oidcAccessToken}`, + 'Content-Type': 'application/json', + ...csrfHeaders(), + }, + }, + ) + const data = response.data as TokenResponse + if (data.access_token) { + this.setAccessToken(data.access_token) + } + if (data.refresh_token) { + this.setRefreshToken(data.refresh_token) + } + return data + } + + async loginWithPassword( + email: string, + password: string, + organizationId?: string, + inviteToken?: string, + ): Promise { + const response = await this.client.post('/api/v1/auth/login', { + email, + password, + ...(organizationId ? { organization_id: organizationId } : {}), + ...(inviteToken ? { invite_token: inviteToken } : {}), + }) + return response.data + } + + async logout( + refreshToken?: string | null, + credentials?: { accessToken?: string | null; apiKey?: string | null }, + ): Promise<{ success: boolean; auth_method: string }> { + return this.revokeUserSession(refreshToken, credentials) + } + + private async revokeUserSession( + refreshToken?: string | null, + credentials?: { accessToken?: string | null; apiKey?: string | null }, + ): Promise<{ success: boolean; auth_method: string }> { + const headers: Record = { + 'Content-Type': 'application/json', + ...csrfHeaders(), + } + const accessToken = credentials?.accessToken ?? this.inMemoryAccessToken + const apiKey = this.cookieSessionEnabled + ? credentials?.apiKey ?? null + : credentials?.apiKey ?? localStorage.getItem('apiKey') + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}` + } else if (apiKey) { + headers['X-API-Key'] = apiKey + } + const response = await axios.post( + `${API_BASE_URL}/api/v1/auth/logout`, + this.cookieSessionEnabled + ? {} + : { + refresh_token: + refreshToken ?? this.inMemoryRefreshToken ?? undefined, + }, + { headers, withCredentials: true }, + ) + return response.data + } + + async refreshSession(refreshToken?: string): Promise { + const response = await axios.post( + `${API_BASE_URL}/api/v1/auth/refresh`, + this.cookieSessionEnabled ? {} : { refresh_token: refreshToken }, + { + withCredentials: true, + headers: { 'Content-Type': 'application/json', ...csrfHeaders() }, + }, + ) + return response.data + } + + async getMe(): Promise { + const response = await this.client.get('/api/v1/auth/me') + return response.data + } + + async switchOrganization(organizationId: string): Promise { + const response = await this.client.post('/api/v1/auth/switch-org', { + organization_id: organizationId, + ...(this.cookieSessionEnabled + ? {} + : { refresh_token: this.inMemoryRefreshToken || undefined }), + }) + return response.data + } + + async setPassword(data: { + new_password: string + current_password?: string + email?: string + }): Promise { + const response = await this.client.post('/api/v1/auth/password', data) + return response.data + } + + async generateApiKey(name?: string): Promise { + const response = await this.client.post('/api/v1/auth/generate-key', { name }) + return response.data + } + + async validateApiKey(): Promise<{ valid: boolean; message: string }> { + const response = await this.client.post('/api/v1/auth/validate') + return response.data + } + + // Settings / API Key Management endpoints + async listApiKeys(): Promise { + const response = await this.client.get('/api/v1/settings/api-keys') + return response.data + } + + async createApiKey(name?: string): Promise { + const response = await this.client.post('/api/v1/settings/api-keys', { name }) + return response.data + } + + async deleteApiKey(keyId: string): Promise { + await this.client.delete(`/api/v1/settings/api-keys/${keyId}`) + } + + async regenerateApiKey(keyId: string): Promise { + const response = await this.client.post(`/api/v1/settings/api-keys/${keyId}/regenerate`) + return response.data + } + + // Audio endpoints + async uploadAudio(file: File): Promise { + const formData = new FormData() + formData.append('file', file) + const response = await this.client.post('/api/v1/audio/upload', formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + return response.data + } + + async getAudio(audioId: string): Promise { + const response = await this.client.get(`/api/v1/audio/${audioId}`) + return response.data + } + + async listAudio(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/audio', { + params: { skip, limit }, + }) + return response.data + } + + async deleteAudio(audioId: string): Promise { + const response = await this.client.delete(`/api/v1/audio/${audioId}`) + return response.data + } + + async downloadAudio(audioId: string): Promise { + const response = await this.client.get(`/api/v1/audio/${audioId}/download`, { + responseType: 'blob', + }) + return response.data + } + + // Evaluation endpoints + async createEvaluation(data: EvaluationCreate): Promise { + const response = await this.client.post('/api/v1/evaluations/create', data) + return response.data + } + + async getEvaluation(evaluationId: string): Promise { + const response = await this.client.get(`/api/v1/evaluations/${evaluationId}`) + return response.data + } + + async listEvaluations( + skip = 0, + limit = 100, + status?: EvaluationStatus + ): Promise { + const response = await this.client.get('/api/v1/evaluations', { + params: { skip, limit, status }, + }) + return response.data + } + + async cancelEvaluation(evaluationId: string): Promise { + const response = await this.client.post(`/api/v1/evaluations/${evaluationId}/cancel`) + return response.data + } + + async deleteEvaluation(evaluationId: string): Promise { + const response = await this.client.delete(`/api/v1/evaluations/${evaluationId}`) + return response.data + } + + // Results endpoints + async getEvaluationResult(evaluationId: string): Promise { + const response = await this.client.get(`/api/v1/results/${evaluationId}`) + return response.data + } + + async getMetrics(evaluationId: string): Promise<{ + evaluation_id: string + metrics: Record + processing_time?: number | null + }> { + const response = await this.client.get(`/api/v1/results/${evaluationId}/metrics`) + return response.data + } + + async getTranscript(evaluationId: string): Promise<{ + evaluation_id: string + transcript: string + }> { + const response = await this.client.get(`/api/v1/results/${evaluationId}/transcript`) + return response.data + } + + async compareEvaluations(evaluationIds: string[]): Promise<{ + evaluations: EvaluationResult[] + comparison_metrics: Record + }> { + const response = await this.client.post('/api/v1/results/compare', { + evaluation_ids: evaluationIds, + }) + return response.data + } + + // Agents endpoints + async getMessagingPublicBaseUrl(): Promise<{ + public_base_url: string + twilio_public_base_url?: string + telnyx_public_base_url?: string + meta_whatsapp_inbound_webhook_url?: string + }> { + const response = await this.client.get('/api/v1/chat/messaging/public-base-url') + return response.data + } + + async simulateAgentMetaWhatsappInbound( + agentId: string, + payload: { messaging_recipient: string; body?: string }, + ): Promise<{ ok: boolean }> { + const response = await this.client.post( + `/api/v1/agents/${agentId}/chat-messaging/simulate-meta-whatsapp-inbound`, + payload, + ) + return response.data + } + + async testAgentTwilioSms( + agentId: string, + overrides?: { + messaging_recipient?: string + twilio_from?: string + twilio_sms_trial_body_template?: string + }, + ): Promise<{ + ok: boolean + message_sid: string + to: string + from: string + body_sent: string + }> { + const response = await this.client.post( + `/api/v1/agents/${agentId}/chat-messaging/test-twilio-sms`, + overrides ?? {}, + ) + return response.data + } + + async testAgentMetaWhatsapp( + agentId: string, + overrides?: { messaging_recipient?: string }, + ): Promise<{ + ok: boolean + message_id: string + to: string + wa_id: string + template_sent: string + }> { + const response = await this.client.post( + `/api/v1/agents/${agentId}/chat-messaging/test-meta-whatsapp`, + overrides ?? {}, + ) + return response.data + } + + async testAgentTelnyxSms( + agentId: string, + overrides?: { + messaging_recipient?: string + telnyx_from?: string + }, + ): Promise<{ + ok: boolean + message_id: string + to: string + from: string + body_sent: string + }> { + const response = await this.client.post( + `/api/v1/agents/${agentId}/chat-messaging/test-telnyx-sms`, + overrides ?? {}, + ) + return response.data + } + + async createAgent(data: { + name: string + phone_number?: string + telephony_phone_number_id?: string + language: string + description?: string | null + call_type: string + call_medium: string + voice_bundle_id?: string + ai_provider_id?: string + voice_ai_integration_id?: string + voice_ai_agent_id?: string + provider_prompt?: string | null + silence_hangup_secs?: number + }): Promise { + const response = await this.client.post('/api/v1/agents', data) + return response.data + } + + async listAgents(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/agents', { + params: { skip, limit }, + }) + return response.data + } + + async setAgentFavorite(agentId: string, favorite: boolean): Promise { + const response = await this.client.put(`/api/v1/agents/${agentId}/favorite`, { favorite }) + return response.data + } + + async getAgent(agentId: string): Promise { + const response = await this.client.get(`/api/v1/agents/${agentId}`) + return response.data + } + + async checkAgentPhoneAssignment(params: { + phoneNumber?: string + telephonyPhoneNumberId?: string + excludeAgentId?: string + }): Promise { + const response = await this.client.get('/api/v1/agents/check-phone-assignment', { + params: { + phone_number: params.phoneNumber, + telephony_phone_number_id: params.telephonyPhoneNumberId, + exclude_agent_id: params.excludeAgentId, + }, + }) + return response.data + } + + async updateAgent(agentId: string, data: { + name?: string + phone_number?: string | null + telephony_phone_number_id?: string | null + language?: string + description?: string | null + call_type?: string + call_medium?: string + silence_hangup_secs?: number + voice_bundle_id?: string | null + ai_provider_id?: string + voice_ai_integration_id?: string | null + voice_ai_agent_id?: string | null + provider_prompt?: string | null + prompt_variables?: Record + }): Promise { + const response = await this.client.put(`/api/v1/agents/${agentId}`, data) + return response.data + } + + async deleteAgent(agentId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/agents/${agentId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + async getAgentDeleteImpact(agentId: string): Promise<{ + agent_id: string + agent_name: string + dependencies: Record + can_delete_without_force: boolean + }> { + const response = await this.client.get(`/api/v1/agents/${agentId}/delete-impact`) + return response.data + } + + async generateAgentDescription(data: { + description: string + tone?: string + format_style?: string + provider?: string + model?: string + agent_id?: string + include_linked_scenarios?: boolean + append_scenarios_to_output?: boolean + }): Promise<{ content: string; provider: string; model: string }> { + const response = await this.client.post('/api/v1/agents/generate-description', data) + return response.data + } + + async generateProductionPromptFromBrief(data: { + agent_name: string + brief: string + language?: string + call_type?: string + provider?: string + model?: string + credential_id?: string + additional_context?: string + }): Promise<{ production_prompt: string; provider: string; model: string }> { + const response = await this.client.post('/api/v1/agents/generate-production-prompt', data) + return response.data + } + + async generateTestPromptFromProduction( + data: GenerateTestPromptParams, + ): Promise<{ + sections: Array<{ key: string; title: string; content: string }> + test_agent_prompt: string + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + test_agent_template: { + sections: Array<{ key: string; title: string; content: string }> + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + } + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/agents/generate-test-prompt', data) + return response.data + } + + async generateScenariosFromPrompt( + data: GenerateScenariosFromPromptParams, + ): Promise<{ + scenarios: Array<{ name: string; description: string; goal?: string | null }> + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/agents/generate-scenarios-from-prompt', data) + return response.data + } + + async generateTestSetupFromProduction( + data: GenerateTestSetupParams, + ): Promise<{ + sections: Array<{ key: string; title: string; content: string }> + test_agent_prompt: string + scenarios: Array<{ name: string; description: string; goal?: string | null }> + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/agents/generate-test-setup', data) + return response.data + } + + // Personas endpoints + async listPersonas(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/personas', { + params: { skip, limit }, + }) + return response.data + } + + async getPersona(personaId: string): Promise { + const response = await this.client.get(`/api/v1/personas/${personaId}`) + return response.data + } + + async createPersona(data: { + name: string + gender: string + tts_provider?: string + tts_voice_id?: string + tts_voice_name?: string + is_custom?: boolean + description?: string + tts_config?: Record + llm_temperature?: number + llm_max_tokens?: number + response_delay_ms?: number + max_turns?: number + allow_interruptions?: boolean + }): Promise { + const response = await this.client.post('/api/v1/personas', data) + return response.data + } + + async updatePersona(personaId: string, data: any): Promise { + const response = await this.client.put(`/api/v1/personas/${personaId}`, data) + return response.data + } + + async deletePersona(personaId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/personas/${personaId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + async clonePersona(personaId: string, name?: string): Promise { + const response = await this.client.post(`/api/v1/personas/${personaId}/clone`, { name }) + return response.data + } + + async seedDemoData(): Promise { + const response = await this.client.post('/api/v1/personas/seed-data') + return response.data + } + + async getPersonaAgentPromptSources(agentId: string): Promise<{ + agent_id: string + agent_name: string + test_agent_prompt: string + agent_prompt: string + }> { + const response = await this.client.get(`/api/v1/personas/agent-prompt-sources/${agentId}`) + return response.data + } + + async generatePersonaPrompt(data: { + agent_id: string + source?: 'test_agent' | 'agent' | 'auto' + persona_name?: string + persona_gender?: string + additional_context?: string + provider?: string + model?: string + credential_id?: string + llm_config?: LLMGenerationConfig | null + }): Promise<{ + persona_prompt: string + source_used: string + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/personas/generate-prompt', data) + return response.data + } + + // Persona voice options (built-in + custom voices, ungated) + async getPersonaVoiceOptions(provider?: string): Promise<{ + providers: Array<{ + id: string + name: string + voices: Array<{ + id: string + name: string + gender: string + is_custom: boolean + custom_voice_id?: string + description?: string | null + }> + }> + }> { + const response = await this.client.get('/api/v1/personas/voice-options', { + params: provider ? { provider } : undefined, + }) + return response.data + } + + // Custom voice CRUD (persona-scoped, ungated) + async listPersonaCustomVoices(provider?: string): Promise { + const response = await this.client.get('/api/v1/personas/custom-voices', { + params: provider ? { provider } : undefined, + }) + return response.data + } + + async createPersonaCustomVoice(data: { + provider: string + voice_id: string + name: string + gender?: string + description?: string + }): Promise { + const response = await this.client.post('/api/v1/personas/custom-voices', data) + return response.data + } + + async updatePersonaCustomVoice(customVoiceId: string, data: { + voice_id?: string + name?: string + gender?: string + description?: string + }): Promise { + const response = await this.client.put(`/api/v1/personas/custom-voices/${customVoiceId}`, data) + return response.data + } + + async deletePersonaCustomVoice(customVoiceId: string): Promise { + const response = await this.client.delete(`/api/v1/personas/custom-voices/${customVoiceId}`) + return response.data + } + + async listAmbientPresets(): Promise<{ presets: { id: string; label: string }[] }> { + const response = await this.client.get('/api/v1/personas/ambient-presets') + return response.data + } + + async listAmbientLibrary(): Promise< + Array<{ + id: string + name: string + original_filename?: string | null + created_at?: string + }> + > { + const response = await this.client.get('/api/v1/personas/ambient-library') + return response.data + } + + async uploadAmbientLibraryAsset(file: File, name?: string): Promise { + const formData = new FormData() + formData.append('file', file) + if (name) { + formData.append('name', name) + } + const response = await this.client.post('/api/v1/personas/ambient-library', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async updateAmbientLibraryAsset(assetId: string, data: { name: string }): Promise { + const response = await this.client.patch(`/api/v1/personas/ambient-library/${assetId}`, data) + return response.data + } + + async deleteAmbientLibraryAsset(assetId: string): Promise { + await this.client.delete(`/api/v1/personas/ambient-library/${assetId}`) + } + + async previewAmbientLibraryAsset(assetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async getAmbientLibraryPreviewUrl( + assetId: string, + expiration: number = 3600, + ): Promise<{ url: string; expires_in: number }> { + const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview-url`, { + params: { expiration }, + }) + return response.data + } + + async previewAmbientPreset(presetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-presets/${presetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async uploadPersonaAmbientAudio(personaId: string, file: File): Promise { + const formData = new FormData() + formData.append('file', file) + const response = await this.client.post(`/api/v1/personas/${personaId}/ambient-audio`, formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async deletePersonaAmbientAudio(personaId: string): Promise { + const response = await this.client.delete(`/api/v1/personas/${personaId}/ambient-audio`) + return response.data + } + + // Scenarios endpoints + async listScenarios(skip = 0, limit = 100, agentId?: string): Promise { + const response = await this.client.get('/api/v1/scenarios', { + params: { skip, limit, ...(agentId ? { agent_id: agentId } : {}) }, + }) + return response.data + } + + async getScenario(scenarioId: string): Promise { + const response = await this.client.get(`/api/v1/scenarios/${scenarioId}`) + return response.data + } + + async createScenario(data: { + name: string + agent_id?: string | null + description?: string | null + required_info: Record + expected_tool_calls?: ExpectedToolCall[] | null + }): Promise { + const response = await this.client.post('/api/v1/scenarios', data) + return response.data + } + + async updateScenario(scenarioId: string, data: { + name?: string + agent_id?: string | null + description?: string | null + required_info?: Record + expected_tool_calls?: ExpectedToolCall[] | null + }): Promise { + const response = await this.client.put(`/api/v1/scenarios/${scenarioId}`, data) + return response.data + } + + async deleteScenario(scenarioId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/scenarios/${scenarioId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + // Harbor simulation runs + async listSimulationRuns(agentId?: string): Promise { + const response = await this.client.get('/api/v1/simulation-runs', { + params: agentId ? { agent_id: agentId } : undefined, + }) + return response.data + } + + async getSimulationRun(runId: string): Promise { + const response = await this.client.get(`/api/v1/simulation-runs/${runId}`) + return response.data + } + + async createSimulationRun(data: SimulationRunCreate): Promise { + const response = await this.client.post('/api/v1/simulation-runs', data) + return response.data + } + + async listSimulationTrials(runId: string): Promise { + const response = await this.client.get(`/api/v1/simulation-runs/${runId}/trials`) + return response.data + } + + async getSimulationTrial(runId: string, trialId: string): Promise { + const response = await this.client.get(`/api/v1/simulation-runs/${runId}/trials/${trialId}`) + return response.data + } + + async renameSimulationRun(runId: string, name: string | null): Promise { + const response = await this.client.patch(`/api/v1/simulation-runs/${runId}`, { name }) + return response.data + } + + async cancelSimulationRun(runId: string): Promise { + const response = await this.client.post(`/api/v1/simulation-runs/${runId}/cancel`) + return response.data + } + + async deleteSimulationRun(runId: string): Promise { + await this.client.delete(`/api/v1/simulation-runs/${runId}`) + } + + async rerunSimulationRun(runId: string, onlyFailed = false): Promise { + const response = await this.client.post(`/api/v1/simulation-runs/${runId}/rerun`, null, { + params: { only_failed: onlyFailed }, + }) + return response.data + } + + async downloadSimulationArtifacts(runId: string): Promise { + const response = await this.client.get(`/api/v1/simulation-runs/${runId}/artifacts`, { + responseType: 'blob', + }) + return response.data + } + + async getSimulationFailureAnalysis(runId: string): Promise { + const response = await this.client.get(`/api/v1/simulation-runs/${runId}/failure-analysis`) + return response.data + } + + async generateSimulationFailureAnalysis( + runId: string, + data: SimulationFailureAnalysisRequest = {}, + ): Promise { + const response = await this.client.post( + `/api/v1/simulation-runs/${runId}/failure-analysis`, + data, + ) + return response.data + } + + async cancelSimulationFailureAnalysis(runId: string): Promise { + const response = await this.client.post( + `/api/v1/simulation-runs/${runId}/failure-analysis/cancel`, + ) + return response.data + } + + async reviewSimulationPromptImprovement( + runId: string, + data: { + suggestion_id: string + review_status: 'pending' | 'accepted' | 'rejected' + edited_text?: string | null + }, + ): Promise { + const response = await this.client.patch( + `/api/v1/simulation-runs/${runId}/prompt-improvements/review`, + data, + ) + return response.data + } + + async getSimulationComposedPrompt(runId: string): Promise { + const response = await this.client.get( + `/api/v1/simulation-runs/${runId}/prompt-improvements/composed`, + ) + return response.data + } + + async downloadSimulationRunPdfReport( + runId: string, + sections: SimulationRunPdfReportSections, + platformBaseUrl?: string, + ): Promise { + const response = await this.client.post( + `/api/v1/simulation-runs/${runId}/pdf-report`, + { sections, platform_base_url: platformBaseUrl || null }, + { responseType: 'blob' }, + ) + return response.data + } + + // Knowledge bases + async listKnowledgeBases(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/knowledge-bases', { params: { skip, limit } }) + return response.data + } + + async getKnowledgeBase(knowledgeBaseId: string): Promise { + const response = await this.client.get(`/api/v1/knowledge-bases/${knowledgeBaseId}`) + return response.data + } + + async listKnowledgeBaseEmbeddingModels(): Promise> { + const response = await this.client.get('/api/v1/knowledge-bases/embedding-models') + return response.data + } + + async getKnowledgeBaseEmbeddingConfig(): Promise<{ + default_provider: string + default_model: string + required_dimensions: number + providers: Array<{ + id: string + label: string + suggested_models: Array<{ id: string; label: string }> + }> + }> { + const response = await this.client.get('/api/v1/knowledge-bases/embedding-config') + return response.data + } + + async createKnowledgeBase(data: { + name: string + description?: string + embedding_provider?: string + embedding_model?: string + embedding_credential_id?: string | null + }): Promise { + const response = await this.client.post('/api/v1/knowledge-bases', data) + return response.data + } + + async updateKnowledgeBase( + knowledgeBaseId: string, + data: { + name?: string + description?: string + status?: string + embedding_provider?: string + embedding_model?: string + embedding_credential_id?: string | null + }, + ): Promise { + const response = await this.client.patch(`/api/v1/knowledge-bases/${knowledgeBaseId}`, data) + return response.data + } + + async deleteKnowledgeBase(knowledgeBaseId: string): Promise { + await this.client.delete(`/api/v1/knowledge-bases/${knowledgeBaseId}`) + } + + async listToolConnections(): Promise { + const response = await this.client.get('/api/v1/tool-connections') + return response.data + } + + async createToolConnection(data: ToolConnectionCreate): Promise { + const response = await this.client.post('/api/v1/tool-connections', data) + return response.data + } + + async updateToolConnection( + connectionId: string, + data: { + name?: string + config?: Record + secrets?: ToolConnectionSecrets + simulation_mode?: ToolSimulationMode + }, + ): Promise { + const response = await this.client.patch(`/api/v1/tool-connections/${connectionId}`, data) + return response.data + } + + async testToolConnection(connectionId: string): Promise { + const response = await this.client.post(`/api/v1/tool-connections/${connectionId}/test`) + return response.data + } + + async deleteToolConnection(connectionId: string): Promise { + await this.client.delete(`/api/v1/tool-connections/${connectionId}`) + } + + async getAgentToolConnections(agentId: string): Promise { + const response = await this.client.get(`/api/v1/tool-connections/agents/${agentId}/links`) + return response.data + } + + async updateAgentToolConnections( + agentId: string, + links: AgentToolConnectionLink[], + ): Promise { + const response = await this.client.put(`/api/v1/tool-connections/agents/${agentId}/links`, { links }) + return response.data + } + + async getAgentToolCatalog(agentId: string): Promise { + const response = await this.client.get(`/api/v1/tool-connections/agents/${agentId}/catalog`) + return response.data + } + + async uploadKnowledgeDocument(knowledgeBaseId: string, file: File): Promise { + const form = new FormData() + form.append('file', file) + const response = await this.client.post( + `/api/v1/knowledge-bases/${knowledgeBaseId}/documents`, + form, + { headers: { 'Content-Type': 'multipart/form-data' } }, + ) + return response.data + } + + async uploadKnowledgeDocuments( + knowledgeBaseId: string, + files: File[], + options?: { + onProgress?: (progress: { uploaded: number; total: number; fileName: string }) => void + }, + ): Promise<{ uploaded: number; failed: Array<{ fileName: string; error: string }> }> { + const failed: Array<{ fileName: string; error: string }> = [] + let uploaded = 0 + const total = files.length + for (const file of files) { + options?.onProgress?.({ uploaded, total, fileName: file.name }) + try { + await this.uploadKnowledgeDocument(knowledgeBaseId, file) + uploaded += 1 + } catch (err: unknown) { + const ax = err as { response?: { data?: { detail?: string } }; message?: string } + const detail = ax?.response?.data?.detail + failed.push({ + fileName: file.name, + error: typeof detail === 'string' ? detail : ax?.message || 'Upload failed', + }) + } + } + options?.onProgress?.({ uploaded, total, fileName: '' }) + return { uploaded, failed } + } + + async retryKnowledgeDocument(knowledgeBaseId: string, documentId: string): Promise { + const response = await this.client.post( + `/api/v1/knowledge-bases/${knowledgeBaseId}/documents/${documentId}/retry`, + ) + return response.data + } + + async deleteKnowledgeDocument(knowledgeBaseId: string, documentId: string): Promise { + await this.client.delete( + `/api/v1/knowledge-bases/${knowledgeBaseId}/documents/${documentId}`, + ) + } + + async getKnowledgeDocumentView(knowledgeBaseId: string, documentId: string): Promise { + const response = await this.client.get( + `/api/v1/knowledge-bases/${knowledgeBaseId}/documents/${documentId}/view`, + ) + return response.data + } + + knowledgeDocumentDownloadUrl(knowledgeBaseId: string, documentId: string): string { + return this.buildAuthenticatedApiUrl( + `/api/v1/knowledge-bases/${knowledgeBaseId}/documents/${documentId}/download`, + ) + } + + // Chat/Inference endpoints + async chatCompletion(data: { + messages: Array<{ role: string; content: string }> + provider: string + model: string + temperature?: number + max_tokens?: number + llm_config?: LLMGenerationConfig | null + }): Promise<{ + text: string + model: string + usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } + processing_time?: number + }> { + const response = await this.client.post('/api/v1/chat/completion', data) + return response.data + } + + // VoiceBundle endpoints + async listVoiceBundles(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/voicebundles', { + params: { skip, limit }, + }) + return response.data + } + + async getVoiceBundle(voicebundleId: string): Promise { + const response = await this.client.get(`/api/v1/voicebundles/${voicebundleId}`) + return response.data + } + + async createVoiceBundle(data: any): Promise { + const response = await this.client.post('/api/v1/voicebundles', data) + return response.data + } + + async updateVoiceBundle(voicebundleId: string, data: any): Promise { + const response = await this.client.put(`/api/v1/voicebundles/${voicebundleId}`, data) + return response.data + } + + async deleteVoiceBundle(voicebundleId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/voicebundles/${voicebundleId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + // AI Provider endpoints + async listAIProviders(): Promise { + const response = await this.client.get('/api/v1/aiproviders') + return response.data + } + + async getAIProvider(aiproviderId: string): Promise { + const response = await this.client.get(`/api/v1/aiproviders/${aiproviderId}`) + return response.data + } + + async createAIProvider(data: any): Promise { + const response = await this.client.post('/api/v1/aiproviders', data) + return response.data + } + + async updateAIProvider(aiproviderId: string, data: any): Promise { + const response = await this.client.put(`/api/v1/aiproviders/${aiproviderId}`, data) + return response.data + } + + async deleteAIProvider(aiproviderId: string): Promise { + await this.client.delete(`/api/v1/aiproviders/${aiproviderId}`) + } + + async setDefaultAIProvider(aiproviderId: string): Promise { + const response = await this.client.post( + `/api/v1/aiproviders/${aiproviderId}/set-default`, + ) + return response.data + } + + async testAIProvider(aiproviderId: string): Promise { + const response = await this.client.post(`/api/v1/aiproviders/${aiproviderId}/test`) + return response.data + } + + async getLLMGatewaySettings(): Promise { + const response = await this.client.get('/api/v1/organizations/llm-gateway') + return response.data + } + + async updateLLMGatewaySettings(data: LLMGatewaySettingsUpdate): Promise { + const response = await this.client.put('/api/v1/organizations/llm-gateway', data) + return response.data + } + + async getAlertingSyncSettings(): Promise { + const response = await this.client.get('/api/v1/organizations/alerting-sync') + return response.data + } + + async updateAlertingSyncSettings(data: AlertingSyncSettingsUpdate): Promise { + const response = await this.client.put('/api/v1/organizations/alerting-sync', data) + return response.data + } + + // IAM endpoints + async getOrganization(): Promise { + const response = await this.client.get('/api/v1/iam/organization') + return response.data + } + + async updateOrganization(data: { name: string }): Promise { + const response = await this.client.patch('/api/v1/iam/organization', data) + return response.data + } + + async listOrganizationUsers(): Promise { + const response = await this.client.get('/api/v1/iam/users') + return response.data + } + + async inviteUser(data: InvitationCreate): Promise { + const response = await this.client.post('/api/v1/iam/invitations', data) + return response.data + } + + async listInvitations(): Promise { + const response = await this.client.get('/api/v1/iam/invitations') + return response.data + } + + async updateUserRole(userId: string, role: Role): Promise { + const response = await this.client.put(`/api/v1/iam/users/${userId}/role`, { role }) + return response.data + } + + async removeUser(userId: string): Promise { + await this.client.delete(`/api/v1/iam/users/${userId}`) + } + + async cancelInvitation(invitationId: string): Promise { + await this.client.delete(`/api/v1/iam/invitations/${invitationId}`) + } + + /** + * Admin-initiated password reset for another organization member. + * + * The caller must be an ADMIN of the same organization as `userId`. The + * backend ignores any current-password concept here (that's for the + * self-service `/auth/password` endpoint). Communicate the new password + * to the user out-of-band. + */ + async adminResetUserPassword( + userId: string, + newPassword: string, + ): Promise<{ user_id: string; email: string; message: string }> { + const response = await this.client.post( + `/api/v1/iam/users/${userId}/reset-password`, + { new_password: newPassword }, + ) + return response.data + } + + // Profile endpoints + async getProfile(): Promise { + const response = await this.client.get('/api/v1/profile') + return response.data + } + + async updateProfile(data: UserUpdate): Promise { + const response = await this.client.put('/api/v1/profile', data) + return response.data + } + + async getMyInvitations(): Promise { + const response = await this.client.get('/api/v1/profile/invitations') + return response.data + } + + async acceptInvitation(invitationId: string): Promise { + const response = await this.client.post(`/api/v1/profile/invitations/${invitationId}/accept`) + return response.data + } + + async declineInvitation(invitationId: string): Promise { + const response = await this.client.post(`/api/v1/profile/invitations/${invitationId}/decline`) + return response.data + } + + // User Preferences endpoints + async getUserPreferences(): Promise { + const response = await this.client.get('/api/v1/profile/preferences') + return response.data + } + + async updateUserPreferences(data: UserPreferencesUpdate): Promise { + const response = await this.client.put('/api/v1/profile/preferences', data) + return response.data + } + + // Integration endpoints + async listIntegrations(): Promise { + const response = await this.client.get('/api/v1/integrations') + return response.data + } + + async getIntegration(integrationId: string): Promise { + const response = await this.client.get(`/api/v1/integrations/${integrationId}`) + return response.data + } + + async previewIntegrationAgentPrompt( + integrationId: string, + voiceAiAgentId: string, + options?: { agentChannel?: 'voice' | 'chat' }, + ): Promise<{ provider_prompt: string }> { + const response = await this.client.post( + `/api/v1/integrations/${integrationId}/preview-agent-prompt`, + { + voice_ai_agent_id: voiceAiAgentId, + ...(options?.agentChannel ? { agent_channel: options.agentChannel } : {}), + }, + ) + return response.data + } + + async listIntegrationVoiceAgents( + integrationId: string, + options?: { refresh?: boolean; search?: string; agentKind?: 'voice' | 'chat' }, + ): Promise { + const response = await this.client.get( + `/api/v1/integrations/${integrationId}/voice-agents`, + { + params: { + ...(options?.refresh ? { refresh: true } : {}), + ...(options?.search ? { search: options.search } : {}), + ...(options?.agentKind ? { agent_kind: options.agentKind } : {}), + }, + }, + ) + return response.data + } + + async createIntegration(data: IntegrationCreate): Promise { + const response = await this.client.post('/api/v1/integrations', data) + return response.data + } + + async updateIntegration(integrationId: string, data: Partial): Promise { + const response = await this.client.put(`/api/v1/integrations/${integrationId}`, data) + return response.data + } + + async deleteIntegration(integrationId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/integrations/${integrationId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + async setDefaultIntegration(integrationId: string): Promise { + const response = await this.client.post( + `/api/v1/integrations/${integrationId}/set-default`, + ) + return response.data + } + + async getIntegrationApiKey(integrationId: string): Promise<{ api_key: string; public_key?: string | null }> { + const response = await this.client.get(`/api/v1/integrations/${integrationId}/api-key`) + return response.data + } + + // Telephony endpoints (provider-agnostic) + async createTelephonyConfig(data: TelephonyIntegrationCreatePayload): Promise { + const response = await this.client.post('/api/v1/telephony/config', data) + return response.data + } + + async getTelephonyConfig(provider: string = 'plivo'): Promise { + const response = await this.client.get('/api/v1/telephony/config', { params: { provider } }) + return response.data + } + + async listTelephonyConfigs(provider?: string): Promise { + const response = await this.client.get('/api/v1/telephony/configs', { + params: provider ? { provider } : undefined, + }) + return response.data + } + + async updateTelephonyConfig(data: TelephonyIntegrationUpdatePayload): Promise { + const response = await this.client.put('/api/v1/telephony/config', data) + return response.data + } + + async setDefaultTelephonyConfig(integrationId: string): Promise { + const response = await this.client.post( + `/api/v1/telephony/config/${integrationId}/set-default`, + ) + return response.data + } + + async deleteTelephonyConfig(integrationId: string): Promise { + await this.client.delete(`/api/v1/telephony/config/${integrationId}`) + } + + async listTelephonyNumbers(provider?: string): Promise { + const response = await this.client.get('/api/v1/telephony/numbers', { + params: provider ? { provider } : undefined, + }) + return response.data + } + + async listAvailableTelephonyNumbers( + provider: string, + credentialId?: string, + ): Promise { + const response = await this.client.get('/api/v1/telephony/numbers/available', { + params: { + provider, + ...(credentialId ? { credential_id: credentialId } : {}), + }, + }) + return response.data + } + + async importTelephonyNumbers( + provider: string, + numbers: string[], + agentId?: string, + credentialId?: string, + ): Promise { + const response = await this.client.post('/api/v1/telephony/numbers/import', { + provider, + numbers, + agent_id: agentId || undefined, + credential_id: credentialId || undefined, + }) + return response.data + } + + async listVobizAvailableNumbers(): Promise { + return this.listAvailableTelephonyNumbers('vobiz') + } + + async importVobizNumbers(numbers: string[], agentId?: string): Promise { + return this.importTelephonyNumbers('vobiz', numbers, agentId) + } + + async deleteTelephonyNumber(numberId: string): Promise { + await this.client.delete(`/api/v1/telephony/numbers/${numberId}`) + } + + async deleteImportedVobizNumber(numberId: string): Promise { + await this.deleteTelephonyNumber(numberId) + } + + async listTelephonyOutboundPool(): Promise { + const response = await this.client.get('/api/v1/telephony/outbound-pool') + return response.data + } + + async listVobizOutboundPool(): Promise { + return this.listTelephonyOutboundPool() + } + + async listTelephonyDialTargets(): Promise { + const response = await this.client.get('/api/v1/telephony/dial-targets') + return response.data + } + + async createVobizOutboundCall(data: VobizOutboundCallRequest): Promise { + const response = await this.client.post('/api/v1/telephony/vobiz/calls/outbound', data) + return response.data + } + + async listDialTargets(): Promise { + const response = await this.client.get('/api/v1/telephony/dial-targets') + return response.data + } + + async createDialTarget(data: { phone_number: string; label?: string }): Promise { + const response = await this.client.post('/api/v1/telephony/dial-targets', data) + return response.data + } + + async updateDialTarget( + targetId: string, + data: { phone_number?: string; label?: string }, + ): Promise { + const response = await this.client.put(`/api/v1/telephony/dial-targets/${targetId}`, data) + return response.data + } + + async deleteDialTarget(targetId: string): Promise { + await this.client.delete(`/api/v1/telephony/dial-targets/${targetId}`) + } + + // Data Sources endpoints + async testS3Connection(): Promise { + const response = await this.client.post('/api/v1/data-sources/s3/test') + return response.data + } + + async listS3Files(prefix?: string, maxKeys = 1000): Promise { + const response = await this.client.get('/api/v1/data-sources/s3/files', { + params: { prefix, max_keys: maxKeys }, + }) + return response.data + } + + async browseS3(path = '', maxKeys = 1000): Promise { + const response = await this.client.get('/api/v1/data-sources/s3/browse', { + params: { path, max_keys: maxKeys }, + }) + return response.data + } + + async uploadToS3(file: File, customFilename?: string): Promise { + const formData = new FormData() + formData.append('file', file) + if (customFilename) { + formData.append('filename', customFilename) + } + const response = await this.client.post('/api/v1/data-sources/s3/upload', formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + return response.data + } + + async getS3Status(): Promise { + const response = await this.client.get('/api/v1/data-sources/s3/status') + return response.data + } + + async downloadFromS3(fileKey: string): Promise { + const response = await this.client.get(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}/download`, { + responseType: 'blob', + }) + // Create a blob URL and trigger download + const url = window.URL.createObjectURL(new Blob([response.data])) + const link = document.createElement('a') + link.href = url + link.setAttribute('download', fileKey.split('/').pop() || 'file') + document.body.appendChild(link) + link.click() + link.remove() + window.URL.revokeObjectURL(url) + } + + async getS3PresignedUrl(fileKey: string, expiration: number = 3600): Promise<{ url: string; expires_in: number }> { + const response = await this.client.get(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}/presigned-url`, { + params: { expiration }, + }) + return response.data + } + + async deleteFromS3(fileKey: string): Promise { + const response = await this.client.delete(`/api/v1/data-sources/s3/files/${encodeURIComponent(fileKey)}`) + return response.data + } + + // Call Imports endpoints + /** + * Inspect an uploaded CSV / Excel file and get its sheets + headers. + * + * Drives the column-mapping UI without forcing the frontend to parse + * CSV / xlsx itself. CSV files come back as a single synthetic sheet; + * Excel workbooks come back with one entry per worksheet. + */ + async previewCallImportFile(file: File): Promise { + return this.callImports.previewCallImportFile(file) + } + + /** @deprecated Use createCallImport → updateCallImportMapping → startCallImport. */ + async uploadCallImport( + file: File, + options: { + provider: string + telephonyIntegrationId: string + /** Reusable Input Parameter schema this upload is mapped against. */ + schemaId: string + /** + * Schema-driven mapping: ``{parameter_name: csv_header}``. Must + * include every required parameter on the schema. Pass an empty + * string / omit a parameter to leave it unmapped (only allowed + * for optional parameters). + */ + parameterMapping: Record + /** + * CSV/Excel headers the user has explicitly skipped. Every source + * column must either appear in ``parameterMapping`` or here, or + * the backend rejects the upload with 400. + */ + skippedColumns?: string[] + dataset?: string | null + tagIds?: string[] + /** + * Worksheet name to import when the file is an Excel workbook. + * Required for .xlsx / .xlsm uploads; must be left undefined / null + * for CSV uploads (the backend rejects a non-empty value for CSV). + */ + sheetName?: string | null + } + ): Promise { + return this.callImports.uploadCallImport(file, options) + } + + /** + * UPLOAD stage of the staged call-import flow. + * + * Persists the CSV / Excel file to S3, captures a sheets snapshot, + * and creates a ``CallImport`` row with ``status='uploaded'``. The + * returned record has no mapping / provider / rows yet — the caller + * follows up with {@link updateCallImportMapping} and + * {@link startCallImport} to advance the batch. + */ + async createCallImport( + file: File, + options: { + dataset: string + tagIds?: string[] + schemaId?: string | null + contentModality?: 'chat' | 'voice' + }, + ): Promise { + return this.callImports.createCallImport(file, options) + } + + /** @deprecated Prefer presigned upload via uploadCallImportAudioChunked. */ + async uploadCallImportAudio( + files: File[], + options: { + dataset: string + tagIds?: string[] + batchName?: string + }, + ): Promise { + return this.callImports.uploadCallImportAudio(files, options) + } + + async appendCallImportAudio( + importId: string, + files: File[], + ): Promise { + return this.callImports.appendCallImportAudio(importId, files) + } + + async uploadCallImportAudioChunked( + files: File[], + options: { + dataset: string + tagIds?: string[] + batchName?: string + onProgress?: (progress: CallImportAudioUploadProgress) => void + }, + ): Promise { + return this.callImports.uploadCallImportAudioChunked(files, options) + } + + async uploadCallImportAudioAppendChunked( + importId: string, + files: File[], + options?: { + onProgress?: (progress: CallImportAudioUploadProgress) => void + }, + ): Promise { + return this.callImports.uploadCallImportAudioAppendChunked(importId, files, options) + } + + /** + * MAP stage of the staged call-import flow. + * + * Persists the schema + sheet + parameter mapping on the batch. The + * backend validates against the sheet snapshot it cached at upload + * time, so this is purely a metadata operation (no S3 fetch). + * Idempotent: safe to call repeatedly while the batch is in + * ``uploaded`` or ``mapped`` state. + */ + async updateCallImportMapping( + id: string, + options: { + schemaId: string + sheetName?: string | null + parameterMapping: Record + skippedColumns?: string[] + }, + ): Promise { + return this.callImports.updateCallImportMapping(id, options) + } + + /** + * IMPORT stage of the staged call-import flow. + * + * Materialises rows from the staged source file using the persisted + * mapping and fans them out to the ``imports`` Celery queue. Returns + * the same shape as the legacy one-shot ``uploadCallImport``. + */ + async startCallImport( + id: string, + options: { + provider: string | null + telephonyIntegrationId: string | null + }, + ): Promise { + return this.callImports.startCallImport(id, options) + } + + // ------------------------------------------------------------------- + // Call Import Schemas (reusable Input Parameter schemas) + // ------------------------------------------------------------------- + async listCallImportSchemas(): Promise { + return this.callImports.listCallImportSchemas() + } + + async getCallImportSchema(id: string): Promise { + return this.callImports.getCallImportSchema(id) + } + + async createCallImportSchema( + payload: CallImportSchemaCreate, + ): Promise { + return this.callImports.createCallImportSchema(payload) + } + + async updateCallImportSchema( + id: string, + payload: CallImportSchemaUpdate, + ): Promise { + return this.callImports.updateCallImportSchema(id, payload) + } + + async deleteCallImportSchema( + id: string, + options?: { force?: boolean }, + ): Promise { + return this.callImports.deleteCallImportSchema(id, options) + } + + async previewPlatformCallImport(payload: { + voice_integration_id: string + window_start: string + window_end: string + window_timezone?: string + dataset?: string + filters?: { + agent_ids?: string[] + min_duration_seconds?: number + speed?: string + import_provider_transcript?: boolean + medium?: 'voice' | 'chat' + agents?: Array<{ id: string; name: string; medium?: string | null }> + } + }): Promise<{ + recordings_in_window: number + recordings_count_complete: boolean + count_pages_scanned?: number + discovery_slices?: number + sample_count: number + samples: Array<{ + conversation_id: string + agent_id?: string | null + started_at?: string | null + status?: string | null + }> + }> { + return this.callImports.previewPlatformCallImport(payload) + } + + async createPlatformCallImport(payload: { + voice_integration_id: string + window_start: string + window_end: string + window_timezone?: string + name?: string + dataset?: string + max_calls?: number + filters?: { + agent_ids?: string[] + min_duration_seconds?: number + speed?: string + import_provider_transcript?: boolean + medium?: 'voice' | 'chat' + agents?: Array<{ id: string; name: string; medium?: string | null }> + } + }, idempotencyKey?: string): Promise<{ id: string; status: string; discovery_status?: string }> { + return this.callImports.createPlatformCallImport(payload, idempotencyKey) + } + + async pausePlatformImport(importId: string): Promise<{ status: string }> { + return this.callImports.pausePlatformImport(importId) + } + + async resumePlatformImport(importId: string): Promise<{ status: string }> { + return this.callImports.resumePlatformImport(importId) + } + + async retryPlatformDiscovery(importId: string): Promise<{ status: string }> { + return this.callImports.retryPlatformDiscovery(importId) + } + + async listPlatformImportAgents( + integrationId: string, + ): Promise> { + return this.callImports.listPlatformImportAgents(integrationId) + } + + async getPlatformImportProgress(importId: string): Promise { + return this.callImports.getPlatformImportProgress(importId) + } + + async listCallImports( + params: { + page?: number + page_size?: number + status?: CallImportStatus + dataset?: string + tag_id?: string[] + source_format?: string + } = {} + ): Promise { + return this.callImports.listCallImports(params) + } + + async listCallImportDatasets(): Promise { + return this.callImports.listCallImportDatasets() + } + + async getCallImport( + id: string, + params: { + row_limit?: number + row_offset?: number + q?: string + /** + * Optional filter on ``CallImportRow.diarised_transcript_status``. + * When set, ``filtered_total_rows`` on the response reflects the + * post-filter row count (combined with ``q`` when both are + * supplied) so the UI can paginate against the same slice. + */ + diarised_status?: 'pending' | 'running' | 'completed' | 'failed' + } = {} + ): Promise { + return this.callImports.getCallImport(id, params) + } + + async getCallImportRow(id: string, rowId: string): Promise { + return this.callImports.getCallImportRow(id, rowId) + } + + async listCallImportRowIds( + id: string, + params: { + q?: string + diarised_status?: 'pending' | 'running' | 'completed' | 'failed' + } = {} + ): Promise<{ ids: string[]; total: number }> { + return this.callImports.listCallImportRowIds(id, params) + } + + async updateCallImport( + id: string, + payload: { + original_filename?: string | null + dataset?: string | null + tag_ids?: string[] + /** + * Reassign the Input Parameter schema. Only honoured while the + * batch is in ``uploaded`` / ``mapped`` state. Switching schemas + * resets the mapping and rewinds the batch to ``uploaded``. + */ + schema_id?: string | null + }, + ): Promise { + return this.callImports.updateCallImport(id, payload) + } + + async deleteCallImport( + id: string, + ): Promise<{ id: string; status: 'accepted' | 'completed' }> { + return this.callImports.deleteCallImport(id) + } + + async deleteCallImportRow(id: string, rowId: string): Promise { + return this.callImports.deleteCallImportRow(id, rowId) + } + + async bulkDeleteCallImportRows( + id: string, + options: + | { rowIds: string[] } + | { rowFilter: CallImportRowSelectionFilter }, + ): Promise<{ deleted: number; status?: 'completed' | 'accepted' }> { + return this.callImports.bulkDeleteCallImportRows(id, options) + } + + async createCallImportEvaluation( + callImportId: string, + payload: { + metric_ids: string[] + name?: string | null + /** + * Which transcript to score against. ``'diarised'`` (default) + * auto-diarizes then scores; ``'production'`` scores the CSV + * transcript directly. + */ + transcript_sources?: Array<'production' | 'diarised'> + /** Run-level LLM provider key. Leave undefined for legacy default. */ + llm_provider?: string | null + llm_model?: string | null + llm_credential_id?: string | null + llm_config?: LLMGenerationConfig | null + /** Per-metric LLM override map keyed by metric UUID. */ + metric_llm_overrides?: Record | null + /** When true, diarize rows missing transcripts before evaluation. */ + auto_transcribe?: boolean + transcribe_overwrite?: boolean + /** + * Diarisation pipeline shape for the auto-transcribe step. + * 'stt_llm' (default) runs STT then an LLM diariser over the + * resulting text. 'llm_only' skips STT and feeds the audio + * directly to the multimodal ``diarization_llm_*`` model — STT + * fields must be omitted in that case. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + stt_provider?: string | null + stt_model?: string | null + stt_credential_id?: string | null + stt_language?: string | null + /** + * LLM diariser config: post-STT, an LLM splits the plain + * transcript into agent/user turns using ``diarization_prompt`` + * (or the canonical default when empty). Required by the server + * when ``auto_transcribe`` is set. Also used in 'llm_only' mode + * where this same LLM receives the audio directly. + */ + diarization_llm_provider?: string | null + diarization_llm_model?: string | null + diarization_llm_credential_id?: string | null + diarization_prompt?: string | null + /** + * Opt into LLM-driven discovery of brand-new top-level metrics + * for this run. Surfaces candidates in the Discovered metrics + * panel on the evaluation detail Flow tab. + */ + discover_new_metrics?: boolean + /** Telephony credentials when starting from a mapped batch. */ + provider?: string | null + telephony_integration_id?: string | null + }, + ): Promise { + return this.callImports.createCallImportEvaluation(callImportId, payload) + } + + /** Aggregated metric distributions for a single evaluation run. */ + async getCallImportEvaluationAggregate( + callImportId: string, + evaluationId: string, + baselineEvaluationId?: string | null, + ): Promise { + return this.callImports.getCallImportEvaluationAggregate(callImportId, evaluationId, baselineEvaluationId) + } + + /** + * Read the cached LLM TLDR for an evaluation. Returns ``null`` when + * the user has never generated one (the empty-state CTA renders + * this case). + */ + async getCallImportEvaluationInsights( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationInsights(callImportId, evaluationId) + } + + /** + * Generate (or return cached) the LLM TLDR for an evaluation run. + * + * - Pass ``regenerate: true`` to force a fresh LLM call even when a + * cached summary exists at the current ``completed_rows`` watermark. + * - Pass ``provider`` + ``model`` to pin a specific LLM. Omit both to + * let the backend auto-detect the org's first active OpenAI / + * Anthropic / Google credential (mirroring Prompt Partials). + */ + async generateCallImportEvaluationInsights( + callImportId: string, + evaluationId: string, + options?: { + regenerate?: boolean + provider?: string | null + model?: string | null + credential_id?: string | null + max_llm_calls?: number | null + }, + ): Promise { + return this.callImports.generateCallImportEvaluationInsights(callImportId, evaluationId, options) + } + + async getCallImportEvaluationUserInsights( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationUserInsights(callImportId, evaluationId) + } + + async getCallImportEvaluationMetricClusters( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationMetricClusters(callImportId, evaluationId) + } + + async getCallImportEvaluationPromptImprovements( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationPromptImprovements(callImportId, evaluationId) + } + + async generateCallImportEvaluationPromptImprovements( + callImportId: string, + evaluationId: string, + data: { + imported_agent_id?: string + regenerate?: boolean + force?: boolean + provider?: string + model?: string + credential_id?: string + }, + ): Promise { + return this.callImports.generateCallImportEvaluationPromptImprovements(callImportId, evaluationId, data) + } + + async reviewCallImportEvaluationPromptImprovement( + callImportId: string, + evaluationId: string, + data: { + suggestion_id: string + review_status: 'pending' | 'accepted' | 'rejected' + edited_text?: string | null + }, + ): Promise { + return this.callImports.reviewCallImportEvaluationPromptImprovement(callImportId, evaluationId, data) + } + + async previewCallImportEvaluationPromptPush( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.previewCallImportEvaluationPromptPush(callImportId, evaluationId) + } + + async pushCallImportEvaluationPrompt( + callImportId: string, + evaluationId: string, + confirm: string, + options?: { pushToLive?: boolean }, + ): Promise { + return this.callImports.pushCallImportEvaluationPrompt(callImportId, evaluationId, confirm, options) + } + + async listCallImportEvaluationMetricClusterEligibleRows( + callImportId: string, + evaluationId: string, + options?: { + limit?: number + count_only?: boolean + }, + ): Promise { + return this.callImports.listCallImportEvaluationMetricClusterEligibleRows(callImportId, evaluationId, options) + } + + async getCallImportEvaluationMetricClusterFailurePolicies( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationMetricClusterFailurePolicies(callImportId, evaluationId) + } + + async saveCallImportEvaluationMetricClusterFailurePolicies( + callImportId: string, + evaluationId: string, + policies: Record, + ): Promise { + return this.callImports.saveCallImportEvaluationMetricClusterFailurePolicies(callImportId, evaluationId, policies) + } + + async generateCallImportEvaluationMetricClusters( + callImportId: string, + evaluationId: string, + options?: { + regenerate?: boolean + force?: boolean + provider?: string | null + model?: string | null + credential_id?: string | null + max_llm_calls?: number | null + evaluation_row_ids?: string[] | null + row_limit?: number | null + failure_policies?: Record< + string, + import('../types/api').MetricFailurePolicy + > | null + }, + ): Promise { + return this.callImports.generateCallImportEvaluationMetricClusters(callImportId, evaluationId, options) + } + + async cancelCallImportEvaluationMetricClusters( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.cancelCallImportEvaluationMetricClusters(callImportId, evaluationId) + } + + async generateCallImportEvaluationUserInsights( + callImportId: string, + evaluationId: string, + options?: { + regenerate?: boolean + force?: boolean + provider?: string | null + model?: string | null + credential_id?: string | null + max_llm_calls?: number | null + }, + ): Promise { + return this.callImports.generateCallImportEvaluationUserInsights(callImportId, evaluationId, options) + } + + /** + * Aggregate flow chart for a parent metric: returns nodes/edges built + * from per-row LLM-inferred ``sequence`` arrays. Used by the React + * Flow visualisation on the evaluation overview. + */ + async getCallImportEvaluationFlow( + callImportId: string, + evaluationId: string, + parentMetricId: string, + ): Promise { + return this.callImports.getCallImportEvaluationFlow(callImportId, evaluationId, parentMetricId) + } + + /** + * List LLM-discovered candidate sub-labels for a parent metric in an + * evaluation. Returned items power the Discovered Labels panel where + * users decide which candidates to promote or merge. + */ + async getCallImportEvaluationDiscoveredLabels( + callImportId: string, + evaluationId: string, + parentMetricId: string, + ): Promise { + return this.callImports.getCallImportEvaluationDiscoveredLabels(callImportId, evaluationId, parentMetricId) + } + + /** + * Merge a discovered slug into another within an evaluation. Rewrites + * every row's discovered_labels and sequence array so the panel + + * flow chart converge on the surviving slug. + */ + async mergeCallImportEvaluationDiscoveredLabels( + callImportId: string, + evaluationId: string, + body: { parent_metric_id: string; from_key: string; to_key: string }, + ): Promise { + return this.callImports.mergeCallImportEvaluationDiscoveredLabels(callImportId, evaluationId, body) + } + + /** + * Delete (tombstone) an LLM-discovered candidate. Strips the slug + * from every row's discovered_labels + sequence and records a + * deletion alias on the evaluation so workers finishing later don't + * resurrect it. Use for gibberish candidates the LLM proposed. + */ + async deleteCallImportEvaluationDiscoveredLabel( + callImportId: string, + evaluationId: string, + body: { parent_metric_id: string; key: string }, + ): Promise { + return this.callImports.deleteCallImportEvaluationDiscoveredLabel(callImportId, evaluationId, body) + } + + /** + * Promote an LLM-discovered candidate into a real child Metric under + * the given parent. ``key`` must equal ``slugify(name)`` so existing + * rows' sequence arrays auto-resolve against the new child. + * + * ``capture_rationale`` defaults to true on the backend (matches the + * Discovered Labels UX: candidates are LLM-suggested with rationales, + * so the user almost always wants future rows that hit them to keep + * capturing rationales). + */ + async promoteDiscoveredChild( + parentMetricId: string, + body: { + key: string + name: string + description?: string | null + capture_rationale?: boolean + }, + ): Promise { + return this.callImports.promoteDiscoveredChild(parentMetricId, body) + } + + /** + * List LLM-discovered candidate TOP-LEVEL metrics for an evaluation. + * Mirrors :func:`getCallImportEvaluationDiscoveredLabels` but is + * scoped to the evaluation as a whole (no ``parent_metric_id``). + * Returns an empty ``items`` list when the evaluation didn't opt + * into top-level metric discovery, so callers can fetch + * unconditionally. + */ + async getCallImportEvaluationDiscoveredMetrics( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluationDiscoveredMetrics(callImportId, evaluationId) + } + + /** + * Merge one discovered top-level metric slug into another. Rewrites + * every row's ``__discovered_metrics__`` list and records the + * alias on the evaluation so workers finishing later converge on + * the surviving slug. + */ + async mergeCallImportEvaluationDiscoveredMetrics( + callImportId: string, + evaluationId: string, + body: { from_key: string; to_key: string }, + ): Promise { + return this.callImports.mergeCallImportEvaluationDiscoveredMetrics(callImportId, evaluationId, body) + } + + /** + * Tombstone an LLM-discovered top-level metric candidate. Strips + * the slug from every row's ``__discovered_metrics__`` list and + * records a deletion alias on the evaluation so workers finishing + * later can't resurrect it. + */ + async deleteCallImportEvaluationDiscoveredMetric( + callImportId: string, + evaluationId: string, + body: { key: string }, + ): Promise { + return this.callImports.deleteCallImportEvaluationDiscoveredMetric(callImportId, evaluationId, body) + } + + /** + * Promote an LLM-discovered top-level metric candidate into a real + * standalone :class:`Metric` row. ``key`` must equal + * ``slugify(name)`` so already-scored rows that referenced the + * candidate keep resolving against the promoted metric. + * + * ``metric_type`` selects how the new metric will be scored on + * future runs (``boolean`` / ``rating`` / ``category``). ``category`` + * creates a ``multi_label`` parent with no children — the user adds + * children via the existing Metrics page. + */ + async promoteDiscoveredMetric(body: { + key: string + name: string + description?: string | null + metric_type?: 'boolean' | 'rating' | 'category' + capture_rationale?: boolean + custom_config?: Record | null + }): Promise { + return this.callImports.promoteDiscoveredMetric(body) + } + + /** Cross-run insights for the call import detail page. */ + async getCallImportInsights( + callImportId: string, + ): Promise { + return this.callImports.getCallImportInsights(callImportId) + } + + /** + * Fetch the canonical LLM diariser prompt. Used by the Transcribe / + * Run Evaluation modals to pre-fill the prompt textarea so the + * operator sees the actual default they'd otherwise get. + */ + async getCallImportDiarisationPromptDefault(): Promise { + return this.callImports.getCallImportDiarisationPromptDefault() + } + + /** Fan out diarization tasks for a batch of rows. */ + async transcribeCallImport( + callImportId: string, + payload: CallImportTranscribeRequest, + ): Promise { + return this.callImports.transcribeCallImport(callImportId, payload) + } + + /** Diarize a single row's recording. */ + async transcribeCallImportRow( + callImportId: string, + rowId: string, + payload: CallImportTranscribeRequest, + ): Promise { + return this.callImports.transcribeCallImportRow(callImportId, rowId, payload) + } + + /** + * Re-enqueue every failed import row in this batch. + * + * Rows are reset to `pending` first so the UI can immediately show that + * the retry sweep started; rows that fail to enqueue again are returned + * in `enqueue_failed` and stay `failed`. + */ + async retryFailedCallImportRows( + callImportId: string, + options?: { + provider: string | null + telephonyIntegrationId: string | null + }, + ): Promise { + return this.callImports.retryFailedCallImportRows(callImportId, options) + } + + /** + * Abort an in-flight (or queued) diarisation for a single row. + * + * Idempotent — calling on a row whose diarisation is already in a + * terminal state (``completed`` / ``failed`` / ``idle``) returns the + * row unchanged so the UI can wire this to a "Stop" button without + * pre-checking state. The backend flips the row's + * ``diarised_transcript_status`` to ``failed`` and stamps + * ``"Diarisation cancelled by user"`` as the error so the existing + * diarisation pill in the row header renders the right state on the + * next poll. + */ + async cancelCallImportRowDiarisation( + callImportId: string, + rowId: string, + ): Promise { + return this.callImports.cancelCallImportRowDiarisation(callImportId, rowId) + } + + /** + * Abort in-flight diarisation for many rows in one call. + * + * Pass ``rowIds`` to cancel a specific subset. Omit it (or pass + * ``null``) to cancel every row in the import whose + * ``diarised_transcript_status`` is currently ``pending`` or + * ``running`` — the "stop everything" affordance. + */ + async cancelCallImportDiarisation( + callImportId: string, + options?: { rowIds?: string[] | null; rowFilter?: CallImportRowSelectionFilter | null }, + ): Promise<{ cancelled: number; skipped: number }> { + return this.callImports.cancelCallImportDiarisation(callImportId, options) + } + + /** + * Flip ``diarised_speaker_swap`` on a row and re-render + * ``diarised_transcript`` from ``diarised_segments``. Returns the + * updated row so the caller can swap it into local state without an + * extra refetch. + * + * The backend rejects this with 409 when the row has no structured + * segments (e.g. legacy diarisations done before turns were + * persisted) — callers should surface that as "re-diarise to enable + * speaker swapping". + */ + async toggleCallImportRowSpeakerSwap( + callImportId: string, + rowId: string, + ): Promise { + return this.callImports.toggleCallImportRowSpeakerSwap(callImportId, rowId) + } + + async updateCallImportEvaluation( + callImportId: string, + evaluationId: string, + payload: { name?: string | null }, + ): Promise { + return this.callImports.updateCallImportEvaluation(callImportId, evaluationId, payload) + } + + async bulkDeleteCallImportEvaluations( + callImportId: string, + evaluationIds: string[], + ): Promise<{ deleted: number }> { + return this.callImports.bulkDeleteCallImportEvaluations(callImportId, evaluationIds) + } + + async deleteCallImportEvaluationRow( + callImportId: string, + evaluationId: string, + evalRowId: string, + ): Promise { + return this.callImports.deleteCallImportEvaluationRow(callImportId, evaluationId, evalRowId) + } + + /** + * Re-enqueue failed evaluation rows. With no body, every row in the + * run whose status is ``failed`` is re-run with the run's saved + * config. Pass ``evalRowIds`` to scope the retry to a subset (used + * by the per-row "Retry" button), and/or the ``llm_*`` / ``stt_*`` + * fields to swap out the LLM / STT provider before re-enqueueing + * (the backend validates + persists them onto the run). + * + * Rows still in progress / already completed are silently skipped + * on the server and reported in the response's ``skipped`` list — + * the mutation never fails just because the caller's selection is + * stale. + */ + async retryCallImportEvaluation( + callImportId: string, + evaluationId: string, + options?: { + evalRowIds?: string[] + rowFilter?: import('../types/api').CallImportEvaluationRowRetryFilter | null + llmProvider?: string | null + llmModel?: string | null + llmCredentialId?: string | null + llmConfig?: LLMGenerationConfig | null + sttProvider?: string | null + sttModel?: string | null + sttCredentialId?: string | null + transcribeOverwrite?: boolean + transcribeMode?: 'stt_llm' | 'llm_only' + diarizationLlmProvider?: string | null + diarizationLlmModel?: string | null + diarizationLlmCredentialId?: string | null + diarizationPrompt?: string | null + /** + * Override telephony credentials on the batch when failed rows must + * re-fetch recordings. Send both fields together; omit to keep the + * batch's existing pinned credentials. + */ + provider?: string | null + telephonyIntegrationId?: string | null + /** + * Metric-subset retry: when set, only these metrics are + * recomputed and merged into the row's existing metric_scores + * (other metrics' previous values are preserved). The backend + * auto-flips ``include_completed`` to true in this case so + * already-successful rows are eligible. + */ + metricIds?: string[] + /** + * When true, rows currently ``completed`` are eligible for + * retry too (otherwise only ``failed`` rows are picked up). + * Required-and-implied when ``metricIds`` is set. + */ + includeCompleted?: boolean + /** + * Per-metric LLM overrides, keyed by leaf metric id. Replaces the + * run's stored overrides, so callers should merge with the existing + * map. Used to route classification metrics to a System 1 model. + */ + metricLlmOverrides?: Record + }, + ): Promise { + return this.callImports.retryCallImportEvaluation(callImportId, evaluationId, options) + } + + /** + * Re-enqueue a single failed evaluation row. Returns the refreshed + * row so the UI can flip the badge to ``pending`` immediately + * without waiting for the next polling tick. Server returns 409 if + * the row is still in progress or already completed. + */ + async retryCallImportEvaluationRow( + callImportId: string, + evaluationId: string, + evalRowId: string, + ): Promise { + return this.callImports.retryCallImportEvaluationRow(callImportId, evaluationId, evalRowId) + } + + /** + * Abort an entire in-flight (or queued) evaluation run. + * + * Idempotent — calling on a run whose rows are already terminal + * returns the run unchanged so the UI can wire this to an "Abort + * run" button without pre-checking state. The backend flips every + * cancellable row's ``status`` to ``failed`` and stamps + * ``"Evaluation cancelled by user"`` as the error so the polling UI + * surfaces the cancel on the next tick. Pairs with the worker's + * cancellation guard which prevents a late-finishing task from + * overwriting the cancelled state. + */ + async cancelCallImportEvaluation( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.cancelCallImportEvaluation(callImportId, evaluationId) + } + + /** + * Force-fail only rows currently in ``pending`` for an evaluation run. + * + * Unlike ``cancelCallImportEvaluation``, this does NOT touch rows that + * are already ``running``. Useful when queued rows are stuck indefinitely + * but active workers should keep progressing. + */ + async forceFailCallImportEvaluationPending( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.forceFailCallImportEvaluationPending(callImportId, evaluationId) + } + + /** + * Abort an in-flight (or queued) evaluation for a single row. + * + * Mirrors :func:`cancelCallImportEvaluation` but scoped to one row, + * intended for the per-row Stop button on the evaluation detail + * rows table. The parent run's counters are rolled up server-side + * so the run-level pill updates on the same response. + */ + async cancelCallImportEvaluationRow( + callImportId: string, + evaluationId: string, + evalRowId: string, + ): Promise { + return this.callImports.cancelCallImportEvaluationRow(callImportId, evaluationId, evalRowId) + } + + async listCallImportEvaluations(callImportId: string): Promise { + return this.callImports.listCallImportEvaluations(callImportId) + } + + async getCallImportEvaluation( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.getCallImportEvaluation(callImportId, evaluationId) + } + + async listCallImportEvaluationRows( + callImportId: string, + evaluationId: string, + params: { + page?: number + page_size?: number + q?: string + metric_id?: string + metric_value?: string + classification_yes_no?: string + classification_choice?: string + classification_level?: string + status?: string + // Flow-chart drilldown: filter to rows whose sequence under the + // given parent contains ``flow_node`` (and optionally is + // immediately followed by ``flow_edge_target`` for edge clicks). + // Accepts a child UUID, a ``disc:`` discovered id, or a + // raw slug. + flow_parent_id?: string + flow_node?: string + flow_edge_target?: string + // Discovered-label filters: either pin to a specific discovered + // slug or surface every row that produced any candidate under + // ``discovered_parent_id``. + discovered_parent_id?: string + discovered_label_key?: string + has_discovered?: boolean + // Column-click sorting from the UI. ``sort_by`` is either a + // built-in column key (``row_index`` / ``conversation_id`` / + // ``status``) or ``metric:`` for per-metric value + // sorting; ``sort_dir`` is ``asc`` (default) or ``desc``. + sort_by?: string + sort_dir?: 'asc' | 'desc' + } = {}, + ): Promise { + return this.callImports.listCallImportEvaluationRows(callImportId, evaluationId, params) + } + + async getCallImportEvaluationRow( + callImportId: string, + evaluationId: string, + rowId: string, + ): Promise { + return this.callImports.getCallImportEvaluationRow(callImportId, evaluationId, rowId) + } + + async getCallImportEvaluationRowWerAlignment( + callImportId: string, + evaluationId: string, + rowId: string, + ) { + return this.callImports.getCallImportEvaluationRowWerAlignment( + callImportId, + evaluationId, + rowId, + ) + } + + async startCallImportEvaluationExport( + callImportId: string, + evaluationId: string, + format: 'csv' | 'xlsx' = 'csv', + includeTranscripts = true, + ): Promise { + return this.callImports.startCallImportEvaluationExport(callImportId, evaluationId, format, includeTranscripts) + } + + async getCallImportEvaluationExport( + callImportId: string, + evaluationId: string, + jobId: string, + ): Promise { + return this.callImports.getCallImportEvaluationExport(callImportId, evaluationId, jobId) + } + + async generateCallImportEvaluationPdfReport( + callImportId: string, + evaluationId: string, + vendorName: string, + reportType: 'external' | 'internal', + includeWeeklyDelta = false, + options?: { + internalBrandImageId?: string | null + externalBrandImageId?: string | null + useCase?: string | null + baselineEvaluationId?: string | null + reportConfig?: Record + platformBaseUrl?: string | null + }, + ): Promise { + return this.callImports.generateCallImportEvaluationPdfReport(callImportId, evaluationId, vendorName, reportType, includeWeeklyDelta, options) + } + + async listCallImportEvaluationPdfReports( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.listCallImportEvaluationPdfReports(callImportId, evaluationId) + } + + async getCallImportEvaluationPdfReport( + callImportId: string, + evaluationId: string, + reportId: string, + ): Promise { + return this.callImports.getCallImportEvaluationPdfReport(callImportId, evaluationId, reportId) + } + + async downloadCallImportEvaluationPdfReport( + callImportId: string, + evaluationId: string, + reportId: string, + ): Promise { + return this.callImports.downloadCallImportEvaluationPdfReport(callImportId, evaluationId, reportId) + } + + async listCallImportEvaluationBaselineCandidates( + callImportId: string, + evaluationId: string, + ): Promise { + return this.callImports.listCallImportEvaluationBaselineCandidates(callImportId, evaluationId) + } + + async deleteCallImportEvaluation(callImportId: string, evaluationId: string): Promise { + return this.callImports.deleteCallImportEvaluation(callImportId, evaluationId) + } + + // Call Import Tags endpoints + async listCallImportTags(): Promise { + return this.callImports.listCallImportTags() + } + + async createCallImportTag(payload: { + name: string + color?: string | null + }): Promise { + return this.callImports.createCallImportTag(payload) + } + + async updateCallImportTag( + tagId: string, + payload: { name?: string; color?: string | null }, + ): Promise { + return this.callImports.updateCallImportTag(tagId, payload) + } + + async deleteCallImportTag(tagId: string): Promise { + return this.callImports.deleteCallImportTag(tagId) + } + + // Model Config endpoints + async getDashboardSummary(): Promise { + const response = await this.client.get('/api/v1/dashboard/summary') + return response.data + } + + async getOrgUsageFxRate(): Promise<{ + base: string + quote: string + rate: number + as_of: string + source: string + }> { + const response = await this.client.get('/api/v1/organizations/usage/fx-rate') + return response.data + } + + async getOrgUsageSummary(params: { + start?: string + end?: string + tz?: string + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + usage_kind?: string + call_import_id?: string + evaluation_id?: string + dataset?: string + tag_id?: string + }, signal?: AbortSignal): Promise<{ + start: string + end: string + totals: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + audio_seconds: number + tts_characters: number + call_count: number + input_cost_micro_usd?: number + output_cost_micro_usd?: number + cache_read_cost_micro_usd?: number + cache_creation_cost_micro_usd?: number + reasoning_cost_micro_usd?: number + audio_cost_micro_usd?: number + tts_cost_micro_usd?: number + total_cost_micro_usd?: number + costs?: { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + currency: string + has_unpriced_usage: boolean + } + } + last_updated_at?: string | null + }> { + const response = await this.client.get('/api/v1/organizations/usage/summary', { + params, + signal, + }) + return response.data + } + + async getOrgUsageBreakdown(params: { + start?: string + end?: string + tz?: string + group_by?: 'workspace' | 'product_section' | 'model' | 'resource' | 'usage_kind' | 'call_import' + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + usage_kind?: string + call_import_id?: string + evaluation_id?: string + dataset?: string + tag_id?: string + limit?: number + offset?: number + omit_call_import_sources?: boolean + }, signal?: AbortSignal): Promise<{ + start: string + end: string + group_by: string + rows: Array<{ + workspace_id?: string | null + workspace_name?: string | null + product_section?: string | null + product_section_label?: string | null + model?: string | null + resource_id?: string | null + resource_type?: string | null + resource_label?: string | null + call_import_id?: string | null + call_import_label?: string | null + usage_kind?: string | null + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + audio_seconds: number + tts_characters: number + call_count: number + input_cost_micro_usd?: number + output_cost_micro_usd?: number + cache_read_cost_micro_usd?: number + cache_creation_cost_micro_usd?: number + reasoning_cost_micro_usd?: number + audio_cost_micro_usd?: number + tts_cost_micro_usd?: number + total_cost_micro_usd?: number + costs?: { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + currency: string + has_unpriced_usage: boolean + } + }> + total_count: number + truncated_at_limit?: boolean + last_updated_at?: string | null + }> { + const response = await this.client.get('/api/v1/organizations/usage/breakdown', { + params, + signal, + }) + return response.data + } + + async listUsagePricingAvailableModels(): Promise<{ models: string[] }> { + const response = await this.client.get( + '/api/v1/organizations/usage/pricing/available-models', + ) + return response.data + } + + async getUsagePricingEffective( + model: string, + params?: { usage_kind?: string; as_of?: string }, + ): Promise<{ + model: string + usage_kind: string + as_of: string + catalog_rates?: Record | null + effective_rates?: Record | null + effective_source?: string | null + has_override: boolean + override?: { + effective_from: string + effective_to?: string | null + rates: Record + } | null + }> { + const response = await this.client.get( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + { params }, + ) + return response.data + } + + async listUsagePricingOverrides(params?: { + model?: string + usage_kind?: string + }): Promise< + Array<{ + id: string + organization_id: string + model: string + usage_kind: string + effective_from: string + effective_to?: string | null + rates: { + input_per_1m?: number | null + output_per_1m?: number | null + cache_read_per_1m?: number | null + cache_write_per_1m?: number | null + reasoning_per_1m?: number | null + audio_per_minute?: number | null + tts_per_1m_characters?: number | null + } + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> + > { + const response = await this.client.get('/api/v1/organizations/usage/pricing/overrides', { + params, + }) + return response.data + } + + async upsertUsagePricingOverride( + model: string, + body: { + usage_kind?: string + effective_from: string + effective_to?: string + rates: { + input_per_1m?: number + output_per_1m?: number + cache_read_per_1m?: number + cache_write_per_1m?: number + reasoning_per_1m?: number + audio_per_minute?: number + tts_per_1m_characters?: number + } + recompute?: boolean + } + ): Promise<{ + id: string + model: string + usage_kind: string + effective_from: string + effective_to?: string | null + rates: Record + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> { + const response = await this.client.put( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + body + ) + return response.data + } + + async deleteUsagePricingOverride( + model: string, + params?: { usage_kind?: string; effective_from?: string; recompute?: boolean } + ): Promise<{ + deleted: boolean + model: string + usage_kind: string + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> { + const response = await this.client.delete( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + { params } + ) + return response.data + } + + async triggerUsageCostRecompute(body?: { + start_date?: string + end_date?: string + model?: string + usage_kind?: string + }): Promise<{ + id: string + organization_id: string + status: string + model?: string | null + usage_kind?: string | null + start_date?: string | null + end_date?: string | null + updated_rows: number + error_message?: string | null + celery_task_id?: string | null + created_at?: string | null + updated_at?: string | null + completed_at?: string | null + }> { + const response = await this.client.post( + '/api/v1/organizations/usage/pricing/recompute', + body ?? {} + ) + return response.data + } + + async getUsageCostRecomputeJob(jobId: string): Promise<{ + id: string + organization_id: string + status: string + model?: string | null + usage_kind?: string | null + start_date?: string | null + end_date?: string | null + updated_rows: number + error_message?: string | null + celery_task_id?: string | null + created_at?: string | null + updated_at?: string | null + completed_at?: string | null + }> { + const response = await this.client.get( + `/api/v1/organizations/usage/pricing/recompute/${jobId}` + ) + return response.data + } + + async getOrgUsageFilters(params?: { + start?: string + end?: string + tz?: string + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + evaluation_id?: string + usage_kind?: string + call_import_id?: string + q?: string + dataset?: string + tag_id?: string + }, signal?: AbortSignal): Promise<{ + workspaces: Array<{ id: string; name: string }> + product_sections: Array<{ id: string; label: string }> + call_imports: Array<{ id: string; label: string }> + evaluations: Array<{ id: string; label: string }> + models: string[] + resources: Array<{ id: string; type?: string; label: string; product_section?: string }> + usage_kinds: Array<{ id: string; label: string }> + datasets: string[] + tags: Array<{ id: string; label: string }> + }> { + const response = await this.client.get('/api/v1/organizations/usage/filters', { + params, + signal, + }) + return response.data + } + + async getAllModels(): Promise> { + const response = await this.client.get('/api/v1/model-config/models') + return response.data + } + + async getModelConfig(modelName: string): Promise { + const response = await this.client.get(`/api/v1/model-config/models/${modelName}`) + return response.data + } + + async getModelsByProvider(provider: string): Promise { + const response = await this.client.get(`/api/v1/model-config/providers/${provider}/models`) + return response.data + } + + async getModelOptions(provider: string): Promise<{ stt: string[]; llm: string[]; tts: string[]; s2s: string[]; tts_voices: Record; tts_sample_rates?: number[] }> { + const response = await this.client.get(`/api/v1/model-config/providers/${provider}/options`) + const data = response.data + // Ensure s2s and tts_voices are always present (for backward compatibility) + return { + ...data, + s2s: data.s2s || [], + tts_voices: data.tts_voices || {}, + } + } + + async getModelsByType(provider: string, modelType: 'stt' | 'llm' | 'tts'): Promise { + const response = await this.client.get(`/api/v1/model-config/providers/${provider}/types/${modelType}/models`) + return response.data + } + + // Manual Evaluations endpoints + async listManualEvaluationAudioFiles(prefix?: string, maxKeys = 1000): Promise { + const response = await this.client.get('/api/v1/manual-evaluations/audio-files', { + params: { prefix, max_keys: maxKeys }, + }) + return response.data + } + + async getAudioPresignedUrl(fileKey: string, expiration = 3600): Promise<{ url: string; expires_in: number }> { + const response = await this.client.get( + `/api/v1/manual-evaluations/audio-files/${encodeURIComponent(fileKey)}/presigned-url`, + { + params: { expiration }, + } + ) + return response.data + } + + async transcribeAudio(data: { + audio_file_key: string + stt_provider: string + stt_model: string + name?: string + language?: string + enable_speaker_diarization?: boolean + }): Promise { + const response = await this.client.post('/api/v1/manual-evaluations/transcribe', data) + return response.data + } + + async listManualTranscriptions(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/manual-evaluations/transcriptions', { + params: { skip, limit }, + }) + return response.data + } + + async getManualTranscription(transcriptionId: string): Promise { + const response = await this.client.get(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`) + return response.data + } + + async updateManualTranscription(transcriptionId: string, name: string): Promise { + const response = await this.client.patch(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`, { name }) + return response.data + } + + async deleteManualTranscription(transcriptionId: string): Promise { + const response = await this.client.delete(`/api/v1/manual-evaluations/transcriptions/${transcriptionId}`) + return response.data + } + + // Test Agent endpoints + async createTestAgentConversation(data: { + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + conversation_metadata?: Record + }): Promise { + const response = await this.client.post('/api/v1/test-agents/conversations', data) + return response.data + } + + async getTestAgentConversation(conversationId: string): Promise { + const response = await this.client.get(`/api/v1/test-agents/conversations/${conversationId}`) + return response.data + } + + async listTestAgentConversations(): Promise { + const response = await this.client.get('/api/v1/test-agents/conversations') + return response.data + } + + async startTestAgentConversation(conversationId: string): Promise { + const response = await this.client.post(`/api/v1/test-agents/conversations/${conversationId}/start`) + return response.data + } + + async processTestAgentAudio(conversationId: string, audioFile: File, chunkTimestamp?: number): Promise { + const formData = new FormData() + formData.append('audio_file', audioFile) + if (chunkTimestamp !== undefined) { + formData.append('chunk_timestamp', chunkTimestamp.toString()) + } + const response = await this.client.post( + `/api/v1/test-agents/conversations/${conversationId}/process-audio`, + formData, + { + headers: { + 'Content-Type': 'multipart/form-data', + }, + } + ) + return response.data + } + + async getTestAgentResponseAudio(conversationId: string): Promise { + const response = await this.client.get( + `/api/v1/test-agents/conversations/${conversationId}/response-audio`, + { + responseType: 'blob', + } + ) + return response.data + } + + async endTestAgentConversation(conversationId: string, finalAudioKey?: string): Promise { + const response = await this.client.post(`/api/v1/test-agents/conversations/${conversationId}/end`, { + final_audio_key: finalAudioKey, + }) + return response.data + } + + async deleteTestAgentConversation(conversationId: string): Promise { + await this.client.delete(`/api/v1/test-agents/conversations/${conversationId}`) + } + + // Voice Agent endpoints + async getVoiceAgentConnection( + query?: Record, + ): Promise<{ ws_url: string; endpoint?: string }> { + const search = new URLSearchParams() + if (query) { + for (const [key, value] of Object.entries(query)) { + if (value !== undefined && value !== null && value !== '') { + search.set(key, String(value)) + } + } + } + const qs = search.toString() + const path = `/api/v1/voice-agent/connect${qs ? `?${qs}` : ''}` + const response = await this.client.post(path) + return response.data + } + + // Playground endpoints + async createWebCall(data: { + agent_id: string + metadata?: Record + retell_llm_dynamic_variables?: Record + custom_sip_headers?: Record + ui_surface?: 'agents_talk' | 'agent_playground' + }): Promise<{ + call_type: string + access_token?: string + call_id?: string + agent_id: string + agent_version?: number + call_status?: string + agent_name?: string + metadata?: Record + retell_llm_dynamic_variables?: Record + sample_rate?: number + call_short_id?: string + signed_url?: string + host?: string + room_name?: string + conversation_id?: string + }> { + const response = await this.client.post('/api/v1/playground/web-call', data) + return response.data + } + + async listCallRecordings(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/playground/call-recordings', { + params: { skip, limit }, + }) + return response.data + } + + async getCallRecording(callShortId: string): Promise { + const response = await this.client.get(`/api/v1/playground/call-recordings/${callShortId}`) + return response.data + } + + getCallRecordingAudioStreamUrl(callShortId: string, options?: { stereo?: boolean }): string { + const params = new URLSearchParams({ proxy: 'true' }) + if (options?.stereo) params.set('stereo', 'true') + return this.buildAuthenticatedApiUrl( + `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, + ) + } + + async getCallRecordingLogs(callShortId: string): Promise<{ + platform: string + entries: Array<{ + time?: string | null + level?: string | null + category?: string | null + summary?: string | null + raw?: Record + }> + count: number + }> { + const response = await this.client.get(`/api/v1/playground/call-recordings/${callShortId}/logs`) + return response.data + } + + async refreshCallRecording( + callShortId: string, + providerCallId?: string | null, + ): Promise<{ message: string }> { + const response = await this.client.post( + `/api/v1/playground/call-recordings/${callShortId}/refresh`, + providerCallId ? { provider_call_id: providerCallId } : undefined, + ) + return response.data + } + + async finalizePlaygroundCallRecording( + callShortId: string, + providerCallId?: string | null, + ): Promise { + const maxAttempts = 6 + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + try { + await this.refreshCallRecording(callShortId, providerCallId) + return + } catch (error: unknown) { + const axiosError = error as { + response?: { status?: number; data?: { detail?: string } } + } + const status = axiosError.response?.status + const detail = String(axiosError.response?.data?.detail || '') + const missingProvider = + status === 400 && detail.toLowerCase().includes('provider') + if (!missingProvider || attempt === maxAttempts - 1) { + throw error + } + await new Promise((resolve) => setTimeout(resolve, 400 * (attempt + 1))) + } + } + } + + async updateCallRecording(callShortId: string, providerCallId: string): Promise<{ message: string; provider_call_id: string }> { + const response = await this.client.put(`/api/v1/playground/call-recordings/${callShortId}`, { + provider_call_id: providerCallId + }) + return response.data + } + + async deleteCallRecording(callShortId: string): Promise<{ message: string }> { + const response = await this.client.delete(`/api/v1/playground/call-recordings/${callShortId}`) + return response.data + } + + async reEvaluateCallRecording(callShortId: string): Promise<{ + message: string + evaluator_result_id: string + result_id: string + audio_s3_key: string + task_id: string + }> { + const response = await this.client.post(`/api/v1/playground/call-recordings/${callShortId}/re-evaluate`) + return response.data + } + + async getCallRecordingAudioBlob( + callShortId: string, + options?: { stereo?: boolean }, + ): Promise { + const params = new URLSearchParams({ proxy: 'true' }) + if (options?.stereo) params.set('stereo', 'true') + const response = await this.client.get( + `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, + { responseType: 'blob' }, + ) + return response.data as Blob + } + + async getCallRecordingAudioUrl(callShortId: string, options?: { stereo?: boolean }): Promise { + const blob = await this.getCallRecordingAudioBlob(callShortId, options) + return URL.createObjectURL(blob) + } + + async getCallRecordingAudioBuffer( + callShortId: string, + options?: { stereo?: boolean }, + ): Promise { + const params = new URLSearchParams({ proxy: 'true' }) + if (options?.stereo) params.set('stereo', 'true') + const response = await this.client.get( + `/api/v1/playground/call-recordings/${callShortId}/audio?${params.toString()}`, + { responseType: 'arraybuffer' }, + ) + return response.data as ArrayBuffer + } + + async createCustomWebsocketSession(data: { + agent_id: string + websocket_url: string + transcript_entries: Array<{ role: 'user' | 'agent'; content: string; timestamp: string }> + started_at?: string + ended_at?: string + call_short_id?: string + audio_file?: File + }): Promise<{ + message: string + call_short_id: string + audio_s3_key?: string | null + evaluator_result_id?: string | null + }> { + const formData = new FormData() + formData.append('agent_id', data.agent_id) + formData.append('websocket_url', data.websocket_url) + formData.append('transcript_entries', JSON.stringify(data.transcript_entries)) + if (data.started_at) { + formData.append('started_at', data.started_at) + } + if (data.ended_at) { + formData.append('ended_at', data.ended_at) + } + if (data.call_short_id) { + formData.append('call_short_id', data.call_short_id) + } + if (data.audio_file) { + formData.append('audio_file', data.audio_file) + } + + const response = await this.client.post('/api/v1/playground/custom-websocket-sessions', formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + return response.data + } + + async evaluateCustomWebsocketSession(callShortId: string): Promise<{ + message: string + evaluator_result_id: string + result_id: string + task_id: string + }> { + const response = await this.client.post(`/api/v1/playground/custom-websocket-sessions/${callShortId}/evaluate`) + return response.data + } + + async getAgentSttConfig(agentId: string): Promise<{ + available: boolean + provider?: string + model?: string + reason?: string + }> { + const response = await this.client.get(`/api/v1/playground/agents/${agentId}/stt-config`) + return response.data + } + + async transcribeTurn( + agentId: string, + channel: 'user' | 'agent', + audioBlob: Blob, + ): Promise<{ transcript: string; channel: string }> { + const formData = new FormData() + formData.append('agent_id', agentId) + formData.append('channel', channel) + formData.append('audio_file', audioBlob, `turn_${channel}_${Date.now()}.wav`) + const response = await this.client.post('/api/v1/playground/transcribe-turn', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async summarizeTranscript(params: { + transcript?: string + entries?: Array<{ role: string; content: string; timestamp?: string }> + callShortId?: string + agentId?: string + force?: boolean + }): Promise<{ + summary: string + provider: string + model: string + source?: 'voice_bundle' | 'org_fallback' + cached?: boolean + generated_at?: string + usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } + }> { + const body: Record = {} + if (params.transcript) body.transcript = params.transcript + if (params.entries && params.entries.length > 0) body.entries = params.entries + if (params.callShortId) body.call_short_id = params.callShortId + if (params.agentId) body.agent_id = params.agentId + if (params.force) body.force = true + const response = await this.client.post('/api/v1/playground/summarize-transcript', body) + return response.data + } + + // Observability endpoints + async listObservabilityCalls(skip = 0, limit = 100): Promise { + const response = await this.client.get('/api/v1/observability/calls', { + params: { skip, limit }, + }) + return response.data + } + + async listCallsHub(params?: { + skip?: number + limit?: number + status?: string + search?: string + event?: string + }): Promise<{ + items: Array<{ kind: 'obs' | 'trace'; sort_at?: string; obs?: ObservabilityCall; trace?: Record }> + total: number + page: number + page_size: number + page_count: number + summary: { + total: number + obs_total: number + obs_live: number + obs_ended: number + obs_started: number + obs_other: number + traces_open: number + traces_closed: number + } + }> { + const response = await this.client.get('/api/v1/observability/calls-hub', { params }) + return response.data + } + + private buildAuthenticatedApiUrl(path: string): string { + const normalizedPath = path.startsWith('/') ? path : `/${path}` + const configuredBase = (this.client.defaults.baseURL || '').replace(/\/$/, '') + const url = configuredBase + ? new URL(`${configuredBase}${normalizedPath}`) + : new URL(normalizedPath, window.location.origin) + + const workspaceId = localStorage.getItem('activeWorkspaceId') + if (!this.cookieSessionEnabled) { + const accessToken = this.inMemoryAccessToken + const apiKey = localStorage.getItem('apiKey') + if (accessToken) { + url.searchParams.set('token', accessToken) + } else if (apiKey) { + url.searchParams.set('api_key', apiKey) + } + } + if (workspaceId) { + url.searchParams.set('workspace_id', workspaceId) + } + return url.toString() + } + + /** SSE / EventSource cannot use axios; must send session cookies explicitly. */ + openAuthenticatedEventSource(path: string): EventSource { + const url = this.buildAuthenticatedApiUrl(path) + return new EventSource(url, { withCredentials: true }) + } + + async getObservabilityCall(callShortId: string): Promise { + const response = await this.client.get(`/api/v1/observability/calls/${callShortId}`) + return response.data + } + + getObservabilityCallLiveEventsUrl(callShortId: string): string { + return this.buildAuthenticatedApiUrl( + `/api/v1/observability/calls/${callShortId}/live-events`, + ) + } + + getObservabilityCallAudioStreamUrl(callShortId: string): string { + return this.buildAuthenticatedApiUrl( + `/api/v1/observability/calls/${callShortId}/audio?proxy=true`, + ) + } + + async getObservabilityCallAudioBuffer(callShortId: string): Promise { + const response = await this.client.get( + `/api/v1/observability/calls/${callShortId}/audio?proxy=true`, + { responseType: 'arraybuffer' }, + ) + return response.data as ArrayBuffer + } + + async getObservabilityCallAudioBlob(callShortId: string): Promise { + const response = await this.client.get( + `/api/v1/observability/calls/${callShortId}/audio`, + { responseType: 'blob' }, + ) + return response.data as Blob + } + + async getObservabilityCallAudioUrl(callShortId: string): Promise { + const blob = await this.getObservabilityCallAudioBlob(callShortId) + return URL.createObjectURL(blob) + } + + async deleteObservabilityCall(callShortId: string): Promise<{ message: string }> { + const response = await this.client.delete(`/api/v1/observability/calls/${callShortId}`) + return response.data + } + + async evaluateObservabilityCall(callShortId: string, evaluatorId: string): Promise { + const response = await this.client.post(`/api/v1/observability/calls/${callShortId}/evaluate`, { + evaluator_id: evaluatorId, + }) + return response.data + } + + // Evaluator endpoints + async createEvaluator(data: { + name?: string + agent_id?: string + persona_id?: string + scenario_id?: string + custom_prompt?: string + metric_ids?: string[] + llm_provider?: string + llm_model?: string + tags?: string[] + }): Promise { + const response = await this.client.post('/api/v1/evaluators', data) + return response.data + } + + async formatCustomPrompt(prompt: string): Promise<{ formatted_prompt: string }> { + const response = await this.client.post('/api/v1/evaluators/format-prompt', { prompt }) + return response.data + } + + async createEvaluatorsBulk(data: { + name?: string + agent_id: string + scenario_id: string + persona_ids: string[] + tags?: string[] + }): Promise { + const response = await this.client.post('/api/v1/evaluators/bulk', data) + return response.data + } + + async listEvaluators(): Promise { + const response = await this.client.get('/api/v1/evaluators') + return response.data + } + + async getEvaluator(evaluatorId: string): Promise { + const response = await this.client.get(`/api/v1/evaluators/${evaluatorId}`) + return response.data + } + + async updateEvaluator(evaluatorId: string, data: { + agent_id?: string + persona_id?: string + scenario_id?: string + name?: string + custom_prompt?: string + metric_ids?: string[] + llm_provider?: string + llm_model?: string + tags?: string[] + }): Promise { + const response = await this.client.put(`/api/v1/evaluators/${evaluatorId}`, data) + return response.data + } + + async deleteEvaluator(evaluatorId: string, force?: boolean): Promise { + const response = await this.client.delete(`/api/v1/evaluators/${evaluatorId}`, { + params: force ? { force: true } : undefined, + }) + return response.data + } + + async runEvaluators(evaluatorIds: string[]): Promise<{ task_ids: string[]; evaluator_results: any[] }> { + const response = await this.client.post('/api/v1/evaluators/run', { evaluator_ids: evaluatorIds }) + return response.data + } + + async createEvaluatorSuite(data: { + name?: string + agent_id: string + persona_id?: string + persona_ids?: string[] + scenario_ids: string[] + metric_ids?: string[] + llm_provider?: string + llm_model?: string + tags?: string[] + default_runs_per_combination?: number + }): Promise { + const response = await this.client.post('/api/v1/evaluator-suites', data) + return response.data + } + + async listEvaluatorSuites(): Promise { + const response = await this.client.get('/api/v1/evaluator-suites') + return response.data + } + + async getEvaluatorSuite(suiteId: string): Promise { + const response = await this.client.get(`/api/v1/evaluator-suites/${suiteId}`) + return response.data + } + + async updateEvaluatorSuite( + suiteId: string, + data: { + name?: string + metric_ids?: string[] | null + llm_provider?: string | null + llm_model?: string | null + tags?: string[] + default_runs_per_combination?: number + }, + ): Promise { + const response = await this.client.put(`/api/v1/evaluator-suites/${suiteId}`, data) + return response.data + } + + async addEvaluatorSuiteScenarios(suiteId: string, scenarioIds: string[]): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/scenarios`, { + scenario_ids: scenarioIds, + }) + return response.data + } + + async removeEvaluatorSuiteScenario(suiteId: string, scenarioId: string): Promise { + const response = await this.client.delete( + `/api/v1/evaluator-suites/${suiteId}/scenarios/${scenarioId}`, + ) + return response.data + } + + async addEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/personas`, { + persona_ids: personaIds, + }) + return response.data + } + + async replaceEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { + const response = await this.client.put(`/api/v1/evaluator-suites/${suiteId}/personas`, { + persona_ids: personaIds, + }) + return response.data + } + + async removeEvaluatorSuitePersona(suiteId: string, personaId: string): Promise { + const response = await this.client.delete( + `/api/v1/evaluator-suites/${suiteId}/personas/${personaId}`, + ) + return response.data + } + + async deleteEvaluatorSuite(suiteId: string): Promise { + await this.client.delete(`/api/v1/evaluator-suites/${suiteId}`) + } + + async runEvaluatorSuite( + suiteId: string, + data: { runs_per_combination: number; to_number?: string; from_number?: string }, + ): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/run`, data) + return response.data + } + + async runNextCombination( + suiteId: string, + data?: { from_number?: string }, + ): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/run-next`, data || {}) + return response.data + } + + async chooseNextCombination(suiteId: string): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/choose-next`) + return response.data + } + + async activateEvaluatorSuite(suiteId: string): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/activate`) + return response.data + } + + // Metric endpoints + async createMetric(data: { + name: string + description?: string + /** + * Optional illustrative example surfaced alongside the description + * in the LLM judge's rubric. Mainly used by categorization label + * children but accepted on every metric. + */ + example?: string | null + metric_type: 'number' | 'boolean' | 'rating' | 'text' + trigger?: 'always' + enabled?: boolean + metric_origin?: 'default' | 'custom' + supported_surfaces?: string[] + enabled_surfaces?: string[] + custom_data_type?: 'boolean' | 'enum' | 'number_range' + custom_config?: Record + tags?: string[] + capture_rationale?: boolean + parent_metric_id?: string | null + selection_mode?: 'single_choice' | 'multi_label' | null + /** + * Scope of the new metric: + * - "workspace" (default): stamped with the active X-Workspace-Id + * header so it only appears inside that workspace. + * - "organization": stored with workspace_id=NULL so it surfaces + * in every workspace of the caller's org. + * Ignored when parent_metric_id is set — children always inherit + * their parent's scope. + */ + scope?: 'workspace' | 'organization' + scenario_ids?: string[] + agent_ids?: string[] + }): Promise { + const response = await this.client.post('/api/v1/metrics', data) + return response.data + } + + /** + * Atomically create a parent (category) metric + N children, used by + * the "Create category" flow on the Metrics page and by the + * /parse-bulk → save path when a hierarchy was requested. + */ + async createMetricWithChildren(data: { + name: string + description?: string | null + selection_mode: 'single_choice' | 'multi_label' + enabled?: boolean + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] | null + allow_discovery?: boolean + /** + * Parent-level "Enable LLM Rationale" toggle. When true the LLM + * judge emits one rationale at the category level (children never + * carry rationales in hierarchical mode). + */ + capture_rationale?: boolean + children: Array<{ + name: string + description?: string | null + /** + * Optional illustrative example for this label, surfaced + * alongside the description in the LLM judge's rubric. + */ + example?: string | null + enabled?: boolean + capture_rationale?: boolean | null + tags?: string[] | null + }> + /** + * Same semantics as {@link createMetric}'s ``scope``: when + * ``"organization"`` the parent + every child are stored with + * ``workspace_id=NULL`` so the whole category subtree is visible + * in every workspace of the org. + */ + scope?: 'workspace' | 'organization' + scenario_ids?: string[] + agent_ids?: string[] + }): Promise { + const response = await this.client.post( + '/api/v1/metrics/with-children', + data, + ) + return response.data + } + + async addMetricChild(parentMetricId: string, data: { + name: string + description?: string | null + /** + * Optional illustrative example for this label, surfaced alongside + * the description in the LLM judge's rubric. + */ + example?: string | null + enabled?: boolean + capture_rationale?: boolean | null + tags?: string[] | null + }): Promise { + const response = await this.client.post( + `/api/v1/metrics/${parentMetricId}/children`, + data, + ) + return response.data + } + + async listMetrics( + surface?: string, + includeChildren: boolean = true, + options?: { includeDrafts?: boolean; draftsOnly?: boolean; enabledOnly?: boolean }, + ): Promise { + const response = await this.client.get('/api/v1/metrics', { + params: { + ...(surface ? { surface } : {}), + include_children: includeChildren, + ...(options?.includeDrafts ? { include_drafts: true } : {}), + ...(options?.draftsOnly ? { drafts_only: true } : {}), + ...(options?.enabledOnly ? { enabled_only: true } : {}), + }, + }) + return response.data + } + + async createMetricDraft(data: { + name: string + description?: string + metric_type: string + trigger: string + metric_origin?: string + studio_notes?: string + parent_metric_id?: string + selection_mode?: string + custom_data_type?: string + custom_config?: Record + capture_rationale?: boolean + compare_transcripts?: boolean + scope?: 'workspace' | 'organization' + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] + enabled?: boolean + }): Promise { + const response = await this.client.post('/api/v1/metrics/drafts', data) + return response.data + } + + async createMetricDraftWithChildren(data: { + name: string + description?: string | null + selection_mode: 'single_choice' | 'multi_label' + allow_discovery?: boolean + capture_rationale?: boolean + supported_surfaces: string[] + enabled_surfaces: string[] + children: Array<{ + name: string + description?: string | null + example?: string | null + capture_rationale?: boolean + enabled?: boolean + }> + scope?: 'workspace' | 'organization' + studio_notes?: string + }): Promise { + const response = await this.client.post('/api/v1/metrics/drafts/with-children', data) + return response.data + } + + async promoteMetric(metricId: string): Promise<{ metric: any; promoted_at: string }> { + const response = await this.client.post(`/api/v1/metrics/${metricId}/promote`) + return response.data + } + + async createMetricStudioRun(data: { + name?: string + metric_ids: string[] + sources: Array<{ + source_kind: 'call_import_row' | 'call_recording' | 'evaluator_result' + source_ref: string + display_label?: string + }> + transcript_source?: 'production' | 'diarised' + llm_provider?: string + llm_model?: string + llm_credential_id?: string + llm_config?: LLMGenerationConfig | null + metric_llm_overrides?: Record + }): Promise { + const response = await this.client.post('/api/v1/metric-studio/runs', data) + return response.data + } + + async listMetricStudioRuns(skip = 0, limit = 50): Promise<{ items: any[]; total: number }> { + const response = await this.client.get('/api/v1/metric-studio/runs', { + params: { skip, limit }, + }) + return response.data + } + + async getMetricStudioRun(runId: string): Promise { + const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}`) + return response.data + } + + async listMetricStudioRunResults( + runId: string, + skip = 0, + limit = 100, + ): Promise<{ items: any[]; total: number }> { + const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}/results`, { + params: { skip, limit }, + }) + return response.data + } + + async retryMetricStudioRun( + runId: string, + resultIds?: string[], + ): Promise { + const response = await this.client.post(`/api/v1/metric-studio/runs/${runId}/retry`, { + result_ids: resultIds, + }) + return response.data + } + + async deleteMetricStudioRun(runId: string): Promise { + await this.client.delete(`/api/v1/metric-studio/runs/${runId}`) + } + + // Evaluator Results endpoints + async listEvaluatorResults( + paramsOrEvaluatorId?: ListEvaluatorResultsParams | string, + playground?: boolean, + testAgentsOnly?: boolean, + agentId?: string + ): Promise { + let params: Record = {} + if (typeof paramsOrEvaluatorId === 'string' || paramsOrEvaluatorId === undefined) { + if (paramsOrEvaluatorId) { + params.evaluator_id = paramsOrEvaluatorId + } + if (agentId) { + params.agent_id = agentId + } + if (playground !== undefined) { + params.playground = playground + } + if (testAgentsOnly !== undefined) { + params.test_agents_only = testAgentsOnly + } + } else { + const p = paramsOrEvaluatorId + if (p.skip !== undefined) params.skip = p.skip + if (p.limit !== undefined) params.limit = p.limit + if (p.evaluatorId) params.evaluator_id = p.evaluatorId + if (p.agentId) params.agent_id = p.agentId + if (p.suiteId) params.suite_id = p.suiteId + if (p.scenarioId) params.scenario_id = p.scenarioId + if (p.status) params.status = p.status + if (p.since) params.since = p.since + if (p.until) params.until = p.until + if (p.unassignedOnly) params.unassigned_only = true + if (p.playground !== undefined) params.playground = p.playground + if (p.testAgentsOnly !== undefined) params.test_agents_only = p.testAgentsOnly + } + const response = await this.client.get('/api/v1/evaluator-results', { params }) + return response.data + } + + async getEvaluatorResultsOverview(params?: { + agentId?: string + suiteId?: string + since?: string + until?: string + }): Promise { + const query: Record = {} + if (params?.agentId) query.agent_id = params.agentId + if (params?.suiteId) query.suite_id = params.suiteId + if (params?.since) query.since = params.since + if (params?.until) query.until = params.until + const response = await this.client.get('/api/v1/evaluator-results/overview', { params: query }) + return response.data + } + + async getEvaluatorResultsAggregate(params: { + suiteId?: string + agentId?: string + scenarioId?: string + since?: string + until?: string + }): Promise { + const query: Record = {} + if (params.suiteId) query.suite_id = params.suiteId + if (params.agentId) query.agent_id = params.agentId + if (params.scenarioId) query.scenario_id = params.scenarioId + if (params.since) query.since = params.since + if (params.until) query.until = params.until + const response = await this.client.get('/api/v1/evaluator-results/aggregate', { params: query }) + return response.data + } + + private _evaluatorResultClusterQuery(scope: { + agentId: string + scenarioIds?: string[] + since?: string + until?: string + suiteId?: string + scenarioId?: string + scopeKey?: string + jobId?: string + }): Record { + const query: Record = { + agent_id: scope.agentId, + } + if (scope.jobId) query.job_id = scope.jobId + else if (scope.scopeKey) query.scope_key = scope.scopeKey + if (scope.scenarioIds?.length) { + query.scenario_ids = scope.scenarioIds + } + if (scope.since) query.since = scope.since + if (scope.until) query.until = scope.until + if (scope.suiteId) query.suite_id = scope.suiteId + if (scope.scenarioId) query.scenario_id = scope.scenarioId + return query + } + + async getEvaluatorResultMetricClusterFailurePolicies(scope: { + agentId: string + scenarioIds?: string[] + since?: string + until?: string + }): Promise { + const response = await this.client.get( + '/api/v1/evaluator-results/metric-clusters/failure-policies', + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + + async saveEvaluatorResultMetricClusterFailurePolicies( + scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, + policies: Record, + ): Promise { + const response = await this.client.put( + '/api/v1/evaluator-results/metric-clusters/failure-policies', + { policies }, + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + + async listEvaluatorResultMetricClusterEligibleRows( + scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, + options?: { limit?: number; count_only?: boolean }, + ): Promise { + const response = await this.client.get( + '/api/v1/evaluator-results/metric-clusters/eligible-rows', + { params: { ...this._evaluatorResultClusterQuery(scope), ...options } }, + ) + return response.data + } + + async listEvaluatorResultMetricClusterScopes(): Promise< + import('../types/api').EvaluatorResultClusterScopeListResponse + > { + const response = await this.client.get( + '/api/v1/evaluator-results/metric-clusters/scopes', + ) + return response.data + } + + async getEvaluatorResultMetricClusters(scope: { + agentId: string + scenarioIds?: string[] + since?: string + until?: string + jobId?: string + scopeKey?: string + }): Promise { + const response = await this.client.get('/api/v1/evaluator-results/metric-clusters', { + params: this._evaluatorResultClusterQuery(scope), + }) + return response.data + } + + async generateEvaluatorResultMetricClusters( + scope: { agentId: string; scenarioIds?: string[]; since?: string; until?: string }, + body: { + force?: boolean + regenerate?: boolean + provider?: string + model?: string + row_limit?: number + failure_policies?: Record + }, + ): Promise { + const response = await this.client.post('/api/v1/evaluator-results/metric-clusters', body, { + params: this._evaluatorResultClusterQuery(scope), + }) + return response.data + } + + async cancelEvaluatorResultMetricClusters(scope: { + agentId: string + scenarioIds?: string[] + since?: string + until?: string + scopeKey?: string + }): Promise { + const response = await this.client.post( + '/api/v1/evaluator-results/metric-clusters/cancel', + {}, + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + + async deleteEvaluatorResultMetricClusters(scope: { + agentId: string + scenarioIds?: string[] + since?: string + until?: string + jobId?: string + scopeKey?: string + }): Promise { + await this.client.delete('/api/v1/evaluator-results/metric-clusters', { + params: this._evaluatorResultClusterQuery(scope), + }) + } + + async getEvaluatorResult(id: string, includeRelations: boolean = true): Promise { + const params = includeRelations ? { include_relations: 'true' } : {} + const response = await this.client.get(`/api/v1/evaluator-results/${id}`, { params }) + return response.data + } + + getEvaluatorResultLiveEventsUrl(resultId: string): string { + return this.buildAuthenticatedApiUrl( + `/api/v1/evaluator-results/${resultId}/live-events`, + ) + } + + async listConversationEvaluations(agentId: string): Promise { + const response = await this.client.get('/api/v1/conversation-evaluations', { + params: { agent_id: agentId }, + }) + return response.data + } + + async createConversationEvaluation(data: { + transcription_id: string + agent_id: string + }): Promise { + const response = await this.client.post('/api/v1/conversation-evaluations', data) + return response.data + } + + async deleteConversationEvaluation(evaluationId: string): Promise { + await this.client.delete(`/api/v1/conversation-evaluations/${evaluationId}`) + } + + async updateConversationEvaluation( + evaluationId: string, + data: Record, + ): Promise { + const response = await this.client.patch( + `/api/v1/conversation-evaluations/${evaluationId}`, + data, + ) + return response.data + } + + async getEvaluatorResultMetrics(id: string): Promise { + const response = await this.client.get(`/api/v1/evaluator-results/${id}/metrics`) + return response.data + } + + async getSyntheticCallTraceForResult( + evaluatorResultId: string, + includeSpans = true, + ): Promise { + const response = await this.client.get( + `/api/v1/observability/traces/results/${evaluatorResultId}`, + { params: { include_spans: includeSpans } }, + ) + return response.data + } + + async listSyntheticCallTraces(params?: { + skip?: number + limit?: number + status?: string + cursor?: string + }): Promise<{ items: any[]; total: number; next_cursor?: string; has_more?: boolean }> { + const response = await this.client.get('/api/v1/observability/traces', { params }) + return response.data + } + + async deleteSyntheticCallTrace(traceId: string): Promise<{ message: string }> { + const response = await this.client.delete(`/api/v1/observability/traces/${traceId}`) + return response.data + } + + async getSyntheticCallTraceSpans(traceId: string): Promise { + const response = await this.client.get(`/api/v1/observability/traces/${traceId}/spans`) + return response.data + } + + async getSyntheticCallTrace(traceId: string, includeSpans = true): Promise { + const response = await this.client.get(`/api/v1/observability/traces/${traceId}`, { + params: { include_spans: includeSpans }, + }) + return response.data + } + + async getSyntheticCallTraceByCallShortId(callShortId: string, includeSpans = true): Promise { + const response = await this.client.get( + `/api/v1/observability/traces/by-call-short-id/${callShortId}`, + { params: { include_spans: includeSpans } }, + ) + return response.data + } + + async getSyntheticCallTraceSetup(): Promise { + const response = await this.client.get('/api/v1/observability/traces/setup') + return response.data + } + + async createSyntheticTraceSession(data: { + transport?: 'webrtc' | 'websocket' | 'phone' | 'custom' + evaluator_result_id?: string + agent_id?: string + }): Promise<{ + trace_id: string + call_short_id: string + workspace_id: string + transport: string + status: string + otel_correlation: Record + }> { + const response = await this.client.post('/api/v1/observability/traces/sessions', data) + return response.data + } + + async closeSyntheticTraceSession(callShortId: string): Promise<{ + trace_id: string + call_short_id: string + status: string + }> { + const response = await this.client.post( + `/api/v1/observability/traces/sessions/${callShortId}/close`, + ) + return response.data + } + + async ingestSyntheticTraceJson(data: { + call_short_id: string + spans: Array<{ + name: string + turn_number: number + ttfb_ms?: number + attributes?: Record + }> + }): Promise<{ + accepted_spans: number + synthetic_call_trace_id?: string + correlated: boolean + }> { + const response = await this.client.post('/api/v1/observability/traces/ingest', data) + return response.data + } + + async getEvaluatorResultOtelCorrelation(id: string): Promise { + const response = await this.client.get(`/api/v1/evaluator-results/${id}/otel-correlation`) + return response.data + } + + getEvaluatorResultAudioStreamUrl(resultId: string): string { + return this.buildAuthenticatedApiUrl( + `/api/v1/evaluator-results/${resultId}/audio`, + ) + } + + async getEvaluatorResultAudioBuffer(resultId: string): Promise { + const response = await this.client.get( + `/api/v1/evaluator-results/${resultId}/audio`, + { responseType: 'arraybuffer' }, + ) + return response.data as ArrayBuffer + } + + async getEvaluatorResultAudioBlob(resultId: string): Promise { + const response = await this.client.get( + `/api/v1/evaluator-results/${resultId}/audio`, + { responseType: 'blob' }, + ) + return response.data as Blob + } + + async getEvaluatorResultAudioUrl(resultId: string): Promise { + const blob = await this.getEvaluatorResultAudioBlob(resultId) + return URL.createObjectURL(blob) + } + + async createEvaluatorResultManual(data: { + evaluator_id: string + audio_s3_key: string + duration_seconds?: number + }): Promise { + const response = await this.client.post('/api/v1/evaluator-results', data) + return response.data + } + + async reEvaluateResult(id: string): Promise { + const response = await this.client.post(`/api/v1/evaluator-results/${id}/re-evaluate`) + return response.data + } + + async deleteEvaluatorResult(id: string): Promise { + await this.client.delete(`/api/v1/evaluator-results/${id}`) + } + + async deleteEvaluatorResultsBulk(ids: string[]): Promise { + // FastAPI expects multiple query parameters with the same name + // Build query string manually: result_ids=id1&result_ids=id2 + const params = new URLSearchParams() + ids.forEach(id => params.append('result_ids', id)) + await this.client.delete(`/api/v1/evaluator-results?${params.toString()}`) + } + + async getMetric(metricId: string): Promise { + const response = await this.client.get(`/api/v1/metrics/${metricId}`) + return response.data + } + + async updateMetric(metricId: string, data: { + name?: string + description?: string + /** + * Pass ``""`` (empty string) to clear the stored example; omit to + * leave the persisted value untouched. + */ + example?: string | null + metric_type?: 'number' | 'boolean' | 'rating' | 'text' + trigger?: 'always' + enabled?: boolean + metric_origin?: 'default' | 'custom' + supported_surfaces?: string[] + enabled_surfaces?: string[] + custom_data_type?: 'boolean' | 'enum' | 'number_range' + custom_config?: Record + tags?: string[] + capture_rationale?: boolean + selection_mode?: 'single_choice' | 'multi_label' | null + /** Only honored on multi_label parents; backend 400s otherwise. */ + allow_discovery?: boolean + scenario_ids?: string[] + agent_ids?: string[] + }): Promise { + const response = await this.client.put(`/api/v1/metrics/${metricId}`, data) + return response.data + } + + async deleteMetric(metricId: string): Promise { + await this.client.delete(`/api/v1/metrics/${metricId}`) + } + + async seedDefaultMetrics(): Promise { + const response = await this.client.post('/api/v1/metrics/seed-defaults') + return response.data + } + + async generateMetric(data: { + mode: 'description' | 'examples' + surface: 'agent' | 'chat_agent' | 'voice_playground' + description?: string + examples?: Array<{ transcript: string; rating: any; notes?: string }> + provider?: string + model?: string + credential_id?: string + llm_config?: Record + }): Promise<{ + name: string + description: string + metric_type: 'rating' | 'boolean' | 'number' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' | null + custom_config: Record + supported_surfaces: string[] + enabled_surfaces: string[] + suggested_tags: string[] + provider?: string + model?: string + }> { + const response = await this.client.post('/api/v1/metrics/generate', data) + return response.data + } + + async generateMetricsFromScenarios(data: { + agent_name: string + production_prompt: string + call_medium?: string | null + scenarios: Array<{ + name: string + description: string + goal?: string + metric_type?: 'auto' | 'boolean' | 'rating' | 'number' | 'text' + }> + provider?: string + model?: string + credential_id?: string + llm_config?: Record + }): Promise<{ + metrics: Array<{ + name: string + description: string + metric_type: 'rating' | 'boolean' | 'number' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' | null + custom_config: Record + supported_surfaces: string[] + enabled_surfaces: string[] + tags: string[] + scenario_name: string + }> + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/metrics/generate-from-scenarios', data) + return response.data + } + + async parseBulkMetric(data: { + prompt: string + surface: 'agent' | 'chat_agent' | 'voice_playground' + /** When set, the response includes a ``parent`` block + all labels are children. */ + parent_name?: string + parent_description?: string + selection_mode?: 'single_choice' | 'multi_label' + }): Promise<{ + metrics: Array<{ + name: string + description: string + metric_type: 'rating' | 'boolean' | 'number' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' | null + custom_config: Record + supported_surfaces: string[] + enabled_surfaces: string[] + capture_rationale: boolean + suggested_tags: string[] + source_label: { label_name: string; definition: string; examples: string } + }> + parent?: { + name: string + description: string | null + selection_mode: 'single_choice' | 'multi_label' + supported_surfaces: string[] + enabled_surfaces: string[] + } | null + }> { + const response = await this.client.post('/api/v1/metrics/parse-bulk', data) + return response.data + } + + // Alert endpoints + async listAlerts(status?: string): Promise { + const params: any = {} + if (status) { + params.status_filter = status + } + const response = await this.client.get('/api/v1/alerts', { params }) + return response.data + } + + async getAlert(alertId: string): Promise { + const response = await this.client.get(`/api/v1/alerts/${alertId}`) + return response.data + } + + async createAlert(data: { + name: string + description?: string | null + data_source?: string + metric_type: string + aggregation: string + operator: string + threshold_value: number + time_window_minutes: number + agent_ids?: string[] | null + notify_frequency: string + notify_emails?: string[] + notify_webhooks?: string[] + notify_pagerduty_routing_keys?: string[] + }): Promise { + const response = await this.client.post('/api/v1/alerts', data) + return response.data + } + + async updateAlert(alertId: string, data: { + name?: string + description?: string | null + data_source?: string + metric_type?: string + aggregation?: string + operator?: string + threshold_value?: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: string + notify_emails?: string[] + notify_webhooks?: string[] + notify_pagerduty_routing_keys?: string[] + status?: string + }): Promise { + const response = await this.client.put(`/api/v1/alerts/${alertId}`, data) + return response.data + } + + async deleteAlert(alertId: string): Promise { + await this.client.delete(`/api/v1/alerts/${alertId}`) + } + + async toggleAlertStatus(alertId: string): Promise { + const response = await this.client.post(`/api/v1/alerts/${alertId}/toggle`) + return response.data + } + + // Alert History endpoints + async getOpenIncidentSummary(): Promise<{ total: number; by_alert_id: Record }> { + const response = await this.client.get('/api/v1/alerts/history/open-summary') + return response.data + } + + async listAlertHistory( + status?: string, + alertId?: string, + skip = 0, + limit = 100, + openOnly = false + ): Promise { + const params: any = { skip, limit } + if (openOnly) { + params.open_only = true + } else if (status) { + params.status_filter = status + } + if (alertId) { + params.alert_id = alertId + } + const response = await this.client.get('/api/v1/alerts/history/all', { params }) + return response.data + } + + async getAlertHistoryItem(historyId: string): Promise { + const response = await this.client.get(`/api/v1/alerts/history/${historyId}`) + return response.data + } + + async acknowledgeAlertHistory(historyId: string): Promise { + const response = await this.client.put(`/api/v1/alerts/history/${historyId}`, { + status: 'acknowledged' + }) + return response.data + } + + async resolveAlertHistory(historyId: string, resolutionNotes?: string): Promise { + const response = await this.client.put(`/api/v1/alerts/history/${historyId}`, { + status: 'resolved', + resolution_notes: resolutionNotes + }) + return response.data + } + + // Alert Evaluation & Notification endpoints + async triggerAlert(alertId: string): Promise { + const response = await this.client.post(`/api/v1/alerts/${alertId}/trigger`) + return response.data + } + + async evaluateAllAlerts(): Promise { + const response = await this.client.post('/api/v1/alerts/evaluate/all') + return response.data + } + + async testAlertNotification(alertId: string, data: { + webhook_url?: string + email?: string + pagerduty_routing_key?: string + }): Promise { + const response = await this.client.post(`/api/v1/alerts/${alertId}/test-notification`, data) + return response.data + } + + // Cron Job endpoints + async listCronJobs(): Promise { + const response = await this.client.get('/api/v1/cron-jobs') + return response.data + } + + async getCronJob(cronJobId: string): Promise { + const response = await this.client.get(`/api/v1/cron-jobs/${cronJobId}`) + return response.data + } + + async createCronJob(data: { + name: string + cron_expression: string + timezone: string + max_runs: number + interval_days?: number | null + evaluator_ids: string[] + }): Promise { + const response = await this.client.post('/api/v1/cron-jobs', data) + return response.data + } + + async updateCronJob(cronJobId: string, data: { + name?: string + cron_expression?: string + timezone?: string + max_runs?: number + interval_days?: number | null + evaluator_ids?: string[] + status?: string + }): Promise { + const response = await this.client.put(`/api/v1/cron-jobs/${cronJobId}`, data) + return response.data + } + + async deleteCronJob(cronJobId: string): Promise { + await this.client.delete(`/api/v1/cron-jobs/${cronJobId}`) + } + + async toggleCronJobStatus(cronJobId: string): Promise { + const response = await this.client.post(`/api/v1/cron-jobs/${cronJobId}/toggle`) + return response.data + } + + async runCronJobNow(cronJobId: string): Promise { + const response = await this.client.post(`/api/v1/cron-jobs/${cronJobId}/run`) + return response.data + } + + // Voice Playground (TTS Comparison) endpoints + async listTTSProviders(): Promise { + const response = await this.client.get('/api/v1/voice-playground/tts-providers') + return response.data + } + + async listCustomTTSVoices(provider?: string): Promise> { + const response = await this.client.get('/api/v1/voice-playground/custom-voices', { + params: provider ? { provider } : undefined, + }) + return response.data + } + + async createCustomTTSVoice(data: { + provider: string + voice_id: string + name: string + gender?: string + accent?: string + description?: string + }): Promise { + const response = await this.client.post('/api/v1/voice-playground/custom-voices', data) + return response.data + } + + async updateCustomTTSVoice(customVoiceId: string, data: { + voice_id?: string + name?: string + gender?: string + accent?: string + description?: string + }): Promise { + const response = await this.client.put(`/api/v1/voice-playground/custom-voices/${customVoiceId}`, data) + return response.data + } + + async deleteCustomTTSVoice(customVoiceId: string): Promise { + const response = await this.client.delete(`/api/v1/voice-playground/custom-voices/${customVoiceId}`) + return response.data + } + + async createTTSComparison(data: { + name?: string + mode?: 'benchmark' | 'blind_test_only' + // Legacy benchmark fields + provider_a?: string + model_a?: string + voices_a?: Array<{ id: string; name: string; sample_rate_hz?: number }> + provider_b?: string + model_b?: string + voices_b?: Array<{ id: string; name: string; sample_rate_hz?: number }> + // New per-side benchmark config + side_a?: VoicePlaygroundSideConfig + side_b?: VoicePlaygroundSideConfig + // Common + sample_texts?: string[] + num_runs?: number + eval_stt_provider?: string + eval_stt_model?: string + // blind_test_only + pairs?: Array + }): Promise { + const response = await this.client.post('/api/v1/voice-playground/comparisons', data) + return response.data + } + + async listVoicePlaygroundCallImportRows(params: { + call_import_id?: string + with_recording?: boolean + skip?: number + limit?: number + } = {}): Promise<{ + items: Array<{ + id: string + call_import_id: string + call_import_filename: string | null + conversation_id: string + transcript: string | null + recording_s3_key: string | null + has_recording: boolean + status: string + created_at: string | null + }> + total: number + skip: number + limit: number + }> { + const response = await this.client.get('/api/v1/voice-playground/call-import-rows', { params }) + return response.data + } + + async uploadVoicePlaygroundAudio(file: File): Promise<{ + s3_key: string + presigned_url: string | null + filename: string + size_bytes: number + content_type: string + }> { + const form = new FormData() + form.append('file', file) + const response = await this.client.post('/api/v1/voice-playground/uploads', form, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async listTTSComparisons(skip = 0, limit = 50): Promise { + const response = await this.client.get('/api/v1/voice-playground/comparisons', { + params: { skip, limit }, + }) + return response.data + } + + async getTTSComparison(comparisonId: string): Promise { + const response = await this.client.get(`/api/v1/voice-playground/comparisons/${comparisonId}`) + return response.data + } + + async generateTTSComparison(comparisonId: string): Promise<{ message: string; task_id: string }> { + const response = await this.client.post(`/api/v1/voice-playground/comparisons/${comparisonId}/generate`) + return response.data + } + + async submitBlindTest(comparisonId: string, results: Array<{ + sample_index: number + preferred: 'A' | 'B' + voice_a_id: string + voice_b_id: string + }>): Promise { + const response = await this.client.post( + `/api/v1/voice-playground/comparisons/${comparisonId}/blind-test`, + { results } + ) + return response.data + } + + async deleteTTSComparison(comparisonId: string): Promise { + await this.client.delete(`/api/v1/voice-playground/comparisons/${comparisonId}`) + } + + // Blind Test Sharing (owner side) + async createBlindTestShare(comparisonId: string, payload: { + title: string + description?: string + creator_notes?: string + custom_metrics: Array<{ key: string; label: string; type: 'rating' | 'comment'; scale?: number }> + }): Promise { + const response = await this.client.post( + `/api/v1/voice-playground/comparisons/${comparisonId}/share`, + payload + ) + return response.data + } + + async getBlindTestShare(comparisonId: string): Promise { + const response = await this.client.get( + `/api/v1/voice-playground/comparisons/${comparisonId}/share` + ) + return response.data + } + + async updateBlindTestShare(shareId: string, payload: { + title?: string + description?: string + creator_notes?: string + custom_metrics?: Array<{ key: string; label: string; type: 'rating' | 'comment'; scale?: number }> + status?: 'open' | 'closed' + }): Promise { + const response = await this.client.patch( + `/api/v1/voice-playground/shares/${shareId}`, + payload + ) + return response.data + } + + async deleteBlindTestShare(shareId: string): Promise { + await this.client.delete(`/api/v1/voice-playground/shares/${shareId}`) + } + + async listBlindTestResponses(shareId: string, skip = 0, limit = 100): Promise<{ + items: any[] + total: number + }> { + const response = await this.client.get( + `/api/v1/voice-playground/shares/${shareId}/responses`, + { params: { skip, limit } } + ) + return response.data + } + + async generateSampleTexts(params: { + voice_bundle_id?: string + provider?: string + model?: string + scenario?: string + count?: number + length?: string + temperature?: number + llm_config?: LLMGenerationConfig | null + }): Promise<{ samples: string[]; provider: string; model: string }> { + const response = await this.client.post('/api/v1/voice-playground/generate-samples', params) + return response.data + } + + async getTTSAnalytics(): Promise> { + const response = await this.client.get('/api/v1/voice-playground/analytics') + return response.data + } + + async downloadTTSComparisonReport( + comparisonId: string, + includeUnfinishedSamples = false, + reportOptions?: TTSReportOptionsPayload + ): Promise { + const response = await this.client.get( + `/api/v1/voice-playground/comparisons/${comparisonId}/report.pdf`, + { + params: { + include_unfinished_samples: includeUnfinishedSamples, + ...(reportOptions ? { report_options: JSON.stringify(reportOptions) } : {}), + }, + responseType: 'blob', + } + ) + return response.data + } + + async createTTSComparisonReportJob( + comparisonId: string, + reportOptions?: TTSReportOptionsPayload + ): Promise<{ + id: string + comparison_id: string + status: string + format: string + task_id?: string + report_options?: TTSReportOptionsPayload + created_at?: string | null + }> { + const response = await this.client.post( + `/api/v1/voice-playground/comparisons/${comparisonId}/reports`, + reportOptions ? { report_options: reportOptions } : {} + ) + return response.data + } + + async getVoicePlaygroundReportThresholdDefaults(): Promise<{ + zone_threshold_overrides: NonNullable + is_custom: boolean + }> { + const response = await this.client.get('/api/v1/voice-playground/report-threshold-defaults') + return response.data + } + + async updateVoicePlaygroundReportThresholdDefaults(data: { + zone_threshold_overrides?: NonNullable + reset_to_system_defaults?: boolean + }): Promise<{ + zone_threshold_overrides: NonNullable + is_custom: boolean + message: string + }> { + const response = await this.client.put('/api/v1/voice-playground/report-threshold-defaults', data) + return response.data + } + + async getTTSComparisonReportJob(reportJobId: string): Promise<{ + id: string + comparison_id: string + status: string + format: string + filename?: string | null + error_message?: string | null + task_id?: string | null + download_url?: string | null + report_options?: TTSReportOptionsPayload + created_at?: string | null + updated_at?: string | null + }> { + const response = await this.client.get(`/api/v1/voice-playground/reports/${reportJobId}`) + return response.data + } + + // Prompt Partials + async listPromptPartials( + skip = 0, + limit = 100, + search?: string, + kind: 'all' | 'partial' | 'imported_agent' | 'metric' = 'all', + ): Promise { + const response = await this.client.get('/api/v1/prompt-partials', { + params: { + skip, + limit, + kind, + ...(search ? { search } : {}), + }, + }) + return response.data + } + + async listImportedAgents(skip = 0, limit = 100, search?: string): Promise { + return this.listPromptPartials(skip, limit, search, 'imported_agent') + } + + async listMetricPartials(skip = 0, limit = 100, search?: string): Promise { + return this.listPromptPartials(skip, limit, search, 'metric') + } + + async createImportedAgent(data: { + name: string + description?: string | null + content: string + }): Promise { + const response = await this.client.post('/api/v1/prompt-partials', { + ...data, + tags: ['__imported_agent__'], + }) + return response.data + } + + async createMetricPartial(data: { + name: string + description?: string | null + content: string + }): Promise { + const response = await this.client.post('/api/v1/prompt-partials', { + ...data, + tags: ['__metric_partial__'], + }) + return response.data + } + + async updateMetricPartial( + partialId: string, + data: { + name?: string + description?: string | null + content?: string + change_summary?: string + }, + ): Promise { + const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, { + ...data, + tags: ['__metric_partial__'], + }) + return response.data + } + + async generateAgentFlowchart( + partialId: string, + options?: { + provider?: string + model?: string + credential_id?: string + regenerate?: boolean + }, + ): Promise { + const response = await this.client.post( + `/api/v1/prompt-partials/${partialId}/flowchart`, + { + provider: options?.provider, + model: options?.model, + credential_id: options?.credential_id, + regenerate: options?.regenerate ?? false, + }, + ) + return response.data + } + + async saveAgentFlowchartLayout( + partialId: string, + nodes: Array<{ id: string; position_x: number; position_y: number }>, + ): Promise { + const response = await this.client.put( + `/api/v1/prompt-partials/${partialId}/flowchart/layout`, + { nodes }, + ) + return response.data + } + + async mapAgentFlowchartPromptSections( + partialId: string, + options?: { provider?: string; model?: string; credential_id?: string }, + ): Promise { + const response = await this.client.post( + `/api/v1/prompt-partials/${partialId}/flowchart/prompt-map`, + { + provider: options?.provider, + model: options?.model, + credential_id: options?.credential_id, + }, + ) + return response.data + } + + async updateImportedAgent( + partialId: string, + data: { + name?: string + description?: string | null + content?: string + change_summary?: string + tags?: string[] + }, + ): Promise { + const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, data) + return response.data + } + + async getPromptPartial(partialId: string): Promise { + const response = await this.client.get(`/api/v1/prompt-partials/${partialId}`) + return response.data + } + + async createPromptPartial(data: { + name: string + description?: string + content: string + tags?: string[] + }): Promise { + const response = await this.client.post('/api/v1/prompt-partials', data) + return response.data + } + + async updatePromptPartial(partialId: string, data: { + name?: string + description?: string | null + content?: string + tags?: string[] + change_summary?: string + }): Promise { + const response = await this.client.put(`/api/v1/prompt-partials/${partialId}`, data) + return response.data + } + + async deletePromptPartial(partialId: string): Promise { + await this.client.delete(`/api/v1/prompt-partials/${partialId}`) + } + + async listPromptPartialVersions(partialId: string): Promise { + const response = await this.client.get(`/api/v1/prompt-partials/${partialId}/versions`) + return response.data + } + + async getPromptPartialVersion(partialId: string, versionNumber: number): Promise { + const response = await this.client.get(`/api/v1/prompt-partials/${partialId}/versions/${versionNumber}`) + return response.data + } + + async revertPromptPartial(partialId: string, versionNumber: number): Promise { + const response = await this.client.post(`/api/v1/prompt-partials/${partialId}/revert/${versionNumber}`) + return response.data + } + + async previewPromptPartialVersionPush( + partialId: string, + versionNumber: number, + ): Promise { + const response = await this.client.get( + `/api/v1/prompt-partials/${partialId}/versions/${versionNumber}/push-preview`, + ) + return response.data + } + + async pushPromptPartialVersion( + partialId: string, + versionNumber: number, + confirm: string, + ): Promise { + const response = await this.client.post( + `/api/v1/prompt-partials/${partialId}/versions/${versionNumber}/push`, + { confirm }, + ) + return response.data + } + + async clonePromptPartial(partialId: string): Promise { + const response = await this.client.post(`/api/v1/prompt-partials/${partialId}/clone`) + return response.data + } + + async generatePromptWithAI(data: { + description: string + tone?: string + format_style?: string + provider?: string + model?: string + credential_id?: string + llm_config?: LLMGenerationConfig | null + }): Promise<{ content: string; provider: string; model: string }> { + const response = await this.client.post('/api/v1/prompt-partials/generate', data) + return response.data + } + + async improvePromptWithAI(data: { + content: string + instructions?: string + provider?: string + model?: string + credential_id?: string + llm_config?: LLMGenerationConfig | null + }): Promise<{ content: string; provider: string; model: string }> { + const response = await this.client.post('/api/v1/prompt-partials/improve', data) + return response.data + } + + // License / Enterprise + async getLicenseInfo(): Promise { + const response = await this.client.get('/api/v1/settings/license-info') + return response.data + } + + async getReportBranding(): Promise { + const response = await this.client.get('/api/v1/settings/report-branding') + return response.data + } + + async updateReportBranding(data: { heading?: string | null }): Promise { + const response = await this.client.patch('/api/v1/settings/report-branding', data) + return response.data + } + + async uploadReportBrandingImages( + files: File[], + role: 'internal' | 'external' | 'generic' = 'generic', + ): Promise { + const formData = new FormData() + for (const file of files) { + formData.append('files', file) + } + formData.append('role', role) + const response = await this.client.post('/api/v1/settings/report-branding/images', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async deleteReportBrandingImage(imageId: string): Promise { + const response = await this.client.delete(`/api/v1/settings/report-branding/images/${imageId}`) + return response.data + } + + // GEPA Prompt Optimization (Enterprise) + async createOptimizationRun(data: { + agent_id: string + evaluator_id?: string + voice_bundle_id?: string + config?: Record + }): Promise { + const response = await this.client.post('/api/v1/prompt-optimization/runs', data) + return response.data + } + + async deleteOptimizationRun(runId: string): Promise { + await this.client.delete(`/api/v1/prompt-optimization/runs/${runId}`) + } + + async listOptimizationRuns(agentId?: string): Promise { + const params = agentId ? { agent_id: agentId } : {} + const response = await this.client.get('/api/v1/prompt-optimization/runs', { params }) + return response.data + } + + async getOptimizationRun(runId: string): Promise { + const response = await this.client.get(`/api/v1/prompt-optimization/runs/${runId}`) + return response.data + } + + async listOptimizationCandidates(runId: string): Promise { + const response = await this.client.get(`/api/v1/prompt-optimization/runs/${runId}/candidates`) + return response.data + } + + async acceptCandidate(runId: string, candidateId: string): Promise { + const response = await this.client.post( + `/api/v1/prompt-optimization/runs/${runId}/candidates/${candidateId}/accept` + ) + return response.data + } + + async pushCandidateToProvider(runId: string, candidateId: string): Promise { + const response = await this.client.post( + `/api/v1/prompt-optimization/runs/${runId}/candidates/${candidateId}/push` + ) + return response.data + } + + async syncProviderPrompt(agentId: string): Promise<{ + synced: boolean + provider_prompt: string | null + provider_prompt_synced_at: string | null + }> { + const response = await this.client.post(`/api/v1/agents/${agentId}/sync-provider-prompt`) + return response.data + } + + // ------------------------------------------------------------------- + // Judge Alignment (AlignEval-style hybrid integration) + // + // Per-org thresholds and judge model selection are sourced from the + // backend (no hardcoded defaults in the frontend). + // ------------------------------------------------------------------- + + async getJudgeAlignmentSettings(): Promise { + const response = await this.client.get('/api/v1/judge-alignment/settings') + return response.data + } + + async updateJudgeAlignmentSettings( + data: { min_labels_to_evaluate: number; min_labels_to_optimize: number } + ): Promise { + const response = await this.client.patch('/api/v1/judge-alignment/settings', data) + return response.data + } + + async listJudgeCapableModels(): Promise { + const response = await this.client.get('/api/v1/judge-alignment/available-models') + return response.data + } + + async listJudgeDatasets(): Promise { + const response = await this.client.get('/api/v1/judge-alignment/datasets') + return response.data + } + + async createJudgeDataset(data: { + name: string + description?: string + source_type: 'transcript' | 'metric_output' | 'csv' + source_config?: Record + input_field?: string + output_field?: string + }): Promise { + const response = await this.client.post('/api/v1/judge-alignment/datasets', data) + return response.data + } + + async uploadJudgeDatasetCsv( + file: File, + name: string, + description?: string + ): Promise { + const formData = new FormData() + formData.append('file', file) + formData.append('name', name) + if (description) formData.append('description', description) + const response = await this.client.post( + '/api/v1/judge-alignment/datasets/upload-csv', + formData, + { headers: { 'Content-Type': 'multipart/form-data' } } + ) + return response.data + } + + async getJudgeDataset(datasetId: string): Promise { + const response = await this.client.get(`/api/v1/judge-alignment/datasets/${datasetId}`) + return response.data + } + + async deleteJudgeDataset(datasetId: string): Promise { + await this.client.delete(`/api/v1/judge-alignment/datasets/${datasetId}`) + } + + async listJudgeSamples( + datasetId: string, + options: { only_labeled?: boolean; skip?: number; limit?: number } = {} + ): Promise { + const response = await this.client.get( + `/api/v1/judge-alignment/datasets/${datasetId}/samples`, + { params: options } + ) + return response.data + } + + async labelJudgeSample( + sampleId: string, + label: 'pass' | 'fail' | null + ): Promise { + const response = await this.client.patch( + `/api/v1/judge-alignment/samples/${sampleId}`, + { label } + ) + return response.data + } + + async bulkLabelJudgeSamples( + datasetId: string, + items: Array<{ sample_id: string; label: 'pass' | 'fail' | null }> + ): Promise<{ updated: number; requested: number }> { + const response = await this.client.post( + `/api/v1/judge-alignment/datasets/${datasetId}/samples/bulk-label`, + { items } + ) + return response.data + } + + async listJudgeRuns(datasetId: string): Promise { + const response = await this.client.get( + `/api/v1/judge-alignment/datasets/${datasetId}/runs` + ) + return response.data + } + + async triggerJudgeRun( + datasetId: string, + data: { evaluator_id: string; split?: 'all' | 'dev' | 'test'; sample_ids?: string[] } + ): Promise { + const response = await this.client.post( + `/api/v1/judge-alignment/datasets/${datasetId}/run`, + data + ) + return response.data + } + + async getJudgeRun(runId: string): Promise { + const response = await this.client.get(`/api/v1/judge-alignment/runs/${runId}`) + return response.data + } + + async recomputeJudgeRunMetrics(runId: string): Promise { + const response = await this.client.post( + `/api/v1/judge-alignment/runs/${runId}/recompute-metrics` + ) + return response.data + } + + async deleteJudgeRun(runId: string): Promise { + await this.client.delete(`/api/v1/judge-alignment/runs/${runId}`) + } + + async optimizeJudge( + datasetId: string, + data: { + evaluator_id: string + dev_ratio?: number + seed?: number + max_metric_calls?: number + minibatch_size?: number + agent_id?: string + } + ): Promise { + const response = await this.client.post( + `/api/v1/judge-alignment/datasets/${datasetId}/optimize`, + data + ) + return response.data + } +} + +// ------------------------------------------------------------------- +// Judge Alignment shared types +// ------------------------------------------------------------------- + +export interface JudgeAlignmentSettings { + min_labels_to_evaluate: number + min_labels_to_optimize: number + defaults: { min_labels_to_evaluate: number; min_labels_to_optimize: number } +} + +export interface JudgeModelCatalogEntry { + provider: string + provider_label: string + model: string + label: string +} + +export interface JudgeDataset { + id: string + name: string + description?: string | null + source_type: 'transcript' | 'metric_output' | 'csv' + source_config: Record + input_field: string + output_field: string + total_samples: number + labeled_samples: number + unlabeled_samples: number + created_at?: string + updated_at?: string +} + +export interface JudgeSample { + id: string + dataset_id: string + external_id?: string | null + input_text: string + output_text: string + label?: 'pass' | 'fail' | null + labeled_by?: string | null + labeled_at?: string | null + extra?: Record | null + created_at?: string +} + +export interface JudgeRunMetrics { + precision: number + recall: number + f1: number + kappa: number + tp: number + fp: number + tn: number + fn: number + n: number +} + +export interface JudgeRun { + id: string + dataset_id: string + evaluator_id?: string | null + split: 'all' | 'dev' | 'test' + llm_provider?: string | null + llm_model?: string | null + status: 'pending' | 'queued' | 'running' | 'completed' | 'failed' + metrics?: JudgeRunMetrics | null + predictions?: Record | null + error_message?: string | null + celery_task_id?: string | null + gepa_optimization_id?: string | null + created_at?: string + updated_at?: string +} + +export interface JudgeOptimizeResponse { + optimization_run_id: string + dev_sample_count: number + test_sample_count: number +} + + +// Factory function to create ApiClient instance +// This ensures TypeScript correctly infers the type as ApiClient, not AxiosInstance +function createApiClient(): ApiClient { + return new ApiClient() +} + +// Create instance +const apiClientInstance = createApiClient() + +// Export with explicit type annotation - CRITICAL for TypeScript to recognize as ApiClient +// Without this explicit type, TypeScript may incorrectly infer AxiosInstance +export const apiClient: ApiClient = apiClientInstance + +// Re-export the type for explicit typing if needed +export type { ApiClient } + + +// ---------------------------------------------------------------------- +// Public (unauthenticated) blind-test API. +// Uses a bare axios instance so no Authorization / X-API-Key headers are +// attached. Anyone with the share_token in the URL can call these. +// ---------------------------------------------------------------------- + +const publicClient = axios.create({ + baseURL: API_BASE_URL, + headers: { 'Content-Type': 'application/json' }, +}) + +export interface PublicBlindTestEntrySubmit { + sample_index: number + preferred: 'X' | 'Y' + ratings_x?: Record + ratings_y?: Record + comment?: string +} + +export const publicBlindTestApi = { + async getForm(shareToken: string): Promise { + const res = await publicClient.get(`/api/v1/public/blind-tests/${shareToken}`) + return res.data + }, + async submit(shareToken: string, payload: { + rater_name: string + rater_email: string + client_token: string + responses: PublicBlindTestEntrySubmit[] + }): Promise { + const res = await publicClient.post( + `/api/v1/public/blind-tests/${shareToken}/responses`, + payload + ) + return res.data + }, +} diff --git a/frontend/src/lib/chatConnectionCapabilities.ts b/frontend/src/lib/chatConnectionCapabilities.ts index a7115993..95a91d3f 100644 --- a/frontend/src/lib/chatConnectionCapabilities.ts +++ b/frontend/src/lib/chatConnectionCapabilities.ts @@ -22,8 +22,8 @@ export type ChatConnectionCapability = { export const CHAT_CONNECTION_CAPABILITIES: ChatConnectionCapability[] = [ { id: 'internal_llm', - label: 'LLM', - description: 'LLM plays production; test agent is the customer.', + label: 'Hosted agent', + description: 'Your prompt and model answer as production.', pattern: 'llm_to_llm', offeredInCreateWizard: false, }, diff --git a/frontend/src/lib/llmModelOptions.ts b/frontend/src/lib/llmModelOptions.ts index f2b41240..3220f1c2 100644 --- a/frontend/src/lib/llmModelOptions.ts +++ b/frontend/src/lib/llmModelOptions.ts @@ -2,24 +2,10 @@ import type { AIProvider } from '../types/api' import type { LLMGenerationConfig } from '../config/llmGenerationParams' import { resolveActiveAIProvider, routesViaGateway, usesGatewayDirectModel } from './gatewayRouting' -/** Substring fingerprints for chat models that accept audio input. */ -export const AUDIO_CAPABLE_MODEL_MATCHERS: Record = { - openai: [/audio/i, /realtime/i], - google: [/^gemini-(1\.5|[2-9])/i], -} - -export const AUDIO_CAPABLE_PROVIDERS = Object.keys(AUDIO_CAPABLE_MODEL_MATCHERS) - export type LLMModelResolution = | { mode: 'gateway_direct'; model: string } | { mode: 'catalog'; models: string[] } -export function isAudioCapableModel(provider: string, model: string): boolean { - const matchers = AUDIO_CAPABLE_MODEL_MATCHERS[provider.toLowerCase()] - if (!matchers) return false - return matchers.some((re) => re.test(model)) -} - /** * System 1 (Jev) models score classification metrics. Mirrors backend * ``_is_jev_model`` minus Tev, which rejects classification questions. @@ -161,44 +147,8 @@ export function resolveLLMModelsForCredential( return { mode: 'catalog', models: catalogModels } } -/** Gateway-routed custom credentials are allowed for multimodal diariser. */ -export function isGatewayAudioCapableCredential( - credential: AIProvider, -): boolean { - const gatewayModel = credential.gateway_model?.trim() - const provider = credential.provider?.toLowerCase() ?? '' - - // Custom Bifrost integrations: any configured gateway model may be - // used for LLM-only diarisation — audio support is operator-owned. - if (provider === 'custom') { - return Boolean(gatewayModel) - } - - if (!gatewayModel) return false - if (!['openai', 'google'].includes(provider)) return false - - if (routesViaGateway(credential)) return true - return isAudioCapableModel(provider, gatewayModel) -} - -export function isAudioCapableSelection( - provider: string, - model: string | null, - credential?: AIProvider | null, -): boolean { - if (credential && usesGatewayDirectModel(credential)) { - return isGatewayAudioCapableCredential(credential) - } - if (credential && isGatewayAudioCapableCredential(credential)) { - return true - } - if (!model) return false - return isAudioCapableModel(provider, model) -} - export interface LLMProviderFilterOptions { providerAllowList?: string[] - audioCapableOnly?: boolean } export function buildLLMProviderKeys( @@ -222,16 +172,6 @@ export function buildLLMProviderKeys( ? new Set(filters.providerAllowList.map((p) => p.toLowerCase())) : null - if (filters?.audioCapableOnly) { - const audioKeys = new Set(AUDIO_CAPABLE_PROVIDERS) - for (const p of aiProviders) { - if (p.is_active && isGatewayAudioCapableCredential(p)) { - audioKeys.add(p.provider.toLowerCase()) - } - } - keys = keys.filter((k) => audioKeys.has(k)) - } - if (allowSet) { keys = keys.filter((k) => allowSet.has(k)) } diff --git a/frontend/src/lib/metricOptionLabels.ts b/frontend/src/lib/metricOptionLabels.ts new file mode 100644 index 00000000..66ba6046 --- /dev/null +++ b/frontend/src/lib/metricOptionLabels.ts @@ -0,0 +1,30 @@ +/** Align with app.services.metrics.option_labels (enum / classification labels). */ + +export function optionLabelFromRaw(opt: unknown): string | null { + if (typeof opt === 'string') { + const trimmed = opt.trim() + return trimmed || null + } + if (typeof opt === 'number') { + if (Number.isInteger(opt)) return String(opt) + if (opt === Math.trunc(opt)) return String(Math.trunc(opt)) + return String(opt) + } + if (typeof opt === 'boolean') return String(opt) + if (opt && typeof opt === 'object' && !Array.isArray(opt)) { + const o = opt as Record + for (const key of ['value', 'label', 'name', 'description'] as const) { + const nested = optionLabelFromRaw(o[key]) + if (nested) return nested + } + } + return null +} + +export function enumOptionsToCsv(options: unknown): string { + if (!Array.isArray(options) || options.length === 0) return '' + return options + .map((opt) => optionLabelFromRaw(opt)) + .filter((label): label is string => Boolean(label)) + .join(', ') +} diff --git a/frontend/src/lib/presignedUpload.ts b/frontend/src/lib/presignedUpload.ts new file mode 100644 index 00000000..e1ed5e40 --- /dev/null +++ b/frontend/src/lib/presignedUpload.ts @@ -0,0 +1,14 @@ +export async function uploadFileToPresignedPutUrl( + uploadUrl: string, + file: File, + contentType: string, +): Promise { + const response = await fetch(uploadUrl, { + method: 'PUT', + body: file, + headers: { 'Content-Type': contentType }, + }) + if (!response.ok) { + throw new Error(`Direct storage upload failed (${response.status}).`) + } +} diff --git a/frontend/src/lib/usageCurrency.ts b/frontend/src/lib/usageCurrency.ts index a110721d..396ea29c 100644 --- a/frontend/src/lib/usageCurrency.ts +++ b/frontend/src/lib/usageCurrency.ts @@ -1,3 +1,5 @@ +import { formatCompactCostMagnitude } from './usageFormatters' + export type UsageDisplayCurrency = 'USD' | 'INR' const STORAGE_KEY = 'efficientai.usage.displayCurrency' @@ -13,31 +15,66 @@ export function setUsageDisplayCurrency(currency: UsageDisplayCurrency): void { window.localStorage.setItem(STORAGE_KEY, currency) } +function displayAmount( + usd: number | null | undefined, + currency: UsageDisplayCurrency, + inrRate: number, +): number | null { + const amountUsd = Number(usd || 0) + if (!amountUsd) return null + return currency === 'INR' ? amountUsd * inrRate : amountUsd +} + function formatUsdAmount(amountUsd: number): string { const abs = Math.abs(amountUsd) const digits = abs >= 1 ? 2 : 4 - return `$${amountUsd.toLocaleString('en-US', { + const sign = amountUsd < 0 ? '-' : '' + return `${sign}$${Math.abs(amountUsd).toLocaleString('en-US', { minimumFractionDigits: digits, maximumFractionDigits: digits, })}` } +function formatInrAmount(amountInr: number): string { + return new Intl.NumberFormat('en-IN', { + style: 'currency', + currency: 'INR', + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }).format(amountInr) +} + +function formatExactMoney(amount: number, currency: UsageDisplayCurrency): string { + return currency === 'INR' ? formatInrAmount(amount) : formatUsdAmount(amount) +} + +function formatCompactMoney(amount: number, currency: UsageDisplayCurrency): string | null { + const compact = formatCompactCostMagnitude(Math.abs(amount)) + if (!compact) return null + const symbol = currency === 'INR' ? '₹' : '$' + return `${amount < 0 ? '-' : ''}${symbol}${compact}` +} + +/** Exact 2-decimal cost below 1 lakh. Above that, compact with more decimals at each unit. */ export function formatUsageCostUsd( usd: number | null | undefined, currency: UsageDisplayCurrency, inrRate: number, ): string { - const amountUsd = Number(usd || 0) - if (!amountUsd) return '—' - if (currency === 'INR') { - return new Intl.NumberFormat('en-IN', { - style: 'currency', - currency: 'INR', - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }).format(amountUsd * inrRate) - } - return formatUsdAmount(amountUsd) + const amount = displayAmount(usd, currency, inrRate) + if (amount == null) return '—' + return formatCompactMoney(amount, currency) ?? formatExactMoney(amount, currency) +} + +/** Full cost for hover tooltips and the breakdown modal. */ +export function formatUsageCostExactUsd( + usd: number | null | undefined, + currency: UsageDisplayCurrency, + inrRate: number, +): string { + const amount = displayAmount(usd, currency, inrRate) + if (amount == null) return '—' + return formatExactMoney(amount, currency) } export function formatFxRateHint( diff --git a/frontend/src/lib/usageFormatters.test.ts b/frontend/src/lib/usageFormatters.test.ts new file mode 100644 index 00000000..39957fa6 --- /dev/null +++ b/frontend/src/lib/usageFormatters.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { + formatAudioDuration, + formatAudioExact, + formatCompactCount, + formatExactCount, +} from './usageFormatters' +import { formatUsageCostExactUsd, formatUsageCostUsd } from './usageCurrency' + +describe('formatCompactCount', () => { + it('keeps small counts exact and compacts the rest', () => { + expect(formatCompactCount(0)).toBe('0') + expect(formatCompactCount(999)).toBe('999') + expect(formatCompactCount(1_000)).toBe('1K') + expect(formatCompactCount(46_640)).toBe('46.7K') + expect(formatCompactCount(194_826)).toBe('194.9K') + expect(formatCompactCount(999_950)).toBe('1M') + expect(formatCompactCount(1_008_146)).toBe('1.01M') + expect(formatCompactCount(2_472_082)).toBe('2.48M') + expect(formatCompactCount(342_616_620)).toBe('342.62M') + expect(formatCompactCount(1_500_000_000)).toBe('1.5B') + expect(formatCompactCount(14_049_396_030)).toBe('14.05B') + }) + + it.each([ + [0, '0'], + [1, '1'], + [42, '42'], + [999, '999'], + [1_000, '1K'], + [1_001, '1.1K'], + [1_500, '1.5K'], + [9_949, '10K'], + [46_640, '46.7K'], + [194_826, '194.9K'], + [999_949, '1M'], + [1_000_000, '1M'], + [1_000_001, '1.01M'], + [1_010_000, '1.01M'], + [2_472_082, '2.48M'], + [342_616_620, '342.62M'], + [999_999_999, '1B'], + [1_000_000_000, '1B'], + [1_000_000_001, '1.001B'], + [1_500_000_000, '1.5B'], + [14_049_396_030, '14.05B'], + [999_999_999_999, '1T'], + [1_000_000_000_000, '1T'], + [1_234_567_890_123, '1.235T'], + ])('shows %d as %s', (value, expected) => { + expect(formatCompactCount(value)).toBe(expected) + expect(formatExactCount(value)).toBe(new Intl.NumberFormat('en-US').format(value)) + }) +}) + +describe('formatExactCount', () => { + it('groups the full integer', () => { + expect(formatExactCount(342_616_620)).toBe('342,616,620') + expect(formatExactCount(0)).toBe('0') + }) +}) + +describe('audio duration', () => { + it('keeps the short clock and spells out leftover seconds on hover', () => { + expect(formatAudioDuration(45)).toBe('45s') + expect(formatAudioDuration(505 * 3600 + 12 * 60 + 18)).toBe('505h 12m') + expect(formatAudioExact(505 * 3600 + 12 * 60 + 18)).toBe('505h 12m 18s (1,818,738s)') + expect(formatAudioExact(45)).toBe('45s (45s)') + }) +}) + +describe('usage cost formatting', () => { + it('compacts large USD and INR amounts and keeps the exact value separate', () => { + expect(formatUsageCostUsd(0.0042, 'USD', 95)).toBe('$0.0042') + expect(formatUsageCostExactUsd(0.0042, 'USD', 95)).toBe('$0.0042') + expect(formatUsageCostUsd(1250, 'USD', 95)).toBe('$1,250.00') + expect(formatUsageCostUsd(9593.46, 'USD', 95)).toBe('$9,593.46') + expect(formatUsageCostUsd(22824.78, 'USD', 95)).toBe('$22,824.78') + expect(formatUsageCostExactUsd(22824.78, 'USD', 95)).toBe('$22,824.78') + expect(formatUsageCostExactUsd(662.675, 'USD', 95)).toBe('$662.68') + + expect(formatUsageCostUsd(62954.12 / 95, 'INR', 95)).toBe('₹62,954.12') + expect(formatUsageCostExactUsd(62954.12 / 95, 'INR', 95)).toBe('₹62,954.12') + expect(formatUsageCostUsd(1_250_000 / 95, 'INR', 95)).toBe('₹1.25M') + expect(formatUsageCostUsd(640.2 / 95, 'INR', 95)).toBe('₹640.20') + }) + + it.each([ + [0.0042, '$0.0042', '$0.0042'], + [0.5, '$0.5000', '$0.5000'], + [1, '$1.00', '$1.00'], + [662.68, '$662.68', '$662.68'], + [1_250, '$1,250.00', '$1,250.00'], + [9_593.46, '$9,593.46', '$9,593.46'], + [9_999.99, '$9,999.99', '$9,999.99'], + [10_000, '$10,000.00', '$10,000.00'], + [22_824.78, '$22,824.78', '$22,824.78'], + [99_999.99, '$99,999.99', '$99,999.99'], + [99_999.996, '$100K', '$100,000.00'], + [100_000, '$100K', '$100,000.00'], + [100_050, '$100.05K', '$100,050.00'], + [250_000.49, '$250K', '$250,000.49'], + [999_999.4, '$1M', '$999,999.40'], + [1_000_000, '$1M', '$1,000,000.00'], + [1_234_567.89, '$1.235M', '$1,234,567.89'], + [1_250_000, '$1.25M', '$1,250,000.00'], + [1_000_000_000, '$1B', '$1,000,000,000.00'], + [1_234_567_890.12, '$1.2346B', '$1,234,567,890.12'], + [1_000_000_000_000, '$1T', '$1,000,000,000,000.00'], + [1_234_567_890_123.45, '$1.2346T', '$1,234,567,890,123.45'], + ])('USD %d shows %s and hovers %s', (usd, compact, exact) => { + expect(formatUsageCostUsd(usd, 'USD', 1)).toBe(compact) + expect(formatUsageCostExactUsd(usd, 'USD', 1)).toBe(exact) + }) + + it.each([ + [0.5, '₹0.50', '₹0.50'], + [640.2, '₹640.20', '₹640.20'], + [9_999.99, '₹9,999.99', '₹9,999.99'], + [62_954.12, '₹62,954.12', '₹62,954.12'], + [99_999.99, '₹99,999.99', '₹99,999.99'], + [100_000, '₹100K', '₹1,00,000.00'], + [1_250_000, '₹1.25M', '₹12,50,000.00'], + [1_000_000_000, '₹1B', '₹1,00,00,00,000.00'], + [1_000_000_000_000, '₹1T', '₹10,00,00,00,00,000.00'], + ])('INR %d shows %s and hovers %s', (inr, compact, exact) => { + expect(formatUsageCostUsd(inr, 'INR', 1)).toBe(compact) + expect(formatUsageCostExactUsd(inr, 'INR', 1)).toBe(exact) + }) +}) diff --git a/frontend/src/lib/usageFormatters.ts b/frontend/src/lib/usageFormatters.ts new file mode 100644 index 00000000..62d46341 --- /dev/null +++ b/frontend/src/lib/usageFormatters.ts @@ -0,0 +1,119 @@ +const TIERS = [ + { value: 1e12, suffix: 'T', decimals: 3 }, + { value: 1e9, suffix: 'B', decimals: 3 }, + { value: 1e6, suffix: 'M', decimals: 2 }, + { value: 1e3, suffix: 'K', decimals: 1 }, +] as const + +function finiteNumber(value: number): number { + const n = Number(value) + return Number.isFinite(n) ? n : 0 +} + +function trimFixed(value: number, digits: number): string { + return value.toFixed(digits).replace(/\.0+$/, '').replace(/(\.\d*[1-9])0+$/, '$1') +} + +/** Round up to `digits` places. Exact values stay put (1.50 does not become 1.51). */ +function ceilTo(value: number, digits: number): number { + const factor = 10 ** digits + const scaled = value * factor + const nearest = Math.round(scaled) + if (Math.abs(scaled - nearest) < 1e-6) return nearest / factor + return Math.ceil(scaled - 1e-8) / factor +} + +/** + * Compact magnitude. Decimals grow with the unit (K 1, M 2, B/T 3) and the + * short figure is rounded up so it is never smaller than the real number. + * Returns null below 1,000. + */ +export function formatCompactMagnitude(abs: number): string | null { + if (!(abs >= 1000)) return null + for (let i = 0; i < TIERS.length; i++) { + const tier = TIERS[i] + if (abs < tier.value) continue + let rounded = ceilTo(abs / tier.value, tier.decimals) + let decimals: number = tier.decimals + let suffix: string = tier.suffix + if (rounded >= 1000 && i > 0) { + const higher = TIERS[i - 1] + rounded = ceilTo(abs / higher.value, higher.decimals) + decimals = higher.decimals + suffix = higher.suffix + } + return `${trimFixed(rounded, decimals)}${suffix}` + } + return null +} + +const COST_COMPACT_FROM = 100_000 +const COST_TIER_DECIMALS = { K: 2, M: 3, B: 4, T: 4 } as const + +/** Nearest value at `digits` places. Exact values are left unchanged. */ +function roundHalfUp(value: number, digits: number): number { + const factor = 10 ** digits + return Math.round((value + Number.EPSILON) * factor) / factor +} + +/** + * Cost compact form. Amounts under 1 lakh stay exact at 2 decimal places. + * Above that: K keeps 2 decimals, millions 3, and billions or more stop at 4. + */ +export function formatCompactCostMagnitude(abs: number): string | null { + if (roundHalfUp(abs, 2) < COST_COMPACT_FROM) return null + for (let i = 0; i < TIERS.length; i++) { + const tier = TIERS[i] + if (abs < tier.value) continue + let decimals: number = COST_TIER_DECIMALS[tier.suffix] + let rounded = roundHalfUp(abs / tier.value, decimals) + let suffix: string = tier.suffix + if (rounded >= 1000 && i > 0) { + const higher = TIERS[i - 1] + decimals = COST_TIER_DECIMALS[higher.suffix] + rounded = roundHalfUp(abs / higher.value, decimals) + suffix = higher.suffix + } + return `${trimFixed(rounded, decimals)}${suffix}` + } + return null +} + +/** Token / call / character counts: exact below 1,000, otherwise `K`/`M`/`B`/`T`. */ +export function formatCompactCount(value: number): string { + const n = Math.round(finiteNumber(value)) + const sign = n < 0 ? '-' : '' + const abs = Math.abs(n) + if (abs < 1000) return `${sign}${abs}` + return `${sign}${formatCompactMagnitude(abs) ?? String(abs)}` +} + +export function formatExactCount(value: number): string { + return new Intl.NumberFormat('en-US').format(Math.round(finiteNumber(value))) +} + +/** Primary STT duration. Hours drop leftover seconds so the cell stays short. */ +export function formatAudioDuration(seconds: number): string { + const total = Math.max(0, Math.floor(finiteNumber(seconds))) + if (total < 60) return `${total}s` + const mins = Math.floor(total / 60) + const secs = total % 60 + if (mins < 60) return secs ? `${mins}m ${secs}s` : `${mins}m` + const hours = Math.floor(mins / 60) + const remMins = mins % 60 + return remMins ? `${hours}h ${remMins}m` : `${hours}h` +} + +/** Exact duration for hover, including leftover seconds and the raw total. */ +export function formatAudioExact(seconds: number): string { + const total = Math.max(0, Math.floor(finiteNumber(seconds))) + const hours = Math.floor(total / 3600) + const mins = Math.floor((total % 3600) / 60) + const secs = total % 60 + const clock = hours + ? `${hours}h ${mins}m ${secs}s` + : mins + ? `${mins}m ${secs}s` + : `${secs}s` + return `${clock} (${formatExactCount(total)}s)` +} diff --git a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx index 41b9b6e4..66613d25 100644 --- a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx +++ b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx @@ -19,6 +19,7 @@ import { CallTypeBadge } from '../evaluators/components/evaluatorUi' import type { AgentDetailTab } from './components/AgentInfoView' import AgentEditForm from './components/AgentEditForm' import AgentTalkSidebar from './components/AgentTalkSidebar' +import AgentSimulationsPanel from '../simulations/AgentSimulationsPanel' import { Save, X } from 'lucide-react' import { extractPhoneConflictDetail } from './components/agentPhoneValidation' import { @@ -50,12 +51,15 @@ import { import { chatEvalModeForConnection } from './components/create/chatPreprodScope' import type { ChatIntegrationOptionId } from './components/create/ChatIntegrationTypeStep' -const VALID_TABS: AgentDetailTab[] = ['overview', 'test_agent', 'voice_ai_agent'] +// 'simulations' is a workspace-only tab (Harbor runs); info/edit views never see it. +type WorkspaceTab = AgentDetailTab | 'simulations' -function parseTabFromSearch(params: URLSearchParams): AgentDetailTab { +const VALID_TABS: WorkspaceTab[] = ['overview', 'test_agent', 'voice_ai_agent', 'simulations'] + +function parseTabFromSearch(params: URLSearchParams): WorkspaceTab { const tab = params.get('tab') - if (tab && VALID_TABS.includes(tab as AgentDetailTab)) { - return tab as AgentDetailTab + if (tab && VALID_TABS.includes(tab as WorkspaceTab)) { + return tab as WorkspaceTab } return 'overview' } @@ -82,6 +86,8 @@ function agentToFormData(agent: NonNullable { + (tab: WorkspaceTab) => { const next = new URLSearchParams(searchParams) if (tab === 'overview') { next.delete('tab') @@ -157,6 +165,8 @@ export default function AgentWorkspaceDetail({ voice_ai_integration_id: '', voice_ai_agent_id: '', provider_prompt: '', + knowledge_base_ids: [], + test_knowledge_base_ids: [], }) const [chatConnection, setChatConnection] = useState({ connectionType: 'internal_llm', @@ -206,6 +216,11 @@ export default function AgentWorkspaceDetail({ queryFn: () => apiClient.listAIProviders(), }) + const { data: knowledgeBases = [] } = useQuery({ + queryKey: ['knowledge-bases'], + queryFn: () => apiClient.listKnowledgeBases(), + }) + const syncChatStateFromAgent = useCallback( (a: NonNullable>>) => { if (!isChatMedium(a.call_medium)) return @@ -312,6 +327,9 @@ export default function AgentWorkspaceDetail({ payload.provider_prompt = data.provider_prompt?.trim() || null } + payload.knowledge_base_ids = data.knowledge_base_ids + payload.test_knowledge_base_ids = data.test_knowledge_base_ids + return apiClient.updateAgent(agentRouteId!, payload) }, onSuccess: () => { @@ -578,11 +596,13 @@ export default function AgentWorkspaceDetail({ { id: 'overview' as const, label: 'Overview' }, { id: 'test_agent' as const, label: 'Test Agent' }, { id: 'voice_ai_agent' as const, label: agentProductionTabLabel('chat') }, + { id: 'simulations' as const, label: 'Simulations' }, ] : [ { id: 'overview' as const, label: 'Overview' }, { id: 'test_agent' as const, label: 'Test Agent' }, { id: 'voice_ai_agent' as const, label: agentProductionTabLabel('voice') }, + { id: 'simulations' as const, label: 'Simulations' }, ] ).map((tab) => (
)} - {activeTab === 'voice_ai_agent' && - isChatAgent && + {activeTab === 'voice_ai_agent' && ( +
+ + + {productionAgentSubTab === 'tools' && ( + onChange({ ...formData, knowledge_base_ids: ids })} + /> + )} + + {productionAgentSubTab === 'prompt' && isChatAgent && chatConnection && onChatConnectionChange && onChatConnectionConfigChange && - onChatEditPlatformChange && ( + onChatEditPlatformChange ? ( - )} + ) : null} - {activeTab === 'voice_ai_agent' && !isChatAgent && hasPlatformLink && ( + {productionAgentSubTab === 'prompt' && !isChatAgent && hasPlatformLink ? (
@@ -833,9 +871,9 @@ export default function AgentEditForm({ />
- )} + ) : null} - {activeTab === 'voice_ai_agent' && !isChatAgent && !hasPlatformLink && ( + {productionAgentSubTab === 'prompt' && !isChatAgent && !hasPlatformLink ? (
@@ -886,6 +924,8 @@ export default function AgentEditForm({
)}
+ ) : null} +
)} ) diff --git a/frontend/src/pages/agents/components/AgentInfoView.tsx b/frontend/src/pages/agents/components/AgentInfoView.tsx index 85a3453b..3d23166a 100644 --- a/frontend/src/pages/agents/components/AgentInfoView.tsx +++ b/frontend/src/pages/agents/components/AgentInfoView.tsx @@ -36,10 +36,17 @@ import VoiceBundleDetailCard from './VoiceBundleDetailCard' import ChatTestAgentLlmDetailCard, { chatTestAgentLlmConfigured, } from './ChatTestAgentLlmDetailCard' +import ChatProductionLlmDetailCard from './ChatProductionLlmDetailCard' import AgentPromptVisualization from './AgentPromptVisualization' import Button from '../../../components/Button' import { agentProviderPromptTag } from './agentFlowchartUtils' import TestAgentSubTabNav, { type TestAgentSubTab } from './TestAgentSubTabNav' +import ProductionAgentSubTabNav, { + type ProductionAgentSubTab, +} from './ProductionAgentSubTabNav' +import AgentKnowledgeBasesPanel from './AgentKnowledgeBasesPanel' +import ProductionAgentToolsSection from './ProductionAgentToolsSection' +import type { KnowledgeBaseSummary } from '../../knowledgeBases/knowledgeBaseTypes' import { PRODUCTION_FIRST_MESSAGE_OPTIONS, callerFirstMessageHelperText, @@ -69,6 +76,7 @@ interface AgentInfoViewProps { /** Live web call to production voice platform (Retell/Vapi) — voice agents only. */ onTalkToProduction?: () => void onEditVoiceBundle?: (bundleId: string) => void + knowledgeBases?: KnowledgeBaseSummary[] } const PROSE = @@ -116,11 +124,16 @@ export default function AgentInfoView({ isSyncingPrompt, onTalkToProduction, onEditVoiceBundle, + knowledgeBases = [], }: AgentInfoViewProps) { const navigate = useNavigate() const [testPromptView, setTestPromptView] = useState<'text' | 'visualization'>('text') const [voiceAiPromptView, setVoiceAiPromptView] = useState<'text' | 'visualization'>('text') const [testAgentSubTab, setTestAgentSubTab] = useState('prompt') + const [productionAgentSubTab, setProductionAgentSubTab] = + useState('prompt') + const attachedKnowledgeBaseIds = agent.knowledge_base_ids ?? [] + const attachedTestKnowledgeBaseIds = agent.test_knowledge_base_ids ?? [] const linkedBundle = agent.voice_bundle_id ? voiceBundles.find((v) => v.id === agent.voice_bundle_id) @@ -360,7 +373,11 @@ export default function AgentInfoView({ title={agentProductionTabLabel(agent.call_medium)} description={ isChatAgent - ? 'Production agent under evaluation.' + ? chatConn === 'internal_llm' + ? agent.main_llm_model + ? `Production replies use ${agent.main_llm_model} with your production prompt.` + : 'Set a production model and prompt for hosted simulation.' + : 'Production agent under evaluation.' : 'Provider agent under evaluation.' } > @@ -388,6 +405,16 @@ export default function AgentInfoView({ ) : ( )} + {isChatAgent && chatConn === 'internal_llm' ? ( + + ) : null} {chatConn === 'provider_chat' || !isChatAgent ? ( )} + + {testAgentSubTab === 'knowledge_bases' && ( + + )}
) } @@ -589,11 +625,21 @@ export default function AgentInfoView({ )} )} + + {testAgentSubTab === 'knowledge_bases' && ( + + )} ) } const isChatProductionTab = isChatMedium(agent.call_medium) + const chatConnType = (agent.chat_connection_type || 'internal_llm').toLowerCase() const chatProdConnectionDisplay = isChatProductionTab ? resolveChatProductionConnectionFromAgent(agent, { telephonyConfigs, @@ -604,6 +650,26 @@ export default function AgentInfoView({ return (
+ + + {productionAgentSubTab === 'tools' && ( + + )} + + {productionAgentSubTab === 'prompt' && ( +
+ {isChatProductionTab && chatConnType === 'internal_llm' ? ( + + ) : null} +

@@ -729,5 +795,7 @@ export default function AgentInfoView({ )}

+ )} +
) } diff --git a/frontend/src/pages/agents/components/AgentKnowledgeBasesPanel.tsx b/frontend/src/pages/agents/components/AgentKnowledgeBasesPanel.tsx new file mode 100644 index 00000000..b9495122 --- /dev/null +++ b/frontend/src/pages/agents/components/AgentKnowledgeBasesPanel.tsx @@ -0,0 +1,140 @@ +import { Link } from 'react-router-dom' +import { BookOpen } from 'lucide-react' +import type { KnowledgeBaseSummary } from '../../knowledgeBases/knowledgeBaseTypes' + +interface AgentKnowledgeBasesPanelProps { + mode: 'view' | 'edit' + attachedIds: string[] + knowledgeBases: KnowledgeBaseSummary[] + onChange?: (ids: string[]) => void + compact?: boolean + /** Which side these bases serve; each side has its own list. */ + side?: 'production' | 'test' +} + +const SIDE_COPY = { + production: { + title: 'Production agent knowledge bases', + compact: 'Searched by the agent under test when it answers.', + full: 'Document libraries the production agent searches during chat simulation, telephony, and eval runs.', + }, + test: { + title: 'Test agent knowledge bases', + compact: 'Facts the simulated customer can look up (order details, account data).', + full: 'Document libraries the simulated customer searches during chat simulation and the voice playground. Independent of the production agent.', + }, +} + +export default function AgentKnowledgeBasesPanel({ + mode, + attachedIds, + knowledgeBases, + onChange, + compact = false, + side = 'production', +}: AgentKnowledgeBasesPanelProps) { + const attached = knowledgeBases.filter((kb) => attachedIds.includes(kb.id)) + const copy = SIDE_COPY[side] + + return ( +
+
+

+ {copy.title} +

+

+ {compact ? copy.compact : copy.full} +

+
+ + {knowledgeBases.length === 0 ? ( +

+ No knowledge bases in this workspace yet.{' '} + + Create one + + . +

+ ) : mode === 'edit' ? ( +
    + {knowledgeBases.map((kb) => { + const checked = attachedIds.includes(kb.id) + return ( +
  • + +
    • + ) + })} +
+ ) : attached.length === 0 ? ( +

+ No knowledge bases attached. Use Edit to link bases with indexed documents. +

+ ) : ( +
+
+ + + + + + + + + + {attached.map((kb) => ( + + + + + + ))} + +
+ Name + + Documents + + Embedding +
+ + + {kb.name} + + + {kb.ready_document_count}/{kb.document_count} ready + {kb.failed_document_count > 0 ? ( + · {kb.failed_document_count} failed + ) : null} + + {(kb.embedding_provider || 'openai')}/{kb.embedding_model || 'text-embedding-3-small'} +
+
+
+ )} +
+ ) +} diff --git a/frontend/src/pages/agents/components/AgentToolConnectionsPanel.tsx b/frontend/src/pages/agents/components/AgentToolConnectionsPanel.tsx new file mode 100644 index 00000000..58111d33 --- /dev/null +++ b/frontend/src/pages/agents/components/AgentToolConnectionsPanel.tsx @@ -0,0 +1,161 @@ +import { useEffect, useMemo, useState } from 'react' +import { Link } from 'react-router-dom' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { FlaskConical, Zap } from 'lucide-react' +import { apiClient } from '../../../lib/api' +import { getApiErrorDetail } from '../../../lib/authSession' +import Button from '../../../components/Button' +import type { AgentToolConnectionLink } from '../../../types/api' +import ToolConnectionLogo from '../../toolConnections/ToolConnectionLogo' + +interface Props { + agentId: string +} + +/** Which tool connections the production agent can call; saved independently of the agent form. */ +export default function AgentToolConnectionsPanel({ agentId }: Props) { + const queryClient = useQueryClient() + const { data: connections = [] } = useQuery({ + queryKey: ['tool-connections'], + queryFn: () => apiClient.listToolConnections(), + }) + const { data: savedLinks } = useQuery({ + queryKey: ['agent-tool-connections', agentId], + queryFn: () => apiClient.getAgentToolConnections(agentId), + }) + const { data: catalog = [] } = useQuery({ + queryKey: ['agent-tool-catalog', agentId], + queryFn: () => apiClient.getAgentToolCatalog(agentId), + }) + const [links, setLinks] = useState([]) + const [error, setError] = useState(null) + + useEffect(() => { + if (savedLinks) setLinks(savedLinks) + }, [savedLinks]) + + const dirty = useMemo(() => JSON.stringify(links) !== JSON.stringify(savedLinks ?? []), [links, savedLinks]) + + const saveMutation = useMutation({ + mutationFn: () => apiClient.updateAgentToolConnections(agentId, links), + onSuccess: () => { + setError(null) + queryClient.invalidateQueries({ queryKey: ['agent-tool-connections', agentId] }) + queryClient.invalidateQueries({ queryKey: ['agent-tool-catalog', agentId] }) + queryClient.invalidateQueries({ queryKey: ['tool-connections'] }) + }, + onError: (err: unknown) => setError(getApiErrorDetail(err) ?? 'Save failed'), + }) + + const linkFor = (connectionId: string) => links.find((l) => l.tool_connection_id === connectionId) + + const toggleConnection = (connectionId: string) => + setLinks((prev) => + linkFor(connectionId) + ? prev.filter((l) => l.tool_connection_id !== connectionId) + : [...prev, { tool_connection_id: connectionId, enabled_tools: null }], + ) + + const toggleTool = (connectionId: string, allTools: string[], tool: string) => + setLinks((prev) => + prev.map((link) => { + if (link.tool_connection_id !== connectionId) return link + const enabled = link.enabled_tools ?? allTools + const next = enabled.includes(tool) ? enabled.filter((t) => t !== tool) : [...enabled, tool] + return { ...link, enabled_tools: next.length === allTools.length ? null : next } + }), + ) + + const kbTools = catalog.filter((entry) => entry.name === 'search_knowledge_base') + + return ( +
+
+
+

Production agent tools

+

+ Tools the agent under test can call during simulations. Calls are recorded per turn and scored. +

+
+ +
+ + {kbTools.length ? ( +

+ Also available: search_knowledge_base over the production agent’s + knowledge bases. +

+ ) : null} + + {connections.length === 0 ? ( +

+ No tool sources in this workspace yet.{' '} + + Connect Slack, Notion, Postgres or an API + + . +

+ ) : ( +
    + {connections.map((connection) => { + const link = linkFor(connection.id) + const allTools = connection.tools.map((t) => t.name) + const enabled = link?.enabled_tools ?? allTools + return ( +
  • + + {link && connection.tools.length ? ( +
      + {connection.tools.map((tool) => { + const mocked = + connection.simulation_mode === 'mock_all' || + (connection.simulation_mode === 'mock_writes' && tool.side_effect === 'write') + return ( +
    • + +
      • + ) + })} +
    + ) : null} +
    • + ) + })} +
+ )} + {error ?

{error}

: null} +
+ ) +} diff --git a/frontend/src/pages/agents/components/AgentsListSidebar.tsx b/frontend/src/pages/agents/components/AgentsListSidebar.tsx index 277e64cd..5023bfee 100644 --- a/frontend/src/pages/agents/components/AgentsListSidebar.tsx +++ b/frontend/src/pages/agents/components/AgentsListSidebar.tsx @@ -1,5 +1,5 @@ import { useMemo, useState } from 'react' -import { Search } from 'lucide-react' +import { Search, Star } from 'lucide-react' import { TestAgent, Integration, IntegrationPlatform } from '../../../types/api' import { getIntegrationPlatformLabel, getIntegrationPlatformLogo } from '../../../config/providers' import { CallTypeBadge } from '../../evaluators/components/evaluatorUi' @@ -22,6 +22,8 @@ interface AgentsListSidebarProps { onSelectAgent: (agent: TestAgent) => void onSelectAgentCheckbox: (agentId: string, checked: boolean) => void onSelectAll: () => void + onToggleFavorite: (agent: TestAgent) => void + favoritePendingId?: string | null } export default function AgentsListSidebar({ @@ -33,6 +35,8 @@ export default function AgentsListSidebar({ onSelectAgent, onSelectAgentCheckbox, onSelectAll, + onToggleFavorite, + favoritePendingId, }: AgentsListSidebarProps) { const [search, setSearch] = useState('') const [mediumFilter, setMediumFilter] = useState<'all' | 'voice' | 'chat'>('all') @@ -45,13 +49,16 @@ export default function AgentsListSidebar({ list = list.filter((a) => !isChatMedium(a.call_medium)) } const q = search.trim().toLowerCase() - if (!q) return list - return list.filter((agent) => { - const id = agent.agent_id?.toLowerCase() || '' - const name = agent.name.toLowerCase() - const internal = agent.id.toLowerCase() - return id.includes(q) || name.includes(q) || internal.includes(q) - }) + if (q) { + list = list.filter((agent) => { + const id = agent.agent_id?.toLowerCase() || '' + const name = agent.name.toLowerCase() + const internal = agent.id.toLowerCase() + return id.includes(q) || name.includes(q) || internal.includes(q) + }) + } + if (mediumFilter !== 'all') return list + return [...list].sort((a, b) => Number(Boolean(b.is_favorite)) - Number(Boolean(a.is_favorite))) }, [agents, search, mediumFilter]) const getIntegrationLogo = (agent: TestAgent) => { @@ -129,7 +136,20 @@ export default function AgentsListSidebar({ isActive ? 'bg-primary-50 ring-1 ring-primary-200' : 'hover:bg-gray-50' }`} > -
e.stopPropagation()}> +
e.stopPropagation()}> + + +

No production model configured.

+

+ Edit this agent and set a credential and model under Production Agent → Prompt. +

+
+ ) + } + + return ( +
+
+

Production language model

+

Agent under test in LLM-to-LLM chat simulations.

+
+
+
+

+ + Language model (LLM) +

+
+
+

Provider

+
+ {provider && getProviderLogo(provider) ? ( + + ) : null} + + {provider ? getProviderLabel(provider) : agent.main_llm_provider || '—'} + +
+
+
+

Model

+
+ {model} +
+
+
+
+
+
+ ) +} diff --git a/frontend/src/pages/agents/components/CreateAgentModal.tsx b/frontend/src/pages/agents/components/CreateAgentModal.tsx index 4074883e..6f28c0be 100644 --- a/frontend/src/pages/agents/components/CreateAgentModal.tsx +++ b/frontend/src/pages/agents/components/CreateAgentModal.tsx @@ -36,6 +36,7 @@ import { type AgentMedium, type CreateWizardPhase, DEFAULT_CREATE_AGENT_FORM, + HOSTED_STEPS, TELEPHONY_STEPS, PLATFORM_STEPS, chatWizardMaxStep, @@ -43,7 +44,7 @@ import { createWizardMaxStep, } from './create/createAgentTypes' import { NATIVE_PROVIDER_TEXT_CHAT_PLATFORMS } from '../../../lib/chatConnectionCapabilities' -import ChatConnectionStep, { +import { type ChatConnectionForm, chatConnectionValidationMessage, validateChatConnection, @@ -53,6 +54,17 @@ import ChatTestAgentLlmStep, { testAgentLlmValidationMessage, validateTestAgentLlm, } from './create/ChatTestAgentLlmStep' +import HostedProductionAgentStep, { + hostedProductionStepValidationMessage, + validateHostedProductionStep, +} from './create/HostedProductionAgentStep' +import HostedTestAgentStep, { + hostedTestStepValidationMessage, + validateHostedTestStep, +} from './create/HostedTestAgentStep' +import HostedProductionLlmStep from './create/HostedProductionLlmStep' +import HostedTestLlmStep from './create/HostedTestLlmStep' +import type { KnowledgeBaseSummary } from '../../knowledgeBases/knowledgeBaseTypes' import { applyGeneratedTemplate } from './TestAgentTemplateEditor' import { type TestAgentTemplateDraft, @@ -82,6 +94,8 @@ export default function CreateAgentModal({ useState('provider_chat') const [formData, setFormData] = useState(DEFAULT_CREATE_AGENT_FORM) const [productionPrompt, setProductionPrompt] = useState('') + const [hostedProductionBrief, setHostedProductionBrief] = useState('') + const [productionPromptAiGenerated, setProductionPromptAiGenerated] = useState(false) const [setupAdditionalContext, setSetupAdditionalContext] = useState('') const [phoneNumberInputMode, setPhoneNumberInputMode] = useState<'provider' | 'custom'>('provider') const [selectedPlatform, setSelectedPlatform] = useState(null) @@ -104,11 +118,13 @@ export default function CreateAgentModal({ }) const steps = - createPath === 'telephony' - ? TELEPHONY_STEPS - : createPath === 'chat' - ? chatWizardSteps(chatIntegrationOption) - : PLATFORM_STEPS + createPath === 'hosted' + ? HOSTED_STEPS + : createPath === 'telephony' + ? TELEPHONY_STEPS + : createPath === 'chat' + ? chatWizardSteps(chatIntegrationOption) + : PLATFORM_STEPS const maxStep = createPath === 'chat' ? chatWizardMaxStep(chatIntegrationOption) @@ -129,6 +145,12 @@ export default function CreateAgentModal({ queryFn: () => apiClient.listAIProviders(), }) + const { data: knowledgeBases = [] } = useQuery({ + queryKey: ['knowledge-bases'], + queryFn: () => apiClient.listKnowledgeBases(), + enabled: isOpen && createPath === 'hosted', + }) + const { isChecking: isCheckingPhoneAssignment, hasConflict: hasPhoneConflict } = useAgentPhoneAssignmentCheck({ enabled: isOpen && createPath === 'telephony', @@ -209,6 +231,36 @@ export default function CreateAgentModal({ ...(aiModel ? { model: aiModel } : {}), }) + const generateProductionPromptMutation = useMutation({ + mutationFn: () => { + if (!formData.name.trim()) { + throw new Error('Agent name is required before generating a production prompt') + } + if (!hostedProductionBrief.trim()) { + throw new Error('Describe what the agent should do before generating') + } + return apiClient.generateProductionPromptFromBrief({ + agent_name: formData.name.trim(), + brief: hostedProductionBrief.trim(), + language: formData.language, + call_type: formData.call_type, + ...buildGenerationParams(), + }) + }, + onSuccess: (data) => { + setProductionPrompt(data.production_prompt) + setProductionPromptAiGenerated(true) + invalidateGeneratedTestTemplate() + showToast('Production prompt generated', 'success') + }, + onError: (err: any) => { + showToast( + err?.message || err?.response?.data?.detail || 'Failed to generate production prompt', + 'error', + ) + }, + }) + const generateTestPromptMutation = useMutation({ mutationFn: () => { if (!formData.name.trim()) { @@ -283,7 +335,7 @@ export default function CreateAgentModal({ call_medium: createPath === 'platform' ? 'web_call' - : createPath === 'chat' + : createPath === 'chat' || createPath === 'hosted' ? 'chat' : 'phone_call', silence_hangup_secs: data.silence_hangup_secs ?? 15, @@ -310,21 +362,29 @@ export default function CreateAgentModal({ } } + if (createPath === 'hosted') { + const prompt = productionPrompt.trim() + payload.provider_prompt = prompt + payload.chat_connection_type = 'internal_llm' + payload.chat_eval_mode = chatEvalModeForConnection('internal_llm') + if (chatConnection.mainLlmProvider && chatConnection.mainLlmModel.trim()) { + payload.main_llm_provider = chatConnection.mainLlmProvider + payload.main_llm_model = chatConnection.mainLlmModel + if (chatConnection.mainLlmCredentialId) { + payload.main_llm_credential_id = chatConnection.mainLlmCredentialId + } + } + applyTestAgentLlmPayload(payload, chatConnection, aiProviders) + payload.knowledge_base_ids = data.knowledge_base_ids + payload.test_knowledge_base_ids = data.test_knowledge_base_ids + } + if (createPath === 'chat') { const prompt = productionPrompt.trim() const connType = chatConnectionTypeFromOption(chatIntegrationOption) payload.provider_prompt = prompt payload.chat_connection_type = connType payload.chat_eval_mode = chatEvalModeForConnection(connType) - if (chatIntegrationOption === 'internal_llm') { - if (chatConnection.mainLlmProvider && chatConnection.mainLlmModel.trim()) { - payload.main_llm_provider = chatConnection.mainLlmProvider - payload.main_llm_model = chatConnection.mainLlmModel - if (chatConnection.mainLlmCredentialId) { - payload.main_llm_credential_id = chatConnection.mainLlmCredentialId - } - } - } if (chatIntegrationOption === 'customer_api') { payload.chat_connection_config = buildChatConnectionConfigPayload( 'customer_api', @@ -350,9 +410,7 @@ export default function CreateAgentModal({ payload.voice_ai_integration_id = formData.voice_ai_integration_id.trim() payload.voice_ai_agent_id = formData.voice_ai_agent_id.trim() } - if (chatIntegrationOption !== 'internal_llm') { - applyTestAgentLlmPayload(payload, chatConnection, aiProviders) - } + applyTestAgentLlmPayload(payload, chatConnection, aiProviders) } return apiClient.createAgent(payload as Parameters[0]) @@ -385,6 +443,8 @@ export default function CreateAgentModal({ setCreatePath('telephony') setCurrentStep(1) setProductionPrompt('') + setHostedProductionBrief('') + setProductionPromptAiGenerated(false) setSetupAdditionalContext('') setPhoneNumberInputMode('provider') setSelectedPlatform(null) @@ -514,6 +574,44 @@ export default function CreateAgentModal({ } } + if (createPath === 'hosted') { + if (currentStep === 1) { + if (!validateHostedProductionStep(formData, productionPrompt, productionPromptAiGenerated)) { + showToast( + hostedProductionStepValidationMessage( + formData, + productionPrompt, + productionPromptAiGenerated, + ), + 'error', + ) + return false + } + return true + } + if (currentStep === 2) { + if (!validateHostedTestStep(formData.test_agent_template)) { + showToast(hostedTestStepValidationMessage(formData.test_agent_template), 'error') + return false + } + return true + } + if (currentStep === 3) { + if (!validateChatConnection(chatConnection, aiProviders)) { + showToast(chatConnectionValidationMessage(chatConnection), 'error') + return false + } + return true + } + if (currentStep === 4) { + if (!validateTestAgentLlm(chatConnection, aiProviders)) { + showToast(testAgentLlmValidationMessage(), 'error') + return false + } + return true + } + } + if (createPath === 'chat') { if (currentStep === 1) { if (!validateChatAgentStep(formData, productionPrompt, { requireProductionPrompt: false })) { @@ -521,7 +619,6 @@ export default function CreateAgentModal({ return false } if ( - chatIntegrationOption !== 'internal_llm' && !validateChatConnectionDetails( chatIntegrationOption, formData, @@ -540,29 +637,9 @@ export default function CreateAgentModal({ } return false } - if (chatIntegrationOption === 'internal_llm') { - if (!isPromptStepValid(productionPrompt, formData.test_agent_template, CREATE_PROMPT_VALIDATION)) { - showToast( - promptStepValidationMessage( - productionPrompt, - formData.test_agent_template, - CREATE_PROMPT_VALIDATION, - ), - 'error', - ) - return false - } - } return true } if (currentStep === 2) { - if (chatIntegrationOption === 'internal_llm') { - if (!validateChatConnection(chatConnection, aiProviders)) { - showToast(chatConnectionValidationMessage(chatConnection), 'error') - return false - } - return true - } if (!isPromptStepValid(productionPrompt, formData.test_agent_template, CREATE_PROMPT_VALIDATION)) { showToast( promptStepValidationMessage( @@ -653,7 +730,7 @@ export default function CreateAgentModal({ return } if (wizardPhase === 'steps' && currentStep === 1) { - setWizardPhase('entry_integration') + setWizardPhase(agentMedium === 'hosted' ? 'entry_medium' : 'entry_integration') return } setCurrentStep((step) => Math.max(step - 1, 1) as CreateStepId) @@ -661,19 +738,34 @@ export default function CreateAgentModal({ const handleAgentMediumChange = (medium: AgentMedium) => { setAgentMedium(medium) + if (medium === 'hosted') { + setCreatePath('hosted') + setChatIntegrationOption('internal_llm') + setFormData((prev) => ({ ...prev, call_medium: 'chat' })) + setChatConnection((prev) => ({ ...prev, connectionType: 'internal_llm' })) + return + } if (medium === 'chat') { setCreatePath('chat') setChatIntegrationOption('provider_chat') return } - if (createPath === 'chat') { + if (createPath === 'chat' || createPath === 'hosted') { setCreatePath('telephony') } } const handleContinueFromMedium = () => { if (!agentMedium) { - showToast('Choose voice or chat to continue.', 'error') + showToast('Choose how you want to create your agent.', 'error') + return + } + if (agentMedium === 'hosted') { + setCreatePath('hosted') + setChatIntegrationOption('internal_llm') + setFormData((prev) => ({ ...prev, call_medium: 'chat' })) + setCurrentStep(1) + setWizardPhase('steps') return } if (agentMedium === 'chat') { @@ -703,6 +795,10 @@ export default function CreateAgentModal({ const handleCreate = () => { if (currentStep !== maxStep || !validateCurrentStep()) return + if (createPath === 'hosted') { + createMutation.mutate(formData) + return + } if (createPath === 'chat') { if (!isPromptStepValid(productionPrompt, formData.test_agent_template, CREATE_PROMPT_VALIDATION)) { showToast( @@ -715,17 +811,7 @@ export default function CreateAgentModal({ ) return } - if ( - chatIntegrationOption === 'internal_llm' && - !validateChatConnection(chatConnection, aiProviders) - ) { - showToast(chatConnectionValidationMessage(chatConnection), 'error') - return - } - if ( - chatIntegrationOption !== 'internal_llm' && - !validateTestAgentLlm(chatConnection, aiProviders) - ) { + if (!validateTestAgentLlm(chatConnection, aiProviders)) { showToast(testAgentLlmValidationMessage(), 'error') return } @@ -791,9 +877,11 @@ export default function CreateAgentModal({ const isEntryPhase = wizardPhase === 'entry_medium' || wizardPhase === 'entry_integration' + const isHostedWizardSteps = createPath === 'hosted' && wizardPhase === 'steps' + const entryModalWidthClass = wizardPhase === 'entry_medium' - ? 'w-[min(92vw,34rem)]' + ? 'w-[min(94vw,64rem)]' : agentMedium === 'chat' ? 'w-[min(92vw,48rem)]' : 'w-[min(92vw,38rem)]' @@ -812,7 +900,7 @@ export default function CreateAgentModal({ phase={wizardPhase} agentMedium={agentMedium} onAgentMediumChange={handleAgentMediumChange} - voicePath={createPath === 'chat' ? 'telephony' : createPath} + voicePath={createPath === 'chat' || createPath === 'hosted' ? 'telephony' : createPath} onVoicePathChange={handleVoicePathChange} chatIntegration={chatIntegrationOption} onChatIntegrationChange={syncChatIntegrationOption} @@ -821,6 +909,78 @@ export default function CreateAgentModal({ ) } + if (createPath === 'hosted') { + const canGenerateProduction = + Boolean(formData.name.trim() && hostedProductionBrief.trim() && aiCredentialId) && + Boolean(aiModel.trim() || gatewayDirectModel) + + const hostedStep = ( +
+ {currentStep === 1 ? ( + setFormData((prev) => ({ ...prev, ...patch }))} + brief={hostedProductionBrief} + onBriefChange={setHostedProductionBrief} + productionPrompt={productionPrompt} + onProductionPromptChange={handleProductionPromptChange} + productionPromptAiGenerated={productionPromptAiGenerated} + knowledgeBases={knowledgeBases} + aiProviders={aiProviders} + aiCredentialId={aiCredentialId} + onAiCredentialIdChange={setAiCredentialId} + aiModel={aiModel} + onAiModelChange={setAiModel} + onGenerateProduction={() => generateProductionPromptMutation.mutate()} + isGenerating={generateProductionPromptMutation.isPending} + canGenerate={canGenerateProduction} + /> + ) : null} + {currentStep === 2 ? ( + setFormData((prev) => ({ ...prev, ...patch }))} + productionPrompt={productionPrompt} + testAgentTemplate={formData.test_agent_template} + onTestAgentTemplateChange={handleTestAgentTemplateChange} + additionalContext={setupAdditionalContext} + onAdditionalContextChange={setSetupAdditionalContext} + knowledgeBases={knowledgeBases} + aiProviders={aiProviders} + aiCredentialId={aiCredentialId} + onAiCredentialIdChange={setAiCredentialId} + aiModel={aiModel} + onAiModelChange={setAiModel} + onGenerateTestPrompt={() => generateTestPromptMutation.mutate()} + isGenerating={generateTestPromptMutation.isPending} + canGenerate={canGenerateTestPrompt} + /> + ) : null} + {currentStep === 3 ? ( + setFormData((prev) => ({ ...prev, ...patch }))} + knowledgeBases={knowledgeBases} + chatConnection={chatConnection} + onChatConnectionChange={(patch) => setChatConnection((prev) => ({ ...prev, ...patch }))} + /> + ) : null} + {currentStep === 4 ? ( + setFormData((prev) => ({ ...prev, ...patch }))} + knowledgeBases={knowledgeBases} + chatConnection={chatConnection} + onChatConnectionChange={(patch) => setChatConnection((prev) => ({ ...prev, ...patch }))} + /> + ) : null} +
+ ) + return hostedStep + } + if (createPath === 'chat') { const chatPromptsStep = ( ) : null} - {chatIntegrationOption !== 'internal_llm' && - chatIntegrationOption !== 'provider_chat' ? ( + {chatIntegrationOption !== 'provider_chat' ? ( ) : null} - {chatIntegrationOption === 'internal_llm' ? chatPromptsStep : null}
) } if (currentStep === 2) { - if (chatIntegrationOption === 'internal_llm') { - return ( -
- setChatConnection((prev) => ({ ...prev, ...patch }))} - /> -
- ) - } return
{chatPromptsStep}
} if (currentStep === 3) { @@ -1044,8 +1192,10 @@ export default function CreateAgentModal({
e.stopPropagation()} role="dialog" @@ -1072,10 +1222,10 @@ export default function CreateAgentModal({

) : (

- Step {wizardPhase === 'entry_medium' ? 1 : 2} of 2 + Step {wizardPhase === 'entry_medium' ? 1 : 2} of {agentMedium === 'hosted' ? 1 : 2}

)} - {isEntryPhase ? ( + {isEntryPhase && agentMedium !== 'hosted' ? (
{isEntryPhase ? ( diff --git a/frontend/src/pages/agents/components/ProductionAgentSubTabNav.tsx b/frontend/src/pages/agents/components/ProductionAgentSubTabNav.tsx new file mode 100644 index 00000000..5a3222f9 --- /dev/null +++ b/frontend/src/pages/agents/components/ProductionAgentSubTabNav.tsx @@ -0,0 +1,39 @@ +export type ProductionAgentSubTab = 'prompt' | 'tools' + +const SUB_TABS: { id: ProductionAgentSubTab; label: string }[] = [ + { id: 'prompt', label: 'Prompt' }, + { id: 'tools', label: 'Tools' }, +] + +export default function ProductionAgentSubTabNav({ + value, + onChange, +}: { + value: ProductionAgentSubTab + onChange: (tab: ProductionAgentSubTab) => void +}) { + return ( +
+ {SUB_TABS.map((tab) => ( + + ))} +
+ ) +} diff --git a/frontend/src/pages/agents/components/ProductionAgentToolsSection.tsx b/frontend/src/pages/agents/components/ProductionAgentToolsSection.tsx new file mode 100644 index 00000000..136dabb7 --- /dev/null +++ b/frontend/src/pages/agents/components/ProductionAgentToolsSection.tsx @@ -0,0 +1,89 @@ +import { useState } from 'react' +import { BookOpen, Plug } from 'lucide-react' +import AgentKnowledgeBasesPanel from './AgentKnowledgeBasesPanel' +import AgentToolConnectionsPanel from './AgentToolConnectionsPanel' +import type { KnowledgeBaseSummary } from '../../knowledgeBases/knowledgeBaseTypes' + +const TABS = [ + { id: 'connections', label: 'Connections', icon: Plug }, + { id: 'knowledge-bases', label: 'Knowledge bases', icon: BookOpen }, +] as const + +type ToolsTabId = (typeof TABS)[number]['id'] + +type ViewProps = { + mode: 'view' + agentId: string + attachedKnowledgeBaseIds: string[] + knowledgeBases: KnowledgeBaseSummary[] +} + +type EditProps = { + mode: 'edit' + agentId?: string + attachedKnowledgeBaseIds: string[] + knowledgeBases: KnowledgeBaseSummary[] + onKnowledgeBaseIdsChange: (ids: string[]) => void +} + +export type ProductionAgentToolsSectionProps = ViewProps | EditProps + +/** Production agent tool sources: external connections and knowledge bases (inner tabs, like Tools page). */ +export default function ProductionAgentToolsSection(props: ProductionAgentToolsSectionProps) { + const [tab, setTab] = useState('connections') + + const { attachedKnowledgeBaseIds, knowledgeBases } = props + + return ( +
+
+ {TABS.map(({ id, label, icon: Icon }) => ( + + ))} +
+ +
+ {props.mode === 'view' || props.agentId ? ( + + ) : ( +

+ Save the agent first, then attach tool connections here. +

+ )} +
+ +
+ {props.mode === 'view' ? ( + + ) : ( + + )} +
+
+ ) +} diff --git a/frontend/src/pages/agents/components/TestAgentSubTabNav.tsx b/frontend/src/pages/agents/components/TestAgentSubTabNav.tsx index 2cb9acc7..16be191c 100644 --- a/frontend/src/pages/agents/components/TestAgentSubTabNav.tsx +++ b/frontend/src/pages/agents/components/TestAgentSubTabNav.tsx @@ -1,8 +1,9 @@ -export type TestAgentSubTab = 'configuration' | 'prompt' +export type TestAgentSubTab = 'configuration' | 'prompt' | 'knowledge_bases' const SUB_TABS: { id: TestAgentSubTab; label: string }[] = [ { id: 'prompt', label: 'Prompt' }, { id: 'configuration', label: 'Configuration' }, + { id: 'knowledge_bases', label: 'Knowledge Bases' }, ] export default function TestAgentSubTabNav({ diff --git a/frontend/src/pages/agents/components/create/AgentMediumStep.tsx b/frontend/src/pages/agents/components/create/AgentMediumStep.tsx index 361dffef..1e42b669 100644 --- a/frontend/src/pages/agents/components/create/AgentMediumStep.tsx +++ b/frontend/src/pages/agents/components/create/AgentMediumStep.tsx @@ -1,4 +1,4 @@ -import { Mic, MessagesSquare } from 'lucide-react' +import { Brain, Mic, MessagesSquare } from 'lucide-react' import type { AgentMedium } from './createAgentTypes' import { WizardStepHeader } from './WizardStepHeader' import WizardIconCard from './WizardIconCard' @@ -16,8 +16,25 @@ const OPTIONS: { subtitle: string icon: typeof Mic }[] = [ - { id: 'voice', label: 'Voice', subtitle: 'Phone or platform voice agents', icon: Mic }, - { id: 'chat', label: 'Chat', subtitle: 'Text evals against live or API chat', icon: MessagesSquare }, + { + id: 'hosted', + label: 'Hosted agent from scratch', + subtitle: + 'Create your production agent from scratch and self improve by simulating them against 100s of scenarios', + icon: Brain, + }, + { + id: 'voice', + label: 'Voice', + subtitle: 'Test your voice agents already hosted in production.', + icon: Mic, + }, + { + id: 'chat', + label: 'Chat', + subtitle: 'Test your chat agents already hosted in production.', + icon: MessagesSquare, + }, ] export default function AgentMediumStep({ value, onChange, embedded, compact }: AgentMediumStepProps) { @@ -26,12 +43,12 @@ export default function AgentMediumStep({ value, onChange, embedded, compact }: {!embedded ? ( ) : null}
{OPTIONS.map((option) => ( = { } const WIZARD_LABELS: Record = { - internal_llm: 'LLM', + internal_llm: 'Hosted agent', provider_chat: WIZARD_EXISTING_PLATFORM_LABEL, customer_api: 'HTTP API', customer_websocket: 'WebSocket', diff --git a/frontend/src/pages/agents/components/create/HostedProductionAgentStep.tsx b/frontend/src/pages/agents/components/create/HostedProductionAgentStep.tsx new file mode 100644 index 00000000..ebcfd26c --- /dev/null +++ b/frontend/src/pages/agents/components/create/HostedProductionAgentStep.tsx @@ -0,0 +1,182 @@ +import { Loader2, Sparkles } from 'lucide-react' + +import LlmCredentialAnchoredFields from '../../../../components/providers/LlmCredentialAnchoredFields' +import type { AIProvider } from '../../../../types/api' +import AgentKnowledgeBasesPanel from '../AgentKnowledgeBasesPanel' +import type { KnowledgeBaseSummary } from '../../../knowledgeBases/knowledgeBaseTypes' +import type { CreateAgentFormData } from './createAgentTypes' +import HostedWizardSplitLayout, { HostedPromptTextarea } from './HostedWizardSplitLayout' +import { CREATE_WIZARD_FIELD_CLASS, CREATE_WIZARD_LABEL_CLASS } from './createWizardUi' + +interface HostedProductionAgentStepProps { + formData: CreateAgentFormData + onFormChange: (patch: Partial) => void + brief: string + onBriefChange: (value: string) => void + productionPrompt: string + onProductionPromptChange: (value: string) => void + productionPromptAiGenerated: boolean + knowledgeBases: KnowledgeBaseSummary[] + aiProviders: AIProvider[] + aiCredentialId: string + onAiCredentialIdChange: (value: string) => void + aiModel: string + onAiModelChange: (value: string) => void + onGenerateProduction: () => void + isGenerating: boolean + canGenerate: boolean +} + +export function validateHostedProductionStep( + formData: CreateAgentFormData, + productionPrompt: string, + productionPromptAiGenerated: boolean, +): boolean { + if (!formData.name.trim()) return false + if (!productionPromptAiGenerated) return false + const words = productionPrompt.trim().split(/\s+/).filter(Boolean).length + return words >= 10 +} + +export function hostedProductionStepValidationMessage( + formData: CreateAgentFormData, + productionPrompt: string, + productionPromptAiGenerated: boolean, +): string { + if (!formData.name.trim()) return 'Agent name is required.' + if (!productionPromptAiGenerated) { + return 'Use “Generate production prompt” before continuing.' + } + const words = productionPrompt.trim().split(/\s+/).filter(Boolean).length + if (words < 10) return 'Production prompt must be at least 10 words.' + return 'Complete the production agent step.' +} + +export default function HostedProductionAgentStep({ + formData, + onFormChange, + brief, + onBriefChange, + productionPrompt, + onProductionPromptChange, + productionPromptAiGenerated, + knowledgeBases, + aiProviders, + aiCredentialId, + onAiCredentialIdChange, + aiModel, + onAiModelChange, + onGenerateProduction, + isGenerating, + canGenerate, +}: HostedProductionAgentStepProps) { + const activeProviders = aiProviders.filter((p) => p.is_active) + + return ( + + } + right={ + <> +
+
+ + onFormChange({ name: e.target.value })} + placeholder="Customer Support Bot" + /> +
+
+ + +
+
+
+ +