Knowledge Intelligence 3.0: enterprise RAG platform

.claude/launch.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.0.1",
+  "configurations": [
+    {
+      "name": "backend",
+      "runtimeExecutable": "backend/.venv/Scripts/python.exe",
+      "runtimeArgs": ["backend/scripts/dev.py"],
+      "port": 8000
+    },
+    {
+      "name": "frontend",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["--prefix", "frontend", "run", "dev"],
+      "port": 3000
+    }
+  ]
+}

.env.example

@@ -0,0 +1,112 @@
+# =============================================================================
+# Enterprise RAG & Knowledge Intelligence Platform — environment template
+# Copy to .env and replace every CHANGE_ME. Never commit .env.
+# =============================================================================
+
+# --- Deployment -------------------------------------------------------------------
+ENVIRONMENT=prod                       # dev | test | prod (prod enforces secure settings at startup)
+DOMAIN=rag.example.com                 # Caddy obtains a TLS certificate for this name
+PUBLIC_APP_URL=https://rag.example.com # used in email links and the OAuth redirect URI
+CORS_ORIGINS=https://rag.example.com
+LOG_LEVEL=INFO
+LOG_JSON=true
+
+# --- Secrets (generate each one; see DEPLOYMENT.md) ----------------------------------
+# python -c "import secrets; print(secrets.token_urlsafe(48))"
+JWT_SECRET=CHANGE_ME
+# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
+# Comma-separated; the FIRST key encrypts, all keys decrypt (append to rotate, then re-encrypt).
+ENCRYPTION_KEYS=CHANGE_ME
+COOKIE_SECURE=true
+
+# --- PostgreSQL 16 + pgvector ------------------------------------------------------------
+POSTGRES_SUPERUSER_PASSWORD=CHANGE_ME  # used only by the init script and backups
+APP_DB_USER=rag_app                    # non-superuser application role (Row-Level Security applies)
+APP_DB_PASSWORD=CHANGE_ME
+APP_DB_NAME=rag
+# Outside docker compose, set DATABASE_URL directly:
+# DATABASE_URL=postgresql+psycopg://rag_app:PASSWORD@db-host:5432/rag
+DB_POOL_SIZE=10
+DB_STATEMENT_TIMEOUT_MS=30000
+
+# --- Object storage (S3 / MinIO) ---------------------------------------------------------
+MINIO_ROOT_USER=CHANGE_ME
+MINIO_ROOT_PASSWORD=CHANGE_ME
+S3_BUCKET=rag-documents
+# For AWS S3 instead of the bundled MinIO: STORAGE_BACKEND=s3, S3_ENDPOINT_URL= (empty), S3_REGION, keys.
+MAX_UPLOAD_MB=100
+MAX_BATCH_FILES=50
+
+# --- Background jobs & events --------------------------------------------------------------
+# JOB_BACKEND: external (separate worker service) | embedded (single process) | eager (tests)
+JOB_MAX_ATTEMPTS=4
+WORKER_CONCURRENCY=4
+SCHEDULER_INTERVAL_SECONDS=30
+
+# --- Email ------------------------------------------------------------------------------------
+EMAIL_BACKEND=smtp                     # smtp | console (dev only, refused in prod)
+SMTP_HOST=smtp.example.com
+SMTP_PORT=587
+SMTP_USERNAME=
+SMTP_PASSWORD=
+SMTP_FROM=no-reply@example.com
+SMTP_STARTTLS=true
+EMAIL_VERIFICATION_REQUIRED=false
+
+# --- Security policy -----------------------------------------------------------------------------
+PASSWORD_MIN_LENGTH=10
+ACCESS_TOKEN_TTL_MINUTES=15
+REFRESH_TOKEN_TTL_DAYS=14
+LOGIN_RATE_LIMIT_PER_MINUTE=10
+API_RATE_LIMIT_PER_MINUTE=600
+# Database connections / AI endpoints on private networks (typical for self-hosted installs):
+ALLOW_PRIVATE_NETWORK_TARGETS=true
+# Website/sitemap/RSS ingestion and self-managed GitLab / Confluence site / IMAP hosts on private
+# networks (keep false for multi-tenant SaaS; or allow-list specific hosts below):
+ALLOW_PRIVATE_WEB_INGESTION=false
+# Hosts that bypass the private-range rule (never cloud metadata), e.g. an internal Ollama:
+SSRF_ALLOWLIST=
+# Directories from which SQLite connections may be opened (read-only):
+SQLITE_ALLOWED_DIRS=/data/sqlite
+
+# --- AI defaults ---------------------------------------------------------------------------------------
+# Organisations configure their own providers in Settings → AI providers. Without one, answers are
+# extractive (quoted from sources). Platform-wide fallback (self-hosted only):
+# Deployment-wide model for organisations that have not configured their own provider.
+# DEFAULT_PROVIDER is tried first; DEFAULT_MODEL is used when it belongs to that provider,
+# otherwise claude-sonnet-5-5 (Anthropic) or gpt-5.4-mini (OpenAI).
+PLATFORM_LLM_FALLBACK=false
+DEFAULT_PROVIDER=anthropic
+DEFAULT_MODEL=claude-sonnet-5-5
+ANTHROPIC_API_KEY=
+# Organisation-level Anthropic keys also need a workspace id.
+ANTHROPIC_WORKSPACE_ID=
+OPENAI_API_KEY=
+OLLAMA_BASE_URL=http://ollama:11434
+LLM_TIMEOUT_SECONDS=120
+
+# --- SaaS integrations: OAuth apps (optional) ------------------------------------------------------------------
+# Register {PUBLIC_APP_URL}/api/v1/integrations/oauth/callback with each vendor (DEPLOYMENT.md §3, "OAuth apps").
+# Unset vendors show "Needs setup" in Integrations; Notion, Confluence and Slack still work with tokens,
+# GitLab and IMAP never need an app. Set these for both api and worker (the worker refreshes tokens).
+OAUTH_GOOGLE_CLIENT_ID=                # Google Drive
+OAUTH_GOOGLE_CLIENT_SECRET=
+OAUTH_MICROSOFT_CLIENT_ID=             # OneDrive, SharePoint, Teams (Microsoft Graph)
+OAUTH_MICROSOFT_CLIENT_SECRET=
+OAUTH_MICROSOFT_TENANT=common          # or your directory id / domain for a single-tenant app registration
+OAUTH_DROPBOX_CLIENT_ID=               # Dropbox "App key"
+OAUTH_DROPBOX_CLIENT_SECRET=           # Dropbox "App secret"
+OAUTH_NOTION_CLIENT_ID=                # Notion public integration
+OAUTH_NOTION_CLIENT_SECRET=
+OAUTH_ATLASSIAN_CLIENT_ID=             # Confluence Cloud (Atlassian OAuth 2.0 3LO)
+OAUTH_ATLASSIAN_CLIENT_SECRET=
+OAUTH_SLACK_CLIENT_ID=                 # Slack app
+OAUTH_SLACK_CLIENT_SECRET=
+
+# --- Billing (optional) ------------------------------------------------------------------------------------
+BILLING_ENABLED=false                  # false = every organisation gets the unlimited 'enterprise' plan
+DEFAULT_PLAN=enterprise
+
+# --- First platform administrator (created by the migrate service) -----------------------------------------
+BOOTSTRAP_ADMIN_EMAIL=
+BOOTSTRAP_ADMIN_PASSWORD=

.github/workflows/ci.yml

@@ -0,0 +1,173 @@
+name: ci
+
+on:
+  push:
+    branches: [main, v2, "platform/**"]
+  pull_request:
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          # Disposable test database: trust auth, so no credentials live in this workflow.
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ["5432:5432"]
+        options: >-
+          --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 20
+      # A separate server plays the customer's database: the SSRF guard (rightly) refuses
+      # connections to the platform's own database endpoint.
+      customer:
+        image: postgres:16
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ["5433:5432"]
+        options: >-
+          --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 20
+    env:
+      TEST_DATABASE_URL: postgresql+psycopg://rag@localhost:5432/rag_test
+      TEST_CUSTOMER_PG_ADMIN_URL: postgresql://postgres@localhost:5433/postgres
+    defaults:
+      run:
+        working-directory: backend
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Prepare databases (the app role must not be a superuser: superusers bypass RLS)"
+        run: |
+          psql -h localhost -U postgres -c "CREATE ROLE rag LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEROLE"
+          for db in rag_test rag_drift; do
+            psql -h localhost -U postgres -c "CREATE DATABASE $db OWNER rag"
+            psql -h localhost -U postgres -d "$db" -c "CREATE EXTENSION IF NOT EXISTS vector"
+          done
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check app tests
+      - name: Tests (PostgreSQL 16 + pgvector service)
+        run: pytest -q
+      - name: Migration drift check
+        env:
+          DATABASE_URL: postgresql+psycopg://rag@localhost:5432/rag_drift
+        run: |
+          alembic upgrade head
+          alembic check
+      - name: Dependency audit
+        run: pip install pip-audit && pip-audit --progress-spinner off || true
+
+  frontend:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npx tsc --noEmit
+      - run: npm run build
+      - run: npm audit --omit=dev --audit-level=high || true
+
+  # Browser end-to-end tests (frontend/e2e) against the real backend + built frontend.
+  # See docs/TESTING.md. Chat answers are extractive here: no model provider keys.
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          # Disposable test database: trust auth, so no credentials live in this workflow.
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ["5432:5432"]
+        options: >-
+          --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 20
+    env:
+      DATABASE_URL: postgresql+psycopg://rag@localhost:5432/rag
+      # The suite signs in once per browser context and registers a few accounts.
+      REGISTER_RATE_LIMIT_PER_HOUR: "1000"
+      LOGIN_RATE_LIMIT_PER_MINUTE: "1000"
+      E2E_BASE_URL: http://localhost:3000
+      # The console email backend prints invitation links here; the tests read them.
+      E2E_BACKEND_LOG: ${{ github.workspace }}/e2e-logs/backend.log
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: backend/pyproject.toml
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: "Prepare databases (the app role must not be a superuser: superusers bypass RLS)"
+        run: |
+          psql -h localhost -U postgres -c "CREATE ROLE rag LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEROLE"
+          for db in rag; do
+            psql -h localhost -U postgres -c "CREATE DATABASE $db OWNER rag"
+            psql -h localhost -U postgres -d "$db" -c "CREATE EXTENSION IF NOT EXISTS vector"
+          done
+      - name: Install backend
+        working-directory: backend
+        run: pip install -e ".[dev]"
+      - name: Start backend (PostgreSQL service via DATABASE_URL)
+        working-directory: backend
+        run: |
+          mkdir -p "$(dirname "$E2E_BACKEND_LOG")"
+          nohup python scripts/dev.py > "$E2E_BACKEND_LOG" 2>&1 &
+      - name: Wait for the API
+        run: |
+          for i in $(seq 1 90); do
+            if curl -fsS http://127.0.0.1:8000/api/v1/health > /dev/null; then echo "backend up"; exit 0; fi
+            sleep 2
+          done
+          echo "backend did not become healthy"; cat "$E2E_BACKEND_LOG"; exit 1
+
+      - name: Install frontend dependencies
+        working-directory: frontend
+        run: npm ci
+      - name: Build frontend
+        working-directory: frontend
+        run: npm run build
+      - name: Start frontend (standalone server)
+        working-directory: frontend
+        run: |
+          cp -r .next/static .next/standalone/.next/static
+          cp -r public .next/standalone/public
+          # HOSTNAME is preset on runners; the standalone server binds to it.
+          PORT=3000 HOSTNAME=0.0.0.0 nohup node .next/standalone/server.js > "$GITHUB_WORKSPACE/e2e-logs/frontend.log" 2>&1 &
+          for i in $(seq 1 60); do
+            if curl -fsS http://localhost:3000/login > /dev/null; then echo "frontend up"; exit 0; fi
+            sleep 2
+          done
+          echo "frontend did not start"; cat "$GITHUB_WORKSPACE/e2e-logs/frontend.log"; exit 1
+
+      - name: Install Playwright Chromium
+        working-directory: frontend
+        run: npx playwright install --with-deps chromium
+      - name: Run end-to-end tests
+        working-directory: frontend
+        run: npm run test:e2e
+      - name: Upload Playwright report and server logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            frontend/playwright-report
+            frontend/test-results
+            e2e-logs
+          retention-days: 14

.gitignore

@@ -0,0 +1,40 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.egg-info/
+.venv/
+venv/
+env/
+.env
+.env.*
+!.env.example
+.streamlit/secrets.toml
+.pytest_cache/
+.coverage
+htmlcov/
+.ruff_cache/
+.mypy_cache/
+.ipynb_checkpoints/
+.DS_Store
+Thumbs.db
+*.log
+build/
+dist/
+
+# Runtime data: embedded PostgreSQL, uploaded files, SQLite sources
+backend/var/
+
+# Frontend build output and dependencies
+frontend/node_modules/
+frontend/.next/
+frontend/out/
+frontend/tsconfig.tsbuildinfo
+frontend/next-env.d.ts
+
+# Playwright end-to-end tests (frontend/e2e)
+frontend/test-results/
+frontend/playwright-report/
+frontend/blob-report/
+frontend/playwright/.cache/
+frontend/e2e/.state/