From 57a6fc8c97891091d04bab5e9ee7f1422b37c517 Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Thu, 29 Oct 2026 12:05:20 -0600 Subject: [PATCH 1/3] Add company prioritization tools and Scarf history to live dashboard --- docs/live-visitors.md | 45 +- src/components/ProductScarf.tsx | 9 +- src/components/admin/CompanyFacetFilter.tsx | 64 + src/components/admin/LiveCompanyCharts.tsx | 182 +-- src/components/admin/LiveCompanySessions.tsx | 175 +++ src/components/admin/LiveVisitors.tsx | 1375 +++++++++++++----- src/routes/admin/live.tsx | 80 +- src/routes/index.tsx | 3 +- src/server/runtime/blob-storage.server.ts | 2 +- src/utils/analytics/live-dashboard.ts | 164 +++ src/utils/analytics/live.functions.ts | 26 +- src/utils/analytics/live.server.ts | 73 +- src/utils/analytics/scarf-history.server.ts | 191 +++ src/utils/analytics/scarf-history.ts | 96 ++ src/utils/analytics/scarf-pixels.ts | 7 + tests/live-company-tools.test.ts | 197 +++ tests/scarf-history.test.ts | 95 ++ 17 files changed, 2242 insertions(+), 542 deletions(-) create mode 100644 src/components/admin/CompanyFacetFilter.tsx create mode 100644 src/components/admin/LiveCompanySessions.tsx create mode 100644 src/utils/analytics/scarf-history.server.ts create mode 100644 src/utils/analytics/scarf-history.ts create mode 100644 src/utils/analytics/scarf-pixels.ts create mode 100644 tests/live-company-tools.test.ts create mode 100644 tests/scarf-history.test.ts diff --git a/docs/live-visitors.md b/docs/live-visitors.md index f3016aa60..c9ad5c002 100644 --- a/docs/live-visitors.md +++ b/docs/live-visitors.md @@ -7,14 +7,41 @@ function. Admin queries refresh every three seconds while the page is visible. The dashboard opens with a company grid. Individual sessions are hidden until a company is expanded, and each session has its own expandable tabs and activity. Unmatched traffic stays in one collapsed row at the bottom. Columns support -sorting, and search matches companies, industries, libraries, and page paths. +multi-column sorting with Shift-click, column visibility, row selection, +pagination, compact density, and CSV export. Search matches companies, industries, +libraries, and page paths. Library, industry, size, and funnel-stage filters can +be combined with minimum website views and recent actions. Values within a +filter match with OR, while different filters match with AND. The overview counts identified companies once per domain. Library breakdowns count a company once per library, so one company can appear in several bars. -Industry breakdowns exclude unmatched traffic. The company trend records real -polling snapshots in the open dashboard tab, up to five minutes, and resets on -reload. Grid action totals count named interactions in the last five minutes, -excluding page views. +Industry and size breakdowns exclude unmatched traffic. Clicking a breakdown +applies its filter. Size buckets use the lower bound of Scarf's employee range. +Grid action totals count named interactions in the last five minutes, excluding +page views. Pause freezes polling, including expanded sessions and history. + +## Scarf company history + +The server uses the existing `ANALYTICS_SCARF_API_TOKEN` bearer token for Scarf's +`/v3/insights/TanStack/aggregations/export` endpoint. Reporting is separate from +the lookup key and requires Scarf reporting access. Exports cover the selected +7, 30, 90, or 365 UTC calendar days, including today. Growth compares website +pixel views with the preceding equal-length window, so today's totals are +incomplete. These are window totals, not lifetime totals. + +Website views include only the site's registered Scarf pixels. Site events +come from the TanStack Site Telemetry package, including browser interactions +and server requests. Events are not added to views. Sources and endpoints are +Scarf's distinct network counts, not people. Company reports join live company +domains exactly, without guessing parent-company relationships. Missing rows or +failed reports show N/A rather than zero. No company data is automatically +unlocked and no Scarf credits are spent on enrichment. + +Successful aggregate exports are cached at the edge for 15 minutes and partial +or unavailable responses for one minute. Live presence still refreshes every +three seconds. Sorting and filtering do not request new exports. Only admins +can request reporting data, and source identifiers and credentials are never +returned to the browser. ## Enable the experiment @@ -46,9 +73,11 @@ contact, not proof that a person is looking at the page. `pageshow` handles restoration from the browser's back/forward cache. Sequence numbers prevent a late heartbeat or departure from overwriting newer tab state. -Each tab retains up to 40 named actions. The admin includes departures from the -last five minutes when requested and returns at most 500 recently seen tabs, -with a visible truncation notice. Sessions disappear from reads at 24 hours and +Each tab retains up to 40 named actions. The company overview aggregates all +eligible tabs without a 500-tab cap. Individual company sessions are fetched +only when expanded, with at most 500 eligible recently seen tabs per company +and a visible truncation notice. The admin includes departures from the last +five minutes when requested. Sessions disappear from reads at 24 hours and are deleted, with their tabs, on the next minute cleanup. ## Company lookup diff --git a/src/components/ProductScarf.tsx b/src/components/ProductScarf.tsx index c37c50416..1f7054551 100644 --- a/src/components/ProductScarf.tsx +++ b/src/components/ProductScarf.tsx @@ -1,18 +1,13 @@ import { useRouterState } from '@tanstack/react-router' import { Scarf } from './Scarf' -const products = [ - { path: '/application-starter', id: 'ed471ec3-aa08-430f-8481-258b3a1f94dc' }, - { path: '/builder', id: 'aa4dcee9-86a2-49b4-9981-40c97cc1f55b' }, - { path: '/stats/npm', id: 'e729f879-e127-48e1-be19-c68f6786c9e8' }, - { path: '/intent/registry', id: '811c34af-38bd-4f06-9762-3f58dea0650a' }, -] +import { productScarfPixels } from '~/utils/analytics/scarf-pixels' export function ProductScarf() { const product = useRouterState({ select: (state) => { const pathname = state.resolvedLocation?.pathname - return products.find( + return productScarfPixels.find( ({ path }) => pathname === path || pathname?.startsWith(`${path}/`), ) }, diff --git a/src/components/admin/CompanyFacetFilter.tsx b/src/components/admin/CompanyFacetFilter.tsx new file mode 100644 index 000000000..78920ade2 --- /dev/null +++ b/src/components/admin/CompanyFacetFilter.tsx @@ -0,0 +1,64 @@ +import { useState } from 'react' +import { CaretDownIcon } from '@phosphor-icons/react' + +export function CompanyFacetFilter({ + label, + items, + selected, + onToggle, +}: { + label: string + items: { name: string; value: number }[] + selected: string[] + onToggle: (values: string[]) => void +}) { + const [search, setSearch] = useState('') + const visible = items.filter((item) => + item.name.toLowerCase().includes(search.toLowerCase()), + ) + return ( +
+ + {label} + {selected.length > 0 && ( + {selected.length} + )} + +
+ {label} + setSearch(event.target.value)} + className="mb-1 w-full rounded border border-gray-500/20 bg-transparent px-2 py-1.5 text-xs" + /> + {visible.map((item) => ( + + ))} + {!visible.length && ( +

No matching values.

+ )} +
+
+ ) +} diff --git a/src/components/admin/LiveCompanyCharts.tsx b/src/components/admin/LiveCompanyCharts.tsx index 93bc4de29..b1124d46b 100644 --- a/src/components/admin/LiveCompanyCharts.tsx +++ b/src/components/admin/LiveCompanyCharts.tsx @@ -1,158 +1,64 @@ -import { useMemo } from 'react' -import { areaY, defineChart, lineY } from '@tanstack/charts' -import { Chart } from '@tanstack/charts/react' -import { scaleLinear, scaleUtc } from 'd3' - -export type CompanySample = { time: number; companies: number } - -export function LiveCompanyTrend({ - samples, - count, -}: { - samples: CompanySample[] - count: number -}) { - const definition = useMemo(() => { - const first = samples[0]?.time ?? Date.now() - const last = Math.max(first + 1_000, samples.at(-1)?.time ?? first) - const points = samples.map((sample) => ({ - ...sample, - date: new Date(sample.time), - })) - return defineChart({ - marks: [ - areaY(points, { - x: 'date', - y1: 0, - y2: 'companies', - fill: '#10b981', - fillOpacity: 0.12, - }), - lineY(points, { - x: 'date', - y: 'companies', - stroke: '#10b981', - strokeWidth: 2, - }), - ], - scales: { - x: { - scale: scaleUtc().domain([new Date(first), new Date(last)]), - axis: { - ticks: { - count: 3, - format: (value) => - new Date(value).toLocaleTimeString([], { - minute: '2-digit', - second: '2-digit', - }), - }, - }, - }, - y: { - scale: scaleLinear() - .domain([ - 0, - Math.max(3, ...samples.map((sample) => sample.companies)), - ]) - .nice(), - grid: true, - axis: { - ticks: { - count: 3, - format: (value) => (Number.isInteger(value) ? String(value) : ''), - }, - }, - }, - }, - margin: { top: 10, right: 10, bottom: 24, left: 25 }, - theme: { - background: 'transparent', - foreground: 'currentColor', - muted: '#6b7280', - grid: 'rgba(107, 114, 128, 0.15)', - }, - }) - }, [samples]) - return ( -
-
-

- Identified companies online -

- - Since opened · up to 5m - -
-

- {count} -

-
- {samples.length < 2 ? ( -

- Trend starts with the next live update. -

- ) : ( - - )} -
-
- ) -} - export function CompanyBreakdownChart({ title, items, color, + selected, + onSelect, note, }: { title: string - items: { name: string; value: number }[] + items: { name: string; value: number; values: string[] }[] color: string + selected: string[] + onSelect: (values: string[]) => void note?: string }) { const maximum = Math.max(1, ...items.map((item) => item.value)) return ( -
-
-

+
+
+

{title}

- Companies + Companies
- {items.length === 0 ? ( -

- No identified companies online yet. + {!items.length ? ( +

+ No companies match.

) : ( -
    - {items.map((item) => ( -
  • -
    - - {item.name} - - {item.value} -
    -
    -
    -
    -
    • - ))} +
      + {items.map((item) => { + const active = item.values.every((value) => + selected.includes(value), + ) + return ( +
    • + +
      • + ) + })}
    )}
diff --git a/src/components/admin/LiveCompanySessions.tsx b/src/components/admin/LiveCompanySessions.tsx new file mode 100644 index 000000000..34ed7614a --- /dev/null +++ b/src/components/admin/LiveCompanySessions.tsx @@ -0,0 +1,175 @@ +import { useQuery } from '@tanstack/react-query' +import { CaretRightIcon, CaretDownIcon } from '@phosphor-icons/react' +import { getLiveCompanySessions } from '~/utils/analytics/live.functions' +import { + groupLiveCompanies, + type LiveTab, +} from '~/utils/analytics/live-dashboard' +import { liveActions } from '~/utils/analytics/live' + +const lookupLabels: Record = { + pending: 'Looking up company', + unknown: 'No cached company match', + failed: 'Company lookup failed', + rate_limited: 'Company lookup rate limited', + unconfigured: 'Company lookup not configured', +} +function liveAge(time: Date | string | number, now: string) { + const seconds = Math.max( + 0, + Math.floor((new Date(now).getTime() - new Date(time).getTime()) / 1000), + ) + return seconds < 5 + ? 'Just now' + : seconds < 60 + ? `${seconds}s ago` + : `${Math.floor(seconds / 60)}m ago` +} +export function LiveCompanySessions({ + domain, + paused, +}: { + domain: string + paused: boolean +}) { + const query = useQuery({ + queryKey: ['admin', 'live-company-sessions', domain], + queryFn: () => getLiveCompanySessions({ data: { domain } }), + refetchInterval: paused ? false : 3000, + refetchIntervalInBackground: false, + refetchOnWindowFocus: !paused, + refetchOnReconnect: !paused, + retry: false, + }) + if (query.isPending) + return

Loading sessions...

+ if (query.isError) + return ( +
+

Could not load sessions.

+ +
+ ) + const sessions = groupLiveCompanies(query.data, true).flatMap( + (company) => company.sessions, + ) + return ( +
+ {query.data.truncated && ( +

+ Showing this company's 500 most recently seen tabs. +

+ )} + {sessions.length ? ( + sessions.map((tabs) => ( + + )) + ) : ( +

No recent sessions remain.

+ )} +
+ ) +} + +function SessionDetails({ tabs, now }: { tabs: LiveTab[]; now: string }) { + const first = tabs[0] + const current = + tabs.find((tab) => tab.active && tab.visible) ?? + tabs.find((tab) => tab.active) ?? + first + const active = tabs.some((tab) => tab.active) + const activity = tabs + .flatMap((tab) => tab.activity) + .sort((a, b) => b.at.localeCompare(a.at)) + .slice(0, 40) + return ( +
+ + +
+
+

+ Tabs {tabs.length} +

+
    + {tabs.map((tab) => ( +
  • + + {tab.active + ? tab.visible + ? 'Visible' + : 'Background' + : 'Left'} + + {tab.page} +
    • + ))} +
+

+ Arrived {liveAge(first.arrivedAt, now)} +

+
+
+

Recent activity

+ {activity.length ? ( +
    + {activity.map((event, index) => ( +
  1. + +
    +

    {liveActions[event.action]}

    +

    + {event.page} +

    +
    +
    2. + ))} +
+ ) : ( +

No actions recorded yet.

+ )} +
+
+
+ ) +} diff --git a/src/components/admin/LiveVisitors.tsx b/src/components/admin/LiveVisitors.tsx index f71d0a06b..c521c9679 100644 --- a/src/components/admin/LiveVisitors.tsx +++ b/src/components/admin/LiveVisitors.tsx @@ -1,224 +1,930 @@ -import { Fragment, useEffect, useMemo, useState } from 'react' +import { Fragment, useEffect, useMemo, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { z } from 'zod' +import { csvFormat } from 'd3' import { createSortedRowModel, + createPaginatedRowModel, rowSortingFeature, + rowPaginationFeature, + rowSelectionFeature, + columnVisibilityFeature, sortFn_alphanumeric, sortFn_basic, tableFeatures, useTable, type ColumnDef, type SortingState, + type PaginationState, + type ColumnVisibilityState, + type RowSelectionState, } from '@tanstack/react-table' import { + ArrowsDownUpIcon, CaretDownIcon, CaretRightIcon, - ArrowsDownUpIcon, MagnifyingGlassIcon, + XIcon, + DownloadSimpleIcon, + ColumnsIcon, } from '@phosphor-icons/react' -import { liveActions } from '~/utils/analytics/live' +import { getScarfCompanyHistory } from '~/utils/analytics/live.functions' +import { historyDaysSchema } from '~/utils/analytics/scarf-history' import { - companyBreakdown, - groupLiveCompanies, - type LiveCompanyRow, - type LiveData, - type LiveTab, + buildCompanyGridRows, + companyFacetCounts, + companyWidgetItems, + filterCompanyRows, + emptyCompanyFilters, + companyGrowth, + csvSafeText, + type CompanyGridRow, + type CompanyFacet, + type LiveOverview, } from '~/utils/analytics/live-dashboard' -import { - CompanyBreakdownChart, - LiveCompanyTrend, - type CompanySample, -} from './LiveCompanyCharts' +import { CompanyBreakdownChart } from './LiveCompanyCharts' +import { CompanyFacetFilter } from './CompanyFacetFilter' +import { LiveCompanySessions } from './LiveCompanySessions' const features = tableFeatures({ rowSortingFeature, sortedRowModel: createSortedRowModel(), + rowPaginationFeature, + paginatedRowModel: createPaginatedRowModel(), + rowSelectionFeature, + columnVisibilityFeature, }) -const initialSorting: SortingState = [{ id: 'name', desc: false }] -const columns: ColumnDef[] = [ - { accessorKey: 'name', header: 'Company', sortFn: sortFn_alphanumeric }, +const initialSorting: SortingState = [ + { id: 'unmatched', desc: false }, + { id: 'siteViews', desc: true }, + { id: 'actions', desc: true }, + { id: 'sessions', desc: true }, +] +const initialPagination: PaginationState = { pageIndex: 0, pageSize: 50 } +const initialVisibility: ColumnVisibilityState = { + unmatched: false, + tabs: false, + recentViews: false, + siteEvents: false, + origins: false, + endpoints: false, + lastScarfSeen: false, +} +const initialSelection: RowSelectionState = {} +const visibilitySchema = z.record(z.string(), z.boolean()) +const facets: CompanyFacet[] = ['library', 'size', 'industry', 'stage'] +const widgetFacets: CompanyFacet[] = ['library', 'size', 'industry'] +const facetLabels: Record = { + library: 'Library', + size: 'Company size', + industry: 'Industry', + stage: 'Funnel stage', +} +const number = new Intl.NumberFormat('en-US') +function numeric(value: number | undefined) { + return value === undefined ? ( + + N/A + + ) : ( + number.format(value) + ) +} +function age(value: Date, now: string) { + const seconds = Math.max( + 0, + Math.floor((new Date(now).getTime() - new Date(value).getTime()) / 1000), + ) + return seconds < 5 + ? 'Now' + : seconds < 60 + ? `${seconds}s` + : `${Math.floor(seconds / 60)}m` +} +function growthLabel(row: CompanyGridRow) { + const growth = companyGrowth(row) + if (row.history?.previousViews === 0 && (row.history.views ?? 0) > 0) + return 'New' + return growth === undefined + ? 'N/A' + : `${growth > 0 ? '+' : ''}${growth.toFixed(0)}%` +} +const columns: ColumnDef[] = [ + { + id: 'unmatched', + accessorFn: (row) => (row.id ? 0 : 1), + sortFn: sortFn_basic, + enableHiding: false, + }, + { id: 'select', header: 'Select', enableSorting: false, enableHiding: false }, + { + id: 'name', + accessorKey: 'name', + header: 'Company', + sortFn: sortFn_alphanumeric, + enableHiding: false, + }, { id: 'industry', - accessorFn: (row) => row.company?.industry ?? '', + accessorKey: 'industry', header: 'Industry', sortFn: sortFn_alphanumeric, + cell: ({ row }) => ( + + {row.original.industry} + + ), }, { id: 'size', - accessorFn: (row) => row.company?.company_size ?? '', - header: 'Company size', - enableSorting: false, + accessorFn: (row) => row.employeeMin, + header: 'Employees', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => row.original.company?.company_size || 'N/A', }, { id: 'libraries', accessorFn: (row) => row.libraries.join(', '), - header: 'On site', - enableSorting: false, + header: 'Libraries', + sortFn: sortFn_alphanumeric, + cell: ({ row }) => ( + + {row.original.libraries.join(', ') || 'None active'} + + ), + }, + { + id: 'sessions', + accessorKey: 'activeSessions', + header: 'Sessions', + sortFn: sortFn_basic, + cell: ({ row }) => numeric(row.original.activeSessions), + }, + { + id: 'tabs', + accessorKey: 'activeTabs', + header: 'Tabs', + sortFn: sortFn_basic, + cell: ({ row }) => numeric(row.original.activeTabs), + }, + { + id: 'actions', + accessorKey: 'actions', + header: 'Actions · 5m', + sortFn: sortFn_basic, + cell: ({ row }) => numeric(row.original.actions), + }, + { + id: 'recentViews', + accessorKey: 'recentViews', + header: 'Views · 5m', + sortFn: sortFn_basic, + cell: ({ row }) => numeric(row.original.recentViews), + }, + { + id: 'siteViews', + accessorFn: (row) => row.history?.views, + header: 'Site views', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => numeric(row.original.history?.views), + }, + { + id: 'growth', + accessorFn: (row) => companyGrowth(row), + header: 'Change', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => ( + 0 + ? 'text-emerald-600 dark:text-emerald-400' + : (companyGrowth(row.original) ?? 0) < 0 + ? 'text-amber-600 dark:text-amber-400' + : 'text-gray-500' + } + > + {growthLabel(row.original)} + + ), + }, + { + id: 'siteEvents', + accessorFn: (row) => row.history?.siteEvents, + header: 'Site events', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => numeric(row.original.history?.siteEvents), }, { - id: 'pages', - accessorFn: (row) => row.pages.length, - header: 'Pages now', + id: 'origins', + accessorFn: (row) => row.history?.origins, + header: 'Sources', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => numeric(row.original.history?.origins), + }, + { + id: 'endpoints', + accessorFn: (row) => row.history?.endpoints, + header: 'Endpoints', + sortFn: sortFn_basic, + sortUndefined: 'last', + cell: ({ row }) => numeric(row.original.history?.endpoints), + }, + { + id: 'stage', + accessorFn: (row) => row.history?.stage, + header: 'Funnel stage', + sortFn: sortFn_alphanumeric, + sortUndefined: 'last', + cell: ({ row }) => ( + + {row.original.history?.stage?.replaceAll('-', ' ') || 'N/A'} + + ), + }, + { + id: 'lastScarfSeen', + accessorFn: (row) => row.history?.lastSeen, + header: 'Scarf last seen', + sortFn: sortFn_alphanumeric, + sortUndefined: 'last', + cell: ({ row }) => + row.original.history?.lastSeen + ? new Date(row.original.history.lastSeen).toLocaleDateString() + : 'N/A', + }, + { + id: 'lastSeen', + accessorFn: (row) => new Date(row.lastSeen).getTime(), + header: 'Seen', sortFn: sortFn_basic, }, - { accessorKey: 'actions', header: 'Actions · 5m', sortFn: sortFn_basic }, - { accessorKey: 'lastSeen', header: 'Last seen', sortFn: sortFn_basic }, ] -const lookupLabels: Record = { - pending: 'Looking up company', - unknown: 'No cached company match', - failed: 'Company lookup failed', - rate_limited: 'Company lookup rate limited', - unconfigured: 'Company lookup not configured', -} -function age(time: Date | string | number, now: string) { - const seconds = Math.max( - 0, - Math.floor((new Date(now).getTime() - new Date(time).getTime()) / 1000), - ) - return seconds < 5 - ? 'Just now' - : seconds < 60 - ? `${seconds}s ago` - : `${Math.floor(seconds / 60)}m ago` +const columnHelp: Record = { + sessions: 'Anonymous sessions with a heartbeat in the last 65 seconds.', + actions: + 'Recorded interactions from the last five minutes, limited to 40 retained events per tab.', + recentViews: + 'Recorded browser page-view events in the last five minutes, from retained tab activity.', + siteViews: + 'Scarf pixel views for the pixels embedded in tanstack.com, in the selected history window. Not unique people.', + growth: + 'Pixel-view change against the previous equal-length window. N/A means a comparison was not reported.', + siteEvents: + 'Events from the dedicated TanStack Site Telemetry package. Includes page views, actions, and server requests. These are not added to pixel views.', + origins: + 'Scarf unique origins across the selected site pixels. These are sources, not identified people.', + endpoints: + 'Scarf unique endpoints across the selected site pixels. These are not individual identities.', } -export function LiveVisitors({ data }: { data: LiveData }) { +export function LiveVisitors({ + data, + paused = false, +}: { + data: LiveOverview + paused?: boolean +}) { const [expanded, setExpanded] = useState(new Set()) - const [showRecent, setShowRecent] = useState(false) - const [search, setSearch] = useState('') + const [filters, setFilters] = useState(emptyCompanyFilters) const [sorting, setSorting] = useState(initialSorting) - const initialSamples: CompanySample[] = [] - const [samples, setSamples] = useState(initialSamples) - const activeCompanies = useMemo(() => groupLiveCompanies(data, false), [data]) - const count = activeCompanies.filter((company) => company.id).length + const [pagination, setPagination] = useState(initialPagination) + const [visibility, setVisibility] = useState(initialVisibility) + const [selection, setSelection] = useState(initialSelection) + const [selectedOnly, setSelectedOnly] = useState(false) + const [comfortable, setComfortable] = useState(false) + const [showBreakdowns, setShowBreakdowns] = useState(true) + const [days, setDays] = useState(historyDaysSchema.parse(30)) + const [preferencesLoaded, setPreferencesLoaded] = useState(false) + const searchInput: { current: HTMLInputElement | null } = useRef(null) + const history = useQuery({ + queryKey: ['admin', 'scarf-company-history', days], + queryFn: () => getScarfCompanyHistory({ data: { days } }), + staleTime: 15 * 60_000, + refetchInterval: paused ? false : 15 * 60_000, + refetchOnWindowFocus: false, + refetchOnReconnect: !paused, + retry: false, + }) useEffect(() => { - const time = new Date(data.now).getTime() - setSamples((previous) => - previous.at(-1)?.time === time - ? previous - : [ - ...previous.filter((sample) => sample.time >= time - 5 * 60_000), - { time, companies: count }, - ], - ) - }, [data.now, count]) - const companies = useMemo( + try { + const saved = localStorage.getItem('tanstack.live-companies.columns.v1') + if (saved) { + const parsed = visibilitySchema.safeParse(JSON.parse(saved)) + if (parsed.success) + setVisibility({ + ...initialVisibility, + ...parsed.data, + unmatched: false, + }) + } + } catch {} + setPreferencesLoaded(true) + }, []) + useEffect(() => { + if (preferencesLoaded) + try { + localStorage.setItem( + 'tanstack.live-companies.columns.v1', + JSON.stringify(visibility), + ) + } catch {} + }, [visibility, preferencesLoaded]) + useEffect(() => { + const onKey = (event: KeyboardEvent) => { + const target = event.target + if ( + event.key === '/' && + !event.metaKey && + !event.ctrlKey && + !( + target instanceof HTMLElement && + (target.matches('input,textarea,select') || target.isContentEditable) + ) + ) { + event.preventDefault() + searchInput.current?.focus() + } + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, []) + const allRows = useMemo( + () => buildCompanyGridRows(data, history.data), + [data, history.data], + ) + const filtered = useMemo( + () => filterCompanyRows(allRows, filters), + [allRows, filters], + ) + const gridRows = useMemo( () => - groupLiveCompanies(data, showRecent).filter((row) => - `${row.name} ${row.id} ${row.company?.industry ?? ''} ${row.libraries.join(' ')} ${row.sessions - .flat() - .map((tab) => tab.page) - .join(' ')}` - .toLowerCase() - .includes(search.trim().toLowerCase()), - ), - [data, showRecent, search], + selectedOnly + ? filtered.filter((row) => selection[row.id || 'unmatched']) + : filtered, + [filtered, selectedOnly, selection], ) + const activeRows = allRows.filter((row) => row.active) + const activeCount = activeRows.filter((row) => row.id).length + const summaryRows = gridRows.filter((row) => row.active) + const selectedRows = allRows.filter((row) => selection[row.id || 'unmatched']) const table = useTable({ features, columns, - data: companies, + data: gridRows, getRowId: (row) => row.id || 'unmatched', - state: { sorting }, - onSortingChange: setSorting, + state: { + sorting, + pagination, + columnVisibility: visibility, + rowSelection: selection, + }, + onSortingChange: (updater) => + setSorting((previous) => [ + { id: 'unmatched', desc: false }, + ...(typeof updater === 'function' ? updater(previous) : updater).filter( + (sort) => sort.id !== 'unmatched', + ), + ]), + onPaginationChange: setPagination, + onColumnVisibilityChange: setVisibility, + onRowSelectionChange: setSelection, + autoResetPageIndex: false, }) - const sorted = table.getRowModel().rows - const rows = [ - ...sorted.filter((row) => row.original.id), - ...sorted.filter((row) => !row.original.id), - ] - const toggle = (id: string) => + useEffect(() => { + const lastPage = Math.max( + 0, + Math.ceil(gridRows.length / pagination.pageSize) - 1, + ) + if (pagination.pageIndex > lastPage) + setPagination((previous) => ({ ...previous, pageIndex: lastPage })) + }, [gridRows.length, pagination.pageSize, pagination.pageIndex]) + useEffect(() => { + setPagination((previous) => + previous.pageIndex === 0 ? previous : { ...previous, pageIndex: 0 }, + ) + }, [filters]) + const toggleFacet = (facet: CompanyFacet, values: string[]) => { + setFilters((previous) => ({ + ...previous, + [facet]: values.every((value) => previous[facet].includes(value)) + ? previous[facet].filter((value) => !values.includes(value)) + : [...new Set([...previous[facet], ...values])], + })) + setPagination((previous) => ({ ...previous, pageIndex: 0 })) + } + const toggleCompany = (id: string) => setExpanded((previous) => { const next = new Set(previous) if (next.has(id)) next.delete(id) else next.add(id) return next }) + const filterCount = + facets.reduce((count, facet) => count + filters[facet].length, 0) + + Number(Boolean(filters.search)) + + Number(Boolean(filters.minViews)) + + Number(Boolean(filters.minActions)) + + Number(filters.identifiedOnly) + + Number(filters.showRecent) + const exportRows = () => { + const records = ( + selectedRows.length + ? selectedRows + : table.getPrePaginatedRowModel().rows.map((row) => row.original) + ).map((row) => ({ + Company: csvSafeText(row.name), + Domain: csvSafeText(row.id), + Industry: csvSafeText(row.industry), + Employees: csvSafeText(row.company?.company_size || ''), + Libraries: csvSafeText(row.libraries.join(', ')), + Sessions: row.activeSessions, + Actions5m: row.actions, + RecentViews5m: row.recentViews, + SitePixelViews: row.history?.views ?? '', + PreviousPixelViews: row.history?.previousViews ?? '', + Change: csvSafeText(growthLabel(row)), + SiteEvents: row.history?.siteEvents ?? '', + Sources: row.history?.origins ?? '', + Endpoints: row.history?.endpoints ?? '', + FunnelStage: csvSafeText(row.history?.stage || ''), + HistoryStart: history.data?.start ?? '', + HistoryEndExclusive: history.data?.end ?? '', + LastSeen: new Date(row.lastSeen).toISOString(), + })) + const url = URL.createObjectURL( + new Blob([csvFormat(records)], { type: 'text/csv;charset=utf-8' }), + ) + const link = document.createElement('a') + link.href = url + link.download = `live-companies-${new Date().toISOString().slice(0, 10)}.csv` + link.click() + URL.revokeObjectURL(url) + } + const rows = table.getRowModel().rows return ( -
+
{!data.enabled && ( -

- Visitor collection is off. +

+ Collection is off.

)} {!data.lookupConfigured && ( -

+

Company matching needs the Scarf lookup key.

)} - {data.truncated && ( -

- Overview and grid cover the 500 most recently seen tabs. -

+
+
+ + + {summaryRows.filter((row) => row.id).length} + {' '} + + companies online + + + + + {number.format( + summaryRows.reduce( + (total, row) => total + row.activeSessions, + 0, + ), + )} + {' '} + sessions + + + + {number.format( + summaryRows.reduce((total, row) => total + row.actions, 0), + )} + {' '} + actions · 5m + +
+
+ + +
+
+ {showBreakdowns && ( +
+ {widgetFacets.map((facet) => ( + toggleFacet(facet, values)} + note={ + facet === 'size' + ? "Grouped by the lower bound of Scarf's reported employee range." + : facet === 'library' + ? 'A company can appear in multiple libraries. Other combines the remaining category counts.' + : undefined + } + /> + ))} +
)} -
- - - +
+ {history.isFetching ? ( + Loading Scarf history... + ) : history.isError ? ( + <> + + Scarf history could not load. + + + + ) : history.data?.status === 'unconfigured' ? ( + + Scarf reporting token is not configured. + + ) : history.data?.status === 'unavailable' ? ( + <> + + Scarf history unavailable. Live activity is still updating. + + + + ) : ( + history.data && ( + <> + + Scarf synced{' '} + {new Date(history.data.fetchedAt).toLocaleTimeString()} + + {history.data.status === 'partial' && ( + + Some metrics unavailable + + )} + + Pixel views and site events are separate measurements. + + + ) + )}
-
-

- Companies on site{' '} - - {companies.filter((row) => row.id).length} - -

-
- -
-
+
+
+
+ {facets.map((facet) => ( + toggleFacet(facet, values)} + /> + ))} + + +
+ +
+ + +
+ Visible columns + {table + .getAllLeafColumns() + .filter((column) => column.getCanHide()) + .map((column) => ( + + ))} +
+
+ +
-
+
+ + {filtered.filter((row) => row.id).length} /{' '} + { + allRows.filter( + (row) => row.id && (filters.showRecent || row.active), + ).length + }{' '} + companies + + + + {(selectedRows.length > 0 || selectedOnly) && ( + <> + + + + )} + {filterCount > 0 && ( + + )} +
+ {facets.some((facet) => filters[facet].length > 0) && ( +
+ {facets.flatMap((facet) => + filters[facet].map((value) => ( + + )), + )} +
+ )} +
- + {table.getHeaderGroups().map((group) => ( {group.headers.map((header) => { const sort = header.column.getIsSorted() + const index = sorting.findIndex( + (item) => item.id === header.column.id, + ) return ( - - - - - - - + + ))} {open && ( )} ) })} - {rows.length === 0 && ( + {!rows.length && ( @@ -380,137 +1118,54 @@ export function LiveVisitors({ data }: { data: LiveData }) {
- {header.column.getCanSort() ? ( + {header.column.id === 'select' ? ( + { + if (element) + element.indeterminate = + table.getIsSomePageRowsSelected() + }} + onChange={(event) => + table.toggleAllPageRowsSelected( + event.target.checked, + ) + } + className="accent-emerald-600" + /> + ) : header.column.getCanSort() ? ( @@ -263,116 +990,127 @@ export function LiveVisitors({ data }: { data: LiveData }) { return (
- - {open ? ( - - - {company.company?.industry || 'Unknown'} - - - {company.company?.company_size || 'Unknown'} - -
- {company.libraries.slice(0, 2).map((library) => ( - - {library} - - ))} - {company.libraries.length > 2 && ( + + ) : cell.column.id === 'lastSeen' ? ( - +{company.libraries.length - 2} + {age(company.lastSeen, data.now)} + ) : ( + )} -
- 		- {company.pages.length} - - {company.actions} - -
- +
+
+ {company.name} + {company.id && ( + + {company.id} + + )} + + {company.activeSessions} active sessions ·{' '} + {company.activeTabs} tabs + + {company.company?.linkedin_url && + /^https:\/\/(www\.)?linkedin\.com\//.test( + company.company.linkedin_url, + ) && ( + + LinkedIn + + )} +
+ +
- {search - ? 'No companies or pages match your search.' + {filterCount || selectedOnly + ? 'No companies match these filters.' : 'No companies on public pages right now.'}
-
- - Company matches describe the network, not a verified employer. - - Missing heartbeats expire after 65 seconds. +
+ Network matches, not verified employers. +
+ + + + {Math.min( + pagination.pageIndex + 1, + Math.max(1, table.getPageCount()), + )}{' '} + / {Math.max(1, table.getPageCount())} + + +
) } - -function CompanySessions({ - company, - now, -}: { - company: LiveCompanyRow - now: string -}) { - return ( -
-
-

Sessions at {company.name}

- - { - company.sessions.filter((tabs) => tabs.some((tab) => tab.active)) - .length - }{' '} - active · {company.sessions.length} in view - -
-
- {company.sessions.map((tabs) => ( - - ))} -
-
- ) -} - -function SessionDetails({ tabs, now }: { tabs: LiveTab[]; now: string }) { - const first = tabs[0] - const current = - tabs.find((tab) => tab.active && tab.visible) ?? - tabs.find((tab) => tab.active) ?? - first - const active = tabs.some((tab) => tab.active) - const activity = tabs - .flatMap((tab) => tab.activity) - .sort((a, b) => b.at.localeCompare(a.at)) - .slice(0, 40) - return ( -
- - -
-
-

- Tabs {tabs.length} -

-
    - {tabs.map((tab) => ( -
  • - - {tab.active - ? tab.visible - ? 'Visible' - : 'Background' - : 'Left'} - - {tab.page} -
    • - ))} -
-

- Arrived {age(first.arrivedAt, now)} -

-
-
-

Recent activity

- {activity.length ? ( -
    - {activity.map((event, index) => ( -
  1. - -
    -

    {liveActions[event.action]}

    -

    - {event.page} -

    -
    -
    2. - ))} -
- ) : ( -

No actions recorded yet.

- )} -
-
-
- ) -} diff --git a/src/routes/admin/live.tsx b/src/routes/admin/live.tsx index 191bb4626..116334d6e 100644 --- a/src/routes/admin/live.tsx +++ b/src/routes/admin/live.tsx @@ -1,53 +1,83 @@ +import { useState } from 'react' import { createFileRoute } from '@tanstack/react-router' import { useQuery } from '@tanstack/react-query' -import { BuildingsIcon } from '@phosphor-icons/react' -import { useAdminGuard } from '~/hooks/useAdminGuard' import { - AdminAccessDenied, - AdminLoading, - AdminPageHeader, -} from '~/components/admin' + BuildingsIcon, + PauseIcon, + PlayIcon, + ArrowClockwiseIcon, +} from '@phosphor-icons/react' +import { useAdminGuard } from '~/hooks/useAdminGuard' +import { AdminAccessDenied, AdminLoading } from '~/components/admin' import { LiveVisitors } from '~/components/admin/LiveVisitors' -import { getLiveVisitors } from '~/utils/analytics/live.functions' +import { getLiveCompanyOverview } from '~/utils/analytics/live.functions' export const Route = createFileRoute('/admin/live')({ component: LiveVisitorsPage, }) - function LiveVisitorsPage() { const guard = useAdminGuard() + const [paused, setPaused] = useState(false) const query = useQuery({ - queryKey: ['admin', 'live-visitors'], - queryFn: () => getLiveVisitors(), + queryKey: ['admin', 'live-company-overview'], + queryFn: () => getLiveCompanyOverview(), enabled: guard.status === 'authorized', - refetchInterval: 3000, + refetchInterval: paused ? false : 3000, refetchIntervalInBackground: false, + refetchOnWindowFocus: !paused, + refetchOnReconnect: !paused, retry: false, }) if (guard.status === 'loading') return if (guard.status === 'denied') return return ( -
- } - title="Live companies" - actions={ - +
+
+

+ + Live companies +

+
+ {query.isError ? 'Updates interrupted' - : query.data - ? `Updated ${new Date(query.data.now).toLocaleTimeString()}` - : 'Connecting'} + : paused + ? 'Paused' + : query.data + ? `Updated ${new Date(query.data.now).toLocaleTimeString()}` + : 'Connecting'} - } - /> + + +
+
{query.isPending && } {query.isError && (
-

Couldn’t load live visitors. The last view may be out of date.

+

Could not refresh companies. The last view may be out of date.

)} - {query.data && } + {query.data && }
) } diff --git a/src/routes/index.tsx b/src/routes/index.tsx index 04c764b89..c97e1666d 100644 --- a/src/routes/index.tsx +++ b/src/routes/index.tsx @@ -1,3 +1,4 @@ +import { homepageScarfPixel } from '~/utils/analytics/scarf-pixels' import * as React from 'react' import { Link, createFileRoute } from '@tanstack/react-router' @@ -74,7 +75,7 @@ function Index() { return ( <> - +
{/* Hero — Figma node 802:2027. Full-bleed palm/gradient photo card: diff --git a/src/server/runtime/blob-storage.server.ts b/src/server/runtime/blob-storage.server.ts index ab4f4a7af..564da32b3 100644 --- a/src/server/runtime/blob-storage.server.ts +++ b/src/server/runtime/blob-storage.server.ts @@ -189,7 +189,7 @@ function getCacheRequest(name: BlobStorageName, key: string) { return new Request(url) } -function getDefaultRuntimeCache() { +export function getDefaultRuntimeCache() { if (!isIsolateRuntime() || typeof caches === 'undefined') { return undefined } diff --git a/src/utils/analytics/live-dashboard.ts b/src/utils/analytics/live-dashboard.ts index 0564407a5..79579bf8f 100644 --- a/src/utils/analytics/live-dashboard.ts +++ b/src/utils/analytics/live-dashboard.ts @@ -109,3 +109,167 @@ export function companyBreakdown( }, ] } + +export type LiveOverview = Awaited< + ReturnType +> +export type CompanyFacet = 'library' | 'industry' | 'size' | 'stage' +export type CompanyFilters = { + search: string + library: string[] + industry: string[] + size: string[] + stage: string[] + minViews: string + minActions: string + showRecent: boolean + identifiedOnly: boolean +} +export const emptyCompanyFilters: CompanyFilters = { + search: '', + library: [], + industry: [], + size: [], + stage: [], + minViews: '', + minActions: '', + showRecent: false, + identifiedOnly: false, +} + +export function employeeLowerBound(value: string | null | undefined) { + const match = value?.match(/(\d[\d,]*(?:\.\d+)?)\s*([km])?/i) + if (!match) return undefined + return ( + Number(match[1].replaceAll(',', '')) * + (match[2]?.toLowerCase() === 'm' + ? 1_000_000 + : match[2]?.toLowerCase() === 'k' + ? 1000 + : 1) + ) +} +export function companySizeBucket(value: string | null | undefined) { + const count = employeeLowerBound(value) + return count === undefined + ? 'Unspecified' + : count < 250 + ? '1-249' + : count < 1000 + ? '250-999' + : count < 10_000 + ? '1K-9,999' + : '10K+' +} +export function buildCompanyGridRows( + data: LiveOverview, + history: import('./scarf-history').ScarfHistory | undefined, +) { + const reports = new Map( + history?.companies.map((company) => [company.domain, company]), + ) + return data.companies.map((company) => { + const report = reports.get(company.id) + return { + ...company, + name: + company.company?.name || + report?.name || + company.id || + 'Unmatched traffic', + libraries: [...new Set(company.pages.map(pageLibrary))], + active: company.activeSessions > 0, + industry: company.company?.industry || 'Unspecified', + sizeBucket: companySizeBucket(company.company?.company_size), + employeeMin: employeeLowerBound(company.company?.company_size), + history: report, + } + }) +} +export type CompanyGridRow = ReturnType[number] +export function companyFacetValues(row: CompanyGridRow, facet: CompanyFacet) { + return facet === 'library' + ? row.libraries + : facet === 'industry' + ? [row.industry] + : facet === 'size' + ? [row.sizeBucket] + : [row.history?.stage || 'Unspecified'] +} +export function filterCompanyRows( + rows: CompanyGridRow[], + filters: CompanyFilters, + omit?: CompanyFacet, +) { + const facets: CompanyFacet[] = ['library', 'industry', 'size', 'stage'] + const search = filters.search.trim().toLowerCase() + return rows.filter( + (row) => + (filters.showRecent || row.active) && + (!filters.identifiedOnly || Boolean(row.id)) && + (!search || + `${row.name} ${row.id} ${row.industry} ${row.libraries.join(' ')} ${row.pages.join(' ')}` + .toLowerCase() + .includes(search)) && + (!filters.minViews || + (row.history?.views !== undefined && + row.history.views >= Number(filters.minViews))) && + (!filters.minActions || row.actions >= Number(filters.minActions)) && + facets.every( + (facet) => + omit === facet || + !filters[facet].length || + filters[facet].some((value) => + companyFacetValues(row, facet).includes(value), + ), + ), + ) +} +export function companyFacetCounts( + rows: CompanyGridRow[], + facet: CompanyFacet, + activeOnly = true, +) { + const counts = new Map() + for (const row of rows) { + if (!row.id || (activeOnly && !row.active)) continue + for (const value of companyFacetValues(row, facet)) + counts.set(value, (counts.get(value) ?? 0) + 1) + } + return [...counts] + .map(([name, value]) => ({ name, value, values: [name] })) + .sort((a, b) => + facet === 'size' + ? ['1-249', '250-999', '1K-9,999', '10K+', 'Unspecified'].indexOf( + a.name, + ) - + ['1-249', '250-999', '1K-9,999', '10K+', 'Unspecified'].indexOf( + b.name, + ) + : b.value - a.value || a.name.localeCompare(b.name), + ) +} +export function companyWidgetItems( + items: ReturnType, +) { + return items.length <= 6 + ? items + : [ + ...items.slice(0, 5), + { + name: 'Other', + value: items.slice(5).reduce((count, item) => count + item.value, 0), + values: items.slice(5).flatMap((item) => item.values), + }, + ] +} +export function companyGrowth(row: CompanyGridRow) { + const current = row.history?.views + const previous = row.history?.previousViews + return current === undefined || previous === undefined || previous === 0 + ? undefined + : ((current - previous) / previous) * 100 +} +export function csvSafeText(value: string) { + return /^\s*[=+\-@]|^[\t\r\n]/.test(value) ? `'${value}` : value +} diff --git a/src/utils/analytics/live.functions.ts b/src/utils/analytics/live.functions.ts index a3d95eba7..b4ec11e01 100644 --- a/src/utils/analytics/live.functions.ts +++ b/src/utils/analytics/live.functions.ts @@ -1,5 +1,9 @@ +import { z } from 'zod' +import { historyDaysSchema } from './scarf-history' +import { loadScarfHistory } from './scarf-history.server' +import { requireAdmin } from '~/utils/roles.server' import { createServerFn } from '@tanstack/react-start' -import { listLiveVisitors } from './live.server' +import { listLiveVisitors, listLiveCompanyOverview } from './live.server' import { getLiveConfig } from './scarf-lookup.server' export const getLiveVisitors = createServerFn({ method: 'GET' }).handler( @@ -9,3 +13,23 @@ export const getLiveVisitors = createServerFn({ method: 'GET' }).handler( export const getLiveVisitorsEnabled = createServerFn({ method: 'GET' }).handler( async () => (await getLiveConfig()).enabled, ) + +export const getLiveCompanyOverview = createServerFn({ method: 'GET' }).handler( + async () => listLiveCompanyOverview(), +) +export const getLiveCompanySessions = createServerFn({ method: 'GET' }) + .validator( + z.object({ + domain: z + .string() + .max(253) + .regex(/^(?:[a-z0-9.-]+)?$/), + }), + ) + .handler(async ({ data }) => listLiveVisitors(data.domain)) +export const getScarfCompanyHistory = createServerFn({ method: 'GET' }) + .validator(z.object({ days: historyDaysSchema })) + .handler(async ({ data }) => { + await requireAdmin() + return loadScarfHistory(data.days) + }) diff --git a/src/utils/analytics/live.server.ts b/src/utils/analytics/live.server.ts index 0bd95c553..879b93999 100644 --- a/src/utils/analytics/live.server.ts +++ b/src/utils/analytics/live.server.ts @@ -1,3 +1,4 @@ +import { z } from 'zod' import { and, desc, eq, gt, lt, sql } from 'drizzle-orm' import { isIP } from 'node:net' import { db } from '~/db/client' @@ -10,6 +11,7 @@ import { import { getLiveConfig, lookupScarfCompany } from './scarf-lookup.server' import { isLivePage, + liveCompanySchema, livePublicPagePattern, liveForbiddenPagePattern, liveEventSchema, @@ -172,7 +174,7 @@ export async function recordLiveEvent( } } -export async function listLiveVisitors() { +export async function listLiveVisitors(domain?: string) { await requireAdmin() const config = await getLiveConfig() const now = Date.now() @@ -203,6 +205,9 @@ export async function listLiveVisitors() { and( gt(liveTabs.lastSeen, new Date(now - 5 * 60_000)), gt(liveSessions.createdAt, new Date(now - LIVE_RETENTION_MS)), + domain === undefined + ? undefined + : sql`coalesce(lower(${liveSessions.company}->>'domain'), '') = ${domain}`, sql`length(${liveTabs.page}) <= 400`, sql`${liveTabs.page} ~ ${livePublicPagePattern.source}`, sql`${liveTabs.page} !~ ${liveForbiddenPagePattern.source}`, @@ -234,3 +239,69 @@ export async function pruneLiveVisitors(now = Date.now()) { .delete(liveSessions) .where(lt(liveSessions.createdAt, new Date(now - LIVE_RETENTION_MS))) } + +export const liveCompanyOverviewRowSchema = z.object({ + id: z.string(), + company: liveCompanySchema.nullable(), + activeSessions: z.coerce.number().int().nonnegative(), + activeTabs: z.coerce.number().int().nonnegative(), + visibleTabs: z.coerce.number().int().nonnegative(), + recentSessions: z.coerce.number().int().nonnegative(), + pages: z.array(z.string()), + actions: z.coerce.number().int().nonnegative(), + recentViews: z.coerce.number().int().nonnegative(), + lastSeen: z.coerce.date(), +}) + +export function liveCompanyOverviewSql(now: number) { + const activeSince = new Date(now - LIVE_TIMEOUT_MS).toISOString() + const recentSince = new Date(now - 5 * 60_000).toISOString() + const retainedSince = new Date(now - LIVE_RETENTION_MS).toISOString() + return sql` + WITH eligible AS ( + SELECT s.id AS session_id, s.company, t.page, t.visible, t.last_seen, t.activity, + (t.left_at IS NULL AND t.last_seen > ${activeSince}::timestamptz) AS active + FROM live_tabs t JOIN live_sessions s ON s.id = t.session_id + WHERE t.last_seen > ${recentSince}::timestamptz AND s.created_at > ${retainedSince}::timestamptz + AND length(t.page) <= 400 AND t.page ~ ${livePublicPagePattern.source} + AND t.page !~ ${liveForbiddenPagePattern.source} + ) + SELECT coalesce(lower(e.company->>'domain'), '') AS id, + max(e.company::text)::jsonb AS company, + count(DISTINCT e.session_id) FILTER (WHERE e.active) AS "activeSessions", + count(*) FILTER (WHERE e.active) AS "activeTabs", + count(*) FILTER (WHERE e.active AND e.visible) AS "visibleTabs", + count(DISTINCT e.session_id) AS "recentSessions", + coalesce(jsonb_agg(DISTINCT e.page ORDER BY e.page) FILTER (WHERE e.active), '[]'::jsonb) AS pages, + coalesce(sum(a.actions), 0) AS actions, coalesce(sum(a.views), 0) AS "recentViews", + max(e.last_seen) AS "lastSeen" + FROM eligible e + CROSS JOIN LATERAL ( + SELECT count(*) FILTER (WHERE event->>'action' <> 'page_view') AS actions, + count(*) FILTER (WHERE event->>'action' = 'page_view') AS views + FROM jsonb_array_elements(e.activity) event + WHERE event->>'at' >= ${recentSince} + AND length(event->>'page') <= 400 AND event->>'page' ~ ${livePublicPagePattern.source} + AND event->>'page' !~ ${liveForbiddenPagePattern.source} + ) a + GROUP BY coalesce(lower(e.company->>'domain'), '') + ORDER BY id + ` +} + +export async function listLiveCompanyOverview() { + await requireAdmin() + const config = await getLiveConfig() + const now = Date.now() + const companies = config.enabled + ? z + .array(liveCompanyOverviewRowSchema) + .parse(await db.execute(liveCompanyOverviewSql(now))) + : [] + return { + enabled: config.enabled, + lookupConfigured: Boolean(config.key), + now: new Date(now).toISOString(), + companies, + } +} diff --git a/src/utils/analytics/scarf-history.server.ts b/src/utils/analytics/scarf-history.server.ts new file mode 100644 index 000000000..3d18d9919 --- /dev/null +++ b/src/utils/analytics/scarf-history.server.ts @@ -0,0 +1,191 @@ +import { libraries } from '~/libraries/libraries' +import { getHostRuntimeEnv } from '~/server/runtime/host.server' +import { getDefaultRuntimeCache } from '~/server/runtime/blob-storage.server' +import { homepageScarfPixel, productScarfPixels } from './scarf-pixels' +import { + companyHistoryFromReports, + historyWindow, + scarfAggregationRowSchema, + type ScarfAggregationRow, + scarfHistorySchema, + type ScarfHistory, +} from './scarf-history' + +// This is the event-only /site-events route verified in Scarf's package settings. +const siteTelemetryPackage = 'b942b596-f90d-4072-a763-86fb5fbe5852' +const sitePixels = [ + ...new Set([ + homepageScarfPixel, + ...productScarfPixels.map((pixel) => pixel.id), + ...libraries.flatMap((library) => + library.scarfId ? [library.scarfId] : [], + ), + ]), +] + +export function scarfReportUrl( + start: string, + end: string, + source: 'views' | 'events', +) { + const url = new URL( + 'https://api.scarf.sh/v3/insights/TanStack/aggregations/export', + ) + for (const [key, value] of Object.entries({ + start_date: start, + end_date: end, + rollup: 'total', + breakdown: 'by-company', + format: 'ndjson', + group_by_artifact: 'false', + group_by_artifact_type: 'false', + include_low_confidence: 'false', + })) + url.searchParams.set(key, value) + url.searchParams.set( + 'package_id', + source === 'events' ? siteTelemetryPackage : 'none', + ) + if (source === 'views') + for (const id of sitePixels) + url.searchParams.append('tracking_pixel_id', id) + else url.searchParams.set('tracking_pixel_id', 'none') + return url +} + +class ScarfReportError extends Error { + constructor(readonly detail: string) { + super(detail) + } +} + +async function report( + start: string, + end: string, + source: 'views' | 'events', + token: string, +) { + const response = await fetch(scarfReportUrl(start, end, source), { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/x-ndjson', + }, + signal: AbortSignal.timeout(30_000), + }) + if (!response.ok) { + await response.body?.cancel() + throw new ScarfReportError(`HTTP ${response.status}`) + } + const rows: ScarfAggregationRow[] = [] + const reader = response.body?.getReader() + if (!reader) throw new ScarfReportError('empty response') + const decoder = new TextDecoder() + let buffer = '' + let bytes = 0 + const addRow = (line: string) => { + if (!line.trim()) return + if (rows.length >= 50_000) + throw new ScarfReportError('export too large; choose a shorter window') + rows.push(scarfAggregationRowSchema.parse(JSON.parse(line))) + } + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + bytes += chunk.value.byteLength + if (bytes > 100 * 1024 * 1024) + throw new ScarfReportError('export too large; choose a shorter window') + buffer += decoder.decode(chunk.value, { stream: true }) + let end = buffer.indexOf('\n') + while (end !== -1) { + addRow(buffer.slice(0, end)) + buffer = buffer.slice(end + 1) + end = buffer.indexOf('\n') + } + if (buffer.length > 64_000) + throw new ScarfReportError('invalid export row') + } + buffer += decoder.decode() + addRow(buffer) + return rows + } catch (error) { + await reader.cancel() + throw error + } finally { + reader.releaseLock() + } +} + +export async function loadScarfHistory( + days: ScarfHistory['days'], +): Promise { + const window = historyWindow(days) + const base = { + days, + start: window.start, + end: window.end, + fetchedAt: new Date().toISOString(), + companies: [], + issues: [], + } + const host = await getHostRuntimeEnv() + const token = + host?.ANALYTICS_SCARF_API_TOKEN ?? process.env.ANALYTICS_SCARF_API_TOKEN + if (typeof token !== 'string' || !token) + return { ...base, status: 'unconfigured' } + const cache = getDefaultRuntimeCache() + const cacheKey = new Request( + `https://scarf-reporting.tanstack.internal/site-history/v1/${days}/${window.end}`, + ) + const cached = await cache?.match(cacheKey).catch(() => undefined) + if (cached) { + const parsed = scarfHistorySchema.safeParse(await cached.json()) + if (parsed.success) return parsed.data + } + const results = await Promise.allSettled([ + report(window.start, window.end, 'views', token), + report(window.previousStart, window.start, 'views', token), + report(window.start, window.end, 'events', token), + ]) + const data = results.map((result) => + result.status === 'fulfilled' ? result.value : undefined, + ) + const issues = results.flatMap((result, index) => + result.status === 'fulfilled' + ? [] + : [ + `${['Site pixel views', 'Previous period', 'Site events'][index]} unavailable${result.reason instanceof ScarfReportError ? ` (${result.reason.detail})` : ' (unexpected response or timeout)'}. Check Scarf reporting access or try again later.`, + ], + ) + let companies: ScarfHistory['companies'] = [] + try { + companies = companyHistoryFromReports(data[0], data[1], data[2]) + } catch { + issues.push( + 'Scarf returned inconsistent company totals. History is unavailable.', + ) + } + const history: ScarfHistory = { + ...base, + companies, + issues, + status: issues.length + ? companies.length + ? 'partial' + : 'unavailable' + : 'ready', + } + // Cache aggregate company metrics only, never credentials or source-level data. + await cache + ?.put( + cacheKey, + new Response(JSON.stringify(history), { + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': `max-age=${history.status === 'ready' ? 900 : 60}`, + }, + }), + ) + .catch(() => undefined) + return history +} diff --git a/src/utils/analytics/scarf-history.ts b/src/utils/analytics/scarf-history.ts new file mode 100644 index 000000000..eb3ebdb23 --- /dev/null +++ b/src/utils/analytics/scarf-history.ts @@ -0,0 +1,96 @@ +import { z } from 'zod' + +export const historyDaysSchema = z.union([ + z.literal(7), + z.literal(30), + z.literal(90), + z.literal(365), +]) +export const scarfCompanyHistorySchema = z.object({ + domain: z.string(), + name: z.string().nullable(), + views: z.number().nonnegative().optional(), + previousViews: z.number().nonnegative().optional(), + siteEvents: z.number().nonnegative().optional(), + origins: z.number().nonnegative().optional(), + endpoints: z.number().nonnegative().optional(), + stage: z.string().nullable(), + lastSeen: z.string().nullable(), +}) +export type ScarfCompanyHistory = z.infer +export const scarfHistorySchema = z.object({ + status: z.enum(['ready', 'partial', 'unconfigured', 'unavailable']), + days: historyDaysSchema, + start: z.string(), + end: z.string(), + fetchedAt: z.string(), + companies: z.array(scarfCompanyHistorySchema), + issues: z.array(z.string()), +}) +export type ScarfHistory = z.infer +export function historyWindow(days: number, now = new Date()) { + const end = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1), + ) + const start = new Date(end.getTime() - days * 86_400_000) + const previousStart = new Date(start.getTime() - days * 86_400_000) + return { + start: start.toISOString().slice(0, 10), + end: end.toISOString().slice(0, 10), + previousStart: previousStart.toISOString().slice(0, 10), + } +} +export const scarfAggregationRowSchema = z.object({ + company_domain: z.string().nullable(), + company_name: z.string().nullable(), + total: z.number().int().nonnegative(), + unique_origins: z.number().int().nonnegative(), + unique_endpoints: z.number().int().nonnegative(), + company_funnel_stage: z.string().nullable(), + last_seen: z.string(), +}) +export type ScarfAggregationRow = z.infer +export const scarfAggregationSchema = z.object({ + data: z.array(scarfAggregationRowSchema).max(50_000), +}) + +export function companyHistoryFromReports( + views: z.infer['data'] | undefined, + previous: z.infer['data'] | undefined, + events: z.infer['data'] | undefined, +) { + const companies = new Map() + const reports: { + kind: 'views' | 'previousViews' | 'siteEvents' + rows: typeof views + }[] = [ + { kind: 'views', rows: views }, + { kind: 'previousViews', rows: previous }, + { kind: 'siteEvents', rows: events }, + ] + for (const { kind, rows } of reports) { + const seen = new Set() + for (const row of rows ?? []) { + const domain = row.company_domain?.trim().toLowerCase() + if (!domain) continue + if (seen.has(domain)) + throw new Error('Scarf returned duplicate company totals') + seen.add(domain) + const company = companies.get(domain) ?? { + domain, + name: row.company_name, + stage: null, + lastSeen: null, + } + company[kind] = row.total + if (kind === 'views') { + company.origins = row.unique_origins + company.endpoints = row.unique_endpoints + company.stage = row.company_funnel_stage + company.lastSeen = row.last_seen + } + companies.set(domain, company) + } + } + return [...companies.values()] +} diff --git a/src/utils/analytics/scarf-pixels.ts b/src/utils/analytics/scarf-pixels.ts new file mode 100644 index 000000000..74580fce3 --- /dev/null +++ b/src/utils/analytics/scarf-pixels.ts @@ -0,0 +1,7 @@ +export const homepageScarfPixel = 'b7a8c111-2305-4d95-a02a-d84c5bfb82ae' +export const productScarfPixels = [ + { path: '/application-starter', id: 'ed471ec3-aa08-430f-8481-258b3a1f94dc' }, + { path: '/builder', id: 'aa4dcee9-86a2-49b4-9981-40c97cc1f55b' }, + { path: '/stats/npm', id: 'e729f879-e127-48e1-be19-c68f6786c9e8' }, + { path: '/intent/registry', id: '811c34af-38bd-4f06-9762-3f58dea0650a' }, +] diff --git a/tests/live-company-tools.test.ts b/tests/live-company-tools.test.ts new file mode 100644 index 000000000..d89c66449 --- /dev/null +++ b/tests/live-company-tools.test.ts @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + buildCompanyGridRows, + filterCompanyRows, + emptyCompanyFilters, + employeeLowerBound, + companyFacetCounts, + companyWidgetItems, + companyGrowth, + csvSafeText, + type LiveOverview, +} from '../src/utils/analytics/live-dashboard.ts' +import { + companyHistoryFromReports, + historyWindow, + type ScarfHistory, +} from '../src/utils/analytics/scarf-history.ts' + +const data: LiveOverview = { + enabled: true, + lookupConfigured: true, + now: '2026-10-29T17:00:00Z', + companies: [ + { + id: 'acme.example', + company: { + name: 'Acme', + domain: 'acme.example', + industry: 'Software', + company_size: '1,000 - 4,999', + linkedin_url: null, + }, + activeSessions: 3, + activeTabs: 5, + visibleTabs: 2, + recentSessions: 4, + pages: ['/query/latest', '/router/latest'], + actions: 9, + recentViews: 12, + lastSeen: new Date('2026-10-29T17:00:00Z'), + }, + { + id: 'other.example', + company: { + name: 'Other', + domain: 'other.example', + industry: 'Retail', + company_size: '100K+', + linkedin_url: null, + }, + activeSessions: 1, + activeTabs: 1, + visibleTabs: 1, + recentSessions: 1, + pages: ['/query/latest'], + actions: 1, + recentViews: 2, + lastSeen: new Date('2026-10-29T17:00:00Z'), + }, + { + id: '', + company: null, + activeSessions: 10, + activeTabs: 10, + visibleTabs: 2, + recentSessions: 10, + pages: ['/form/latest'], + actions: 100, + recentViews: 100, + lastSeen: new Date('2026-10-29T17:00:00Z'), + }, + ], +} +const history: ScarfHistory = { + status: 'ready', + days: 30, + start: '2026-09-30', + end: '2026-10-30', + fetchedAt: data.now, + issues: [], + companies: [ + { + domain: 'acme.example', + name: 'Acme', + views: 500, + previousViews: 250, + siteEvents: 2000, + origins: 12, + endpoints: 18, + stage: 'ongoing-usage', + lastSeen: data.now, + }, + ], +} + +test('facets use OR within a dimension and AND across dimensions', () => { + const rows = buildCompanyGridRows(data, history) + assert.deepEqual( + filterCompanyRows(rows, { + ...emptyCompanyFilters, + library: ['Query', 'Form'], + industry: ['Software'], + }).map((row) => row.id), + ['acme.example'], + ) + assert.deepEqual( + filterCompanyRows(rows, { + ...emptyCompanyFilters, + minViews: '100', + minActions: '5', + }).map((row) => row.id), + ['acme.example'], + ) + assert.equal( + filterCompanyRows(rows, { ...emptyCompanyFilters, identifiedOnly: true }) + .length, + 2, + ) +}) +test('facet counts exclude unmatched traffic and count each company once per library', () => { + const rows = buildCompanyGridRows(data, history) + assert.deepEqual( + companyFacetCounts(rows, 'library').map((item) => [item.name, item.value]), + [ + ['Query', 2], + ['Router', 1], + ], + ) + assert.equal( + companyFacetCounts( + filterCompanyRows( + rows, + { ...emptyCompanyFilters, industry: ['Software'] }, + 'industry', + ), + 'industry', + ).length, + 2, + ) +}) +test('Other retains the actual facet values so a click can filter the grouped categories', () => { + const items = Array.from({ length: 8 }, (_, index) => ({ + name: `Industry ${index}`, + value: 8 - index, + values: [`Industry ${index}`], + })) + const other = companyWidgetItems(items).at(-1) + assert.deepEqual(other, { + name: 'Other', + value: 6, + values: ['Industry 5', 'Industry 6', 'Industry 7'], + }) +}) +test('size sorting handles provider range variants, K suffixes, and unknown values', () => { + assert.equal(employeeLowerBound('1,000 - 4,999'), 1000) + assert.equal(employeeLowerBound('5K-10K'), 5000) + assert.equal(employeeLowerBound('100K+'), 100000) + assert.equal(employeeLowerBound(null), undefined) +}) +test('missing historical metrics stay unknown, zero stays zero, and change uses equal windows', () => { + const rows = buildCompanyGridRows(data, history) + assert.equal(companyGrowth(rows[0]), 100) + assert.equal(rows[1].history, undefined) + assert.equal(companyGrowth(rows[1]), undefined) + assert.deepEqual(historyWindow(30, new Date('2026-10-29T17:00:00Z')), { + start: '2026-09-30', + end: '2026-10-30', + previousStart: '2026-08-31', + }) +}) +test('report joins keep site pixel views separate from telemetry events and do not invent missing values', () => { + const report = { + company_domain: 'ACME.EXAMPLE', + company_name: 'Acme', + total: 12, + unique_origins: 2, + unique_endpoints: 3, + company_funnel_stage: null, + last_seen: data.now, + } + const rows = companyHistoryFromReports([report], undefined, [ + { ...report, total: 90 }, + ]) + assert.equal(rows[0].views, 12) + assert.equal(rows[0].siteEvents, 90) + assert.equal(rows[0].previousViews, undefined) + assert.throws( + () => companyHistoryFromReports([report, report], undefined, undefined), + /duplicate/, + ) +}) +test('CSV text neutralizes spreadsheet formulas while preserving numeric values at their source', () => { + assert.equal(csvSafeText('=SUM(1,2)'), "'=SUM(1,2)") + assert.equal(csvSafeText(' +1'), "' +1") + assert.equal(csvSafeText('Acme'), 'Acme') +}) diff --git a/tests/scarf-history.test.ts b/tests/scarf-history.test.ts new file mode 100644 index 000000000..617218222 --- /dev/null +++ b/tests/scarf-history.test.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { + loadScarfHistory, + scarfReportUrl, +} from '../src/utils/analytics/scarf-history.server.ts' + +test('site reports select only the website pixels or its dedicated telemetry package', () => { + const views = scarfReportUrl('2026-09-30', '2026-10-30', 'views') + assert.equal(views.pathname, '/v3/insights/TanStack/aggregations/export') + assert.equal(views.searchParams.get('package_id'), 'none') + assert.ok( + views.searchParams + .getAll('tracking_pixel_id') + .includes('b7a8c111-2305-4d95-a02a-d84c5bfb82ae'), + ) + assert.equal(views.searchParams.get('group_by_artifact'), 'false') + assert.equal(views.searchParams.get('group_by_artifact_type'), 'false') + const events = scarfReportUrl('2026-09-30', '2026-10-30', 'events') + assert.equal( + events.searchParams.get('package_id'), + 'b942b596-f90d-4072-a763-86fb5fbe5852', + ) + assert.equal(events.searchParams.get('tracking_pixel_id'), 'none') +}) +test('unavailable reporting access does not pretend the company has zero history', async (t) => { + const previous = process.env.ANALYTICS_SCARF_API_TOKEN + process.env.ANALYTICS_SCARF_API_TOKEN = 'test-token' + t.after(() => { + if (previous === undefined) delete process.env.ANALYTICS_SCARF_API_TOKEN + else process.env.ANALYTICS_SCARF_API_TOKEN = previous + }) + t.mock.method( + globalThis, + 'fetch', + async () => new Response(null, { status: 403 }), + ) + const result = await loadScarfHistory(30) + assert.equal(result.status, 'unavailable') + assert.deepEqual(result.companies, []) + assert.equal(result.issues.length, 3) +}) + +test('streamed history keeps aggregate views separate and strips source identifiers', async (t) => { + const previous = process.env.ANALYTICS_SCARF_API_TOKEN + process.env.ANALYTICS_SCARF_API_TOKEN = 'test-token' + t.after(() => { + if (previous === undefined) delete process.env.ANALYTICS_SCARF_API_TOKEN + else process.env.ANALYTICS_SCARF_API_TOKEN = previous + }) + const { historyWindow } = + await import('../src/utils/analytics/scarf-history.ts') + const window = historyWindow(30) + t.mock.method( + globalThis, + 'fetch', + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + assert.equal( + new Headers(init?.headers).get('Authorization'), + 'Bearer test-token', + ) + assert.equal(url.searchParams.get('format'), 'ndjson') + const total = + url.searchParams.get('package_id') !== 'none' + ? 90 + : url.searchParams.get('start_date') === window.start + ? 12 + : 6 + const row = { + company_domain: 'example.com', + company_name: 'Example', + total, + unique_origins: 2, + unique_endpoints: 3, + company_funnel_stage: 'ongoing-usage', + last_seen: new Date().toISOString(), + origin_id: 'source-id-must-not-be-returned', + } + return new Response(JSON.stringify(row), { + headers: { 'Content-Type': 'application/x-ndjson' }, + }) + }, + ) + const result = await loadScarfHistory(30) + assert.equal(result.status, 'ready') + assert.equal(result.companies[0].views, 12) + assert.equal(result.companies[0].previousViews, 6) + assert.equal(result.companies[0].siteEvents, 90) + assert.equal( + JSON.stringify(result).includes('source-id-must-not-be-returned'), + false, + ) + assert.equal(JSON.stringify(result).includes('test-token'), false) +}) From 74fb3a7bc4774d0aba0ad95ea8f1626e4d0a950e Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Thu, 29 Oct 2026 12:15:14 -0600 Subject: [PATCH 2/3] Add active and background company status filters and column --- docs/live-visitors.md | 5 ++ src/components/admin/LiveVisitors.tsx | 69 ++++++++++++++++++++------- src/utils/analytics/live-dashboard.ts | 34 +++++++++---- tests/live-company-tools.test.ts | 59 +++++++++++++++++++++++ 4 files changed, 140 insertions(+), 27 deletions(-) diff --git a/docs/live-visitors.md b/docs/live-visitors.md index c9ad5c002..147170e6f 100644 --- a/docs/live-visitors.md +++ b/docs/live-visitors.md @@ -17,6 +17,11 @@ The overview counts identified companies once per domain. Library breakdowns count a company once per library, so one company can appear in several bars. Industry and size breakdowns exclude unmatched traffic. Clicking a breakdown applies its filter. Size buckets use the lower bound of Scarf's employee range. +The status widget and sortable Status column classify a company as Active when +at least one online tab is visible, Background when all online tabs are hidden, +or Left when no tabs remain online. Each company occupies one status, even when +it has both visible and hidden tabs. Active describes visibility, not confirmed +attention or interaction. Status filters combine with the other company filters. Grid action totals count named interactions in the last five minutes, excluding page views. Pause freezes polling, including expanded sessions and history. diff --git a/src/components/admin/LiveVisitors.tsx b/src/components/admin/LiveVisitors.tsx index c521c9679..1a4068532 100644 --- a/src/components/admin/LiveVisitors.tsx +++ b/src/components/admin/LiveVisitors.tsx @@ -72,13 +72,20 @@ const initialVisibility: ColumnVisibilityState = { } const initialSelection: RowSelectionState = {} const visibilitySchema = z.record(z.string(), z.boolean()) -const facets: CompanyFacet[] = ['library', 'size', 'industry', 'stage'] -const widgetFacets: CompanyFacet[] = ['library', 'size', 'industry'] +const facets: CompanyFacet[] = [ + 'status', + 'library', + 'size', + 'industry', + 'stage', +] +const widgetFacets: CompanyFacet[] = ['status', 'library', 'size', 'industry'] const facetLabels: Record = { library: 'Library', size: 'Company size', industry: 'Industry', stage: 'Funnel stage', + status: 'Status', } const number = new Intl.NumberFormat('en-US') function numeric(value: number | undefined) { @@ -124,6 +131,23 @@ const columns: ColumnDef[] = [ sortFn: sortFn_alphanumeric, enableHiding: false, }, + { + id: 'status', + accessorKey: 'status', + header: 'Status', + sortFn: sortFn_alphanumeric, + cell: ({ row }) => ( + + {row.original.status} + + ), + }, { id: 'industry', accessorKey: 'industry', @@ -268,6 +292,8 @@ const columns: ColumnDef[] = [ }, ] const columnHelp: Record = { + status: + 'Active: at least one online tab is visible. Background: every online tab is hidden. Left: no online tabs. Visibility does not prove attention or interaction.', sessions: 'Anonymous sessions with a heartbeat in the last 65 seconds.', actions: 'Recorded interactions from the last five minutes, limited to 40 retained events per tab.', @@ -446,6 +472,7 @@ export function LiveVisitors({ Employees: csvSafeText(row.company?.company_size || ''), Libraries: csvSafeText(row.libraries.join(', ')), Sessions: row.activeSessions, + Status: row.status, Actions5m: row.actions, RecentViews5m: row.recentViews, SitePixelViews: row.history?.views ?? '', @@ -549,17 +576,19 @@ export function LiveVisitors({ {showBreakdowns && (
{widgetFacets.map((facet) => ( toggleFacet(facet, values)} note={ - facet === 'size' - ? "Grouped by the lower bound of Scarf's reported employee range." - : facet === 'library' - ? 'A company can appear in multiple libraries. Other combines the remaining category counts.' - : undefined + facet === 'status' + ? 'Active means at least one visible tab. Background means all online tabs are hidden. Each company is counted once.' + : facet === 'size' + ? "Grouped by the lower bound of Scarf's reported employee range." + : facet === 'library' + ? 'A company can appear in multiple libraries. Other combines the remaining category counts.' + : undefined } /> ))} diff --git a/src/utils/analytics/live-dashboard.ts b/src/utils/analytics/live-dashboard.ts index 79579bf8f..fc5e8671e 100644 --- a/src/utils/analytics/live-dashboard.ts +++ b/src/utils/analytics/live-dashboard.ts @@ -113,13 +113,14 @@ export function companyBreakdown( export type LiveOverview = Awaited< ReturnType > -export type CompanyFacet = 'library' | 'industry' | 'size' | 'stage' +export type CompanyFacet = 'library' | 'industry' | 'size' | 'stage' | 'status' export type CompanyFilters = { search: string library: string[] industry: string[] size: string[] stage: string[] + status: string[] minViews: string minActions: string showRecent: boolean @@ -131,6 +132,7 @@ export const emptyCompanyFilters: CompanyFilters = { industry: [], size: [], stage: [], + status: [], minViews: '', minActions: '', showRecent: false, @@ -179,6 +181,12 @@ export function buildCompanyGridRows( 'Unmatched traffic', libraries: [...new Set(company.pages.map(pageLibrary))], active: company.activeSessions > 0, + status: + company.visibleTabs > 0 + ? 'Active' + : company.activeTabs > 0 + ? 'Background' + : 'Left', industry: company.company?.industry || 'Unspecified', sizeBucket: companySizeBucket(company.company?.company_size), employeeMin: employeeLowerBound(company.company?.company_size), @@ -188,20 +196,28 @@ export function buildCompanyGridRows( } export type CompanyGridRow = ReturnType[number] export function companyFacetValues(row: CompanyGridRow, facet: CompanyFacet) { - return facet === 'library' - ? row.libraries - : facet === 'industry' - ? [row.industry] - : facet === 'size' - ? [row.sizeBucket] - : [row.history?.stage || 'Unspecified'] + return facet === 'status' + ? [row.status] + : facet === 'library' + ? row.libraries + : facet === 'industry' + ? [row.industry] + : facet === 'size' + ? [row.sizeBucket] + : [row.history?.stage || 'Unspecified'] } export function filterCompanyRows( rows: CompanyGridRow[], filters: CompanyFilters, omit?: CompanyFacet, ) { - const facets: CompanyFacet[] = ['library', 'industry', 'size', 'stage'] + const facets: CompanyFacet[] = [ + 'library', + 'industry', + 'size', + 'stage', + 'status', + ] const search = filters.search.trim().toLowerCase() return rows.filter( (row) => diff --git a/tests/live-company-tools.test.ts b/tests/live-company-tools.test.ts index d89c66449..29df4b3e8 100644 --- a/tests/live-company-tools.test.ts +++ b/tests/live-company-tools.test.ts @@ -195,3 +195,62 @@ test('CSV text neutralizes spreadsheet formulas while preserving numeric values assert.equal(csvSafeText(' +1'), "' +1") assert.equal(csvSafeText('Acme'), 'Acme') }) + +test('company status prefers any visible tab and filters background companies without counting them twice', () => { + const rows = buildCompanyGridRows( + { + ...data, + companies: data.companies.map((company, index) => + index === 1 + ? { ...company, visibleTabs: 0 } + : index === 2 + ? { ...company, activeTabs: 0, activeSessions: 0, visibleTabs: 0 } + : company, + ), + }, + history, + ) + assert.deepEqual( + rows.map((row) => row.status), + ['Active', 'Background', 'Left'], + ) + assert.deepEqual( + companyFacetCounts(rows, 'status').map(({ name, value }) => ({ + name, + value, + })), + [ + { name: 'Active', value: 1 }, + { name: 'Background', value: 1 }, + ], + ) + assert.deepEqual( + filterCompanyRows(rows, { + ...emptyCompanyFilters, + status: ['Background'], + library: ['Query'], + industry: ['Retail'], + }).map((row) => row.id), + ['other.example'], + ) + assert.deepEqual( + filterCompanyRows(rows, { + ...emptyCompanyFilters, + status: ['Active', 'Background'], + }).map((row) => row.id), + ['acme.example', 'other.example'], + ) + assert.equal( + filterCompanyRows(rows, { ...emptyCompanyFilters, status: ['Left'] }) + .length, + 0, + ) + assert.equal( + filterCompanyRows(rows, { + ...emptyCompanyFilters, + status: ['Left'], + showRecent: true, + }).length, + 1, + ) +}) From 8c9ae72a4a53b1a47b8e3becaf3c52b0ef41fb9d Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Thu, 29 Oct 2026 14:01:37 -0600 Subject: [PATCH 3/3] Include recently left companies in dashboard breakdowns --- src/components/admin/LiveVisitors.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/admin/LiveVisitors.tsx b/src/components/admin/LiveVisitors.tsx index 1a4068532..21dd1944c 100644 --- a/src/components/admin/LiveVisitors.tsx +++ b/src/components/admin/LiveVisitors.tsx @@ -594,6 +594,7 @@ export function LiveVisitors({ companyFacetCounts( filterCompanyRows(allRows, filters, facet), facet, + !filters.showRecent, ), )} color={