diff --git a/apps/extension/src/omnibox.ts b/apps/extension/src/omnibox.ts index 56c2e39be62..af6a68cea80 100644 --- a/apps/extension/src/omnibox.ts +++ b/apps/extension/src/omnibox.ts @@ -53,9 +53,16 @@ function escapeXml(text: string): string { .replace(/[&<>"']/g, (character) => `&#${character.charCodeAt(0)};`) } -/** `encodeURIComponent`, except a lone surrogate (which it refuses) is dropped. */ +/** + * A typed value encoded for the path as the address bar would send it: raw characters are + * escaped, an escape already typed (`a%2Fb`) is kept rather than escaped again, and a lone + * surrogate (which `encodeURIComponent` refuses) is dropped. + */ function encodeSegment(segment: string): string { - return encodeURIComponent(segment.replace(LONE_SURROGATE, '')) + return encodeURIComponent(segment.replace(LONE_SURROGATE, '')).replace( + /%25([0-9A-Fa-f]{2})/g, + '%$1' + ) } /** diff --git a/apps/sim/app/o/[organizationId]/people/people-import-modal.tsx b/apps/sim/app/o/[organizationId]/people/people-import-modal.tsx index ac2f7ae4082..384a4daf808 100644 --- a/apps/sim/app/o/[organizationId]/people/people-import-modal.tsx +++ b/apps/sim/app/o/[organizationId]/people/people-import-modal.tsx @@ -14,76 +14,18 @@ import { useScrollEdges, } from '@sim/emcn' import { getErrorMessage } from '@sim/utils/errors' -import { PEOPLE_IMPORT_MAX_ROWS, type PeopleImportResult } from '@/lib/api/contracts/people' -import { assertOoxmlPreviewWithinLimits } from '@/lib/file-parsers/ooxml-preview-guard' +import type { PeopleImportResult } from '@/lib/api/contracts/people' import { saveBlob } from '@/lib/uploads/client/download' +import { + type PeopleSheet, + readPeopleSheet, +} from '@/app/o/[organizationId]/people/read-people-sheet' import { useImportPeople } from '@/hooks/queries/people' -const MAX_FILE_BYTES = 5 * 1024 * 1024 const ACCEPTED_FILES = '.csv,.tsv,.xlsx,.xls' const TEMPLATE_CSV = 'email,title,department,location,start_date,manager_email\nada@example.com,Engineer,Platform,Berlin,2024-01-31,grace@example.com\n' -interface PeopleSheet { - fileName: string - headers: string[] - /** Each row's cells, in `headers` order. */ - rows: string[][] -} - -/** - * The first sheet's named columns and its rows of cells, one per sheet row (blank ones included) so the - * row numbers in the preview match the spreadsheet. Delimited text is read verbatim so the server - * interprets dates the same way for every file. A spreadsheet date cell is read from its stored - * value, not the text its locale displays, so `31/01/2024` and `1/31/24` both arrive as - * `2024-01-31`. - */ -async function readPeopleSheet(file: File): Promise { - if (file.size > MAX_FILE_BYTES) throw new Error('Choose a file under 5 MB') - const data = new Uint8Array(await file.arrayBuffer()) - if (/\.xlsx$/i.test(file.name)) await assertOoxmlPreviewWithinLimits(data) - const XLSX = await import('xlsx') - const workbook = XLSX.read(data, { - type: 'array', - raw: /\.(csv|tsv)$/i.test(file.name), - dense: true, - }) - const sheet = workbook.Sheets[workbook.SheetNames[0] ?? ''] - if (!sheet?.['!ref']) throw new Error('The file has no rows') - const range = XLSX.utils.decode_range(sheet['!ref']) - const grid = sheet['!data'] ?? [] - const text = (row: number, column: number): string => { - const cell = grid[row]?.[column] - if (!cell || cell.v === undefined || cell.v === null) return '' - if (cell.t === 'n' && XLSX.SSF.is_date(cell.z ?? '')) { - const date = XLSX.SSF.parse_date_code(Number(cell.v)) - return `${date.y}-${String(date.m).padStart(2, '0')}-${String(date.d).padStart(2, '0')}` - } - return String(cell.w ?? cell.v).trim() - } - const columns: { header: string; index: number }[] = [] - for (let index = range.s.c; index <= range.e.c; index++) { - const header = text(range.s.r, index) - if (!header) continue - if (columns.some((column) => column.header === header)) { - throw new Error(`The ${header} column appears more than once`) - } - columns.push({ header, index }) - } - if (!columns.length) throw new Error('The first row should name the columns') - const rowCount = range.e.r - range.s.r - if (rowCount > PEOPLE_IMPORT_MAX_ROWS) { - throw new Error(`Import at most ${PEOPLE_IMPORT_MAX_ROWS} rows at a time`) - } - return { - fileName: file.name, - headers: columns.map(({ header }) => header), - rows: Array.from({ length: rowCount }, (_, offset) => - columns.map(({ index }) => text(range.s.r + 1 + offset, index)) - ), - } -} - function downloadTemplate() { saveBlob(new Blob([TEMPLATE_CSV], { type: 'text/csv' }), 'people-template.csv') } diff --git a/apps/sim/app/o/[organizationId]/people/read-people-sheet.test.ts b/apps/sim/app/o/[organizationId]/people/read-people-sheet.test.ts new file mode 100644 index 00000000000..2f4c5b5877e --- /dev/null +++ b/apps/sim/app/o/[organizationId]/people/read-people-sheet.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import * as XLSX from 'xlsx' +import { readPeopleSheet } from '@/app/o/[organizationId]/people/read-people-sheet' + +/** January 31, 2024 as a date serial in each of Excel's two date systems. */ +const SERIAL_1900 = 45322 +const SERIAL_1904 = SERIAL_1900 - 1462 + +function workbookFile(rows: unknown[][], { format = 'm/d/yy', date1904 = false } = {}): File { + const sheet = XLSX.utils.aoa_to_sheet(rows, { dense: false }) + for (const [address, cell] of Object.entries(sheet)) { + if (!address.startsWith('!') && (cell as XLSX.CellObject).t === 'n') { + ;(cell as XLSX.CellObject).z = format + } + } + const workbook = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(workbook, sheet, 'People') + workbook.Workbook = { WBProps: { date1904 } } + const bytes = XLSX.write(workbook, { type: 'array', bookType: 'xlsx' }) + return new File([bytes], 'people.xlsx') +} + +describe('readPeopleSheet', () => { + it.each(['m/d/yy', 'dd/mm/yyyy', 'yyyy-mm-dd'])( + 'reads a date cell formatted %s as its calendar date, not its displayed text', + async (format) => { + const sheet = await readPeopleSheet( + workbookFile( + [ + ['email', 'start date'], + ['ada@example.com', SERIAL_1900], + ], + { format } + ) + ) + expect(sheet.rows).toEqual([['ada@example.com', '2024-01-31']]) + } + ) + + it('reads dates from a workbook saved in the 1904 date system', async () => { + const sheet = await readPeopleSheet( + workbookFile( + [ + ['email', 'start date'], + ['ada@example.com', SERIAL_1904], + ], + { date1904: true } + ) + ) + expect(sheet.rows).toEqual([['ada@example.com', '2024-01-31']]) + }) + + it('passes an out-of-range date serial through as text for the server to refuse', async () => { + const sheet = await readPeopleSheet( + workbookFile([ + ['email', 'start date'], + ['ada@example.com', -5], + ]) + ) + expect(sheet.rows[0]?.[0]).toBe('ada@example.com') + expect(sheet.rows[0]?.[1]).not.toMatch(/^\d{4}-\d{2}-\d{2}$/) + }) + + it('keeps blank rows in place so issue row numbers match the sheet', async () => { + const sheet = await readPeopleSheet( + workbookFile([ + ['email', 'title'], + ['ada@example.com', 'Engineer'], + [], + ['bo@example.com', 'Lead'], + ]) + ) + expect(sheet.rows).toEqual([ + ['ada@example.com', 'Engineer'], + ['', ''], + ['bo@example.com', 'Lead'], + ]) + }) + + it('refuses a header row that repeats a column name', async () => { + await expect( + readPeopleSheet( + workbookFile([ + ['email', 'title', 'title'], + ['ada@example.com', 'a', 'b'], + ]) + ) + ).rejects.toThrow('The title column appears more than once') + }) +}) diff --git a/apps/sim/app/o/[organizationId]/people/read-people-sheet.ts b/apps/sim/app/o/[organizationId]/people/read-people-sheet.ts new file mode 100644 index 00000000000..0607514fe9c --- /dev/null +++ b/apps/sim/app/o/[organizationId]/people/read-people-sheet.ts @@ -0,0 +1,70 @@ +import { PEOPLE_IMPORT_MAX_ROWS } from '@/lib/api/contracts/people' +import { assertOoxmlPreviewWithinLimits } from '@/lib/file-parsers/ooxml-preview-guard' + +const MAX_FILE_BYTES = 5 * 1024 * 1024 + +export interface PeopleSheet { + fileName: string + headers: string[] + /** Each row's cells, in `headers` order. */ + rows: string[][] +} + +/** + * The first sheet's named columns and its rows of cells, one per sheet row (blank ones included) so the + * row numbers in the preview match the spreadsheet. Delimited text is read verbatim so the server + * interprets dates the same way for every file. A spreadsheet date cell is read from its stored + * value, not the text its locale displays, so `31/01/2024` and `1/31/24` both arrive as + * `2024-01-31`. + */ +export async function readPeopleSheet(file: File): Promise { + if (file.size > MAX_FILE_BYTES) throw new Error('Choose a file under 5 MB') + const data = new Uint8Array(await file.arrayBuffer()) + if (/\.xlsx$/i.test(file.name)) await assertOoxmlPreviewWithinLimits(data) + const XLSX = await import('xlsx') + const workbook = XLSX.read(data, { + type: 'array', + raw: /\.(csv|tsv)$/i.test(file.name), + dense: true, + // Keeps each cell's number format, which is how a date cell is told from a number. + cellNF: true, + }) + const sheet = workbook.Sheets[workbook.SheetNames[0] ?? ''] + // Workbooks saved in the 1904 date system count their date serials from a different epoch. + const dateSystem = { date1904: Boolean(workbook.Workbook?.WBProps?.date1904) } + if (!sheet?.['!ref']) throw new Error('The file has no rows') + const range = XLSX.utils.decode_range(sheet['!ref']) + const grid = sheet['!data'] ?? [] + const text = (row: number, column: number): string => { + const cell = grid[row]?.[column] + if (!cell || cell.v === undefined || cell.v === null) return '' + if (cell.t === 'n' && XLSX.SSF.is_date(cell.z ?? '')) { + // A serial outside Excel's calendar has no date; its text goes on for the server to refuse. + const date = XLSX.SSF.parse_date_code(Number(cell.v), dateSystem) + if (date) + return `${date.y}-${String(date.m).padStart(2, '0')}-${String(date.d).padStart(2, '0')}` + } + return String(cell.w ?? cell.v).trim() + } + const columns: { header: string; index: number }[] = [] + for (let index = range.s.c; index <= range.e.c; index++) { + const header = text(range.s.r, index) + if (!header) continue + if (columns.some((column) => column.header === header)) { + throw new Error(`The ${header} column appears more than once`) + } + columns.push({ header, index }) + } + if (!columns.length) throw new Error('The first row should name the columns') + const rowCount = range.e.r - range.s.r + if (rowCount > PEOPLE_IMPORT_MAX_ROWS) { + throw new Error(`Import at most ${PEOPLE_IMPORT_MAX_ROWS} rows at a time`) + } + return { + fileName: file.name, + headers: columns.map(({ header }) => header), + rows: Array.from({ length: rowCount }, (_, offset) => + columns.map(({ index }) => text(range.s.r + 1 + offset, index)) + ), + } +} diff --git a/apps/sim/lib/people/import.test.ts b/apps/sim/lib/people/import.test.ts index cf08baa2ff9..30a80856d74 100644 --- a/apps/sim/lib/people/import.test.ts +++ b/apps/sim/lib/people/import.test.ts @@ -65,6 +65,13 @@ describe('matchPeopleColumns', () => { }) }) + it('refuses two columns that name the same field, rather than ignoring one', () => { + expect(matchPeopleColumns(['Email', 'e-mail', 'Title'])).toEqual({ + ok: false, + message: 'The Email and e-mail columns name the same field; keep one', + }) + }) + it('refuses a file without an email column, which is how people are matched', () => { expect(matchPeopleColumns(['name', 'title'])).toEqual({ ok: false, @@ -215,6 +222,7 @@ describe('planPeopleImport', () => { ['2021/3/4', '2021-03-04'], ['3/4/2021', '2021-03-04'], ['2021-03-04T09:00:00Z', '2021-03-04'], + ['2021-03-04T09:00:00.123+02:00', '2021-03-04'], ['2021-03-04 09:00', '2021-03-04'], ])('reads the start date %s', (value, startDate) => { const result = plan( @@ -224,22 +232,30 @@ describe('planPeopleImport', () => { expect(result.changes).toEqual([{ memberId: 'm-ada', profile: profile({ startDate }) }]) }) - it.each(['2021-02-30', '13/45/2021', '04.03.2021', 'March 4', '1/2/21', '2021-03-04Tsoon'])( - 'skips the row with the unreadable start date %s instead of guessing', - (value) => { - const result = plan( - ['email', 'start date'], - [{ email: 'ada@example.com', 'start date': value }] - ) - expect(result.changes).toEqual([]) - expect(result.issues).toEqual([ - expect.objectContaining({ - severity: 'skipped', - message: 'Use a start date like 2024-01-31 or 1/31/2024', - }), - ]) - } - ) + it.each([ + '2021-02-30', + '13/45/2021', + '04.03.2021', + 'March 4', + '1/2/21', + '2021-03-04Tsoon', + '2021-03-04T99:99', + '2021-03-04T24:00', + '2021-03-04T09:00+99:99', + '2021-03/04', + ])('skips the row with the unreadable start date %s instead of guessing', (value) => { + const result = plan( + ['email', 'start date'], + [{ email: 'ada@example.com', 'start date': value }] + ) + expect(result.changes).toEqual([]) + expect(result.issues).toEqual([ + expect.objectContaining({ + severity: 'skipped', + message: 'Use a start date like 2024-01-31 or 1/31/2024', + }), + ]) + }) it('leaves the manager as it was when the manager is not a member, and says so', () => { const result = plan( diff --git a/apps/sim/lib/people/import.ts b/apps/sim/lib/people/import.ts index b04b3cd78ef..4763b9f64be 100644 --- a/apps/sim/lib/people/import.ts +++ b/apps/sim/lib/people/import.ts @@ -56,10 +56,35 @@ function findColumn(headers: readonly string[], aliases: readonly string[]): str return undefined } +const RECOGNIZED_KEYS = new Set([ + ...Object.values(COLUMN_ALIASES).flat(), + ...LOCATION_PARTS, +]) + +/** Two headers that spell the same recognized column differently (`Email`, `e-mail`), if any. */ +function repeatedColumn(headers: readonly string[]): [string, string] | null { + const seen = new Map() + for (const header of headers) { + const key = headerKey(header) + if (!RECOGNIZED_KEYS.has(key)) continue + const earlier = seen.get(key) + if (earlier !== undefined) return [earlier, header] + seen.set(key, header) + } + return null +} + /** Which header holds each field, or why the file can't be imported. */ export function matchPeopleColumns( headers: readonly string[] ): { ok: true; columns: PeopleColumns } | { ok: false; message: string } { + const repeated = repeatedColumn(headers) + if (repeated) { + return { + ok: false, + message: `The ${repeated[0]} and ${repeated[1]} columns name the same field; keep one`, + } + } const email = findColumn(headers, COLUMN_ALIASES.email) if (!email) { return { ok: false, message: 'Add an email column so each row can be matched to a member' } diff --git a/apps/sim/lib/people/validation.ts b/apps/sim/lib/people/validation.ts index f77596ea7c1..46735d13ae3 100644 --- a/apps/sim/lib/people/validation.ts +++ b/apps/sim/lib/people/validation.ts @@ -2,9 +2,12 @@ import { z } from 'zod' export const PROFILE_TEXT_MAX_LENGTH = 200 -/** A date, optionally with a time of day and offset, which are dropped. */ +/** + * A date with one separator throughout, optionally followed by a real time of day and offset, + * which are checked and then dropped. + */ const ISO_DATE = - /^(\d{4})[-/](\d{1,2})[-/](\d{1,2})(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/ + /^(\d{4})([-/])(\d{1,2})\2(\d{1,2})(?:[T ](?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|[+-](?:[01]\d|2[0-3]):?[0-5]\d)?)?$/ const US_DATE = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/ function calendarDate(year: number, month: number, day: number): string | null { @@ -22,7 +25,7 @@ function calendarDate(year: number, month: number, day: number): string | null { */ export function parseStartDate(value: string): string | null { const iso = ISO_DATE.exec(value.trim()) - if (iso) return calendarDate(Number(iso[1]), Number(iso[2]), Number(iso[3])) + if (iso) return calendarDate(Number(iso[1]), Number(iso[3]), Number(iso[4])) const us = US_DATE.exec(value.trim()) if (us) return calendarDate(Number(us[3]), Number(us[1]), Number(us[2])) return null diff --git a/apps/sim/lib/sim-links/application/use-cases.ts b/apps/sim/lib/sim-links/application/use-cases.ts index 6fc3ffafb2c..8b68bb2dbd0 100644 --- a/apps/sim/lib/sim-links/application/use-cases.ts +++ b/apps/sim/lib/sim-links/application/use-cases.ts @@ -20,11 +20,9 @@ import { requireSimLinksEnabled } from '@/lib/sim-links/feature-flag' import type { SimLinkAccess, SimLinkRecord, SimLinkViewer } from '@/lib/sim-links/repository' import * as repository from '@/lib/sim-links/repository' import { - normalizeSimLinkName, - parseTypedSimLink, type SimLinkFields, type SimLinkSort, - stripSimLinkPrefix, + typedSimLinkPrefix, } from '@/lib/sim-links/validation' import type { SortDirection } from '@/lib/url-state' @@ -255,12 +253,6 @@ export const resolveSimLink = defineAuthorizedOrganizationUseCase({ }, }) -/** The name in what was typed after the `sim` keyword: `sim/jira/ABC-1?x` suggests from `jira`. */ -function typedNamePrefix(query: string): string { - const [path = ''] = stripSimLinkPrefix(query.trim()).split('?') - return normalizeSimLinkName(parseTypedSimLink(path.split('/')).name) -} - /** * Address-bar suggestions from the organization `sim/` opens in for this session, so every * suggestion resolves where it is followed. Nothing is suggested where Sim links are off. @@ -269,7 +261,8 @@ export const suggestSimLinks = defineAuthorizedCredentialUserUseCase({ operation: simLinkOperations.suggest, async execute({ principal, input }: { principal: SessionPrincipal; input: { query: string } }) { const organizationId = await resolveSimLinkOrganization(principal) - if (!organizationId) return [] - return repository.suggestSimLinks(organizationId, typedNamePrefix(input.query)) + const prefix = typedSimLinkPrefix(input.query) + if (!organizationId || prefix === null) return [] + return repository.suggestSimLinks(organizationId, prefix) }, }) diff --git a/apps/sim/lib/sim-links/destination.test.ts b/apps/sim/lib/sim-links/destination.test.ts index b3727299100..0845473f4b8 100644 --- a/apps/sim/lib/sim-links/destination.test.ts +++ b/apps/sim/lib/sim-links/destination.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest' import { resolveSimLinkTarget } from '@/lib/sim-links/destination' -import { parseTypedSimLink, simLinkFieldsSchema } from '@/lib/sim-links/validation' +import { + parseTypedSimLink, + simLinkFieldsSchema, + typedSimLinkPrefix, +} from '@/lib/sim-links/validation' const fields = { name: 'docs', @@ -163,6 +167,19 @@ describe('Sim link destinations', () => { }) }) +describe('typedSimLinkPrefix', () => { + it.each([ + ['', ''], + ['sim/Team-Docs/ABC-1?x=1', 'teamdocs'], + ])('reads the name prefix of %j', (typed, prefix) => { + expect(typedSimLinkPrefix(typed)).toBe(prefix) + }) + + it.each(['...', '--', '.:-'])('has no prefix for %j, which no link name can match', (typed) => { + expect(typedSimLinkPrefix(typed)).toBeNull() + }) +}) + describe('parseTypedSimLink', () => { it.each([ [' onboarding', 'onboarding'], diff --git a/apps/sim/lib/sim-links/validation.ts b/apps/sim/lib/sim-links/validation.ts index 38ac06393cc..a7d4011b11c 100644 --- a/apps/sim/lib/sim-links/validation.ts +++ b/apps/sim/lib/sim-links/validation.ts @@ -20,7 +20,7 @@ export function normalizeSimLinkName(name: string): string { } /** Text as typed after `sim/`, whether or not the prefix itself was typed too. */ -export function stripSimLinkPrefix(input: string): string { +function stripSimLinkPrefix(input: string): string { return input.replace(/^sim\//i, '') } @@ -55,6 +55,18 @@ export function parseTypedSimLink(segments: readonly string[]): { return { name: name.replace(PASTED_TRAILING_PUNCTUATION, '').trimEnd() || name, values } } +/** + * The normalized name prefix to suggest links from, out of what was typed after the `sim` + * keyword: `sim/jira/ABC-1?x` suggests from `jira`, and nothing typed browses every link. `null` + * when a name was typed that no link can match, such as only punctuation. + */ +export function typedSimLinkPrefix(query: string): string | null { + const [path = ''] = stripSimLinkPrefix(query.trim()).split('?') + const { name } = parseTypedSimLink(path.split('/')) + const prefix = normalizeSimLinkName(name) + return name && !prefix ? null : prefix +} + export function isVariableSimLinkUrl(url: string): boolean { return url.includes(SIM_LINK_VARIABLE) }