From 826d7b1d1a366b47fcf18f492ca1b29db77054dd Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 12:12:05 +0700 Subject: [PATCH 1/2] feat(import): import rows as Update, Insert or Update, or Insert, Skip Existing --- .../SQLStatementGenerator.swift | 40 +++ .../Core/Plugins/ImportDataSinkAdapter.swift | 226 +++++++++++- .../Services/Export/ImportRowWriting.swift | 152 ++++++++ .../Core/Services/Export/ImportService.swift | 10 +- .../TablePro/Resources/Localizable.xcstrings | 63 ++++ .../Components/TransferResultAlert.swift | 16 +- .../Views/Import/RowImportMapping.swift | 4 + .../Views/Import/RowImportSheet.swift | 83 ++++- .../SQLStatementGeneratorImportTests.swift | 57 +++ ...ImportDataSinkAdapterRowWritingTests.swift | 328 ++++++++++++++++++ .../Export/ImportRowWritingTests.swift | 124 +++++++ .../Views/ImportSummaryTextTests.swift | 30 ++ .../RowImportUpdateUITests.swift | 66 ++++ 13 files changed, 1168 insertions(+), 31 deletions(-) create mode 100644 apple/TablePro/Core/Services/Export/ImportRowWriting.swift create mode 100644 apple/TableProTests/Core/Plugins/ImportDataSinkAdapterRowWritingTests.swift create mode 100644 apple/TableProTests/Core/Services/Export/ImportRowWritingTests.swift create mode 100644 apple/TableProTests/Views/ImportSummaryTextTests.swift create mode 100644 apple/TableProUITests/RowImportUpdateUITests.swift diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 8972df3fb5..51733abfa7 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -273,6 +273,46 @@ struct SQLStatementGenerator { return ParameterizedStatement(sql: sql, parameters: bindParameters) } + // The server compares, so its casting and collation decide a match, as they do for the UPDATE that follows. + func keyMatchCountStatement(keyColumns: [String], keys: [[PluginCellValue]]) -> ParameterizedStatement? { + guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } + + var bindParameters: [Any?] = [] + let quotedKey = keyColumns.map(quoteIdentifierFn) + let selects = keys.enumerated().map { position, values -> String in + let conditions = zip(quotedKey, values).map { column, value -> String in + bindParameters.append(value.asAny) + return "\(column) = \(placeholder(at: bindParameters.count - 1))" + }.joined(separator: " AND ") + return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" + } + + return ParameterizedStatement(sql: selects.joined(separator: " UNION ALL "), parameters: bindParameters) + } + + func keyedUpdateStatement( + columns setColumns: [String], + values: [PluginCellValue], + keyColumns: [String], + keyValues: [PluginCellValue] + ) -> ParameterizedStatement? { + guard !setColumns.isEmpty, setColumns.count == values.count, + !keyColumns.isEmpty, keyColumns.count == keyValues.count else { return nil } + + var bindParameters: [Any?] = [] + func bound(_ column: String, _ value: PluginCellValue) -> String { + bindParameters.append(value.asAny) + return "\(quoteIdentifierFn(column)) = \(placeholder(at: bindParameters.count - 1))" + } + let assignments = zip(setColumns, values).map(bound).joined(separator: ", ") + let conditions = zip(keyColumns, keyValues).map(bound).joined(separator: " AND ") + + return ParameterizedStatement( + sql: "UPDATE \(qualifiedTableName) SET \(assignments) WHERE \(conditions)", + parameters: bindParameters + ) + } + var maxBindParameters: Int { switch databaseType { case .sqlite: 32_766 diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 4134d7c08a..32c09276f7 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -36,6 +36,8 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let sourceFields: Set private let sourceFieldsPerFoldedName: [String: Int] private let rowGenerator: SQLStatementGenerator? + private let rowWriting: ImportRowWriting + private let tally: OSAllocatedUnfairLock /// Asked before every statement this sink sends, because one `insertRows` call is no longer one /// statement: a byte-heavy batch, or an Oracle target that takes one row per statement, splits @@ -78,9 +80,12 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { targetTable: String? = nil, columnMapping: [String: String] = [:], sourceFields: Set = [], + rowWriting: ImportRowWriting = .insert, isCancelled: @escaping @Sendable () -> Bool = { false } ) { self.isCancelled = isCancelled + self.rowWriting = rowWriting + self.tally = OSAllocatedUnfairLock(initialState: ImportRowTally(mode: rowWriting.mode)) self.driver = driver self.databaseType = databaseType self.grammar = databaseType.lexicalGrammar @@ -108,6 +113,10 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { try await execute(statement: statement, line: 1) } + var rowTally: ImportRowTally { + tally.withLock { $0 } + } + var transactionOutcome: ImportTransactionOutcome { transactionRecord.withLock { record in switch record.ending { @@ -184,20 +193,25 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for \(targetTable)") } - let (columns, bindValues) = mappedColumnsAndValues(values) - guard !columns.isEmpty else { + let row = mappedRow(values) + guard !row.columns.isEmpty else { guard values.isEmpty else { throw PluginImportError.importFailed(Self.unmappedRowMessage) } return } - guard let statement = rowGenerator.insertStatement(columns: columns, values: bindValues) else { + guard !rowWriting.mode.matchesExistingRows else { + try await writeMatching([row], generator: rowGenerator) + return + } + guard let statement = rowGenerator.insertStatement(columns: row.columns, values: row.values) else { throw PluginImportError.importFailed( String(format: String(localized: "Could not build an INSERT for %@"), targetTable) ) } _ = try await driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + tally.withLock { $0.inserted += 1 } } func insertRows(_ rows: [[String: PluginCellValue]]) async throws { @@ -208,29 +222,42 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") } - if rows.contains(where: { !$0.isEmpty && mappedColumnsAndValues($0).0.isEmpty }) { + let mapped = rows.map(mappedRow) + if zip(rows, mapped).contains(where: { !$0.0.isEmpty && $0.1.columns.isEmpty }) { throw PluginImportError.importFailed(Self.unmappedRowMessage) } + let writable = mapped.filter { !$0.columns.isEmpty } + + if rowWriting.mode.matchesExistingRows { + try await writeMatching(writable, generator: rowGenerator) + } else { + try await insertGrouped(writable, generator: rowGenerator) + } + } + private struct MappedRow { + let columns: [String] + let values: [PluginCellValue] + } + + private func mappedRow(_ values: [String: PluginCellValue]) -> MappedRow { + let (columns, mappedValues) = mappedColumnsAndValues(values) + return MappedRow(columns: columns, values: mappedValues) + } + + private func insertGrouped(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { var index = 0 while index < rows.count { - let (columns, values) = mappedColumnsAndValues(rows[index]) - guard !columns.isEmpty else { - index += 1 - continue - } - - var groupValues: [[PluginCellValue]] = [values] + let columns = rows[index].columns + var groupValues: [[PluginCellValue]] = [rows[index].values] var next = index + 1 - while next < rows.count { - let (nextColumns, nextValues) = mappedColumnsAndValues(rows[next]) - guard nextColumns == columns else { break } - groupValues.append(nextValues) + while next < rows.count, rows[next].columns == columns { + groupValues.append(rows[next].values) next += 1 } index = next - try await insertGroup(groupValues, columns: columns, generator: rowGenerator) + try await insertGroup(groupValues, columns: columns, generator: generator) } } @@ -269,6 +296,170 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { ) } _ = try await driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + tally.withLock { $0.inserted += batch.count } + } + + // MARK: - Matching existing rows + + private struct KeyedRow { + let row: MappedRow + let key: [PluginCellValue] + } + + // Not the engines' upsert forms: they refuse a NOT NULL column the file leaves out even when the row exists. + private func writeMatching(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { + var run: [KeyedRow] = [] + var keysInRun = Set<[String]>() + for row in rows { + let key = try keyValues(of: row) + // A repeated key has to see the write of its first occurrence, so the run ends before it. + let folded = key.contains(where: \.isNull) ? nil : key.map(Self.foldedKeyValue) + if let folded, keysInRun.contains(folded) { + try await writeMatchedRun(run, generator: generator) + run.removeAll(keepingCapacity: true) + keysInRun.removeAll(keepingCapacity: true) + } + if let folded { keysInRun.insert(folded) } + run.append(KeyedRow(row: row, key: key)) + } + try await writeMatchedRun(run, generator: generator) + } + + // Folds more than a collation or a numeric comparison does: two keys the database takes as one must not share a run. + static func foldedKeyValue(_ value: PluginCellValue) -> String { + switch value { + case .null: + return "null" + case .bytes(let data): + return "bytes:" + data.base64EncodedString() + case .text(let text): + let folded = text.trimmingCharacters(in: .whitespacesAndNewlines) + .folding(options: [.caseInsensitive, .diacriticInsensitive, .widthInsensitive], locale: nil) + // A Double, not a Decimal: a REAL column holds 0.1 and 0.100000000000000001 as one value. + guard folded.contains(where: { $0.isASCII && $0.isNumber }), + folded.allSatisfy({ numberCharacters.contains($0) }), + let number = Double(folded) else { + return "text:" + folded + } + return "number:\(number == 0 ? 0 : number)" + } + } + + private static let numberCharacters = Set("0123456789+-.e") + + private func keyValues(of row: MappedRow) throws -> [PluginCellValue] { + try rowWriting.keyColumns.map { column in + guard let index = row.columns.firstIndex(of: column) else { + throw PluginImportError.importFailed(String( + format: String(localized: "This row has no value for the key column %@, so it cannot be matched."), + column + )) + } + return row.values[index] + } + } + + private func writeMatchedRun(_ run: [KeyedRow], generator: SQLStatementGenerator) async throws { + guard !run.isEmpty else { return } + let counts = try await matchCounts(of: run.map(\.key), generator: generator) + if let ambiguous = zip(run, counts).first(where: { $0.1 > 1 }) { + throw PluginImportError.importFailed(String( + format: String(localized: "The key %1$@ matched %2$@ rows, so the row was not written."), + keyDescription(ambiguous.0.key), + ambiguous.1.formatted() + )) + } + + // Writes go out in file order, so a new parent row lands before an update that points at it. + var inserts: [MappedRow] = [] + for (entry, count) in zip(run, counts) { + switch (count, rowWriting.mode) { + case (0, .update): + tally.withLock { $0.unmatched += 1 } + case (0, _): + inserts.append(entry.row) + case (_, .update), (_, .upsert): + try await insertGrouped(inserts, generator: generator) + inserts.removeAll(keepingCapacity: true) + try await update(entry, generator: generator) + default: + tally.withLock { $0.existing += 1 } + } + } + try await insertGrouped(inserts, generator: generator) + } + + private func update(_ entry: KeyedRow, generator: SQLStatementGenerator) async throws { + let keyColumns = Set(rowWriting.keyColumns) + let assignments = zip(entry.row.columns, entry.row.values).filter { !keyColumns.contains($0.0) } + guard !assignments.isEmpty else { + tally.withLock { $0.existing += 1 } + return + } + guard !isCancelled() else { throw PluginImportCancellationError() } + guard let statement = generator.keyedUpdateStatement( + columns: assignments.map(\.0), + values: assignments.map(\.1), + keyColumns: rowWriting.keyColumns, + keyValues: entry.key + ) else { + throw PluginImportError.importFailed(String(localized: "Could not build an UPDATE for the mapped columns")) + } + _ = try await driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + tally.withLock { $0.updated += 1 } + } + + private static let keysPerProbe = 100 + + private func matchCounts(of keys: [[PluginCellValue]], generator: SQLStatementGenerator) async throws -> [Int] { + let keyWidth = max(1, rowWriting.keyColumns.count) + let perStatement = max(1, min(Self.keysPerProbe, generator.maxBindParameters / keyWidth)) + var counts: [Int] = [] + counts.reserveCapacity(keys.count) + var start = 0 + while start < keys.count { + let chunk = Array(keys[start.. [Int] { + var counts = [Int?](repeating: nil, count: expected) + for row in result.rows { + guard row.count >= 2, let position = integer(row[0]), counts.indices.contains(position), + let count = integer(row[1]) else { continue } + counts[position] = count + } + let read = counts.compactMap { $0 } + guard read.count == expected else { + throw PluginImportError.importFailed(String(localized: "Could not read which rows are already in the table")) + } + return read + } + + private static func integer(_ value: PluginCellValue) -> Int? { + guard let text = value.asText?.trimmingCharacters(in: .whitespaces) else { return nil } + return Int(text) ?? Double(text).flatMap { $0.rounded() == $0 ? Int(exactly: $0) : nil } + } + + private func keyDescription(_ key: [PluginCellValue]) -> String { + zip(rowWriting.keyColumns, key) + .map { "\($0.0) = \($0.1.isNull ? "NULL" : $0.1.sortKey)" } + .joined(separator: ", ") + } + + private static var matchingRefusesClearingMessage: String { + String(localized: """ + Delete existing rows before import empties the table, so no row would match. Turn it off, or import as \ + Insert. + """) } /// A row carrying values none of which reach a mapped column writes nothing. Reporting it as @@ -306,6 +497,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { guard targetTable != nil, let rowGenerator else { throw PluginImportError.importFailed("No target table configured for row import") } + guard !rowWriting.mode.matchesExistingRows else { + throw PluginImportError.importFailed(Self.matchingRefusesClearingMessage) + } _ = try await driver.execute(query: rowGenerator.deleteAllRowsStatement()) } diff --git a/apple/TablePro/Core/Services/Export/ImportRowWriting.swift b/apple/TablePro/Core/Services/Export/ImportRowWriting.swift new file mode 100644 index 0000000000..243f0d6192 --- /dev/null +++ b/apple/TablePro/Core/Services/Export/ImportRowWriting.swift @@ -0,0 +1,152 @@ +// +// ImportRowWriting.swift +// TablePro +// + +import Foundation + +internal enum ImportWriteMode: String, CaseIterable, Identifiable, Sendable { + case insert + case update + case upsert + case insertSkippingExisting + + internal var id: String { rawValue } + + internal var label: String { + switch self { + case .insert: return String(localized: "Insert") + case .update: return String(localized: "Update") + case .upsert: return String(localized: "Insert or Update") + case .insertSkippingExisting: return String(localized: "Insert, Skip Existing") + } + } + + internal var matchesExistingRows: Bool { + self != .insert + } + + internal var updatesMatchedRows: Bool { + self == .update || self == .upsert + } +} + +internal struct ImportMatchKey: Hashable, Identifiable, Sendable { + internal let indexName: String? + internal let columns: [String] + + internal var id: [String] { columns } + + internal var isPrimaryKey: Bool { indexName == nil } + + internal var label: String { + let list = columns.joined(separator: ", ") + guard let indexName else { + return String(format: String(localized: "Primary Key (%@)"), list) + } + return "\(indexName) (\(list))" + } + + // A partial or expression index leaves the plain columns free to repeat, so it cannot pick out one row. + internal static func options(columns: [ColumnInfo], indexes: [IndexInfo]) -> [ImportMatchKey] { + let exact = Set(columns.map(\.name)) + // A spelling two columns share once case is folded names neither: PostgreSQL keeps "ID" and "id" apart. + let folded = Dictionary(grouping: columns.map(\.name), by: { $0.lowercased() }) + .compactMapValues { $0.count == 1 ? $0.first : nil } + func tableColumns(_ indexColumns: [String]) -> [String]? { + let resolved = indexColumns.compactMap { exact.contains($0) ? $0 : folded[$0.lowercased()] } + return !resolved.isEmpty && resolved.count == indexColumns.count ? resolved : nil + } + + var keys: [ImportMatchKey] = [] + let flagged = columns.filter(\.isPrimaryKey).map(\.name) + if let primary = indexes.first(where: \.isPrimary).flatMap({ tableColumns($0.columns) }) + ?? (flagged.isEmpty ? nil : flagged) { + keys.append(ImportMatchKey(indexName: nil, columns: primary)) + } + for index in indexes where index.isUnique && !index.isPrimary && index.isValid && index.whereClause == nil { + guard (index.expressions ?? []).isEmpty, + let keyColumns = tableColumns(index.columns), + !keys.contains(where: { Set($0.columns) == Set(keyColumns) }) else { continue } + keys.append(ImportMatchKey(indexName: index.name, columns: keyColumns)) + } + return keys + } +} + +internal struct ImportRowWriting: Equatable, Sendable { + internal let mode: ImportWriteMode + internal let keyColumns: [String] + + internal static let insert = ImportRowWriting(mode: .insert, keyColumns: []) +} + +internal enum ImportRowWritingProblem: Equatable { + case noKey + case unmappedKeyColumns([String]) + case nothingToUpdate + + internal static func of( + mode: ImportWriteMode, + key: ImportMatchKey?, + mappedColumns: [String] + ) -> ImportRowWritingProblem? { + guard mode.matchesExistingRows else { return nil } + guard let key else { return .noKey } + let mapped = Set(mappedColumns) + let missing = key.columns.filter { !mapped.contains($0) } + guard missing.isEmpty else { return .unmappedKeyColumns(missing) } + if mode.updatesMatchedRows, mapped.subtracting(key.columns).isEmpty { + return .nothingToUpdate + } + return nil + } + + internal var message: String { + switch self { + case .noKey: + return String(localized: "This table has no primary key or unique index to match rows on.") + case .unmappedKeyColumns(let columns): + return String( + format: String(localized: "Map a field to %@ to match rows on it."), + columns.joined(separator: ", ") + ) + case .nothingToUpdate: + return String(localized: "Map a field to a column outside the key to have something to update.") + } + } +} + +// Counted per row as it lands, so a row the database refused is in none of them. +internal struct ImportRowTally: Equatable, Sendable { + internal let mode: ImportWriteMode + internal var inserted = 0 + internal var updated = 0 + // Matched no row, so Update left it out. + internal var unmatched = 0 + // Matched a row left as it was: Insert, Skip Existing, or a row with no value outside the key. + internal var existing = 0 + + internal init(mode: ImportWriteMode) { + self.mode = mode + } + + internal var matched: Int { updated + existing } + internal var skipped: Int { unmatched + existing } + + internal func summaryLines(failed: Int) -> [String] { + var lines = [String(format: String(localized: "Inserted: %@"), inserted.formatted())] + if mode.matchesExistingRows { + lines = [ + String(format: String(localized: "Matched: %@"), matched.formatted()), + String(format: String(localized: "Updated: %@"), updated.formatted()), + String(format: String(localized: "Inserted: %@"), inserted.formatted()), + String(format: String(localized: "Skipped: %@"), skipped.formatted()), + ] + } + if failed > 0 { + lines.append(String(format: String(localized: "Failed: %@"), failed.formatted())) + } + return lines + } +} diff --git a/apple/TablePro/Core/Services/Export/ImportService.swift b/apple/TablePro/Core/Services/Export/ImportService.swift index 501436f233..c85f8e9739 100644 --- a/apple/TablePro/Core/Services/Export/ImportService.swift +++ b/apple/TablePro/Core/Services/Export/ImportService.swift @@ -19,6 +19,7 @@ struct ImportState { var statusMessage: String = "" var errorMessage: String? var transactionOutcome: ImportTransactionOutcome = .untracked + var rowTally: ImportRowTally? } // MARK: - Import Service @@ -67,6 +68,7 @@ final class ImportService: ObservableObject { targetTable: String? = nil, columnMapping: [String: String] = [:], sourceFields: Set = [], + rowWriting: ImportRowWriting = .insert, authorization: OperationReceipt? = nil ) async throws -> PluginImportResult { guard let plugin = PluginManager.shared.importPlugin(forFormat: formatId) else { @@ -153,7 +155,8 @@ final class ImportService: ObservableObject { progress: progress, targetTable: targetTable, columnMapping: columnMapping, - sourceFields: sourceFields + sourceFields: sourceFields, + rowWriting: rowWriting ) } } catch { @@ -244,7 +247,8 @@ final class ImportService: ObservableObject { progress: PluginImportProgress, targetTable: String?, columnMapping: [String: String], - sourceFields: Set + sourceFields: Set, + rowWriting: ImportRowWriting ) async throws -> PluginImportResult { let sink = ImportDataSinkAdapter( driver: driver, @@ -252,11 +256,13 @@ final class ImportService: ObservableObject { targetTable: targetTable, columnMapping: columnMapping, sourceFields: sourceFields, + rowWriting: rowWriting, isCancelled: { progress.isCancelled } ) defer { reportNameHazards(sink.nameHazardStatements) state.transactionOutcome = sink.transactionOutcome + state.rowTally = targetTable == nil ? nil : sink.rowTally } let sessionScope = await openPreservingScope(on: driver) let result: PluginImportResult diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index c260c5ff27..0e47b9bf5f 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -51309,6 +51309,9 @@ } } } + }, + "Could not build an UPDATE for the mapped columns" : { + }, "Could not build the %1$@ endpoint for the AWS region \"%2$@\"." : { "extractionState" : "manual", @@ -51371,6 +51374,9 @@ } } } + }, + "Could not build the query that matches rows" : { + }, "Could not build the table rebuild for this change. If this engine's driver was installed before foreign key editing shipped, update it in Settings > Plugins." : { "localizations" : { @@ -52885,6 +52891,9 @@ } } } + }, + "Could not read which rows are already in the table" : { + }, "Could not release the file" : { "localizations" : { @@ -60824,6 +60833,9 @@ } } } + }, + "Delete existing rows before import empties the table, so no row would match. Turn it off, or import as Insert." : { + }, "Delete existing rows first" : { "localizations" : { @@ -89570,6 +89582,12 @@ } } } + }, + "Import as" : { + + }, + "Import as:" : { + }, "Import Commands" : { "localizations" : { @@ -92951,6 +92969,9 @@ "INSERT only sets cells. Leave %@ out, or delete it with PUT." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Insert or Update" : { + }, "Insert Row" : { "localizations" : { @@ -93289,6 +93310,9 @@ } } } + }, + "Insert, Skip Existing" : { + }, "INSERT, UPDATE, or DELETE" : { "localizations" : { @@ -93371,6 +93395,9 @@ } } } + }, + "Inserted: %@" : { + }, "inserting %d row(s)" : { "localizations" : { @@ -104853,6 +104880,12 @@ } } } + }, + "Map a field to %@ to match rows on it." : { + + }, + "Map a field to a column outside the key to have something to update." : { + }, "Map of %1$d shapes from %2$d rows." : { "localizations" : { @@ -105252,6 +105285,12 @@ } } } + }, + "Match on" : { + + }, + "Match on:" : { + }, "Match System" : { "comment" : "Text for a sidebar row size preference that matches the system's sidebar row size.", @@ -105400,6 +105439,9 @@ } } } + }, + "Matched: %@" : { + }, "Matches “%@”" : { @@ -134546,6 +134588,9 @@ } } } + }, + "Primary Key (%@)" : { + }, "Primary key change" : { "localizations" : { @@ -165642,6 +165687,9 @@ } } } + }, + "Skipped: %@" : { + }, "slash" : { "extractionState" : "manual", @@ -184339,6 +184387,9 @@ } } } + }, + "The key %1$@ matched %2$@ rows, so the row was not written." : { + }, "The key that protects your save history is not available." : { "localizations" : { @@ -195573,6 +195624,9 @@ } } } + }, + "This row has no value for the key column %@, so it cannot be matched." : { + }, "This row has no value for this field." : { @@ -196214,6 +196268,9 @@ } } } + }, + "This table has no primary key or unique index to match rows on." : { + }, "This table has no primary key, and %@ cannot be compared to find the row." : { "extractionState" : "manual", @@ -203763,6 +203820,9 @@ } } } + }, + "Update changes the rows whose key is in the file. Insert or Update also adds the rest, and Insert, Skip Existing adds only rows whose key is not in the table. Columns not mapped keep their values." : { + }, "Update existing" : { "extractionState" : "manual", @@ -204190,6 +204250,9 @@ } } } + }, + "Updated: %@" : { + }, "Updates when TablePro restarts" : { diff --git a/apple/TablePro/Views/Components/TransferResultAlert.swift b/apple/TablePro/Views/Components/TransferResultAlert.swift index 8d3a82c75c..7f3a3d7eaf 100644 --- a/apple/TablePro/Views/Components/TransferResultAlert.swift +++ b/apple/TablePro/Views/Components/TransferResultAlert.swift @@ -135,6 +135,7 @@ internal enum TransferResultAlert { internal static func presentImportSuccess( result: PluginImportResult?, + rowTally: ImportRowTally? = nil, window: NSWindow?, sourceFileName: String = "", targetTable: String? = nil, @@ -146,7 +147,7 @@ internal enum TransferResultAlert { ? String(localized: "Import completed with errors") : String(localized: "Import completed") alert.alertStyle = skipped > 0 ? .warning : .informational - alert.informativeText = importSummary(result) + alert.informativeText = importSummary(result, rowTally: rowTally) AlertHelper.addCancelButton(to: alert, title: String(localized: "Done")) let errors = result?.errors ?? [] @@ -305,8 +306,15 @@ internal enum TransferResultAlert { .joined(separator: "\n\n") } - private static func importSummary(_ result: PluginImportResult?) -> String { + internal static func importSummary(_ result: PluginImportResult?, rowTally: ImportRowTally? = nil) -> String { guard let result else { return "" } + let seconds = String( + format: String(localized: "%@ seconds"), + String(format: "%.2f", result.executionTime) + ) + if let rowTally { + return (rowTally.summaryLines(failed: result.skippedStatements) + [seconds]).joined(separator: "\n") + } let counts = result.skippedStatements > 0 ? String( format: String(localized: "%1$lld statements executed, %2$lld failed"), @@ -317,10 +325,6 @@ internal enum TransferResultAlert { format: String(localized: "%lld statements executed"), Int64(result.executedStatements) ) - let seconds = String( - format: String(localized: "%@ seconds"), - String(format: "%.2f", result.executionTime) - ) return "\(counts)\n\(seconds)" } } diff --git a/apple/TablePro/Views/Import/RowImportMapping.swift b/apple/TablePro/Views/Import/RowImportMapping.swift index 9ea8630370..370c78e588 100644 --- a/apple/TablePro/Views/Import/RowImportMapping.swift +++ b/apple/TablePro/Views/Import/RowImportMapping.swift @@ -18,6 +18,7 @@ internal final class RowImportMapping: ObservableObject { } @Published internal private(set) var columns: [String] = [] + @Published internal private(set) var matchKeys: [ImportMatchKey] = [] @Published internal var rows: [Row] = [] @Published internal private(set) var table: TableScope? @@ -72,6 +73,7 @@ internal final class RowImportMapping: ObservableObject { internal func clear() { table = nil columns = [] + matchKeys = [] rows = [] saved = [:] showsSavedMapping = false @@ -82,6 +84,7 @@ internal final class RowImportMapping: ObservableObject { internal func load( fields: [PluginImportField], columns: [String], + matchKeys: [ImportMatchKey] = [], for scope: TableScope, read: AnyHashable? = nil ) { @@ -95,6 +98,7 @@ internal final class RowImportMapping: ObservableObject { table = scope saved = remembered self.columns = columns + self.matchKeys = matchKeys let names = fields.map(\.name) apply(ImportColumnMatcher.resolve(fields: names, columns: columns, preferring: tiers), to: fields) showsSavedMapping = (reloading ? showsSavedMapping : true) && appliesSavedChoice diff --git a/apple/TablePro/Views/Import/RowImportSheet.swift b/apple/TablePro/Views/Import/RowImportSheet.swift index 881591ffae..62b7171711 100644 --- a/apple/TablePro/Views/Import/RowImportSheet.swift +++ b/apple/TablePro/Views/Import/RowImportSheet.swift @@ -47,9 +47,12 @@ struct RowImportSheet: View { let columns: [String] let columnMapping: [String: String] let newTable: PluginCreateTableDefinition? + let rowWriting: ImportRowWriting } @State private var destination: Destination = .existingTable + @State private var writeMode: ImportWriteMode = .insert + @State private var chosenMatchKey: ImportMatchKey? /// The database and schema every read and write of this sheet goes to: where the connection was /// browsing when the sheet opened. The sheet is modal to its window, but another window on the @@ -105,6 +108,7 @@ struct RowImportSheet: View { @State private var importService: ImportService? @State private var importResult: PluginImportResult? + @State private var importTally: ImportRowTally? @State private var importedRows: DatabaseObjectChange? @State private var importError: (any Error)? @State private var showProgressDialog = false @@ -227,6 +231,7 @@ struct RowImportSheet: View { guard isShowing else { return } TransferResultAlert.presentImportSuccess( result: importResult, + rowTally: importTally, window: hostWindow, sourceFileName: sourceName, targetTable: selectedTargetTable @@ -293,6 +298,36 @@ struct RowImportSheet: View { .labelsHidden() .frame(maxWidth: 280, alignment: .leading) } + + GridRow { + Text("Import as:") + Picker(String(localized: "Import as"), selection: $writeMode) { + ForEach(ImportWriteMode.allCases) { mode in + Text(mode.label).tag(mode) + } + } + .labelsHidden() + .fixedSize() + .help(String(localized: """ + Update changes the rows whose key is in the file. Insert or Update also adds the rest, and \ + Insert, Skip Existing adds only rows whose key is not in the table. Columns not mapped keep \ + their values. + """)) + } + + if writeMode.matchesExistingRows { + GridRow { + Text("Match on:") + Picker(String(localized: "Match on"), selection: matchKeyBinding) { + ForEach(mapping.matchKeys) { key in + Text(key.label).tag(ImportMatchKey?.some(key)) + } + } + .labelsHidden() + .frame(maxWidth: 280, alignment: .leading) + .disabled(mapping.matchKeys.isEmpty) + } + } } else { GridRow { Text("New table:") @@ -621,6 +656,22 @@ struct RowImportSheet: View { ) } + private var matchKey: ImportMatchKey? { + chosenMatchKey.flatMap { chosen in mapping.matchKeys.first { $0 == chosen } } ?? mapping.matchKeys.first + } + + private var matchKeyBinding: Binding { + Binding( + get: { matchKey }, + set: { chosenMatchKey = $0 } + ) + } + + private var rowWriting: ImportRowWriting { + guard writeMode.matchesExistingRows, let matchKey else { return .insert } + return ImportRowWriting(mode: writeMode, keyColumns: matchKey.columns) + } + private var allColumnsIncluded: Binding { Binding( get: { newTable.includesEveryColumn }, @@ -634,7 +685,12 @@ struct RowImportSheet: View { if mapping.mapsOneColumnTwice { return String(localized: "Each column can be mapped from only one field.") } - return nil + guard selectedTargetTable != nil, currentReadHasLanded else { return nil } + return ImportRowWritingProblem.of( + mode: writeMode, + key: matchKey, + mappedColumns: Array(mapping.columnMapping.values) + )?.message case .newTable: if let problem = newTableNameProblem { switch problem { @@ -861,13 +917,22 @@ struct RowImportSheet: View { isLoadingContext = true loadError = nil do { - let columns = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in - try await driver.fetchColumns(table: table) - }.map(\.name) + let (columns, indexes) = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in + (try await driver.fetchColumns(table: table), try? await driver.fetchIndexes(table: table)) + } guard !Task.isCancelled else { return } + if indexes == nil { + Self.logger.warning("Could not read the target's indexes, so only its primary key can be matched on") + } let fields = try await ImportFieldDetection.detectFields(plugin: plugin, at: fileURL, targetTable: table) guard !Task.isCancelled else { return } - mapping.load(fields: fields, columns: columns, for: TableScope(table: table, in: scope), read: request) + mapping.load( + fields: fields, + columns: columns.map(\.name), + matchKeys: ImportMatchKey.options(columns: columns, indexes: indexes ?? []), + for: TableScope(table: table, in: scope), + read: request + ) } catch { guard !Task.isCancelled else { return } loadError = error.localizedDescription @@ -899,7 +964,8 @@ struct RowImportSheet: View { fields: mapping.fields, columns: mapping.columns, columnMapping: mapping.columnMapping, - newTable: nil + newTable: nil, + rowWriting: rowWriting ), scope: scope ) @@ -918,7 +984,8 @@ struct RowImportSheet: View { fields: fields, columns: fields.compactMap { columnMapping[$0] }, columnMapping: columnMapping, - newTable: definition + newTable: definition, + rowWriting: .insert ), scope: scope ) @@ -959,12 +1026,14 @@ struct RowImportSheet: View { targetTable: targetTable, columnMapping: plan.columnMapping, sourceFields: Set(plan.fields), + rowWriting: plan.rowWriting, authorization: authorization ) await MainActor.run { showProgressDialog = false importSucceeded = true importResult = result + importTally = service.state.rowTally importedRows = DatabaseObjectChange( connectionId: connection.id, scope: scope, diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift index 2b2bf3df07..6945aab201 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift @@ -102,6 +102,63 @@ struct SQLStatementGeneratorImportTests { #expect(try makeGenerator(databaseType: .mysql).maxBindParameters == 65_535) } + @Test("keyMatchCountStatement asks one count per key, tagged with its position") + func testKeyMatchCountStatement() throws { + let stmt = try #require(try makeGenerator(databaseType: .postgresql).keyMatchCountStatement( + keyColumns: ["tenant", "id"], + keys: [[.text("t1"), .text("1")], [.text("t1"), .null]] + )) + #expect(stmt.sql == """ + SELECT 0, COUNT(*) FROM "users" WHERE "tenant" = $1 AND "id" = $2 \ + UNION ALL SELECT 1, COUNT(*) FROM "users" WHERE "tenant" = $3 AND "id" = $4 + """) + #expect(stmt.parameters.count == 4) + #expect(stmt.parameters[1] as? String == "1") + #expect(stmt.parameters[3] == nil) + } + + @Test("keyMatchCountStatement returns nil for no keys or a key of the wrong width") + func testKeyMatchCountStatementGuards() throws { + let generator = try makeGenerator() + #expect(generator.keyMatchCountStatement(keyColumns: ["id"], keys: []) == nil) + #expect(generator.keyMatchCountStatement(keyColumns: [], keys: [[]]) == nil) + #expect(generator.keyMatchCountStatement(keyColumns: ["id"], keys: [["1", "2"]]) == nil) + } + + @Test("keyedUpdateStatement sets the given columns and matches on the key") + func testKeyedUpdateStatement() throws { + let stmt = try #require(try makeGenerator().keyedUpdateStatement( + columns: ["name", "note"], + values: [.text("Ada"), .null], + keyColumns: ["id"], + keyValues: ["7"] + )) + #expect(stmt.sql == "UPDATE `users` SET `name` = ?, `note` = ? WHERE `id` = ?") + #expect(stmt.parameters.count == 3) + #expect(stmt.parameters[0] as? String == "Ada") + #expect(stmt.parameters[1] == nil) + #expect(stmt.parameters[2] as? String == "7") + } + + @Test("keyedUpdateStatement numbers PostgreSQL placeholders across SET and WHERE") + func testKeyedUpdateStatementPostgres() throws { + let stmt = try #require(try makeGenerator(databaseType: .postgresql).keyedUpdateStatement( + columns: ["name"], + values: [.text("Ada")], + keyColumns: ["tenant", "id"], + keyValues: [.text("t1"), "7"] + )) + #expect(stmt.sql == "UPDATE \"users\" SET \"name\" = $1 WHERE \"tenant\" = $2 AND \"id\" = $3") + } + + @Test("keyedUpdateStatement returns nil with nothing to set or no key") + func testKeyedUpdateStatementGuards() throws { + let generator = try makeGenerator() + #expect(generator.keyedUpdateStatement(columns: [], values: [], keyColumns: ["id"], keyValues: ["1"]) == nil) + #expect(generator.keyedUpdateStatement(columns: ["a"], values: ["1"], keyColumns: [], keyValues: []) == nil) + #expect(generator.keyedUpdateStatement(columns: ["a"], values: [], keyColumns: ["id"], keyValues: ["1"]) == nil) + } + @Test("deleteAllRowsStatement quotes the table identifier per dialect") func testDeleteAllRows() throws { #expect(try makeGenerator(databaseType: .mysql).deleteAllRowsStatement() == "DELETE FROM `users`") diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterRowWritingTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterRowWritingTests.swift new file mode 100644 index 0000000000..132800d437 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterRowWritingTests.swift @@ -0,0 +1,328 @@ +// +// ImportDataSinkAdapterRowWritingTests.swift +// TableProTests +// + +import Foundation +import SQLite3 +@testable import TablePro +import TableProPluginKit +import Testing + +private struct SQLiteTestError: Error { + let message: String +} + +private final class InMemorySQLiteDriver: PluginDriverStub, @unchecked Sendable { + private var handle: OpaquePointer? + private(set) var statements: [String] = [] + + init(_ setup: String) throws { + guard sqlite3_open(":memory:", &handle) == SQLITE_OK else { throw SQLiteTestError(message: "open") } + _ = try run(setup, parameters: []) + } + + deinit { + sqlite3_close_v2(handle) + } + + func execute(query: String) async throws -> PluginQueryResult { + try run(query, parameters: []) + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try run(query, parameters: parameters) + } + + func rows(_ sql: String) throws -> [[String?]] { + try run(sql, parameters: []).rows.map { row in row.map(\.asText) } + } + + private func run(_ sql: String, parameters: [PluginCellValue]) throws -> PluginQueryResult { + statements.append(sql) + var rows: [[PluginCellValue]] = [] + var remaining = sql + while !remaining.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + var statement: OpaquePointer? + var tail: UnsafePointer? + let prepared = remaining.withCString { text -> Int32 in + let code = sqlite3_prepare_v2(handle, text, -1, &statement, &tail) + remaining = tail.map { String(cString: $0) } ?? "" + return code + } + guard prepared == SQLITE_OK else { throw SQLiteTestError(message: String(cString: sqlite3_errmsg(handle))) } + guard let statement else { continue } + defer { sqlite3_finalize(statement) } + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + for (offset, value) in parameters.enumerated() { + let position = Int32(offset + 1) + switch value { + case .null: sqlite3_bind_null(statement, position) + case .text(let text): sqlite3_bind_text(statement, position, text, -1, transient) + case .bytes(let data): + _ = data.withUnsafeBytes { sqlite3_bind_blob(statement, position, $0.baseAddress, Int32(data.count), transient) } + } + } + var code = sqlite3_step(statement) + while code == SQLITE_ROW { + rows.append((0.. ImportDataSinkAdapter { + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .sqlite), pluginDriver: driver) + return ImportDataSinkAdapter( + driver: adapter, + databaseType: .sqlite, + targetTable: "people", + columnMapping: mapping, + sourceFields: Set(mapping.keys), + rowWriting: mode == .insert ? .insert : ImportRowWriting(mode: mode, keyColumns: key) + ) + } + + private func people(_ driver: InMemorySQLiteDriver) throws -> [[String?]] { + try driver.rows("SELECT id, name, note FROM people ORDER BY id") + } + + private static let changedAndNew: [[String: PluginCellValue]] = [ + ["id": .text("2"), "name": .text("b2")], + ["id": .text("4"), "name": .text("d")], + ] + + @Test("Update changes the mapped columns of matched rows and leaves new rows out") + func updateWritesMatchedRowsOnly() async throws { + let driver = try InMemorySQLiteDriver(""" + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT NOT NULL); + INSERT INTO people VALUES (1, 'a', 'n1'), (2, 'b', 'n2'), (3, 'c', 'n3'); + """) + let sink = sink(on: driver, mode: .update) + + try await sink.insertRows(Self.changedAndNew) + + #expect(try people(driver) == [["1", "a", "n1"], ["2", "b2", "n2"], ["3", "c", "n3"]]) + var expected = ImportRowTally(mode: .update) + expected.updated = 1 + expected.unmatched = 1 + #expect(sink.rowTally == expected) + } + + @Test("Insert or Update updates matched rows and inserts the rest") + func upsertUpdatesAndInserts() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .upsert) + + try await sink.insertRows(Self.changedAndNew) + + #expect(try people(driver) == [["1", "a", "n1"], ["2", "b2", "n2"], ["3", "c", "n3"], ["4", "d", "none"]]) + #expect(sink.rowTally.updated == 1) + #expect(sink.rowTally.inserted == 1) + #expect(sink.rowTally.matched == 1) + #expect(sink.rowTally.skipped == 0) + } + + @Test("Insert, Skip Existing inserts only the rows whose key is new") + func skipExistingLeavesMatchedRowsAlone() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .insertSkippingExisting) + + try await sink.insertRows(Self.changedAndNew) + + #expect(try people(driver) == [["1", "a", "n1"], ["2", "b", "n2"], ["3", "c", "n3"], ["4", "d", "none"]]) + #expect(sink.rowTally.existing == 1) + #expect(sink.rowTally.inserted == 1) + #expect(sink.rowTally.updated == 0) + } + + @Test("A key repeated in one batch updates the row its first occurrence inserted") + func repeatedKeyInBatchSeesEarlierWrite() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .upsert) + + try await sink.insertRows([ + ["id": .text("5"), "name": .text("x")], + ["id": .text("5"), "name": .text("y")], + ]) + + #expect(try driver.rows("SELECT name FROM people WHERE id = 5") == [["y"]]) + #expect(sink.rowTally.inserted == 1) + #expect(sink.rowTally.updated == 1) + } + + @Test("Keys the database takes as one still see each other's write within a batch") + func equivalentKeysInBatchSeeEarlierWrite() async throws { + let driver = try InMemorySQLiteDriver(""" + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT NOT NULL DEFAULT 'none'); + CREATE TABLE codes (code TEXT NOT NULL UNIQUE COLLATE NOCASE, note TEXT); + """) + let upsert = sink(on: driver, mode: .upsert) + try await upsert.insertRows([ + ["id": .text("7"), "name": .text("x")], + ["id": .text("07"), "name": .text("y")], + ]) + #expect(try driver.rows("SELECT id, name FROM people") == [["7", "y"]]) + + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .sqlite), pluginDriver: driver) + let skip = ImportDataSinkAdapter( + driver: adapter, + databaseType: .sqlite, + targetTable: "codes", + columnMapping: ["code": "code", "note": "note"], + sourceFields: ["code", "note"], + rowWriting: ImportRowWriting(mode: .insertSkippingExisting, keyColumns: ["code"]) + ) + try await skip.insertRows([ + ["code": .text("a"), "note": .text("first")], + ["code": .text("A"), "note": .text("second")], + ]) + #expect(try driver.rows("SELECT code, note FROM codes") == [["a", "first"]]) + #expect(skip.rowTally.inserted == 1) + #expect(skip.rowTally.existing == 1) + } + + @Test("A new row is written before a later update that refers to it") + func writesKeepFileOrder() async throws { + let driver = try InMemorySQLiteDriver(""" + PRAGMA foreign_keys = ON; + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT REFERENCES people(id)); + INSERT INTO people VALUES (1, 'a', NULL), (2, 'b', 1); + """) + let sink = sink(on: driver, mode: .upsert, mapping: ["id": "id", "name": "name", "note": "note"]) + + try await sink.insertRows([ + ["id": .text("4"), "name": .text("d")], + ["id": .text("2"), "note": .text("4")], + ]) + + #expect(try people(driver) == [["1", "a", nil], ["2", "b", "4"], ["4", "d", nil]]) + } + + @Test("Key folding joins the spellings a collation or a number comparison treats as equal") + func keyFolding() { + func folded(_ text: String) -> String { ImportDataSinkAdapter.foldedKeyValue(.text(text)) } + #expect(folded("01") == folded("1")) + #expect(folded("1.0") == folded("1")) + #expect(folded("0.1") == folded("0.100000000000000001")) + #expect(folded("-0") == folded("0")) + #expect(folded(" A ") == folded("a")) + #expect(folded("é") == folded("e")) + #expect(folded("１") == folded("1")) + #expect(folded("12abc") != folded("12")) + #expect(folded("a") != folded("b")) + } + + @Test("Matches across several probe statements land on the right rows") + func manyKeysSpanSeveralProbes() async throws { + let driver = try InMemorySQLiteDriver(""" + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT NOT NULL DEFAULT 'none'); + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 2 FROM n WHERE i < 249) + INSERT INTO people (id, name) SELECT i, 'old' FROM n; + """) + let sink = sink(on: driver, mode: .upsert) + + try await sink.insertRows((1...250).map { ["id": .text(String($0)), "name": .text("new")] }) + + #expect(try driver.rows("SELECT COUNT(*), SUM(name = 'new') FROM people") == [["250", "250"]]) + #expect(sink.rowTally.updated == 125) + #expect(sink.rowTally.inserted == 125) + } + + @Test("A key that matches more than one row is refused and nothing is written") + func ambiguousKeyIsRefused() async throws { + let driver = try InMemorySQLiteDriver(""" + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT NOT NULL DEFAULT 'none'); + INSERT INTO people VALUES (1, 'dup', 'n1'), (2, 'dup', 'n2'); + """) + let sink = sink(on: driver, mode: .update, key: ["name"], mapping: ["name": "name", "note": "note"]) + + await #expect(throws: PluginImportError.self) { + try await sink.insertRow(["name": .text("dup"), "note": .text("changed")]) + } + #expect(try driver.rows("SELECT note FROM people ORDER BY id") == [["n1"], ["n2"]]) + #expect(sink.rowTally == ImportRowTally(mode: .update)) + } + + @Test("A row with no value for the key is refused") + func rowWithoutKeyIsRefused() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .update) + + await #expect(throws: PluginImportError.self) { + try await sink.insertRow(["name": .text("nobody")]) + } + } + + @Test("A matching mode refuses to empty the table first") + func matchingModeRefusesClearing() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .upsert) + + await #expect(throws: PluginImportError.self) { + try await sink.deleteAllRowsFromTargetTable() + } + #expect(try people(driver).count == 3) + } + + @Test("Insert counts the rows it wrote") + func insertTalliesRows() async throws { + let driver = try InMemorySQLiteDriver(Self.people) + let sink = sink(on: driver, mode: .insert) + + try await sink.insertRows([["id": .text("7"), "name": .text("g")], ["id": .text("8"), "name": .text("h")]]) + try await sink.insertRow(["id": .text("9"), "name": .text("i")]) + + #expect(sink.rowTally.inserted == 3) + #expect(try people(driver).count == 6) + } + + @Test("Skip and Continue records an ambiguous key against its line and writes the other rows") + func skipAndContinueRecordsAmbiguousKey() async throws { + let driver = try InMemorySQLiteDriver(""" + CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL, note TEXT NOT NULL DEFAULT 'none'); + INSERT INTO people VALUES (1, 'dup', 'n1'), (2, 'dup', 'n2'), (3, 'solo', 'n3'); + """) + let sink = sink(on: driver, mode: .upsert, key: ["name"], mapping: ["name": "name", "note": "note"]) + var batches: [[RowImportRunner.Entry]] = [[ + (line: 2, row: ["name": .text("dup"), "note": .text("x")]), + (line: 3, row: ["name": .text("solo"), "note": .text("updated")]), + (line: 4, row: ["name": .text("fresh"), "note": .text("added")]), + ]] + + let outcome = try await RowImportRunner.run( + configuration: .init(errorHandling: .skipAndContinue, wrapInTransaction: false, deleteExistingRows: false), + sink: sink, + progress: PluginImportProgress(progress: Progress(totalUnitCount: 3)) + ) { + batches.isEmpty ? nil : batches.removeFirst() + } + + #expect(outcome.skipped == 1) + #expect(outcome.errors.map { $0.line } == [2]) + #expect(try driver.rows("SELECT name, note FROM people ORDER BY id") == [ + ["dup", "n1"], ["dup", "n2"], ["solo", "updated"], ["fresh", "added"], + ]) + #expect(sink.rowTally.updated == 1) + #expect(sink.rowTally.inserted == 1) + } +} diff --git a/apple/TableProTests/Core/Services/Export/ImportRowWritingTests.swift b/apple/TableProTests/Core/Services/Export/ImportRowWritingTests.swift new file mode 100644 index 0000000000..aaa04bcffb --- /dev/null +++ b/apple/TableProTests/Core/Services/Export/ImportRowWritingTests.swift @@ -0,0 +1,124 @@ +// +// ImportRowWritingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ImportRowWritingTests { + private func column(_ name: String, primary: Bool = false) -> ColumnInfo { + TestFixtures.makeColumnInfo(name: name, isPrimaryKey: primary) + } + + private func index( + _ name: String, + _ columns: [String], + unique: Bool = true, + primary: Bool = false, + whereClause: String? = nil, + expressions: [String]? = nil + ) -> IndexInfo { + IndexInfo( + name: name, columns: columns, isUnique: unique, isPrimary: primary, type: "BTREE", + whereClause: whereClause, expressions: expressions + ) + } + + @Test("The primary key comes first, then each unique index over plain columns") + func keyOptions() { + let columns = [column("id", primary: true), column("email"), column("tenant"), column("code")] + let keys = ImportMatchKey.options(columns: columns, indexes: [ + index("PRIMARY", ["id"], primary: true), + index("people_email", ["EMAIL"]), + index("people_tenant_code", ["tenant", "code"]), + index("people_code", ["code"], unique: false), + index("people_active_email", ["email"], whereClause: "active"), + index("people_lower_email", ["lower(email)"], expressions: ["lower(email)"]), + ]) + + #expect(keys.map { $0.columns } == [["id"], ["email"], ["tenant", "code"]]) + #expect(keys.map { $0.indexName } == [nil, "people_email", "people_tenant_code"]) + #expect(keys.first?.isPrimaryKey == true) + } + + @Test("An index column resolves to its exact spelling before a case-folded one") + func exactSpellingWins() { + let columns = [column("ID"), column("id", primary: true), column("Email")] + let keys = ImportMatchKey.options(columns: columns, indexes: [ + index("pkey", ["id"], primary: true), + index("email_key", ["email"]), + index("upper_key", ["Id"]), + ]) + #expect(keys.map { $0.columns } == [["id"], ["Email"]]) + } + + @Test("Columns flagged as the primary key stand in when no index says so") + func primaryKeyFromColumns() { + let keys = ImportMatchKey.options(columns: [column("a", primary: true), column("b", primary: true)], indexes: []) + #expect(keys == [ImportMatchKey(indexName: nil, columns: ["a", "b"])]) + } + + @Test("A unique index over the primary key's columns is not offered twice") + func duplicateOfPrimaryKeyIsDropped() { + let keys = ImportMatchKey.options( + columns: [column("id", primary: true)], + indexes: [index("id_unique", ["id"])] + ) + #expect(keys.count == 1) + } + + @Test("A table with no key offers nothing to match on") + func noKey() { + #expect(ImportMatchKey.options(columns: [column("a")], indexes: [index("a_idx", ["a"], unique: false)]).isEmpty) + } + + @Test("Insert needs no key") + func insertHasNoProblem() { + #expect(ImportRowWritingProblem.of(mode: .insert, key: nil, mappedColumns: ["a"]) == nil) + } + + @Test("A matching mode needs a key, mapped, and an update needs a column outside it") + func matchingProblems() { + let key = ImportMatchKey(indexName: nil, columns: ["tenant", "id"]) + #expect(ImportRowWritingProblem.of(mode: .update, key: nil, mappedColumns: ["a"]) == .noKey) + #expect(ImportRowWritingProblem.of(mode: .upsert, key: key, mappedColumns: ["id", "name"]) + == .unmappedKeyColumns(["tenant"])) + #expect(ImportRowWritingProblem.of(mode: .update, key: key, mappedColumns: ["id", "tenant"]) == .nothingToUpdate) + #expect(ImportRowWritingProblem.of(mode: .upsert, key: key, mappedColumns: ["id", "tenant"]) == .nothingToUpdate) + #expect(ImportRowWritingProblem.of(mode: .insertSkippingExisting, key: key, mappedColumns: ["id", "tenant"]) == nil) + #expect(ImportRowWritingProblem.of(mode: .update, key: key, mappedColumns: ["id", "tenant", "name"]) == nil) + } + + @Test("A matching mode reports matched, updated, inserted and skipped rows") + func matchingSummary() { + var tally = ImportRowTally(mode: .update) + tally.updated = 9_500 + tally.unmatched = 500 + + #expect(tally.summaryLines(failed: 2) == [ + String(format: String(localized: "Matched: %@"), 9_500.formatted()), + String(format: String(localized: "Updated: %@"), 9_500.formatted()), + String(format: String(localized: "Inserted: %@"), 0.formatted()), + String(format: String(localized: "Skipped: %@"), 500.formatted()), + String(format: String(localized: "Failed: %@"), 2.formatted()), + ]) + } + + @Test("Skip Existing counts a matched row as matched and skipped") + func skipExistingCounts() { + var tally = ImportRowTally(mode: .insertSkippingExisting) + tally.existing = 3 + tally.inserted = 4 + #expect(tally.matched == 3) + #expect(tally.skipped == 3) + } + + @Test("Insert reports only what it inserted") + func insertSummary() { + var tally = ImportRowTally(mode: .insert) + tally.inserted = 12 + #expect(tally.summaryLines(failed: 0) == [String(format: String(localized: "Inserted: %@"), 12.formatted())]) + } +} diff --git a/apple/TableProTests/Views/ImportSummaryTextTests.swift b/apple/TableProTests/Views/ImportSummaryTextTests.swift new file mode 100644 index 0000000000..acbb525d4b --- /dev/null +++ b/apple/TableProTests/Views/ImportSummaryTextTests.swift @@ -0,0 +1,30 @@ +// +// ImportSummaryTextTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct ImportSummaryTextTests { + @Test("A row import that matched rows lists each count and the failures") + func rowImportSummary() { + var tally = ImportRowTally(mode: .upsert) + tally.updated = 9_500 + tally.inserted = 500 + let result = PluginImportResult(executedStatements: 10_000, executionTime: 1.5, skippedStatements: 1) + + let lines = TransferResultAlert.importSummary(result, rowTally: tally).components(separatedBy: "\n") + + #expect(lines == tally.summaryLines(failed: 1) + ["1.50 seconds"]) + } + + @Test("A SQL file import still counts statements") + func statementSummary() { + let result = PluginImportResult(executedStatements: 12, executionTime: 0.25) + #expect(TransferResultAlert.importSummary(result) == "12 statements executed\n0.25 seconds") + } +} diff --git a/apple/TableProUITests/RowImportUpdateUITests.swift b/apple/TableProUITests/RowImportUpdateUITests.swift new file mode 100644 index 0000000000..23133b7b30 --- /dev/null +++ b/apple/TableProUITests/RowImportUpdateUITests.swift @@ -0,0 +1,66 @@ +import XCTest + +// Import as Update matches the file's rows on the primary key: a matched row takes the file's value, a row whose key +// the table lacks is skipped, and the summary counts each. +final class RowImportUpdateUITests: UITestCase { + private let fileName = "genre-changes.csv" + private let changes = Data("id,name\n2,Bebop\n9,Polka\n".utf8) + + func testImportingAsUpdateChangesOnlyTheMatchedRows() throws { + let launch = try launchWithFixture(openingDataFile: fileName, contents: changes) + let app = launch.app + let dataWindow = dataFileWindow(in: app) + XCTAssertTrue(dataWindow.waitToExist(timeout: 30), "\(fileName) produced no data file window") + let sheet = openRowImportSheet(from: dataWindow, titled: fileName, into: launch.window, in: app) + + choose("genre", from: element(sheet.popUpButtons, labelEndingWith: "Import into"), in: app) + choose("Update", from: element(sheet.popUpButtons, labelEndingWith: "Import as"), in: app) + let key = element(sheet.popUpButtons, labelEndingWith: "Match on") + XCTAssertTrue( + waitForPredicate(timeout: 15) { (key.value as? String) == "Primary Key (id)" }, + "Update must match on genre's primary key, the menu shows \(key.value ?? "nothing")" + ) + + let importButton = sheet.buttons["Import"].firstMatch + XCTAssertTrue( + waitForPredicate(timeout: 15) { importButton.exists && importButton.isEnabled }, + "A mapped key must be ready to import" + ) + importButton.click() + + let query = "SELECT id, name FROM genre ORDER BY id" + XCTAssertTrue( + waitForPredicate(timeout: 30) { + FixtureFile.rows(query, in: launch.database) == [["1", "Rock"], ["2", "Bebop"], ["3", "Metal"]] + }, + "Only genre 2 may change and 9 must stay out, genre holds " + + "\(FixtureFile.rows(query, in: launch.database) ?? [])" + ) + let summary = sheet.descendants(matching: .staticText) + .matching(NSPredicate(format: "value CONTAINS %@", "Updated: 1")) + .firstMatch + XCTAssertTrue(summary.waitToExist(timeout: 15), "The summary must count the updated row") + XCTAssertTrue( + (summary.value as? String)?.contains("Skipped: 1") == true, + "The summary must count the skipped row, it reads \(summary.value ?? "nothing")" + ) + } + + // A pop-up's menu is built when it opens, so a list still loading is opened again. + private func choose(_ title: String, from popUp: XCUIElement, in app: XCUIApplication) { + XCTAssertTrue(popUp.waitToExist(timeout: 15), "The sheet must offer a menu for \(title)") + let item = app.menuItems.matching(NSPredicate(format: "title == %@", title)).firstMatch + let opened = waitForPredicate(timeout: 20) { + popUp.click() + if item.waitToExist(timeout: 2) { return true } + app.typeKey(.escape, modifierFlags: []) + return false + } + XCTAssertTrue(opened, "The menu must offer \(title)") + item.click() + XCTAssertTrue( + waitForPredicate(timeout: 10) { (popUp.value as? String) == title }, + "The menu must show \(title), it shows \(popUp.value ?? "nothing")" + ) + } +} From b427314f18b752e8c8ed5548bd97e5b34125a32c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 21 Oct 2026 12:12:29 +0700 Subject: [PATCH 2/2] docs(changelog): note row import update modes --- apple/changelog.d/3743.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/3743.added.md diff --git a/apple/changelog.d/3743.added.md b/apple/changelog.d/3743.added.md new file mode 100644 index 0000000000..433fc89097 --- /dev/null +++ b/apple/changelog.d/3743.added.md @@ -0,0 +1 @@ +Import a CSV, JSON or XLSX file into an existing table as Update, Insert or Update, or Insert, Skip Existing.