From e6e2f75f22f3617fe5d72ed43cfb44f257ed921c Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 15:56:54 +1100 Subject: [PATCH 01/13] Add Regex+ engines, with rr as the default engine --- README.md | 3 +- biome.json | 2 +- src/cli.ts | 6 +- src/engine.ts | 146 +++++++++-- src/libs/README.md | 28 +++ src/libs/regex-utilities/LICENSE | 21 ++ src/libs/regex-utilities/index.js | 170 +++++++++++++ src/libs/regex/LICENSE | 21 ++ src/libs/regex/atomic.js | 224 +++++++++++++++++ src/libs/regex/backcompat.js | 64 +++++ src/libs/regex/flag-n.js | 49 ++++ src/libs/regex/flag-x.js | 198 +++++++++++++++ src/libs/regex/internals.js | 4 + src/libs/regex/pattern.js | 45 ++++ src/libs/regex/regex.js | 390 +++++++++++++++++++++++++++++ src/libs/regex/subclass.js | 98 ++++++++ src/libs/regex/subroutines.js | 375 ++++++++++++++++++++++++++++ src/libs/regex/utils-internals.js | 33 +++ src/libs/regex/utils.js | 401 ++++++++++++++++++++++++++++++ test/cli/debug.test.js | 2 + test/cli/run.sh | 5 +- test/engine/engine.test.js | 4 +- test/fix/705.test.js | 2 +- test/fix/742.test.js | 2 + test/fix/813.test.js | 2 +- test/fix/854.test.js | 139 +++++++++++ test/helper.js | 2 +- 27 files changed, 2405 insertions(+), 31 deletions(-) create mode 100644 src/libs/README.md create mode 100644 src/libs/regex-utilities/LICENSE create mode 100644 src/libs/regex-utilities/index.js create mode 100644 src/libs/regex/LICENSE create mode 100644 src/libs/regex/atomic.js create mode 100644 src/libs/regex/backcompat.js create mode 100644 src/libs/regex/flag-n.js create mode 100644 src/libs/regex/flag-x.js create mode 100644 src/libs/regex/internals.js create mode 100644 src/libs/regex/pattern.js create mode 100644 src/libs/regex/regex.js create mode 100644 src/libs/regex/subclass.js create mode 100644 src/libs/regex/subroutines.js create mode 100644 src/libs/regex/utils-internals.js create mode 100644 src/libs/regex/utils.js create mode 100644 test/fix/854.test.js diff --git a/README.md b/README.md index 97cae92c..5d4149d4 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | -| `-E` | **`--engine`** What regex engine to use. `default` is the engine of the runtime: `V8` with Node and `JSC` (JavaScriptCore) with Bun. The engine of the other runtime can not be picked. [choices: "default", "V8", "JSC"] [default: "default"] | +| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+v` (also `regex+` and `V`) is full Regex+ with the `v` flag, free spacing (`x`) and only named groups capturing (`n`). `regex+u` (also `U`) is the same with the `u` flag. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+v", "regex+", "V", "regex+u", "U", "runtime", "V8", "JSC"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | @@ -389,6 +389,7 @@ Files identified for potential processing are identified by: - Patterns are described as javascript notation regex - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern +- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same. The one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ is copied into the code, so it is no extra dependency. - Supports backreference to matching groups in the replacement - Data can be piped in, both as the text to search and as the replacement - The names of the changed files are printed to stdout, so they can be piped on (to stderr with `--output` and `--output-match`) diff --git a/biome.json b/biome.json index 4043a395..cf537517 100644 --- a/biome.json +++ b/biome.json @@ -63,6 +63,6 @@ } ], "files": { - "includes": ["**", "!**/dist", "!**/bin", "!**/node_modules"] + "includes": ["**", "!**/dist", "!**/bin", "!**/node_modules", "!src/libs"] } } diff --git a/src/cli.ts b/src/cli.ts index c50d68e5..1f6bad6e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -141,10 +141,10 @@ const yargs = require('yargs') .alias('E', 'engine') .describe( 'E', - 'What regex engine to use. `default` is the default engine of the runtime (`V8` for Node and `JSC` (JavaScriptCore) for Bun.' + 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+v` (also `regex+` and `V`) is full Regex+ with the `v` flag, free spacing (`x`) and only named groups capturing (`n`). `regex+u` (also `U`) is the same with the `u` flag. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' ) - .choices('E', ['default', 'V8', 'JSC' /*'RE2' /*'sd', 'stream'*/]) - .default('E', 'default') + .choices('E', ['rr', 'default', 'regex+v', 'regex+', 'V', 'regex+u', 'U', 'runtime', 'V8', 'JSC']) + .default('E', 'rr') .boolean('q') .describe( diff --git a/src/engine.ts b/src/engine.ts index 7b961097..9c127760 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -24,6 +24,10 @@ const re = { // With a name the location is in parentheses at the end, like `at fn (/path/file.js:2:15)`. stackLineWithoutName: /^\s*at (?:file:\/\/)?((?:\/|[A-Za-z]:[\\/]).*?)(:\d+:\d+)\s*$/, stackLineWithName: /^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/, + // Subroutines of Regex+, which the rr engine only looks for when the pattern has them + subroutineSyntax: /\\g<|\(\?\(DEFINE\)/, + // A $ in a replacement: $$, $&, $`, $', $, or a group number of one or two digits + replacementToken: /\$(\$|&|`|'|<[^>]*>|(\d)(\d?))/g, // Only the chars with a meaning outside a character class. Unicode mode does not allow to escape the others (#686). regexSpecialChars: /[\[\]{}()*+?.\/\\^$|]/g, byteOrSize: /bytes|size/, @@ -39,6 +43,10 @@ const re = { // Only the version ends up in the bundle import {version} from '../package.json'; + +// Regex+ by Steven Levithan, copied into src/libs until it exports the subroutines (see src/libs/README.md) +import {atomic, possessive, subroutines} from './libs/regex/internals.js'; +import {rewrite} from './libs/regex/regex.js'; export {version}; // The regex engine comes with the runtime: Node has V8 and Bun has JSC (JavaScriptCore) @@ -47,7 +55,7 @@ const runtimeEngine = process.versions.bun ? 'JSC' : 'V8'; // Resolves with the status of the run when all files are done. // When the run can not go on, a panic is thrown with the status so far. -export async function engine(config: any = {engine: 'default'}) { +export async function engine(config: any = {engine: 'rr'}) { config.statusInfo = newStatus(config); // A new run has had no panic yet, has read no replacement file, and has its own time, also when the same config is used again config.plzStop = false; @@ -154,7 +162,24 @@ function unpackConfig(config) { // --output-match wins over --output config.output = config.output && !config.outputMatch; - config.regex = getFinalRegex(config.pattern, config); + const {regex, hiddenCaptures} = compileRegex(config.pattern, config); + config.regex = regex; + config.hiddenCaptures = hiddenCaptures; + + // The groups Regex+ adds for atomic groups and possessive quantifiers are not counted in the numbers of the replacement + if ( + hiddenCaptures.length && + 'string' === typeof config.replacement && + !config.replacementJs && + !config.outputMatch && + !config.literalReplacement + ) { + config.replacement = visibleGroupNumbers( + config.replacement, + hiddenCaptures, + emptyMatch(regex).length - 1 + ); + } config.codeUses = config.replacementJs ? codeUses(config.replacementOri, config.regex) : null; @@ -171,6 +196,8 @@ function unpackConfig(config) { // What the regex engine is asked to do. With --replacement-js and --output-match it gets a function. if (config.statusInfo.engine) { config.statusInfo.engine.execution = { + engine: engineName(config.engine), + hiddenCaptures, search: config.regex.source, flags: config.regex.flags, replacement: config.replacementJs || config.outputMatch ? null : config.replacement, @@ -634,7 +661,7 @@ function matchPrinter(config) { warn('The replacement file is not read with --output-match, as only the matches are printed.'); } return (...args) => { - const [match, ...groups] = splitMatch(args).groups; + const [match, ...groups] = splitMatch(args, config.hiddenCaptures).groups; print( groups.length ? groups.join('') : match, config.contentDelimiter, @@ -689,29 +716,98 @@ function codeReplacement(file, config, text, numbers) { }; } -function getFinalRegex(pattern, config) { - const engine = 'default' === config.engine ? runtimeEngine : config.engine; - // RE2 is not in yet, so it uses the engine of the runtime - if (engine !== runtimeEngine && 'RE2' !== engine) { +// -E: default is another name for rr, regex+ and V for regex+v, and U for regex+u +const engineNames = {default: 'rr', 'regex+': 'regex+v', V: 'regex+v', U: 'regex+u'}; +const engineName = name => engineNames[name] ?? name ?? 'rr'; + +// The regex and the numbers of the groups the engine added to it, which the user does not know about +function compileRegex(pattern, config) { + const engine = engineName(config.engine); + if (!['rr', 'regex+v', 'regex+u', 'V8', 'JSC', 'runtime'].includes(engine)) { + panic(`Engine ${config.engine} not supported`); + } + if (('V8' === engine || 'JSC' === engine) && engine !== runtimeEngine) { panic( - 'V8' === engine || 'JSC' === engine - ? `When running via the ${runtime} runtime you can only use default or ${runtimeEngine} as engine` - : `Engine ${config.engine} not supported` + `When running via the ${runtime} runtime the native engine is ${runtimeEngine}, not ${engine}` ); } + const regexPlus = 'regex+v' === engine || 'regex+u' === engine; + // Regex+ always uses u or v itself, and it is an error to ask for them const flags = (config.voidGlobal ? '' : 'g') + (config.voidIgnoreCase ? '' : 'i') + (config.voidMultiline ? '' : 'm') + (config.dotAll ? 's' : '') + - (config.unicode ? 'u' : ''); + (config.unicode && !regexPlus ? 'u' : ''); try { - return new RegExp(pattern, flags); + if (config.literal) { + // The text as it is, without the free spacing of Regex+ + return { + regex: new RegExp(pattern, flags + (regexPlus ? ('regex+u' === engine ? 'u' : 'v') : '')), + hiddenCaptures: [], + }; + } + if (regexPlus) { + const result = rewrite(pattern, {flags, disable: 'regex+u' === engine ? {v: true} : {}}); + return { + regex: new RegExp(result.pattern, result.flags), + hiddenCaptures: result.hiddenCaptures, + }; + } + if ('rr' === engine) { + return compileRr(pattern, flags); + } + return {regex: new RegExp(pattern, flags), hiddenCaptures: []}; } catch (e) { return panic(e.message); } } +// The rr engine: the native engine with the flags of today, and the extended syntax of Regex+ on top. +// Only the transforms of Regex+ run, in its own order, and not its flags, so a pattern that works natively gives the same regex. +function compileRr(pattern, flags) { + let hiddenCaptures = []; + const transform = plugin => { + const result = plugin(pattern, {flags, hiddenCaptures}); + pattern = result.pattern; + hiddenCaptures = result.hiddenCaptures ?? hiddenCaptures; + }; + // The subroutines do not return early by themselves when the pattern has none + if (re.subroutineSyntax.test(pattern)) { + transform(subroutines); + } + transform(possessive); + transform(atomic); + return {regex: new RegExp(pattern, flags), hiddenCaptures}; +} + +// The group numbers in a replacement, like $1, count only the groups of the user, so they are moved past the hidden ones. +// It follows the rules of javascript for $n and $nn with the visible groups, and always gives two digits, +// so a digit after it is not read as part of the number. A number that is no visible group stays as text. +function visibleGroupNumbers(replacement, hiddenCaptures, total) { + const real = [0]; + for (let group = 1; group <= total; group++) { + if (!hiddenCaptures.includes(group)) { + real.push(group); + } + } + const visible = real.length - 1; + return replacement.replace(re.replacementToken, (token, kind, first, second) => { + if (undefined === first) { + return token; + } + const two = '' !== second ? Number(first + second) : 0; + if (1 <= two && two <= visible) { + return '$' + String(real[two]).padStart(2, '0'); + } + const one = Number(first); + if (1 <= one && one <= visible) { + return '$' + String(real[one]).padStart(2, '0') + second; + } + return '$$' + first + second; + }); +} + function readableSize(size) { if (1 === size) { return '1 Byte'; @@ -757,12 +853,15 @@ function codeUses(code, regex) { }; } +// With | at the end the regex matches an empty text, and the match has all the groups +function emptyMatch(regex) { + return new RegExp(regex.source + '|', regex.flags.replace(/[gy]/g, '')).exec(''); +} + // The names of the named groups of a regex, like year for (?…). A name can have any letter, like año. // They come from the regex itself, so text that only looks like a group, like \(? or [(?], is not one. -// With | at the end the regex matches an empty text, and the match has all the groups. function namedGroups(regex) { - const match = new RegExp(regex.source + '|', regex.flags.replace(/[gy]/g, '')).exec(''); - return Object.keys(match?.groups ?? {}); + return Object.keys(emptyMatch(regex)?.groups ?? {}); } // The numbers of --js and their cousins: $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, @@ -850,11 +949,18 @@ function cut(text, length = 300) { return chars.length > length ? chars.slice(0, length).join('') + '…' : text; } -// The arguments of a replace function: the match and the groups, then the offset and the text, and the named groups if the pattern has any -function splitMatch(args) { +// The arguments of a replace function: the match and the groups, then the offset and the text, and the named groups if the pattern has any. +// The groups the engine added are left out, so $1 is the first group of the user. +function splitMatch(args, hiddenCaptures) { const named = args[args.length - 1]; const hasNamed = 'object' === typeof named && null !== named; - return {groups: args.slice(0, hasNamed ? -3 : -2), named: hasNamed ? named : {}}; + const groups = args.slice(0, hasNamed ? -3 : -2); + return { + groups: hiddenCaptures.length + ? groups.filter((_, group) => !hiddenCaptures.includes(group)) + : groups, + named: hasNamed ? named : {}, + }; } // The function that runs the code of --js. It is made once, as the names are the same for every file. @@ -998,7 +1104,7 @@ function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: foundMatch(); match++; config.numbers.matches++; - const {groups, named} = splitMatch(matchArgs); + const {groups, named} = splitMatch(matchArgs, config.hiddenCaptures); let variables = ''; groups.forEach((value, i) => { variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; @@ -1041,7 +1147,7 @@ function getFilePaths(conf) { } excludeRe - .map(pattern => getFinalRegex(regexPattern(pattern, conf), conf)) + .map(pattern => compileRegex(regexPattern(pattern, conf), conf).regex) .forEach(exclude => { files = files.filter(file => !file.match(exclude)); }); diff --git a/src/libs/README.md b/src/libs/README.md new file mode 100644 index 00000000..56e21a11 --- /dev/null +++ b/src/libs/README.md @@ -0,0 +1,28 @@ +# Copied libraries + +These are copies of other packages, bundled into RexReplace, so they are not runtime dependencies. +Keep them as they are upstream, so they are easy to compare. Biome leaves this folder alone. + +| Folder | Package | Upstream | +| --- | --- | --- | +| `regex/` | [`regex`](https://www.npmjs.com/package/regex) 6.1.0, the `src/` folder | https://github.com/slevithan/regex | +| `regex-utilities/` | [`regex-utilities`](https://www.npmjs.com/package/regex-utilities) 2.3.0, `src/index.js` | https://github.com/slevithan/regex-utilities | + +Both are MIT, and their `LICENSE` files are next to the code. + +## Changes from upstream + +Changes 2 and 3 are marked with a `// rexreplace patch:` comment. + +1. `regex/*.js` import `regex-utilities` as `../regex-utilities/index.js`, as the build leaves bare imports out of the bundle. +2. `regex/internals.js` also exports `subroutines`. Asked for upstream. +3. `rewrite()` in `regex/regex.js` also gives `hiddenCaptures`, the numbers of the groups Regex+ added, so RexReplace can skip them without `RegExpSubclass`, which makes `.replace()` 3 to 5 times slower. + +## Removal plan + +When upstream exports `subroutines`: + +1. Add `regex` as a dependency. +2. Import from `regex/internals` and `regex` in place of `./libs/regex/internals.js` and `./libs/regex/regex.js`. +3. If `rewrite()` does not give `hiddenCaptures` upstream, ask for it first, as change 3 is still needed. +4. Delete `src/libs/` and its line in `biome.json`. The tests must pass without changes. diff --git a/src/libs/regex-utilities/LICENSE b/src/libs/regex-utilities/LICENSE new file mode 100644 index 00000000..2790a22c --- /dev/null +++ b/src/libs/regex-utilities/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Steven Levithan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/libs/regex-utilities/index.js b/src/libs/regex-utilities/index.js new file mode 100644 index 00000000..5cf3c4b5 --- /dev/null +++ b/src/libs/regex-utilities/index.js @@ -0,0 +1,170 @@ +// Constant properties for tracking regex syntax context +export const Context = Object.freeze({ + DEFAULT: 'DEFAULT', + CHAR_CLASS: 'CHAR_CLASS', +}); + +/** +Replaces all unescaped instances of a regex pattern in the given context, using a replacement +string or callback. + +Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used +with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. +@param {string} expression Search target +@param {string} needle Search as a regex pattern, with flags `su` applied +@param {string | (match: RegExpExecArray, details: { + context: 'DEFAULT' | 'CHAR_CLASS'; + negated: boolean; +}) => string} replacement +@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified +@returns {string} Updated expression +@example +const str = '.\\.\\\\.[[\\.].].'; +replaceUnescaped(str, '\\.', '@'); +// → '@\\.\\\\@[[\\.]@]@' +replaceUnescaped(str, '\\.', '@', Context.DEFAULT); +// → '@\\.\\\\@[[\\.].]@' +replaceUnescaped(str, '\\.', '@', Context.CHAR_CLASS); +// → '.\\.\\\\.[[\\.]@].' +*/ +export function replaceUnescaped(expression, needle, replacement, context) { + const re = new RegExp(String.raw`${needle}|(?<$skip>\[\^?|\\?.)`, 'gsu'); + const negated = [false]; + let numCharClassesOpen = 0; + let result = ''; + for (const match of expression.matchAll(re)) { + const {0: m, groups: {$skip}} = match; + if (!$skip && (!context || (context === Context.DEFAULT) === !numCharClassesOpen)) { + if (replacement instanceof Function) { + result += replacement(match, { + context: numCharClassesOpen ? Context.CHAR_CLASS : Context.DEFAULT, + negated: negated[negated.length - 1], + }); + } else { + result += replacement; + } + continue; + } + if (m[0] === '[') { + numCharClassesOpen++; + negated.push(m[1] === '^'); + } else if (m === ']' && numCharClassesOpen) { + numCharClassesOpen--; + negated.pop(); + } + result += m; + } + return result; +} + +/** +Runs a callback for each unescaped instance of a regex pattern in the given context. + +Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used +with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. +@param {string} expression Search target +@param {string} needle Search as a regex pattern, with flags `su` applied +@param {(match: RegExpExecArray, details: { + context: 'DEFAULT' | 'CHAR_CLASS'; + negated: boolean; +}) => void} callback +@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified +*/ +export function forEachUnescaped(expression, needle, callback, context) { + // Do this the easy way + replaceUnescaped(expression, needle, callback, context); +} + +/** +Returns a match object for the first unescaped instance of a regex pattern in the given context, or +`null`. + +Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used +with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. +@param {string} expression Search target +@param {string} needle Search as a regex pattern, with flags `su` applied +@param {number} [pos] Offset to start the search +@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified +@returns {RegExpExecArray | null} +*/ +export function execUnescaped(expression, needle, pos = 0, context) { + // Quick partial test; avoid the loop if not needed + if (!(new RegExp(needle, 'su').test(expression))) { + return null; + } + const re = new RegExp(`${needle}|(?<$skip>\\\\?.)`, 'gsu'); + re.lastIndex = pos; + let numCharClassesOpen = 0; + let match; + while (match = re.exec(expression)) { + const {0: m, groups: {$skip}} = match; + if (!$skip && (!context || (context === Context.DEFAULT) === !numCharClassesOpen)) { + return match; + } + if (m === '[') { + numCharClassesOpen++; + } else if (m === ']' && numCharClassesOpen) { + numCharClassesOpen--; + } + // Avoid an infinite loop on zero-length matches + if (re.lastIndex == match.index) { + re.lastIndex++; + } + } + return null; +} + +/** +Checks whether an unescaped instance of a regex pattern appears in the given context. + +Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used +with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. +@param {string} expression Search target +@param {string} needle Search as a regex pattern, with flags `su` applied +@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified +@returns {boolean} Whether the pattern was found +*/ +export function hasUnescaped(expression, needle, context) { + // Do this the easy way + return !!execUnescaped(expression, needle, 0, context); +} + +/** +Extracts the full contents of a group (subpattern) from the given expression, accounting for +escaped characters, nested groups, and character classes. The group is identified by the position +where its contents start (the string index just after the group's opening delimiter). Returns the +rest of the string if the group is unclosed. + +Assumes UnicodeSets-mode syntax. +@param {string} expression Search target +@param {number} contentsStartPos +@returns {string} +*/ +export function getGroupContents(expression, contentsStartPos) { + const token = /\\?./gsu; + token.lastIndex = contentsStartPos; + let contentsEndPos = expression.length; + let numCharClassesOpen = 0; + // Starting search within an open group, after the group's opening + let numGroupsOpen = 1; + let match; + while (match = token.exec(expression)) { + const [m] = match; + if (m === '[') { + numCharClassesOpen++; + } else if (!numCharClassesOpen) { + if (m === '(') { + numGroupsOpen++; + } else if (m === ')') { + numGroupsOpen--; + if (!numGroupsOpen) { + contentsEndPos = match.index; + break; + } + } + } else if (m === ']') { + numCharClassesOpen--; + } + } + return expression.slice(contentsStartPos, contentsEndPos); +} diff --git a/src/libs/regex/LICENSE b/src/libs/regex/LICENSE new file mode 100644 index 00000000..42eabea2 --- /dev/null +++ b/src/libs/regex/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Steven Levithan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/libs/regex/atomic.js b/src/libs/regex/atomic.js new file mode 100644 index 00000000..1254003d --- /dev/null +++ b/src/libs/regex/atomic.js @@ -0,0 +1,224 @@ +import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js'; +import {Context, replaceUnescaped} from '../regex-utilities/index.js'; +/** +@import {PluginData, PluginResult} from './regex.js'; +*/ + +const atomicPluginToken = new RegExp(String.raw`(?${noncapturingDelim})|(?\((?:\?<[^>]+>)?)|\\?.`, 'gsu'); + +/** +Apply transformations for atomic groups: `(?>…)`. +@param {string} expression +@param {PluginData} [data] +@returns {Required} +*/ +function atomic(expression, data) { + const hiddenCaptures = data?.hiddenCaptures ?? []; + // Capture transfer is used by + let captureTransfers = data?.captureTransfers ?? new Map(); + if (!/\(\?>/.test(expression)) { + return { + pattern: expression, + captureTransfers, + hiddenCaptures, + }; + } + + const aGDelim = '(?>'; + const emulatedAGDelim = '(?:(?=('; + const captureNumMap = [0]; + const addedHiddenCaptures = []; + let numCapturesBeforeAG = 0; + let numAGs = 0; + let aGPos = NaN; + let hasProcessedAG; + do { + hasProcessedAG = false; + let numCharClassesOpen = 0; + let numGroupsOpenInAG = 0; + let inAG = false; + let match; + atomicPluginToken.lastIndex = Number.isNaN(aGPos) ? 0 : aGPos + emulatedAGDelim.length; + while (match = atomicPluginToken.exec(expression)) { + const {0: m, index, groups: {capturingStart, noncapturingStart}} = match; + if (m === '[') { + numCharClassesOpen++; + } else if (!numCharClassesOpen) { + + if (m === aGDelim && !inAG) { + aGPos = index; + inAG = true; + } else if (inAG && noncapturingStart) { + numGroupsOpenInAG++; + } else if (capturingStart) { + if (inAG) { + numGroupsOpenInAG++; + } else { + numCapturesBeforeAG++; + captureNumMap.push(numCapturesBeforeAG + numAGs); + } + } else if (m === ')' && inAG) { + if (!numGroupsOpenInAG) { + numAGs++; + const addedCaptureNum = numCapturesBeforeAG + numAGs; + // Replace `expression` and use `<$$N>` as a temporary wrapper for the backref so it + // can avoid backref renumbering afterward. Wrap the whole substitution (including the + // lookahead and following backref) in a noncapturing group to handle following + // quantifiers and literal digits + expression = `${expression.slice(0, aGPos)}${emulatedAGDelim}${ + expression.slice(aGPos + aGDelim.length, index) + }))<$$${addedCaptureNum}>)${expression.slice(index + 1)}`; + hasProcessedAG = true; + addedHiddenCaptures.push(addedCaptureNum); + incrementIfAtLeast(hiddenCaptures, addedCaptureNum); + if (captureTransfers.size) { + const newCaptureTransfers = new Map(); + captureTransfers.forEach((from, to) => { + newCaptureTransfers.set( + to >= addedCaptureNum ? to + 1 : to, + from.map(f => f >= addedCaptureNum ? f + 1 : f) + ); + }); + captureTransfers = newCaptureTransfers; + } + break; + } + numGroupsOpenInAG--; + } + + } else if (m === ']') { + numCharClassesOpen--; + } + } + // Start over from the beginning of the atomic group's contents, in case the processed group + // contains additional atomic groups + } while (hasProcessedAG); + + hiddenCaptures.push(...addedHiddenCaptures); + + // Second pass to adjust numbered backrefs + expression = replaceUnescaped( + expression, + String.raw`\\(?[1-9]\d*)|<\$\$(?\d+)>`, + ({0: m, groups: {backrefNum, wrappedBackrefNum}}) => { + if (backrefNum) { + const bNum = +backrefNum; + if (bNum > captureNumMap.length - 1) { + throw new Error(`Backref "${m}" greater than number of captures`); + } + return `\\${captureNumMap[bNum]}`; + } + return `\\${wrappedBackrefNum}`; + }, + Context.DEFAULT + ); + + return { + pattern: expression, + captureTransfers, + hiddenCaptures, + }; +} + +const baseQuantifier = String.raw`(?:[?*+]|\{\d+(?:,\d*)?\})`; +// Complete tokenizer for base syntax; doesn't (need to) know about character-class-only syntax +const possessivePluginToken = new RegExp(String.raw` +\\(?: \d+ + | c[A-Za-z] + | [gk]<[^>]+> + | [pPu]\{[^\}]+\} + | u[A-Fa-f\d]{4} + | x[A-Fa-f\d]{2} + ) +| \((?: \? (?: [:=!>] + | <(?:[=!]|[^>]+>) + | [A-Za-z\-]+: + | \(DEFINE\) + ))? +| (?${baseQuantifier})(?[?+]?)(?[?*+\{]?) +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +Transform posessive quantifiers into atomic groups. The posessessive quantifiers are: +`?+`, `*+`, `++`, `{N}+`, `{N,}+`, `{N,N}+`. +This follows Java, PCRE, Perl, and Python. +Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`. +@param {string} expression +@returns {PluginResult} +*/ +function possessive(expression) { + if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) { + return { + pattern: expression, + }; + } + + const openGroupIndices = []; + let lastGroupIndex = null; + let lastCharClassIndex = null; + let lastToken = ''; + let numCharClassesOpen = 0; + let match; + possessivePluginToken.lastIndex = 0; + while (match = possessivePluginToken.exec(expression)) { + const {0: m, index, groups: {qBase, qMod, invalidQ}} = match; + if (m === '[') { + if (!numCharClassesOpen) { + lastCharClassIndex = index; + } + numCharClassesOpen++; + } else if (m === ']') { + if (numCharClassesOpen) { + numCharClassesOpen--; + // Unmatched `]` + } else { + lastCharClassIndex = null; + } + } else if (!numCharClassesOpen) { + + if (qMod === '+' && lastToken && !lastToken.startsWith('(')) { + // Invalid following quantifier would become valid via the wrapping group + if (invalidQ) { + throw new Error(`Invalid quantifier "${m}"`); + } + let charsAdded = -1; // -1 for removed trailing `+` + // Possessivizing fixed repetition quantifiers like `{2}` does't change their behavior, so + // avoid doing so (convert them to greedy) + if (/^\{\d+\}$/.test(qBase)) { + expression = spliceStr(expression, index + qBase.length, qMod, ''); + } else { + if (lastToken === ')' || lastToken === ']') { + const nodeIndex = lastToken === ')' ? lastGroupIndex : lastCharClassIndex; + // Unmatched `)` would break out of the wrapping group and mess with handling. + // Unmatched `]` wouldn't be a problem, but it's unnecessary to have dedicated support + // for unescaped `]++` since this won't work with flag u or v anyway + if (nodeIndex === null) { + throw new Error(`Invalid unmatched "${lastToken}"`); + } + expression = `${expression.slice(0, nodeIndex)}(?>${expression.slice(nodeIndex, index)}${qBase})${expression.slice(index + m.length)}`; + } else { + expression = `${expression.slice(0, index - lastToken.length)}(?>${lastToken}${qBase})${expression.slice(index + m.length)}`; + } + charsAdded += 4; // `(?>)` + } + possessivePluginToken.lastIndex += charsAdded; + } else if (m[0] === '(') { + openGroupIndices.push(index); + } else if (m === ')') { + lastGroupIndex = openGroupIndices.length ? openGroupIndices.pop() : null; + } + + } + lastToken = m; + } + + return { + pattern: expression, + }; +} + +export { + atomic, + possessive, +}; diff --git a/src/libs/regex/backcompat.js b/src/libs/regex/backcompat.js new file mode 100644 index 00000000..2a5a0d45 --- /dev/null +++ b/src/libs/regex/backcompat.js @@ -0,0 +1,64 @@ +import {doublePunctuatorChars} from './utils.js'; +/** +@import {PluginResult} from './regex.js'; +*/ + +const incompatibleEscapeChars = '&!#%,:;<=>@`~'; +const token = new RegExp(String.raw` +\[\^?-? +| --?\] +| (?[${doublePunctuatorChars}])\k +| -- +| \\(?[${incompatibleEscapeChars}]) +| \\[pPu]\{[^}]+\} +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +Applies flag v rules when using flag u, for forward compatibility. +Assumes flag u and doesn't worry about syntax errors that are caught by it. +@param {string} expression +@returns {PluginResult} +*/ +function backcompatPlugin(expression) { + const unescapedLiteralHyphenMsg = 'Invalid unescaped "-" in character class'; + let inCharClass = false; + let result = ''; + for (const {0: m, groups: {dp, vOnlyEscape}} of expression.matchAll(token)) { + if (m[0] === '[') { + if (inCharClass) { + throw new Error('Invalid nested character class when flag v not supported; possibly from interpolation'); + } + if (m.endsWith('-')) { + throw new Error(unescapedLiteralHyphenMsg); + } + inCharClass = true; + } else if (m.endsWith(']')) { + if (m[0] === '-') { + throw new Error(unescapedLiteralHyphenMsg); + } + inCharClass = false; + } else if (inCharClass) { + if (m === '&&' || m === '--') { + throw new Error(`Invalid set operator "${m}" when flag v not supported`); + } else if (dp) { + throw new Error(`Invalid double punctuator "${m}", reserved by flag v`); + } else if ('(){}/|'.includes(m)) { + throw new Error(`Invalid unescaped "${m}" in character class`); + } else if (vOnlyEscape) { + // Remove the escaping backslash to emulate flag v rules, since this character is allowed + // to be escaped within character classes with flag v but not with flag u + result += vOnlyEscape; + continue; + } + } + result += m; + } + return { + pattern: result, + }; +} + +export { + backcompatPlugin, +}; diff --git a/src/libs/regex/flag-n.js b/src/libs/regex/flag-n.js new file mode 100644 index 00000000..288fc747 --- /dev/null +++ b/src/libs/regex/flag-n.js @@ -0,0 +1,49 @@ +import {getEndContextForIncompleteExpression, RegexContext} from './utils.js'; +import {noncapturingDelim} from './utils-internals.js'; +/** +@import {Preprocessor} from './utils.js'; +*/ + +const token = new RegExp(String.raw` +${noncapturingDelim} +| \(\?< +| (?\\[1-9]\d*) +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +Apply transformations for flag n (named capture only). + +Preprocessors are applied to the outer regex and interpolated patterns, but not interpolated +regexes or strings. +@type {Preprocessor} +*/ +function flagNPreprocessor(value, runningContext) { + value = String(value); + let expression = ''; + let transformed = ''; + for (const {0: m, groups: {backrefNum}} of value.matchAll(token)) { + expression += m; + runningContext = getEndContextForIncompleteExpression(expression, runningContext); + const {regexContext} = runningContext; + if (regexContext === RegexContext.DEFAULT) { + if (m === '(') { + transformed += '(?:'; + } else if (backrefNum) { + throw new Error(`Invalid decimal escape "${m}" with implicit flag n; replace with named backreference`); + } else { + transformed += m; + } + } else { + transformed += m; + } + } + return { + transformed, + runningContext, + }; +} + +export { + flagNPreprocessor, +}; diff --git a/src/libs/regex/flag-x.js b/src/libs/regex/flag-x.js new file mode 100644 index 00000000..043595b4 --- /dev/null +++ b/src/libs/regex/flag-x.js @@ -0,0 +1,198 @@ +import {CharClassContext, doublePunctuatorChars, getEndContextForIncompleteExpression, RegexContext, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js'; +import {noncapturingDelim} from './utils-internals.js'; +import {Context, replaceUnescaped} from '../regex-utilities/index.js'; +/** +@import {PluginResult} from './regex.js'; +@import {Preprocessor} from './utils.js'; +*/ + +const ws = /^\s$/; +const escapedWsOrHash = /^\\[\s#]$/; +const charClassWs = /^[ \t]$/; +const escapedCharClassWs = /^\\[ \t]$/; +const token = new RegExp(String.raw` +\\(?: [gk]< + | [pPu]\{ + | c[A-Za-z] + | u[A-Fa-f\d]{4} + | x[A-Fa-f\d]{2} + | 0\d+ +) +| \[\^ +| ${noncapturingDelim} +| \(\?< +| (?[${doublePunctuatorChars}])\k +| -- +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +Apply transformations for flag x (insignificant whitespace and line comments). + +Preprocessors are applied to the outer regex and interpolated patterns, but not interpolated +regexes or strings. +@type {Preprocessor} +*/ +function flagXPreprocessor(value, runningContext, options) { + value = String(value); + let ignoringWs = false; + let ignoringCharClassWs = false; + let ignoringComment = false; + let expression = ''; + let transformed = ''; + let lastSignificantToken = ''; + let lastSignificantCharClassContext = ''; + let separatorNeeded = false; + const update = (str, options) => { + const opts = { + prefix: true, + postfix: false, + ...options, + }; + str = (separatorNeeded && opts.prefix ? '(?:)' : '') + str + (opts.postfix ? '(?:)' : ''); + separatorNeeded = false; + return str; + }; + for (const {0: m, index} of value.matchAll(token)) { + if (ignoringComment) { + if (m === '\n') { + ignoringComment = false; + separatorNeeded = true; + } + continue; + } + if (ignoringWs) { + if (ws.test(m)) { + continue; + } + ignoringWs = false; + separatorNeeded = true; + } else if (ignoringCharClassWs) { + if (charClassWs.test(m)) { + continue; + } + ignoringCharClassWs = false; + } + + expression += m; + runningContext = getEndContextForIncompleteExpression(expression, runningContext); + const {regexContext, charClassContext} = runningContext; + if ( + // `--` is matched in one step, so boundary chars aren't `-` unless separated by whitespace + m === '-' && + regexContext === RegexContext.CHAR_CLASS && + lastSignificantCharClassContext === CharClassContext.RANGE && + (options.flags.includes('v') || options.unicodeSetsPlugin) + ) { + // Need to handle this here since the main regex-parsing code would think the hyphen forms + // part of a subtraction operator since we've removed preceding whitespace + throw new Error('Invalid unescaped hyphen as the end value for a range'); + } + if ( + // `??` is matched in one step by the double punctuator token + (regexContext === RegexContext.DEFAULT && /^(?:[?*+]|\?\?)$/.test(m)) || + (regexContext === RegexContext.INTERVAL_QUANTIFIER && m === '{') + ) { + // Skip the separator prefix and connect the quantifier to the previous token. This also + // allows whitespace between a quantifier and the `?` that makes it lazy. Add a postfix + // separator if `m` is `?` and we're following token `(`, to sandbox the `?` from following + // tokens (since `?` can be a group-type marker). Ex: `( ?:)` becomes `(?(?:):)` and throws. + // The loop we're in matches valid group openings in one step, so we won't arrive here if + // matching e.g. `(?:`. Flag n could prevent the need for the postfix since bare `(` is + // converted to `(?:`, but flag x handling always comes first and flag n can be turned off + transformed += update(m, {prefix: false, postfix: lastSignificantToken === '(' && m === '?'}); + } else if (regexContext === RegexContext.DEFAULT) { + if (ws.test(m)) { + ignoringWs = true; + } else if (m.startsWith('#')) { + ignoringComment = true; + } else if (escapedWsOrHash.test(m)) { + transformed += update(m[1], {prefix: false}); + } else { + transformed += update(m); + } + } else if (regexContext === RegexContext.CHAR_CLASS && m !== '[' && m !== '[^') { + if ( + charClassWs.test(m) && + ( charClassContext === CharClassContext.DEFAULT || + charClassContext === CharClassContext.ENCLOSED_Q || + charClassContext === CharClassContext.RANGE + ) + ) { + ignoringCharClassWs = true; + } else if (charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN) { + // Need to handle this here since the main regex-parsing code wouldn't know where the token + // ends if we removed whitespace after an incomplete token that is followed by something + // that completes the token + throw new Error(`Invalid incomplete token in character class: "${m}"`); + } else if ( + escapedCharClassWs.test(m) && + (charClassContext === CharClassContext.DEFAULT || charClassContext === CharClassContext.ENCLOSED_Q) + ) { + transformed += update(m[1], {prefix: false}); + } else if (charClassContext === CharClassContext.DEFAULT) { + const nextChar = value[index + 1] ?? ''; + let updated = sandboxUnsafeNulls(m); + // Avoid escaping lone double punctuators unless required, since some of them are not + // allowed to be escaped with flag u (the `unicodeSetsPlugin` already unescapes them when + // using flag u, but it can be set to `null` via an option) + if (charClassWs.test(nextChar) || m === '^') { + updated = sandboxLoneDoublePunctuatorChar(updated); + } + transformed += update(updated); + } else { + transformed += update(m); + } + } else { + transformed += update(m); + } + if (!(ignoringWs || ignoringCharClassWs || ignoringComment)) { + lastSignificantToken = m; + lastSignificantCharClassContext = charClassContext; + } + } + return { + transformed, + runningContext, + }; +} + +/** +Remove `(?:)` token separators (most likely added by flag x) in cases where it's safe to do so. +@param {string} expression +@returns {PluginResult} +*/ +function clean(expression) { + const sep = String.raw`\(\?:\)`; + // No need for repeated separators + expression = replaceUnescaped(expression, `(?:${sep}){2,}`, '(?:)', Context.DEFAULT); + // No need for separators at: + // - The beginning, if not followed by a quantifier. + // - The end. + // - Outside of character classes: + // - If followed by one of `)|.[$\\`, or `(` if that's not followed by `DEFINE)`. + // - Technically we shouldn't remove `(?:)` if preceded by `(?(DEFINE` and followed by `)`, + // but in this case flag x injects a sandboxing `(?:)` after the preceding invalid `(?`, + // so we already get an error from that. + // - If preceded by one of `()|.]^>`, `\\[bBdDfnrsStvwW]`, `(?:`, or a lookaround opening. + // - So long as the separator is not followed by a quantifier. + // Examples of things that are not safe to remove `(?:)` at the boundaries of: + // - Anywhere: Letters, numbers, or any of `-=_,`. + // - If preceded by any of `\\[cgkpPux]`. + // - Anything inside character classes. + expression = replaceUnescaped( + expression, + String.raw`${sep}(?=[)|.[$\\]|\((?!DEFINE)|$)|(?<=[()|.\]^>]|\\[bBdDfnrsStvwW]|\(\?(?:[:=!]|<[=!])|^)${sep}(?![?*+{])`, + '', + Context.DEFAULT + ); + return { + pattern: expression, + }; +} + +export { + clean, + flagXPreprocessor, +}; diff --git a/src/libs/regex/internals.js b/src/libs/regex/internals.js new file mode 100644 index 00000000..27f51bba --- /dev/null +++ b/src/libs/regex/internals.js @@ -0,0 +1,4 @@ +export {atomic, possessive} from './atomic.js'; +export {RegExpSubclass} from './subclass.js'; +// rexreplace patch: export subroutines (requested upstream) +export {subroutines} from './subroutines.js'; diff --git a/src/libs/regex/pattern.js b/src/libs/regex/pattern.js new file mode 100644 index 00000000..19cb677d --- /dev/null +++ b/src/libs/regex/pattern.js @@ -0,0 +1,45 @@ +class Pattern { + #value; + /** @param {string} value */ + constructor(value) { + this.#value = value; + } + /** @returns {string} */ + toString() { + return String(this.#value); + } +} + +/** +Returns a value that can be interpolated into a `regex` template string without having its special +characters escaped. + +Can be called as a function or template tag: +- `pattern(value)` - String or value coerced to string. +- `` pattern`…` `` - Same as ``pattern(String.raw`…`)``. + +@overload +@param {string | number} value +@returns {Pattern} + +@overload +@param {TemplateStringsArray} template +@param {...string} substitutions +@returns {Pattern} +*/ +function pattern(first, ...substitutions) { + if (Array.isArray(first?.raw)) { + return new Pattern( + // Intersperse raw template strings and substitutions + first.raw.flatMap((raw, i) => i < first.raw.length - 1 ? [raw, substitutions[i]] : raw).join('') + ); + } else if (!substitutions.length) { + return new Pattern(first === undefined ? '' : first); + } + throw new Error(`Unexpected arguments: ${JSON.stringify([first, ...substitutions])}`); +} + +export { + Pattern, + pattern, +}; diff --git a/src/libs/regex/regex.js b/src/libs/regex/regex.js new file mode 100644 index 00000000..cfa5841c --- /dev/null +++ b/src/libs/regex/regex.js @@ -0,0 +1,390 @@ +import {atomic, possessive} from './atomic.js'; +import {backcompatPlugin} from './backcompat.js'; +import {flagNPreprocessor} from './flag-n.js'; +import {clean, flagXPreprocessor} from './flag-x.js'; +import {Pattern, pattern} from './pattern.js'; +import {RegExpSubclass} from './subclass.js'; +import {subroutines} from './subroutines.js'; +import {adjustNumberedBackrefs, CharClassContext, containsCharClassUnion, countCaptures, enclosedTokenCharClassContexts, enclosedTokenRegexContexts, envSupportsFlagGroups, envSupportsFlagV, escapeV, getBreakoutChar, getEndContextForIncompleteExpression, preprocess, RegexContext, sandboxLoneCharClassCaret, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js'; +import {Context, hasUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; + +/** +@typedef {string | RegExp | Pattern | number} InterpolatedValue +@typedef {{ + flags?: string; + captureTransfers?: Map>; + hiddenCaptures?: Array; +}} PluginData +@typedef {{ + pattern: string; + captureTransfers?: Map>; + hiddenCaptures?: Array; +}} PluginResult +@typedef {TemplateStringsArray | {raw: Array}} RawTemplate +@typedef {{ + flags?: string; + subclass?: boolean; + plugins?: Array<(expression: string, data: PluginData) => PluginResult>; + unicodeSetsPlugin?: ((expression: string, data: PluginData) => PluginResult) | null; + disable?: { + x?: boolean; + n?: boolean; + v?: boolean; + atomic?: boolean; + subroutines?: boolean; + }; + force?: { + v?: boolean; + }; +}} RegexTagOptions +*/ +/** +@template T +@typedef RegexTag +@type {{ + (template: RawTemplate, ...substitutions: ReadonlyArray): T; + (flags?: string): RegexTag; + (options: RegexTagOptions & {subclass?: false}): RegexTag; + (options: RegexTagOptions & {subclass: true}): RegexTag; +}} +*/ +/** +Template tag for constructing a regex with extended syntax and context-aware interpolation of +regexes, strings, and patterns. + +Can be called in several ways: +1. `` regex`…` `` - Regex pattern as a raw string. +2. `` regex('gi')`…` `` - To specify flags. +3. `` regex({flags: 'gi'})`…` `` - With options. +@type {RegexTag} +*/ +const regex = (first, ...substitutions) => { + // Given a template + if (Array.isArray(first?.raw)) { + return regexFromTemplate({}, first, ...substitutions); + // Given flags + } else if ((typeof first === 'string' || first === undefined) && !substitutions.length) { + return regexFromTemplate.bind(null, {flags: first ?? ''}); + // Given an options object + } else if ({}.toString.call(first) === '[object Object]' && !substitutions.length) { + return regexFromTemplate.bind(null, first); + } + throw new Error(`Unexpected arguments: ${JSON.stringify([first, ...substitutions])}`); +}; + +/** +@template T +@typedef RegexFromTemplate +@type {{ + (options: RegexTagOptions, template: RawTemplate, ...substitutions: ReadonlyArray) : T; +}} +*/ +/** +Returns a RegExp from a template and substitutions to fill the template holes. +@type {RegexFromTemplate} +*/ +const regexFromTemplate = (options, template, ...substitutions) => { + const opts = getOptions(options); + const prepped = runPreprocessors(template, substitutions, opts); + + let precedingCaptures = 0; + let expression = ''; + let runningContext; + // Intersperse raw template strings and substitutions + prepped.template.raw.forEach((raw, i) => { + const hasNonEmptyRawBoundary = !!(prepped.template.raw[i] || prepped.template.raw[i + 1]); + // Even with flag n enabled, we might have named captures + precedingCaptures += countCaptures(raw); + // Sandbox `\0` in character classes. Not needed outside character classes because in other + // cases a following interpolated value would always be atomized + expression += sandboxUnsafeNulls(raw, Context.CHAR_CLASS); + runningContext = getEndContextForIncompleteExpression(expression, runningContext); + const {regexContext, charClassContext} = runningContext; + if (i < prepped.template.raw.length - 1) { + const substitution = prepped.substitutions[i]; + expression += interpolate( + substitution, + opts.flags, + regexContext, + charClassContext, + hasNonEmptyRawBoundary, + precedingCaptures + ); + if (substitution instanceof RegExp) { + precedingCaptures += countCaptures(substitution.source); + } else if (substitution instanceof Pattern) { + precedingCaptures += countCaptures(String(substitution)); + } + } + }); + + const plugged = runPlugins(expression, opts); + expression = plugged.pattern; + try { + return opts.subclass ? + new RegExpSubclass(expression, opts.flags, {hiddenCaptures: plugged.hiddenCaptures}) : + new RegExp(expression, opts.flags); + } catch (err) { + // Improve DX by always including the generated source in the error message. Some browsers + // include it automatically, but not Firefox or Safari + const stripped = err.message.replace(/ \/.+\/[a-z]*:/, ''); + err.message = `${stripped}: /${expression}/${opts.flags}`; + throw err; + } +}; + +/** +Returns the processed expression and flags as strings. +@param {string} expression +@param {RegexTagOptions} [options] +@returns {{ + pattern: string; + flags: string; +}} +*/ +function rewrite(expression = '', options) { + const opts = getOptions(options); + if (opts.subclass) { + throw new Error('Cannot use option subclass'); + } + return { + // NOTE: Since `pattern` is a Regex+ export with special meaning, the term `expression` is used + // in code to refer to regex source strings, except in the public API + // rexreplace patch: also return hiddenCaptures + ...runPlugins( + runPreprocessors({raw: [expression]}, [], opts).template.raw[0], + opts + ), + flags: opts.flags, + }; +} + +/** +Returns a complete set of options, with default values set for options that weren't provided, and +some options augmented for use. +@param {RegexTagOptions} [options] +@returns {Required} +*/ +function getOptions(options) { + const opts = { + flags: '', + subclass: false, + plugins: [], + unicodeSetsPlugin: backcompatPlugin, + disable: {/* n, v, x, atomic, subroutines */}, + force: {/* v */}, + ...options, + }; + if (/[nuvx]/.test(opts.flags)) { + throw new Error('Implicit flags v/u/x/n cannot be explicitly added'); + } + const useFlagV = opts.force.v || (opts.disable.v ? false : envSupportsFlagV); + opts.flags += useFlagV ? 'v' : 'u'; + if (useFlagV) { + opts.unicodeSetsPlugin = null; + } + return opts; +} + +/** +@param {RawTemplate} template +@param {ReadonlyArray} substitutions +@param {Required} options +@returns {{ + template: RawTemplate; + substitutions: ReadonlyArray; +}} +*/ +function runPreprocessors(template, substitutions, options) { + const preprocessors = []; + // Implicit flag x is handled first because otherwise some regex syntax (if unescaped) within + // comments could cause problems when parsing + if (!options.disable.x) { + preprocessors.push(flagXPreprocessor); + } + // Implicit flag n is a preprocessor because capturing groups affect backreference rewriting in + // both interpolation and plugins + if (!options.disable.n) { + preprocessors.push(flagNPreprocessor); + } + for (const pp of preprocessors) { + ({template, substitutions} = preprocess(template, substitutions, pp, options)); + } + return { + template, + substitutions, + }; +} + +/** +@param {string} expression +@param {Required} options +@returns {Required} +*/ +function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) { + let hiddenCaptures = []; + [ ...plugins, // Run first, so provided plugins can output extended syntax + ...(disable.subroutines ? [] : [subroutines]), + ...(disable.atomic ? [] : [possessive, atomic]), + ...(disable.x ? [] : [clean]), + // Run last, so it doesn't have to worry about parsing extended syntax + ...(!unicodeSetsPlugin ? [] : [unicodeSetsPlugin]), + ].forEach(plugin => { + const result = plugin(expression, {flags, hiddenCaptures}); + if (typeof result?.pattern !== 'string') { + throw new Error('Plugin must return an object with a string property "pattern"'); + } + expression = result.pattern; + if (result.hiddenCaptures) { + hiddenCaptures = result.hiddenCaptures; + } + }); + return { + pattern: expression, + hiddenCaptures, + }; +} + +/** +@param {InterpolatedValue} value +@param {string} flags +@param {string} regexContext +@param {string} charClassContext +@param {boolean} hasNonEmptyRawBoundary +@param {number} precedingCaptures +@returns {string} +*/ +function interpolate(value, flags, regexContext, charClassContext, hasNonEmptyRawBoundary, precedingCaptures) { + if (value instanceof RegExp) { + if (regexContext !== RegexContext.DEFAULT) { + throw new Error('Cannot interpolate a RegExp at this position because the syntax context does not match'); + } + const transformed = transformForLocalFlags(value, flags); + const backrefsAdjusted = adjustNumberedBackrefs(transformed, precedingCaptures); + // Sandbox and atomize + return wrapIfUnsafe(backrefsAdjusted); + } + + if ( + regexContext === RegexContext.INVALID_INCOMPLETE_TOKEN || + charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN + ) { + // Throw in all cases, but only *need* to handle a preceding unescaped backslash (which would + // break sandboxing) since other errors would be handled by the invalid generated regex syntax + throw new Error('Interpolation preceded by invalid incomplete token'); + } + + if ( + typeof value === 'number' && + (regexContext === RegexContext.ENCLOSED_U || charClassContext === CharClassContext.ENCLOSED_U) + ) { + return value.toString(16); + } + + const isPattern = value instanceof Pattern; + value = String(value); + const escapedValue = isPattern ? null : escapeV( + value, + regexContext === RegexContext.CHAR_CLASS ? Context.CHAR_CLASS : Context.DEFAULT + ); + // Checks `escapedValue` in addition to patterns since potential breakout char `>` isn't escaped + const breakoutChar = getBreakoutChar(escapedValue || value, regexContext, charClassContext); + if (breakoutChar) { + throw new Error(`Unescaped stray "${breakoutChar}" in the interpolated value would have side effects outside it`); + } + + if ( + regexContext === RegexContext.INTERVAL_QUANTIFIER || + regexContext === RegexContext.GROUP_NAME || + enclosedTokenRegexContexts.has(regexContext) || + enclosedTokenCharClassContexts.has(charClassContext) + ) { + return isPattern ? value : escapedValue; + } + if (regexContext === RegexContext.CHAR_CLASS) { + if (isPattern) { + if (hasUnescaped(value, '^-|^&&|-$|&&$')) { + // Sandboxing so we don't change the chars outside the pattern into being part of an + // operation they didn't initiate. Same problem as starting a pattern with a quantifier + throw new Error('Cannot use range or set operator at boundary of interpolated pattern; move the operation into the pattern or the operator outside of it'); + } + const sandboxedValue = sandboxLoneCharClassCaret(sandboxLoneDoublePunctuatorChar(value)); + // Atomize via nested character class `[…]` if it contains implicit or explicit union (check + // the unadjusted value) + return containsCharClassUnion(value) ? `[${sandboxedValue}]` : sandboxUnsafeNulls(sandboxedValue); + } + // Atomize via nested character class `[…]` if more than one node + return containsCharClassUnion(escapedValue) ? `[${escapedValue}]` : escapedValue; + } + if (isPattern) { + // Sandbox and atomize + return wrapIfUnsafe(value); + } + // Sandbox and atomize, but avoid noise from noncapturing group if unnecessary (safe to avoid + // because interpolated patterns and regexes are always wrapped) + return hasNonEmptyRawBoundary ? `(?:${escapedValue})` : escapedValue; +} + +/** +@param {string} str +@returns {string} +*/ +function wrapIfUnsafe(str) { + return (str.startsWith('(') && str.endsWith(')')) ? str : `(?:${str})`; +} + +/** +@param {RegExp} re +@param {string} outerFlags +@returns {string} +*/ +function transformForLocalFlags(re, outerFlags) { + /** @type {{i: boolean | null; m: boolean | null; s: boolean | null;}} */ + const modFlagsObj = { + i: null, + m: null, + s: null, + }; + const newlines = '\\n\\r\\u2028\\u2029'; + let value = re.source; + if (re.ignoreCase !== outerFlags.includes('i')) { + if (envSupportsFlagGroups) { + modFlagsObj.i = re.ignoreCase; + } else { + throw new Error('Flag groups not supported, so flag i on the outer and interpolated regex must match'); + } + } + if (re.dotAll !== outerFlags.includes('s')) { + if (envSupportsFlagGroups) { + modFlagsObj.s = re.dotAll; + } else { + value = replaceUnescaped(value, '\\.', (re.dotAll ? '[^]' : `[^${newlines}]`), Context.DEFAULT); + } + } + if (re.multiline !== outerFlags.includes('m')) { + if (envSupportsFlagGroups) { + modFlagsObj.m = re.multiline; + } else { + value = replaceUnescaped(value, '\\^', (re.multiline ? `(?<=^|[${newlines}])` : '(? modFlagsObj[k] === true).join(''); + const modOff = keys.filter(k => modFlagsObj[k] === false).join(''); + if (modOff) { + modifier += `-${modOff}`; + } + if (modifier) { + return `(?${modifier}:${value})`; + } + } + return value; +} + +export { + pattern, + regex, + rewrite, +}; diff --git a/src/libs/regex/subclass.js b/src/libs/regex/subclass.js new file mode 100644 index 00000000..46f17b4f --- /dev/null +++ b/src/libs/regex/subclass.js @@ -0,0 +1,98 @@ +/** +Works the same as JavaScript's native `RegExp` constructor in all contexts, but automatically +adjusts subpattern matches and indices (with flag `d`) to account for captures added as part of +emulating extended syntax. +*/ +class RegExpSubclass extends RegExp { + // Avoid `#private` to allow for subclassing + /** + @private + @type {Map} + */ + _captureMap; + /** + @overload + @param {string} expression + @param {string} [flags] + @param {{ + hiddenCaptures?: Array; + }} [options] + */ + /** + @overload + @param {RegExpSubclass} expression + @param {string} [flags] + */ + constructor(expression, flags, options) { + // Argument `options` isn't provided when regexes are copied via `new RegExpSubclass(regexp)`, + // including as part of the internal handling of string methods `matchAll` and `split` + if (expression instanceof RegExp) { + if (options) { + throw new Error('Cannot provide options when copying a regexp'); + } + super(expression, flags); + if (expression instanceof RegExpSubclass) { + this._captureMap = expression._captureMap; + } else { + this._captureMap = new Map(); + } + } else { + super(expression, flags); + const hiddenCaptures = options?.hiddenCaptures ?? []; + this._captureMap = createCaptureMap(hiddenCaptures); + } + } + /** + Called internally by all String/RegExp methods that use regexes. + @override + @param {string} str + @returns {RegExpExecArray | null} + */ + exec(str) { + const match = super.exec(str); + if (!match || !this._captureMap.size) { + return match; + } + const matchCopy = [...match]; + // Empty all but the first value of the array while preserving its other properties + match.length = 1; + let indicesCopy; + if (this.hasIndices) { + indicesCopy = [...match.indices]; + match.indices.length = 1; + } + for (let i = 1; i < matchCopy.length; i++) { + if (!this._captureMap.get(i)?.hidden) { + match.push(matchCopy[i]); + if (this.hasIndices) { + match.indices.push(indicesCopy[i]); + } + } + } + return match; + } +} + +/** +Build the capturing group map, with hidden captures marked to indicate their submatches shouldn't +appear in match results. +@param {Array} hiddenCaptures +@returns {Map} +*/ +function createCaptureMap(hiddenCaptures) { + const captureMap = new Map(); + for (const num of hiddenCaptures) { + captureMap.set(num, { + hidden: true, + }); + } + return captureMap; +} + +export { + RegExpSubclass, +}; diff --git a/src/libs/regex/subroutines.js b/src/libs/regex/subroutines.js new file mode 100644 index 00000000..8af34bf8 --- /dev/null +++ b/src/libs/regex/subroutines.js @@ -0,0 +1,375 @@ +import {capturingDelim, countCaptures, namedCapturingDelim} from './utils.js'; +import {incrementIfAtLeast, spliceStr} from './utils-internals.js'; +import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; +/** +@import {PluginData, PluginResult} from './regex.js'; +*/ + +/** +@param {string} expression +@param {PluginData} [data] +@returns {PluginResult} +*/ +function subroutines(expression, data) { + // NOTE: subroutines and definition groups fully support numbered backreferences and unnamed + // captures (from interpolated regexes or from turning implicit flag n off), and all of the + // complex forward and backward backreference adjustments that can result + const namedGroups = getNamedCapturingGroups(expression, {includeContents: true}); + const transformed = processSubroutines(expression, namedGroups, data?.hiddenCaptures ?? []); + return { + pattern: processDefinitionGroup(transformed.pattern, namedGroups), + hiddenCaptures: transformed.hiddenCaptures, + }; +} + +// Explicitly exclude `&` from subroutine name chars because it's used by extension +// `regex-recursion` for recursive subroutines via `\g` +const subroutinePattern = String.raw`\\g<(?[^>&]+)>`; +const token = new RegExp(String.raw` +${subroutinePattern} +| (?${capturingDelim}) +| \\(?[1-9]\d*) +| \\k<(?[^>]+)> +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +@typedef { + Map} NamedCapturingGroupsMap +*/ +/** +Apply transformations for subroutines: `\g`. +@param {string} expression +@param {NamedCapturingGroupsMap} namedGroups +@param {Array} hiddenCaptures +@returns {PluginResult} +*/ +function processSubroutines(expression, namedGroups, hiddenCaptures) { + if (!/\\g]+>)', Context.DEFAULT); + const subroutineWrapper = hasBackrefs ? '(' : '(?:'; + const openSubroutines = new Map(); + const openSubroutinesStack = []; + const captureNumMap = [0]; + const addedHiddenCaptures = []; + let numCapturesPassedOutsideSubroutines = 0; + let numCapturesPassedInsideSubroutines = 0; + let numCapturesPassedInsideThisSubroutine = 0; + let numSubroutineCapturesTrackedInRemap = 0; + let numCharClassesOpen = 0; + let match; + token.lastIndex = 0; + while (match = token.exec(expression)) { + const {0: m, index, groups: {subroutineName, capturingStart, backrefNum, backrefName}} = match; + if (m === '[') { + numCharClassesOpen++; + } else if (!numCharClassesOpen) { + + if (subroutineName) { + if (!namedGroups.has(subroutineName)) { + throw new Error(`Invalid named capture referenced by subroutine ${m}`); + } + if (openSubroutines.has(subroutineName)) { + throw new Error(`Subroutine ${m} followed a recursive reference`); + } + const contents = namedGroups.get(subroutineName).contents; + // Wrap value in case it has top-level alternation or is followed by a quantifier. The + // wrapper also marks the end of the expanded contents, which we'll track using + // `unclosedGroupCount`. If there are any backrefs in the expression, wrap with `()` + // instead of `(?:)` in case there are backrefs inside the subroutine that refer to their + // containing capturing group + const subroutineValue = `${subroutineWrapper}${contents})`; + if (hasBackrefs) { + numCapturesPassedInsideThisSubroutine = 0; + numCapturesPassedInsideSubroutines++; + updateHiddenCaptureTracking( + hiddenCaptures, + addedHiddenCaptures, + numCapturesPassedOutsideSubroutines + numCapturesPassedInsideSubroutines + ); + } + openSubroutines.set(subroutineName, { + // Incrementally decremented to track when we've left the group + unclosedGroupCount: countOpenParens(subroutineValue), + }); + openSubroutinesStack.push(subroutineName); + // Expand the subroutine's contents into the pattern we're looping over + expression = spliceStr(expression, index, m, subroutineValue); + token.lastIndex -= m.length - subroutineWrapper.length; + } else if (capturingStart) { + // Somewhere within an expanded subroutine + if (openSubroutines.size) { + if (hasBackrefs) { + numCapturesPassedInsideThisSubroutine++; + numCapturesPassedInsideSubroutines++; + updateHiddenCaptureTracking( + hiddenCaptures, + addedHiddenCaptures, + numCapturesPassedOutsideSubroutines + numCapturesPassedInsideSubroutines + ); + } + // Named capturing group + if (m !== '(') { + // Replace named with unnamed capture. Subroutines ideally wouldn't create any new + // captures, but it can't be helped since we need any backrefs to this capture to work. + // Given that flag n prevents unnamed capture and thereby requires you to rely on named + // backrefs and `groups`, switching to unnamed essentially accomplishes not creating a + // capture. Can fully avoid capturing if there are no backrefs in the expression + expression = spliceStr(expression, index, m, subroutineWrapper); + token.lastIndex -= m.length - subroutineWrapper.length; + } + } else if (hasBackrefs) { + captureNumMap.push( + lastOf(captureNumMap) + 1 + + numCapturesPassedInsideSubroutines - + numSubroutineCapturesTrackedInRemap + ); + numSubroutineCapturesTrackedInRemap = numCapturesPassedInsideSubroutines; + numCapturesPassedOutsideSubroutines++; + } + } else if ((backrefNum || backrefName) && openSubroutines.size) { + // Unify handling for named and unnamed by always using the backref num + const num = backrefNum ? +backrefNum : namedGroups.get(backrefName)?.groupNum; + let isGroupFromThisSubroutine = false; + // Search for the group in the contents of the subroutine stack + for (const s of openSubroutinesStack) { + const group = namedGroups.get(s); + if (num >= group.groupNum && num <= (group.groupNum + group.numCaptures)) { + isGroupFromThisSubroutine = true; + break; + } + } + if (isGroupFromThisSubroutine) { + const group = namedGroups.get(lastOf(openSubroutinesStack)); + // Replace the backref with metadata we'll need to rewrite it later, using + // `\k<$$bNsNrNcN>` as a temporary wrapper: + // - b: The unmodified matched backref num, or the corresponding num of a named backref + // - s: The capture num of the subroutine we're most deeply nested in, including captures + // added by expanding the contents of preceding subroutines + // - r: The original capture num of the group that the subroutine we're most deeply + // nested in references, not counting the effects of subroutines + // - c: The number of captures within `r`, not counting the effects of subroutines + const subroutineNum = numCapturesPassedOutsideSubroutines + numCapturesPassedInsideSubroutines - numCapturesPassedInsideThisSubroutine; + const metadata = `\\k<$$b${num}s${subroutineNum}r${group.groupNum}c${group.numCaptures}>`; + expression = spliceStr(expression, index, m, metadata); + token.lastIndex += metadata.length - m.length; + } + } else if (m === ')') { + if (openSubroutines.size) { + const subroutine = openSubroutines.get(lastOf(openSubroutinesStack)); + subroutine.unclosedGroupCount--; + if (!subroutine.unclosedGroupCount) { + openSubroutines.delete(openSubroutinesStack.pop()); + } + } + } + + } else if (m === ']') { + numCharClassesOpen--; + } + } + + hiddenCaptures.push(...addedHiddenCaptures); + + if (hasBackrefs) { + // Second pass to adjust backrefs + expression = replaceUnescaped( + expression, + String.raw`\\(?:(?[1-9]\d*)|k<\$\$b(?\d+)s(?\d+)r(?\d+)c(?\d+)>)`, + ({0: m, groups: {bNum, bNumSub, subNum, refNum, refCaps}}) => { + if (bNum) { + const backrefNum = +bNum; + if (backrefNum > captureNumMap.length - 1) { + throw new Error(`Backref "${m}" greater than number of captures`); + } + return `\\${captureNumMap[backrefNum]}`; + } + const backrefNumInSubroutine = +bNumSub; + const subroutineGroupNum = +subNum; + const refGroupNum = +refNum; + const numCapturesInRef = +refCaps; + if (backrefNumInSubroutine < refGroupNum || backrefNumInSubroutine > (refGroupNum + numCapturesInRef)) { + return `\\${captureNumMap[backrefNumInSubroutine]}`; + } + return `\\${subroutineGroupNum - refGroupNum + backrefNumInSubroutine}`; + }, + Context.DEFAULT + ); + } + + return { + pattern: expression, + hiddenCaptures, + }; +} + +// `(?:)` allowed because it can be added by flag x's preprocessing of whitespace and comments +const defineGroupToken = new RegExp(String.raw`${namedCapturingDelim}|\(\?:\)|(?\\?.)`, 'gsu'); + +/** +Remove valid subroutine definition groups: `(?(DEFINE)…)`. +@param {string} expression +@param {NamedCapturingGroupsMap} namedGroups +IMPORTANT: Avoid using the `contents` property of `namedGroups` objects, because at this point +subroutine substitution has been performed on the corresponding substrings in `expression` +@returns {string} +*/ +function processDefinitionGroup(expression, namedGroups) { + const defineMatch = execUnescaped(expression, String.raw`\(\?\(DEFINE\)`, 0, Context.DEFAULT); + if (!defineMatch) { + return expression; + } + const defineGroup = getGroup(expression, defineMatch); + if (defineGroup.afterPos < expression.length) { + // Supporting DEFINE at positions other than the end would complicate backref handling. + // NOTE: Flag x's preprocessing permits trailing whitespace and comments + throw new Error('DEFINE group allowed only at the end of a regex'); + } else if (defineGroup.afterPos > expression.length) { + throw new Error('DEFINE group is unclosed'); + } + let match; + defineGroupToken.lastIndex = 0; + while (match = defineGroupToken.exec(defineGroup.contents)) { + const {captureName, invalid} = match.groups; + if (captureName) { + const group = getGroup(defineGroup.contents, match); + let duplicateName; + if (!namedGroups.get(captureName).isUnique) { + duplicateName = captureName; + } else { + const nestedNamedGroups = getNamedCapturingGroups(group.contents, {includeContents: false}); + for (const name of nestedNamedGroups.keys()) { + if (!namedGroups.get(name).isUnique) { + duplicateName = name; + break; + } + } + } + if (duplicateName) { + throw new Error(`Duplicate group name "${duplicateName}" within DEFINE`); + } + defineGroupToken.lastIndex = group.afterPos; + } else if (invalid) { + // Since a DEFINE group is stripped from its expression, we can't easily determine whether + // unreferenced top-level syntax within it is valid. Such syntax serves no purpose, so it's + // easiest to not allow it + throw new Error(`DEFINE group includes unsupported syntax at top level`); + } + } + return expression.slice(0, defineMatch.index); +} + +/** +Counts unescaped open parens outside of character classes, regardless of group type +@param {string} expression +@returns {number} +*/ +function countOpenParens(expression) { + let num = 0; + forEachUnescaped(expression, '\\(', () => num++, Context.DEFAULT); + return num; +} + +/** +@param {string} expression +@param {string} groupName +@returns {number} +*/ +function getCaptureNum(expression, groupName) { + let num = 0; + let pos = 0; + let match; + while (match = execUnescaped(expression, capturingDelim, pos, Context.DEFAULT)) { + const {0: m, index, groups: {captureName}} = match; + num++; + if (captureName === groupName) { + break; + } + pos = index + m.length; + } + return num; +} + +/** +@param {string} expression +@param {RegExpExecArray} delimMatch +@returns {{contents: string; afterPos: number}} +*/ +function getGroup(expression, delimMatch) { + const contentsStart = delimMatch.index + delimMatch[0].length; + const contents = getGroupContents(expression, contentsStart); + const afterPos = contentsStart + contents.length + 1; + return { + contents, + afterPos, + }; +} + +/** +@param {string} expression +@param {{includeContents: boolean}} options +@returns {NamedCapturingGroupsMap} +*/ +function getNamedCapturingGroups(expression, {includeContents}) { + const namedGroups = new Map(); + forEachUnescaped( + expression, + namedCapturingDelim, + ({0: m, index, groups: {captureName}}) => { + // If there are duplicate capture names, subroutines refer to the first instance of the given + // group (matching the behavior of PCRE and Perl) + if (namedGroups.has(captureName)) { + namedGroups.get(captureName).isUnique = false; + } else { + const group = {isUnique: true}; + if (includeContents) { + const contents = getGroupContents(expression, index + m.length); + Object.assign(group, { + contents, + groupNum: getCaptureNum(expression, captureName), + numCaptures: countCaptures(contents), + }); + } + namedGroups.set(captureName, group); + } + }, + Context.DEFAULT + ); + return namedGroups; +} + +/** +@param {Array} arr +@returns {any} +*/ +function lastOf(arr) { + // Remove when support for ES2022 array method `at` (Node.js 16.6) is no longer an issue: + // + return arr[arr.length - 1]; +} + +/** +@param {Array} hiddenCaptures +@param {Array} addedHiddenCaptures +@param {number} addedCaptureNum +*/ +function updateHiddenCaptureTracking(hiddenCaptures, addedHiddenCaptures, addedCaptureNum) { + addedHiddenCaptures.push(addedCaptureNum); + incrementIfAtLeast(hiddenCaptures, addedCaptureNum); +} + +export { + subroutines, +}; diff --git a/src/libs/regex/utils-internals.js b/src/libs/regex/utils-internals.js new file mode 100644 index 00000000..5fe3581c --- /dev/null +++ b/src/libs/regex/utils-internals.js @@ -0,0 +1,33 @@ +// Separating some utils for improved tree shaking of the `./internals` export + +const noncapturingDelim = String.raw`\(\?(?:[:=!>A-Za-z\-]|<[=!]|\(DEFINE\))`; + +/** +Updates the array in place by incrementing each value greater than or equal to the threshold. +@param {Array} arr +@param {number} threshold +*/ +function incrementIfAtLeast(arr, threshold) { + for (let i = 0; i < arr.length; i++) { + if (arr[i] >= threshold) { + arr[i]++; + } + } +} + +/** +@param {string} str +@param {number} pos +@param {string} oldValue +@param {string} newValue +@returns {string} +*/ +function spliceStr(str, pos, oldValue, newValue) { + return str.slice(0, pos) + newValue + str.slice(pos + oldValue.length); +} + +export { + incrementIfAtLeast, + noncapturingDelim, + spliceStr, +}; diff --git a/src/libs/regex/utils.js b/src/libs/regex/utils.js new file mode 100644 index 00000000..cd88ce97 --- /dev/null +++ b/src/libs/regex/utils.js @@ -0,0 +1,401 @@ +import {Pattern, pattern} from './pattern.js'; +import {Context, forEachUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; +/** +@import {InterpolatedValue, RawTemplate, RegexTagOptions} from './regex.js'; +*/ + +const RegexContext = { + DEFAULT: 'DEFAULT', + CHAR_CLASS: 'CHAR_CLASS', + ENCLOSED_P: 'ENCLOSED_P', + ENCLOSED_U: 'ENCLOSED_U', + GROUP_NAME: 'GROUP_NAME', + INTERVAL_QUANTIFIER: 'INTERVAL_QUANTIFIER', + INVALID_INCOMPLETE_TOKEN: 'INVALID_INCOMPLETE_TOKEN', +}; + +const CharClassContext = { + DEFAULT: 'DEFAULT', + ENCLOSED_P: 'ENCLOSED_P', + ENCLOSED_Q: 'ENCLOSED_Q', + ENCLOSED_U: 'ENCLOSED_U', + INVALID_INCOMPLETE_TOKEN: 'INVALID_INCOMPLETE_TOKEN', + RANGE: 'RANGE', +}; + +const enclosedTokenRegexContexts = new Set([ + RegexContext.ENCLOSED_P, + RegexContext.ENCLOSED_U, +]); + +const enclosedTokenCharClassContexts = new Set([ + CharClassContext.ENCLOSED_P, + CharClassContext.ENCLOSED_Q, + CharClassContext.ENCLOSED_U, +]); + +const envSupportsFlagGroups = (() => { + try { + new RegExp('(?i:)'); + } catch { + return false; + } + return true; +})(); + +const envSupportsFlagV = (() => { + try { + new RegExp('', 'v'); + } catch { + return false; + } + return true; +})(); + +const doublePunctuatorChars = '&!#$%*+,.:;<=>?@^`~'; +const namedCapturingDelim = String.raw`\(\?<(?![=!])(?[^>]+)>`; +const capturingDelim = String.raw`\((?!\?)(?!(?<=\(\?\()DEFINE\))|${namedCapturingDelim}`; + +/** +@param {string} expression +@param {number} precedingCaptures +@returns {string} +*/ +function adjustNumberedBackrefs(expression, precedingCaptures) { + return replaceUnescaped( + expression, + String.raw`\\(?[1-9]\d*)`, + ({groups: {num}}) => `\\${+num + precedingCaptures}`, + Context.DEFAULT + ); +} + +// Properties of strings as of ES2024 +const stringPropertyNames = [ + 'Basic_Emoji', + 'Emoji_Keycap_Sequence', + 'RGI_Emoji_Modifier_Sequence', + 'RGI_Emoji_Flag_Sequence', + 'RGI_Emoji_Tag_Sequence', + 'RGI_Emoji_ZWJ_Sequence', + 'RGI_Emoji', +].join('|'); +const charClassUnionToken = new RegExp(String.raw` +\\(?: c[A-Za-z] + | p\{(?${stringPropertyNames})\} + | [pP]\{[^\}]+\} + | (?q) + | u(?:[A-Fa-f\d]{4}|\{[A-Fa-f\d]+\}) + | x[A-Fa-f\d]{2} + | . +) +| -- +| && +| . +`.replace(/\s+/g, ''), 'gsu'); + +// Assumes flag v and doesn't worry about syntax errors that are caught by it +function containsCharClassUnion(charClassPattern) { + // Return `true` if it contains: + // - `\p` (lowercase only) and the name is a property of strings (case sensitive). + // - `\q`. + // - Two single-char-matching tokens in sequence. + // - One single-char-matching token followed immediately by unescaped `[`. + // - One single-char-matching token preceded immediately by unescaped `]`. + // Else, `false`. + // Ranges with `-` create a single token. + // Subtraction and intersection with `--` and `&&` create a single token. + // Supports any number of nested classes + let hasFirst = false; + let lastM; + for (const {0: m, groups} of charClassPattern.matchAll(charClassUnionToken)) { + if (groups.pStrProp || groups.qStrProp) { + return true; + } + if (m === '[' && hasFirst) { + return true; + } + if (['-', '--', '&&'].includes(m)) { + hasFirst = false; + } else if (m !== '[' && m !== ']') { + if (hasFirst || lastM === ']') { + return true; + } + hasFirst = true; + } + lastM = m; + } + return false; +} + +/** +@param {string} expression +@returns {number} +*/ +function countCaptures(expression) { + let num = 0; + forEachUnescaped(expression, capturingDelim, () => num++, Context.DEFAULT); + return num; +} + +/** +Escape special characters for the given context, assuming flag v. +@param {string} str String to escape +@param {'DEFAULT' | 'CHAR_CLASS'} context `Context` option from lib `regex-utilities` +@returns {string} Escaped string +*/ +function escapeV(str, context) { + if (context === Context.CHAR_CLASS) { + // Escape all double punctuators (including ^, which is special on its own in the first + // position) in case they're bordered by the same character in or outside of the escaped string + return str.replace(new RegExp(String.raw`[()\[\]{}|\\/\-${doublePunctuatorChars}]`, 'g'), '\\$&'); + } + return str.replace(/[()\[\]{}|\\^$*+?.]/g, '\\$&'); +} + +// Look for characters that would change the meaning of subsequent tokens outside an interpolated value +function getBreakoutChar(expression, regexContext, charClassContext) { + const escapesRemoved = expression.replace(/\\./gsu, ''); + // Trailing unescaped `\`; checking `.includes('\\')` would also work + if (escapesRemoved.endsWith('\\')) { + return '\\'; + } + if (regexContext === RegexContext.DEFAULT) { + // Unbalanced `[` or `]` are also errors but don't breakout; they're caught by the wrapper + return getUnbalancedChar(escapesRemoved, '(', ')'); + } else if ( + regexContext === RegexContext.CHAR_CLASS && + !enclosedTokenCharClassContexts.has(charClassContext) + ) { + return getUnbalancedChar(escapesRemoved, '[', ']'); + } else if ( + regexContext === RegexContext.INTERVAL_QUANTIFIER || + enclosedTokenRegexContexts.has(regexContext) || + enclosedTokenCharClassContexts.has(charClassContext) + ) { + if (escapesRemoved.includes('}')) { + return '}'; + } + } else if (regexContext === RegexContext.GROUP_NAME) { + if (escapesRemoved.includes('>')) { + return '>'; + } + } + return ''; +} + +const contextToken = new RegExp(String.raw` +(?\(\?<(?![=!])|\\[gk]<) +| (?\\[pPu]\{) +| (?\\q\{) +| (?\{) +| (?\\(?: $ + | c(?![A-Za-z]) + | u(?![A-Fa-f\d]{4})[A-Fa-f\d]{0,3} + | x(?![A-Fa-f\d]{2})[A-Fa-f\d]? + ) +) +| -- +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +@typedef {{ + regexContext: string; + charClassContext: string; + charClassDepth: number; + lastPos: number; +}} RunningContext +*/ +/** +Accepts and returns its full state so it doesn't have to reprocess parts that have already been +seen. Assumes flag v and doesn't worry about syntax errors that are caught by it. +@param {string} incompleteExpression +@param {Partial} [runningContext] +@returns {RunningContext} +*/ +function getEndContextForIncompleteExpression(incompleteExpression, runningContext) { + let {regexContext, charClassContext, charClassDepth, lastPos} = { + regexContext: RegexContext.DEFAULT, + charClassContext: CharClassContext.DEFAULT, + charClassDepth: 0, + lastPos: 0, + ...runningContext, + }; + contextToken.lastIndex = lastPos; + let match; + while (match = contextToken.exec(incompleteExpression)) { + const {0: m, groups: {groupN, enclosedPU, enclosedQ, intervalQ, incompleteT}} = match; + if (m === '[') { + charClassDepth++; + regexContext = RegexContext.CHAR_CLASS; + charClassContext = CharClassContext.DEFAULT; + } else if (m === ']' && regexContext === RegexContext.CHAR_CLASS) { + if (charClassDepth) { + charClassDepth--; + } + if (!charClassDepth) { + regexContext = RegexContext.DEFAULT; + } + charClassContext = CharClassContext.DEFAULT; + } else if (regexContext === RegexContext.CHAR_CLASS) { + if (incompleteT) { + charClassContext = CharClassContext.INVALID_INCOMPLETE_TOKEN; + } else if (m === '-') { + charClassContext = CharClassContext.RANGE; + } else if (enclosedPU) { + charClassContext = m[1] === 'u' ? CharClassContext.ENCLOSED_U : CharClassContext.ENCLOSED_P; + } else if (enclosedQ) { + charClassContext = CharClassContext.ENCLOSED_Q; + } else if ( + (m === '}' && enclosedTokenCharClassContexts.has(charClassContext)) || + // Don't continue in these contexts since we've advanced another token + charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN || + charClassContext === CharClassContext.RANGE + ) { + charClassContext = CharClassContext.DEFAULT; + } + } else { + if (incompleteT) { + regexContext = RegexContext.INVALID_INCOMPLETE_TOKEN; + } else if (groupN) { + regexContext = RegexContext.GROUP_NAME; + } else if (enclosedPU) { + regexContext = m[1] === 'u' ? RegexContext.ENCLOSED_U : RegexContext.ENCLOSED_P; + } else if (intervalQ) { + regexContext = RegexContext.INTERVAL_QUANTIFIER; + } else if ( + (m === '>' && regexContext === RegexContext.GROUP_NAME) || + (m === '}' && (regexContext === RegexContext.INTERVAL_QUANTIFIER || enclosedTokenRegexContexts.has(regexContext))) || + // Don't continue in this context since we've advanced another token + regexContext === RegexContext.INVALID_INCOMPLETE_TOKEN + ) { + regexContext = RegexContext.DEFAULT; + } + } + } + return { + regexContext, + charClassContext, + charClassDepth, + lastPos: incompleteExpression.length, + }; +} + +// No special handling for escaped versions of the characters +function getUnbalancedChar(expression, leftChar, rightChar) { + let numOpen = 0; + for (const [m] of expression.matchAll(new RegExp(`[${escapeV(leftChar + rightChar, Context.CHAR_CLASS)}]`, 'g'))) { + numOpen += m === leftChar ? 1 : -1; + if (numOpen < 0) { + return rightChar; + } + } + if (numOpen > 0) { + return leftChar; + } + return ''; +} + +/** +@typedef {InterpolatedValue} InterpolatedValue +@typedef {RawTemplate} RawTemplate +@typedef {RegexTagOptions} RegexTagOptions +@typedef {( + value: InterpolatedValue, + runningContext: RunningContext, + options: Required +) => { + transformed: string; + runningContext: RunningContext; +}} Preprocessor +*/ +/** +Returns transformed versions of a template and substitutions, using the given preprocessor. Only +processes substitutions that are instanceof `Pattern`. +@param {RawTemplate} template +@param {ReadonlyArray} substitutions +@param {Preprocessor} preprocessor +@param {Required} options +@returns {{template: RawTemplate; substitutions: ReadonlyArray;}} +*/ +function preprocess(template, substitutions, preprocessor, options) { + let /** @type {RawTemplate} */ newTemplate = {raw: []}; + let newSubstitutions = []; + let runningContext; + template.raw.forEach((raw, i) => { + const result = preprocessor(raw, {...runningContext, lastPos: 0}, options); + newTemplate.raw.push(result.transformed); + runningContext = result.runningContext; + if (i < template.raw.length - 1) { + const substitution = substitutions[i]; + if (substitution instanceof Pattern) { + const result = preprocessor(substitution, {...runningContext, lastPos: 0}, options); + newSubstitutions.push(pattern(result.transformed)); + runningContext = result.runningContext; + } else { + newSubstitutions.push(substitution); + } + } + }); + return { + template: newTemplate, + substitutions: newSubstitutions, + }; +} + +// Sandbox `^` if relevant, done so it can't change the meaning of the surrounding character class +// if we happen to be at the first position. See `sandboxLoneDoublePunctuatorChar` for more details +function sandboxLoneCharClassCaret(str) { + return str.replace(/^\^/, '\\^^'); +} + +// Sandbox without escaping by repeating the character and escaping only the first one. The second +// one is so that, if followed by the same symbol, the resulting double punctuator will still throw +// as expected. Details: +// - Only need to check the first position because, if it's part of an implicit union, +// interpolation handling will wrap it in nested `[…]`. +// - Can't just wrap in nested `[…]` here, since the value might be used in a range. +// - Can't add a second unescaped symbol if a lone symbol is the entire string because it might be +// followed by the same unescaped symbol outside an interpolation, and since it won't be wrapped, +// the second symbol wouldn't be sandboxed from the one following it. +function sandboxLoneDoublePunctuatorChar(str) { + return str.replace(new RegExp(`^([${doublePunctuatorChars}])(?!\\1)`), (m, _, pos) => { + return `\\${m}${pos + 1 === str.length ? '' : m}`; + }); +} + +/** +Converts `\0` tokens to `\x00` in the given context. +@param {string} str +@param {'DEFAULT' | 'CHAR_CLASS'} [context] `Context` option from lib `regex-utilities` +@returns {string} +*/ +function sandboxUnsafeNulls(str, context) { + // regex`[\0${0}]` and regex`[${pattern`\0`}0]` can't be guarded against via nested `[…]` + // sandboxing in character classes if the interpolated value doesn't contain union (since it + // might be placed on a range boundary). So escape `\0` in character classes as `\x00` + return replaceUnescaped(str, String.raw`\\0(?!\d)`, '\\x00', context); +} + +export { + adjustNumberedBackrefs, + capturingDelim, + CharClassContext, + containsCharClassUnion, + countCaptures, + doublePunctuatorChars, + enclosedTokenCharClassContexts, + enclosedTokenRegexContexts, + envSupportsFlagGroups, + envSupportsFlagV, + escapeV, + getBreakoutChar, + getEndContextForIncompleteExpression, + namedCapturingDelim, + preprocess, + RegexContext, + sandboxLoneCharClassCaret, + sandboxLoneDoublePunctuatorChar, + sandboxUnsafeNulls, +}; diff --git a/test/cli/debug.test.js b/test/cli/debug.test.js index f5754ab6..534b7535 100644 --- a/test/cli/debug.test.js +++ b/test/cli/debug.test.js @@ -16,6 +16,8 @@ describe('--debug', () => { expect(status.files).toEqual({searched: ['my.file'], updated: ['my.file']}); expect(status.engine.config).toMatchObject({pattern: 'o', replacement: 'x'}); expect(status.engine.execution).toEqual({ + engine: 'rr', + hiddenCaptures: [], search: 'o', flags: 'gim', replacement: 'x', diff --git a/test/cli/run.sh b/test/cli/run.sh index 88777bdf..c2cbfd9e 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -74,9 +74,12 @@ assert "rexreplace o x my.file --output" "fxxbar" assert "cat my.file" "foobar" -# --engine: `default` is the engine of the runtime: V8 with Node and JSC with Bun +# --engine: `rr` (also `default`) reads patterns like the native engine, with the extended syntax of Regex+, like atomic groups. +# `runtime` is the native engine: V8 with Node and JSC with Bun. reset assert "rexreplace o x my.file --output --engine default" "fxxbar" +assert "rexreplace '(?>o+)(b)' '[\$1]' my.file --output" "f[b]ar" +assert "rexreplace o x my.file --output --engine runtime" "fxxbar" if [[ "$(rexreplace --version)" == *+bun ]]; then assert_failure "rexreplace o x my.file --output --engine V8" else diff --git a/test/engine/engine.test.js b/test/engine/engine.test.js index 7e8d71d8..96016ec0 100644 --- a/test/engine/engine.test.js +++ b/test/engine/engine.test.js @@ -3,7 +3,7 @@ import fs from 'fs'; import {rr} from '../helper.js'; describe('--engine', () => { - test('default is the engine of the runtime', async () => { + test('default is rr', async () => { expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'default'})).out).toBe('fxxbar'); }); @@ -17,7 +17,7 @@ describe('--engine', () => { const result = await rr('o', 'x', {includeGlob: ['my.file'], engine: 'V8'}); expect(result.exit).toBe(3); expect(result.err).toContain( - 'When running via the Bun runtime you can only use default or JSC as engine' + 'When running via the Bun runtime the native engine is JSC, not V8' ); expect(fs.readFileSync('my.file', 'utf8')).toBe('foobar'); }); diff --git a/test/fix/705.test.js b/test/fix/705.test.js index 7baeea14..ea709163 100644 --- a/test/fix/705.test.js +++ b/test/fix/705.test.js @@ -55,7 +55,7 @@ describe('#705 the engine returns a promise that resolves when all files are don {includeGlob: ['nope.file'], quietTotal: true}, ]) { const done = engine({ - engine: 'default', + engine: 'rr', encoding: 'utf8', pipedData: null, includeGlob: ['my.file'], diff --git a/test/fix/742.test.js b/test/fix/742.test.js index bc93c2a9..5693cbcb 100644 --- a/test/fix/742.test.js +++ b/test/fix/742.test.js @@ -141,6 +141,8 @@ describe('#742 the engine part of the status', () => { ...flags, }); expect(status.engine.execution).toEqual({ + engine: 'rr', + hiddenCaptures: [], search: 'o+', flags: 'gi', replacement, diff --git a/test/fix/813.test.js b/test/fix/813.test.js index 2a76e2ca..d782252a 100644 --- a/test/fix/813.test.js +++ b/test/fix/813.test.js @@ -30,7 +30,7 @@ describe('#813 a run that stops', () => { test('tells the panic of a second run with the same config again (#844)', async () => { const config = { - engine: 'default', + engine: 'rr', encoding: 'utf8', pipedData: 'foo', includeGlob: [], diff --git a/test/fix/854.test.js b/test/fix/854.test.js new file mode 100644 index 00000000..b4d8b378 --- /dev/null +++ b/test/fix/854.test.js @@ -0,0 +1,139 @@ +// https://github.com/mathiasrw/rexreplace/issues/854 +// Regex+ is behind the engines. rr, the default, reads patterns like the native engine, with the extended syntax of Regex+ on top. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr, rrStatus} from '../helper.js'; + +// What the regex engine was asked to do: the source and the flags of the regex +async function compiled(pattern, config = {}) { + const status = await rrStatus(pattern, 'x', { + pipedData: '', + verbose: true, + status: true, + ...config, + }); + return status.engine.execution; +} + +const out = async (pattern, replacement, text, config = {}) => + (await rr(pattern, replacement, {pipedData: text, ...config})).out; + +describe('#854 the rr engine', () => { + test.each([ + [`(['"])([^\\1]+)\\1`], + [`\\"`], + ['a]'], + ['a{'], + ['\\01'], + ['[(]'], + ['[a-z-]'], + ['\\++'], + ['[+]+'], + ['\\{2}+'], + ['[(?>]'], + ['[a++]'], + ['\\(?>'], + ['(?a)\\k'], + ['\\k'], + ['a+?'], + ['x{1,2}?'], + ['(foo)(.*)'], + ['[\\]+]+'], + ['\\c'], + ['\\x4'], + ['(?x)\\1'], + ['$'], + ['[[]'], + ])('gives the same regex as the native engine for %s', async pattern => { + const native = new RegExp(pattern, 'gim'); + expect(await compiled(pattern)).toMatchObject({search: native.source, flags: 'gim'}); + }); + + test('only uses the u flag with -u', async () => { + expect((await compiled('a', {unicode: true})).flags).toBe('gimu'); + }); + + test.each([ + ['an atomic group', '(?>a+)(b)', '[$1]', 'aab', '[b]'], + ['a possessive quantifier', 'a++b', 'X', 'aaab', 'X'], + ['a possessive quantifier that gives nothing back', 'a++a', 'X', 'aaa', 'aaa'], + ['a subroutine', '(?\\d)-\\g', 'X', '1-2', 'X'], + [ + 'a DEFINE group', + '\\b(?\\g\\.\\g\\.\\g)(?(DEFINE)(?0|[1-9]\\d*))', + 'v$', + '3.12.0 and 01.2.3', + 'v3.12.0 and 01.2.3', + ], + ])('reads %s', async (name, pattern, replacement, text, result) => { + expect(await out(pattern, replacement, text)).toBe(result); + }); + + test('is the engine for default too', async () => { + expect(await out('(?>a+)(b)', '[$1]', 'aab', {engine: 'default'})).toBe('[b]'); + }); +}); + +describe('#854 the groups Regex+ adds', () => { + // (?>a+) adds a group of its own in front of (b) and (c) + test.each([ + ['$1', 'b'], + ['$2$1', 'cb'], + ['$3', '$3'], + ['$10', 'b0'], + ['$$1', '$1'], + ['$&', 'aabc'], + ])('are not counted in the replacement %s', async (replacement, result) => { + expect(await out('(?>a+)(b)(c)?', replacement, 'aabc')).toBe(result); + }); + + test('are not counted in --js', async () => { + expect(await out('(?>a+)(b)', "$1 + '!'", 'aab', {replacementJs: true})).toBe('b!'); + }); + + test('are not printed by --output-match', async () => { + expect(await out('(?>a+)(b)', 'x', 'aab', {outputMatch: true})).toBe('b\n'); + }); + + test('are told in the status report', async () => { + expect(await compiled('(?>a+)(b)')).toMatchObject({engine: 'rr', hiddenCaptures: [1]}); + }); +}); + +describe('#854 the other engines', () => { + // The tests are run by Bun, so the native engine is JSC + test.each([['JSC'], ['runtime']])( + '%s is the native engine, without the extended syntax', + async engine => { + expect(await out('(?a)\\g', 'X', 'ag', {engine})).toBe('X'); + } + ); + + test.each([['regex+v'], ['regex+'], ['V']])( + '%s has free spacing, and only named groups capture', + async engine => { + expect(await out('foo bar', 'X', 'foobar', {engine})).toBe('X'); + expect(await out('(foo)(.*)', '$2$1', 'foox', {engine})).toBe('$2$1'); + expect((await compiled('a', {engine})).flags).toContain('v'); + } + ); + + test.each([['regex+u'], ['U']])('%s uses the u flag and not v', async engine => { + const {flags} = await compiled('a', {engine}); + expect(flags).toContain('u'); + expect(flags).not.toContain('v'); + }); + + test('regex+ stops the run on an escape that is not strict, and the file is not changed', async () => { + fs.writeFileSync('a.txt', 'a"b'); + const result = await rr('\\"', 'x', {includeGlob: ['a.txt'], engine: 'regex+'}); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.txt', 'utf8')).toBe('a"b'); + }); + + test('--literal gives the text as it is, also with the free spacing of regex+', async () => { + expect(await out('a b #c (d)', 'X', 'a b #c (d)!', {engine: 'regex+', literal: true})).toBe( + 'X!' + ); + }); +}); diff --git a/test/helper.js b/test/helper.js index 041a53d2..6807c6fc 100644 --- a/test/helper.js +++ b/test/helper.js @@ -56,7 +56,7 @@ async function run(pattern, replacement, config) { try { const options = { - engine: 'default', + engine: 'rr', encoding: 'utf8', pipedData: null, includeGlob: [], From f5daec1914ed3596e1b4ce1873c90f1db1ddc885 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sun, 25 Oct 2026 08:43:15 +1100 Subject: [PATCH 02/13] Make regex+ the Regex+ engine with v, x and n, add regex+n and regex+x, and drop regex+u --- README.md | 2 +- src/cli.ts | 18 +++++++++++++++--- src/engine.ts | 24 +++++++++++++++++------- test/fix/854.test.js | 26 +++++++++++++++++++------- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5d4149d4..cab2690a 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | -| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+v` (also `regex+` and `V`) is full Regex+ with the `v` flag, free spacing (`x`) and only named groups capturing (`n`). `regex+u` (also `U`) is the same with the `u` flag. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+v", "regex+", "V", "regex+u", "U", "runtime", "V8", "JSC"] [default: "rr"] | +| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. `regex+n` (also `N`) is the same without free spacing, and `regex+x` (also `X`) the same where all groups capture. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+", "V", "regex+n", "N", "regex+x", "X", "runtime", "V8", "JSC"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 1f6bad6e..75f781d4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -141,9 +141,21 @@ const yargs = require('yargs') .alias('E', 'engine') .describe( 'E', - 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+v` (also `regex+` and `V`) is full Regex+ with the `v` flag, free spacing (`x`) and only named groups capturing (`n`). `regex+u` (also `U`) is the same with the `u` flag. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' - ) - .choices('E', ['rr', 'default', 'regex+v', 'regex+', 'V', 'regex+u', 'U', 'runtime', 'V8', 'JSC']) + 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. `regex+n` (also `N`) is the same without free spacing, and `regex+x` (also `X`) the same where all groups capture. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' + ) + .choices('E', [ + 'rr', + 'default', + 'regex+', + 'V', + 'regex+n', + 'N', + 'regex+x', + 'X', + 'runtime', + 'V8', + 'JSC', + ]) .default('E', 'rr') .boolean('q') diff --git a/src/engine.ts b/src/engine.ts index 9c127760..5ade79a3 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -52,6 +52,14 @@ export {version}; // The regex engine comes with the runtime: Node has V8 and Bun has JSC (JavaScriptCore) const runtime = process.versions.bun ? 'Bun' : 'Node'; const runtimeEngine = process.versions.bun ? 'JSC' : 'V8'; +// Node 18 has no v flag, and then Regex+ uses u +const supportsFlagV = (() => { + try { + return Boolean(new RegExp('', 'v')); + } catch (e) { + return false; + } +})(); // Resolves with the status of the run when all files are done. // When the run can not go on, a panic is thrown with the status so far. @@ -716,14 +724,16 @@ function codeReplacement(file, config, text, numbers) { }; } -// -E: default is another name for rr, regex+ and V for regex+v, and U for regex+u -const engineNames = {default: 'rr', 'regex+': 'regex+v', V: 'regex+v', U: 'regex+u'}; +// -E: default is another name for rr, V for regex+, N for regex+n and X for regex+x +const engineNames = {default: 'rr', V: 'regex+', N: 'regex+n', X: 'regex+x'}; +// regex+n leaves out free spacing (x), and regex+x leaves out that only named groups capture (n) +const regexPlusDisable = {'regex+': {}, 'regex+n': {x: true}, 'regex+x': {n: true}}; const engineName = name => engineNames[name] ?? name ?? 'rr'; // The regex and the numbers of the groups the engine added to it, which the user does not know about function compileRegex(pattern, config) { const engine = engineName(config.engine); - if (!['rr', 'regex+v', 'regex+u', 'V8', 'JSC', 'runtime'].includes(engine)) { + if (!['rr', 'regex+', 'regex+n', 'regex+x', 'V8', 'JSC', 'runtime'].includes(engine)) { panic(`Engine ${config.engine} not supported`); } if (('V8' === engine || 'JSC' === engine) && engine !== runtimeEngine) { @@ -731,8 +741,8 @@ function compileRegex(pattern, config) { `When running via the ${runtime} runtime the native engine is ${runtimeEngine}, not ${engine}` ); } - const regexPlus = 'regex+v' === engine || 'regex+u' === engine; - // Regex+ always uses u or v itself, and it is an error to ask for them + const regexPlus = engine in regexPlusDisable; + // Regex+ always uses v itself, or u on a runtime without v, and it is an error to ask for them const flags = (config.voidGlobal ? '' : 'g') + (config.voidIgnoreCase ? '' : 'i') + @@ -743,12 +753,12 @@ function compileRegex(pattern, config) { if (config.literal) { // The text as it is, without the free spacing of Regex+ return { - regex: new RegExp(pattern, flags + (regexPlus ? ('regex+u' === engine ? 'u' : 'v') : '')), + regex: new RegExp(pattern, flags + (regexPlus ? (supportsFlagV ? 'v' : 'u') : '')), hiddenCaptures: [], }; } if (regexPlus) { - const result = rewrite(pattern, {flags, disable: 'regex+u' === engine ? {v: true} : {}}); + const result = rewrite(pattern, {flags, disable: regexPlusDisable[engine]}); return { regex: new RegExp(result.pattern, result.flags), hiddenCaptures: result.hiddenCaptures, diff --git a/test/fix/854.test.js b/test/fix/854.test.js index b4d8b378..d5a4d336 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -1,5 +1,6 @@ // https://github.com/mathiasrw/rexreplace/issues/854 // Regex+ is behind the engines. rr, the default, reads patterns like the native engine, with the extended syntax of Regex+ on top. +// regex+ is Regex+ with the flags v, x and n, regex+n leaves out x, and regex+x leaves out n. import {describe, test, expect} from 'bun:test'; import fs from 'fs'; import {rr, rrStatus} from '../helper.js'; @@ -109,19 +110,30 @@ describe('#854 the other engines', () => { } ); - test.each([['regex+v'], ['regex+'], ['V']])( - '%s has free spacing, and only named groups capture', + test.each([['regex+'], ['V']])( + '%s has free spacing, only named groups capture, and the v flag', async engine => { expect(await out('foo bar', 'X', 'foobar', {engine})).toBe('X'); - expect(await out('(foo)(.*)', '$2$1', 'foox', {engine})).toBe('$2$1'); + expect(await out('(foo)(?.*)', '$1', 'foox', {engine})).toBe('x'); expect((await compiled('a', {engine})).flags).toContain('v'); } ); - test.each([['regex+u'], ['U']])('%s uses the u flag and not v', async engine => { - const {flags} = await compiled('a', {engine}); - expect(flags).toContain('u'); - expect(flags).not.toContain('v'); + test.each([['regex+n'], ['N']])( + '%s has no free spacing, and only named groups capture', + async engine => { + expect(await out('foo bar', 'X', 'foo bar', {engine})).toBe('X'); + expect(await out('(foo)(?.*)', '$1', 'foox', {engine})).toBe('x'); + } + ); + + test.each([['regex+x'], ['X']])('%s has free spacing, and all groups capture', async engine => { + expect(await out('foo bar', 'X', 'foobar', {engine})).toBe('X'); + expect(await out('(foo)(?.*)', '$1', 'foox', {engine})).toBe('foo'); + }); + + test.each([['regex+u'], ['U'], ['regex+v']])('%s is not an engine', async engine => { + expect((await rr('a', 'b', {pipedData: 'a', engine})).exit).toBe(3); }); test('regex+ stops the run on an escape that is not strict, and the file is not changed', async () => { From 7f4fbed34ab8fc9741715c0edc11ddab316fc5b6 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sun, 25 Oct 2026 09:25:19 +1100 Subject: [PATCH 03/13] Add --flags, keep only regex+ with v, x and n, and name the groups of --js $year --- README.md | 5 ++- src/cli.ts | 26 +++++------- src/engine.ts | 97 ++++++++++++++++++++++++++++---------------- test/fix/799.test.js | 8 ++-- test/fix/815.test.js | 2 +- test/fix/816.test.js | 30 +++++++------- test/fix/825.test.js | 48 ++++++++-------------- test/fix/854.test.js | 65 ++++++++++++++++++++++------- 8 files changed, 162 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index cab2690a..32ed06fa 100644 --- a/README.md +++ b/README.md @@ -323,8 +323,9 @@ If planned renames clash, the process is halted and no change is applied to any | `-s` | **`--dot-all`** Have `.` also match newline. [boolean] | | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | +| | **`--flags`** Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which also needs `v` or `u`. [string] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | -| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. `regex+n` (also `N`) is the same without free spacing, and `regex+x` (also `X`) the same where all groups capture. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+", "V", "regex+n", "N", "regex+x", "X", "runtime", "V8", "JSC"] [default: "rr"] | +| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. Use --flags to leave out x or n. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+", "V", "runtime", "V8", "JSC"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | @@ -366,7 +367,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group via the name prepended with `rr_`. `(?...)` gives you `rr_year`. A named group name can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so there is no `$1`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 75f781d4..2b89692b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -118,6 +118,12 @@ const yargs = require('yargs') .describe('u', 'Treat pattern as a sequence of unicode code points.') .alias('u', 'unicode') + .string('flags') + .describe( + 'flags', + 'Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which also needs `v` or `u`.' + ) + .default('e', 'utf8') .alias('e', 'encoding') .describe('e', 'Encoding of files/piped data.') @@ -141,21 +147,9 @@ const yargs = require('yargs') .alias('E', 'engine') .describe( 'E', - 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. `regex+n` (also `N`) is the same without free spacing, and `regex+x` (also `X`) the same where all groups capture. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' - ) - .choices('E', [ - 'rr', - 'default', - 'regex+', - 'V', - 'regex+n', - 'N', - 'regex+x', - 'X', - 'runtime', - 'V8', - 'JSC', - ]) + 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. Use --flags to leave out x or n. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' + ) + .choices('E', ['rr', 'default', 'regex+', 'V', 'runtime', 'V8', 'JSC']) .default('E', 'rr') .boolean('q') @@ -426,7 +420,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). For speed, the code runs once per file unless it names the match, a group, \`$m\` or \`$M\`. -Access a named group via the name prepended with \`rr_\`. \`(?...)\` gives you \`rr_year\`. A named group name can not end with \`_\`. +Access a named group as \`$\` and the name: \`(?...)\` gives you \`$year\`. A group named like a number, like \`n\`, wins over the number. With the flag \`n\` of \`-E regex+\` only named groups capture, so there is no \`$1\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 5ade79a3..99c63454 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -26,6 +26,10 @@ const re = { stackLineWithName: /^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/, // Subroutines of Regex+, which the rr engine only looks for when the pattern has them subroutineSyntax: /\\g<|\(\?\(DEFINE\)/, + // The flags only Regex+ knows: free spacing (x) and only named groups capture (n) + regexPlusOnlyFlags: /[xn]/, + // The flags Regex+ takes as options, and not as flags + regexPlusFlags: /[uvxn]/g, // A $ in a replacement: $$, $&, $`, $', $, or a group number of one or two digits replacementToken: /\$(\$|&|`|'|<[^>]*>|(\d)(\d?))/g, // Only the chars with a meaning outside a character class. Unicode mode does not allow to escape the others (#686). @@ -170,9 +174,11 @@ function unpackConfig(config) { // --output-match wins over --output config.output = config.output && !config.outputMatch; - const {regex, hiddenCaptures} = compileRegex(config.pattern, config); + const {regex, hiddenCaptures, namedOnly} = compileRegex(config.pattern, config); config.regex = regex; config.hiddenCaptures = hiddenCaptures; + // With the flag n of Regex+ only named groups capture, so --js gets no $1 and on + config.namedOnly = namedOnly; // The groups Regex+ adds for atomic groups and possessive quantifiers are not counted in the numbers of the replacement if ( @@ -191,16 +197,6 @@ function unpackConfig(config) { config.codeUses = config.replacementJs ? codeUses(config.replacementOri, config.regex) : null; - // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. - // A group that ends with _ would have the same name as another group with a space, so it can not be used. - // With --output-match the code is not run, so there are no variables. - const endsWith_ = config.outputMatch - ? null - : config.codeUses?.groups.find(name => name.endsWith('_')); - if (endsWith_) { - panic('A named group can not end with _', endsWith_); - } - // What the regex engine is asked to do. With --replacement-js and --output-match it gets a function. if (config.statusInfo.engine) { config.statusInfo.engine.execution = { @@ -724,16 +720,39 @@ function codeReplacement(file, config, text, numbers) { }; } -// -E: default is another name for rr, V for regex+, N for regex+n and X for regex+x -const engineNames = {default: 'rr', V: 'regex+', N: 'regex+n', X: 'regex+x'}; -// regex+n leaves out free spacing (x), and regex+x leaves out that only named groups capture (n) -const regexPlusDisable = {'regex+': {}, 'regex+n': {x: true}, 'regex+x': {n: true}}; +// -E: default is another name for rr, and V for regex+ +const engineNames = {default: 'rr', V: 'regex+'}; const engineName = name => engineNames[name] ?? name ?? 'rr'; -// The regex and the numbers of the groups the engine added to it, which the user does not know about +// The flags of the regex: --flags as they are, or the default flags with -G, -I, -M, -s and -u. +// regex+ also has v, free spacing (x) and only named groups capture (n) by default, and v makes -u needless. +function regexFlags(config, regexPlus) { + if (null != config.flags) { + if ( + config.voidGlobal || + config.voidIgnoreCase || + config.voidMultiline || + config.dotAll || + config.unicode + ) { + panic('--flags can not be used with -G, -I, -M, -s or -u, as it sets all the flags'); + } + return String(config.flags); + } + return ( + (config.voidGlobal ? '' : 'g') + + (config.voidIgnoreCase ? '' : 'i') + + (config.voidMultiline ? '' : 'm') + + (config.dotAll ? 's' : '') + + (regexPlus ? 'vxn' : config.unicode ? 'u' : '') + ); +} + +// The regex, the numbers of the groups the engine added to it, which the user does not know about, +// and if only named groups capture function compileRegex(pattern, config) { const engine = engineName(config.engine); - if (!['rr', 'regex+', 'regex+n', 'regex+x', 'V8', 'JSC', 'runtime'].includes(engine)) { + if (!['rr', 'regex+', 'V8', 'JSC', 'runtime'].includes(engine)) { panic(`Engine ${config.engine} not supported`); } if (('V8' === engine || 'JSC' === engine) && engine !== runtimeEngine) { @@ -741,33 +760,39 @@ function compileRegex(pattern, config) { `When running via the ${runtime} runtime the native engine is ${runtimeEngine}, not ${engine}` ); } - const regexPlus = engine in regexPlusDisable; - // Regex+ always uses v itself, or u on a runtime without v, and it is an error to ask for them - const flags = - (config.voidGlobal ? '' : 'g') + - (config.voidIgnoreCase ? '' : 'i') + - (config.voidMultiline ? '' : 'm') + - (config.dotAll ? 's' : '') + - (config.unicode && !regexPlus ? 'u' : ''); + const regexPlus = 'regex+' === engine; + const flags = regexFlags(config, regexPlus); + if (!regexPlus && re.regexPlusOnlyFlags.test(flags)) { + panic('The flags x and n need -E regex+', flags); + } + if (regexPlus && !flags.includes('v') && !flags.includes('u')) { + panic('-E regex+ needs the flag v or u', flags); + } + // Regex+ takes v, u, x and n as options. It uses v, or u on a runtime without v. + const nativeFlags = regexPlus ? flags.replace(re.regexPlusFlags, '') : flags; + const unicode = flags.includes('v') && supportsFlagV ? 'v' : 'u'; try { if (config.literal) { // The text as it is, without the free spacing of Regex+ return { - regex: new RegExp(pattern, flags + (regexPlus ? (supportsFlagV ? 'v' : 'u') : '')), + regex: new RegExp(pattern, nativeFlags + (regexPlus ? unicode : '')), hiddenCaptures: [], + namedOnly: false, }; } if (regexPlus) { - const result = rewrite(pattern, {flags, disable: regexPlusDisable[engine]}); + const disable = {v: !flags.includes('v'), x: !flags.includes('x'), n: !flags.includes('n')}; + const result = rewrite(pattern, {flags: nativeFlags, disable}); return { regex: new RegExp(result.pattern, result.flags), hiddenCaptures: result.hiddenCaptures, + namedOnly: flags.includes('n'), }; } if ('rr' === engine) { - return compileRr(pattern, flags); + return {...compileRr(pattern, flags), namedOnly: false}; } - return {regex: new RegExp(pattern, flags), hiddenCaptures: []}; + return {regex: new RegExp(pattern, flags), hiddenCaptures: [], namedOnly: false}; } catch (e) { return panic(e.message); } @@ -845,10 +870,10 @@ function countFile(config, text, result) { function codeUses(code, regex) { const named = name => new RegExp('\\$' + name + '(?![A-Za-z])').test(code); const groups = namedGroups(regex); - // A named group is used as rr_year, or as rr_year_ with a space at the end + // A named group is used as $year, or as $year_ with a space at the end const usesGroup = name => new RegExp( - '(^|[^\\p{ID_Continue}$])rr_' + name.replace(/\$/g, '\\$') + '_?(?![\\p{ID_Continue}$])', + '(^|[^\\p{ID_Continue}$])\\$' + name.replace(/\$/g, '\\$') + '_?(?![\\p{ID_Continue}$])', 'u' ).test(code); const matchNumbers = named('m') || named('M'); @@ -1116,12 +1141,16 @@ function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: config.numbers.matches++; const {groups, named} = splitMatch(matchArgs, config.hiddenCaptures); let variables = ''; + // $0 is the match. With the flag n the groups have no numbers, only names. groups.forEach((value, i) => { - variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; + if (!i || !config.namedOnly) { + variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; + } }); - // A named group is a variable with rr_ in front, like rr_year for (?…), and rr_year_ has a space at the end + // A named group is $ and the name, like $year for (?…), and $year_ has a space at the end. + // It comes after the numbers, so a group named like a number, like n, wins over $n. for (const [name, value] of Object.entries(named)) { - variables += `var rr_${name} = ${JSON.stringify(value)}, rr_${name}_ = rr_${name} + ' '; `; + variables += `var $${name} = ${JSON.stringify(value)}, $${name}_ = $${name} + ' '; `; } return runCode({m: match, M: config.numbers.matches}, variables); }; diff --git a/test/fix/799.test.js b/test/fix/799.test.js index d7c4ddf8..4c097f08 100644 --- a/test/fix/799.test.js +++ b/test/fix/799.test.js @@ -9,16 +9,16 @@ describe('#799 groups in --js', () => { expect((await rr('(?a)(b)', '$1 + $2 + typeof $3', config)).out).toBe('abundefined'); }); - test('a named group is a variable with rr_ in front, and _ at the end has a space', async () => { + test('a named group is a variable with $ in front, and _ at the end has a space', async () => { const config = {pipedData: '2024-05 1999-12', replacementJs: true}; const pattern = '(?\\d{4})-(?\\d\\d)'; - expect((await rr(pattern, 'rr_month + "/" + rr_year', config)).out).toBe('05/2024 12/1999'); - expect((await rr(pattern, 'rr_month_ + rr_year', config)).out).toBe('05 2024 12 1999'); + expect((await rr(pattern, '$month + "/" + $year', config)).out).toBe('05/2024 12/1999'); + expect((await rr(pattern, '$month_ + $year', config)).out).toBe('05 2024 12 1999'); }); test('a named group that took no part in the match is undefined', async () => { const config = {pipedData: 'ab', replacementJs: true}; - expect((await rr('(?a)|(?b)', 'String(rr_x) + String(rr_y) + ","', config)).out).toBe( + expect((await rr('(?a)|(?b)', 'String($x) + String($y) + ","', config)).out).toBe( 'aundefined,undefinedb,' ); }); diff --git a/test/fix/815.test.js b/test/fix/815.test.js index 9a9028e0..18504a75 100644 --- a/test/fix/815.test.js +++ b/test/fix/815.test.js @@ -10,7 +10,7 @@ describe('#815 --js code can declare its own variables', () => { ['a function', 'o', 'function $t() { return "F"; } $t()', 'fFF'], ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], - ['a named group with let', '(?o)', 'let rr_o = "N"; rr_o', 'fNN'], + ['a named group with let', '(?o)', 'let $o = "N"; $o', 'fNN'], ])('with the name of %s', async (name, pattern, code, out) => { expect((await rr(pattern, code, {pipedData: 'foo', replacementJs: true})).out).toBe(out); }); diff --git a/test/fix/816.test.js b/test/fix/816.test.js index 40bae980..e5c7ea67 100644 --- a/test/fix/816.test.js +++ b/test/fix/816.test.js @@ -5,21 +5,19 @@ import {rr} from '../helper.js'; describe('#816 named groups with letters that are not ASCII', () => { test.each([ - ['año', '(?\\d)', '"[" + rr_año + "]"', 'a[1]b[2]'], - ['日', '(?<日>\\d)', 'rr_日 * 2', 'a2b4'], - ['año with a space at the end', '(?\\d)', 'rr_año_ + "|"', 'a1 |b2 |'], - ['𝒜, outside the BMP', '(?<𝒜>\\d)', 'rr_𝒜 * 2', 'a2b4'], - ['$é, with $', '(?<$é>\\d)', 'rr_$é * 2', 'a2b4'], - ['Æble, in uppercase', '(?<Æble>\\d)', 'rr_Æble * 2', 'a2b4'], - ['x٣, with a digit that is not ASCII', '(?\\d)', 'rr_x٣ * 2', 'a2b4'], + ['año', '(?\\d)', '"[" + $año + "]"', 'a[1]b[2]'], + ['日', '(?<日>\\d)', '$日 * 2', 'a2b4'], + ['año with a space at the end', '(?\\d)', '$año_ + "|"', 'a1 |b2 |'], + ['𝒜, outside the BMP', '(?<𝒜>\\d)', '$𝒜 * 2', 'a2b4'], + ['$é, with $', '(?<$é>\\d)', '$$é * 2', 'a2b4'], + ['Æble, in uppercase', '(?<Æble>\\d)', '$Æble * 2', 'a2b4'], + ['x٣, with a digit that is not ASCII', '(?\\d)', '$x٣ * 2', 'a2b4'], ])('can be used, like %s', async (name, pattern, code, out) => { expect((await rr(pattern, code, {pipedData: 'a1b2', replacementJs: true})).out).toBe(out); }); - test('can not end with _ either', async () => { - const result = await rr('(?x)', 'rr_año', {pipedData: 'x', replacementJs: true}); - expect(result.err).toStartWith(' ❌ A named group can not end with _\n año_'); - expect(result.exit).toBe(3); + test('can end with _', async () => { + expect((await rr('(?x)', '$año_', {pipedData: 'x', replacementJs: true})).out).toBe('x'); }); test('are read from the regex, so text that only looks like a group is not one', async () => { @@ -28,14 +26,14 @@ describe('#816 named groups with letters that are not ASCII', () => { expect((await rr('[(?]', '"z"', {...config, pipedData: 'ab'})).out).toBe('zb'); }); - test('are only seen as the whole name, so rr_rr_year is not the group year', async () => { - const result = await rr('(?\\d)', 'rr_rr_year', {pipedData: '1', replacementJs: true}); - expect(result.err).toContain('ReferenceError: rr_rr_year is not defined'); + test('are only seen as the whole name, so $years is not the group year', async () => { + const result = await rr('(?\\d)', '$years', {pipedData: '1', replacementJs: true}); + expect(result.err).toContain('ReferenceError: $years is not defined'); expect(result.exit).toBe(3); }); - test('can be used with a space at the end on its own, like rr_year_', async () => { + test('can be used with a space at the end on its own, like $year_', async () => { const config = {pipedData: '2024', replacementJs: true}; - expect((await rr('(?\\d+)', 'rr_year_ + "|"', config)).out).toBe('2024 |'); + expect((await rr('(?\\d+)', '$year_ + "|"', config)).out).toBe('2024 |'); }); }); diff --git a/test/fix/825.test.js b/test/fix/825.test.js index d9e9e4f8..cd668437 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -2,30 +2,29 @@ // A named group was a variable like _year, which could clash: with groups a and a_, _a_ was both a_ and a with a space, // and a group named _require_ broke r(). import {describe, test, expect} from 'bun:test'; -import fs from 'fs'; import {rr} from '../helper.js'; const js = {pipedData: 'in 1999 and 2024', replacementJs: true}; describe('#825 named groups in --js', () => { test.each([ - ['a name in lowercase', '(?\\d+)', '"<" + rr_year + ">"', 'in <1999> and <2024>'], + ['a name in lowercase', '(?\\d+)', '"<" + $year + ">"', 'in <1999> and <2024>'], [ 'the cousin with a space at the end', '(?\\d+)', - 'rr_year + "," + rr_year_', + '$year + "," + $year_', 'in 1999,1999 and 2024,2024 ', ], - ['a name in uppercase', '(?\\d+)', '"Y" + rr_Year', 'in Y1999 and Y2024'], - ['a name with a number', '(?\\d+)', '"Y" + rr_y2k', 'in Y1999 and Y2024'], - ['a name with $', '(?\\d+)', '"Y" + rr_a$', 'in Y1999 and Y2024'], - ['two names that only differ by case', '(?\\d)(?\\d)', 'rr_Y + rr_y', 'in 9199 and 0242'], - ['a name next to its number', '(?\\d+)', '$1 + rr_year', 'in 19991999 and 20242024'], + ['a name in uppercase', '(?\\d+)', '"Y" + $Year', 'in Y1999 and Y2024'], + ['a name with a number', '(?\\d+)', '"Y" + $y2k', 'in Y1999 and Y2024'], + ['a name with $', '(?\\d+)', '"Y" + $a$', 'in Y1999 and Y2024'], + ['two names that only differ by case', '(?\\d)(?\\d)', '$Y + $y', 'in 9199 and 0242'], + ['a name next to its number', '(?\\d+)', '$1 + $year', 'in 19991999 and 20242024'], ])('gives %s', async (name, pattern, code, out) => { expect(await rr(pattern, code, js)).toEqual({out, err: '', exit: null}); }); - test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { + test('have $ in front, so they can not clash with what RexReplace uses', async () => { // With _ in front, the group _require with a space was __require_, which r() uses itself. // $1 makes the code run once per match, where the variables of the groups are made. const config = {pipedData: 'ab', replacementJs: true}; @@ -37,30 +36,15 @@ describe('#825 named groups in --js', () => { }); }); -describe('#825 a named group that ends with _', () => { - test.each([['a_'], ['Year_'], ['_'], ['y2_']])( - 'panics for %s, as that is the name of another group with a space, and nothing is changed', - async name => { - fs.writeFileSync('a.md', 'in 1999'); - const result = await rr(`(?<${name}>\\d+)`, '"x"', { - includeGlob: ['a.md'], - replacementJs: true, - }); - expect(result).toEqual({ - out: '', - err: ` ❌ A named group can not end with _\n ${name}\n No further processing.\n\n`, - exit: 3, - }); - expect(fs.readFileSync('a.md', 'utf8')).toBe('in 1999'); - } - ); - - test('is fine with --output-match, as the code is not run', async () => { - const config = {pipedData: 'ab', replacementJs: true, outputMatch: true}; - expect((await rr('(?a)', 'x', config)).exit).toBe(null); +describe('#825 a named group with the name of another variable', () => { + // The variables of the groups come after the others, so the group wins + test('wins over the number, like n over $n, but not over its cousins with zeros', async () => { + expect((await rr('(?\\w)', '$n + $n_ + $n00', {...js, pipedData: 'ab'})).out).toBe( + 'aa 001bb 001' + ); }); - test('is fine without --js, as there are no variables then', async () => { - expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); + test('can end with _, and then wins over the cousin with a space of the group without it', async () => { + expect((await rr('(?\\d)(?\\w)', '$a + $a_', {...js, pipedData: '1x'})).out).toBe('1x'); }); }); diff --git a/test/fix/854.test.js b/test/fix/854.test.js index d5a4d336..249f90d2 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -1,6 +1,6 @@ // https://github.com/mathiasrw/rexreplace/issues/854 // Regex+ is behind the engines. rr, the default, reads patterns like the native engine, with the extended syntax of Regex+ on top. -// regex+ is Regex+ with the flags v, x and n, regex+n leaves out x, and regex+x leaves out n. +// regex+ is Regex+ with the flags v, x and n, and --flags sets all the flags of any engine. import {describe, test, expect} from 'bun:test'; import fs from 'fs'; import {rr, rrStatus} from '../helper.js'; @@ -119,23 +119,13 @@ describe('#854 the other engines', () => { } ); - test.each([['regex+n'], ['N']])( - '%s has no free spacing, and only named groups capture', + test.each([['regex+n'], ['N'], ['regex+x'], ['X'], ['regex+u'], ['U'], ['regex+v']])( + '%s is not an engine', async engine => { - expect(await out('foo bar', 'X', 'foo bar', {engine})).toBe('X'); - expect(await out('(foo)(?.*)', '$1', 'foox', {engine})).toBe('x'); + expect((await rr('a', 'b', {pipedData: 'a', engine})).exit).toBe(3); } ); - test.each([['regex+x'], ['X']])('%s has free spacing, and all groups capture', async engine => { - expect(await out('foo bar', 'X', 'foobar', {engine})).toBe('X'); - expect(await out('(foo)(?.*)', '$1', 'foox', {engine})).toBe('foo'); - }); - - test.each([['regex+u'], ['U'], ['regex+v']])('%s is not an engine', async engine => { - expect((await rr('a', 'b', {pipedData: 'a', engine})).exit).toBe(3); - }); - test('regex+ stops the run on an escape that is not strict, and the file is not changed', async () => { fs.writeFileSync('a.txt', 'a"b'); const result = await rr('\\"', 'x', {includeGlob: ['a.txt'], engine: 'regex+'}); @@ -149,3 +139,50 @@ describe('#854 the other engines', () => { ); }); }); + +describe('#854 --flags', () => { + const plus = {engine: 'regex+'}; + + test('leaves free spacing out of regex+ without x', async () => { + expect(await out('foo bar', 'X', 'foo bar', {...plus, flags: 'gimvn'})).toBe('X'); + }); + + test('lets all groups of regex+ capture without n', async () => { + expect(await out('(foo)', '[$1]', 'foo', {...plus, flags: 'gimvx'})).toBe('[foo]'); + }); + + test('sets all the flags, so without i the search is case sensitive and without g only the first match', async () => { + expect(await out('o', 'x', 'oOo', {flags: 'm'})).toBe('xOo'); + }); + + test('gives regex+ the flag u in place of v', async () => { + const {flags} = await compiled('a', {...plus, flags: 'gu'}); + expect(flags).toContain('u'); + expect(flags).not.toContain('v'); + }); + + test.each([ + ['-I', {voidIgnoreCase: true, flags: 'g'}], + ['x without regex+', {flags: 'gx'}], + ['n without regex+', {flags: 'gn'}], + ['regex+ without v or u', {...plus, flags: 'gx'}], + ['a flag that is not a flag', {flags: 'gq'}], + ])('stops the run with %s', async (name, config) => { + expect((await rr('a', 'b', {pipedData: 'a', ...config})).exit).toBe(3); + }); +}); + +describe('#854 --js with the flag n of regex+', () => { + const config = {engine: 'regex+', replacementJs: true}; + + test('has the named groups and $0, but no group numbers', async () => { + expect(await out('(?\\d)(?\\d)', '$m + $y + $0', '12', config)).toBe('2112'); + expect((await rr('(?\\d)', '$1', {pipedData: '1', ...config})).err).toContain( + 'ReferenceError: $1 is not defined' + ); + }); + + test('has the group numbers again without n', async () => { + expect(await out('(\\d)(\\d)', '$2 + $1', '12', {...config, flags: 'gimvx'})).toBe('21'); + }); +}); From 5b6d035a1355b9c2acd648d50b8b3755f1979a31 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sun, 25 Oct 2026 09:31:10 +1100 Subject: [PATCH 04/13] Check --flags against the flag options in the CLI, take v for regex+ by default, and warn on $1 under n --- README.md | 2 +- src/cli.ts | 4 +++- src/engine.ts | 33 ++++++++++++++++++--------------- test/fix/854.test.js | 21 ++++++++++++++++++--- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 32ed06fa..48116406 100644 --- a/README.md +++ b/README.md @@ -323,7 +323,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-s` | **`--dot-all`** Have `.` also match newline. [boolean] | | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | -| | **`--flags`** Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which also needs `v` or `u`. [string] | +| | **`--flags`** Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which uses `v`, or `u` when it is given. [string] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | | `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. Use --flags to leave out x or n. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+", "V", "runtime", "V8", "JSC"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 2b89692b..d4b8cefe 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -119,9 +119,11 @@ const yargs = require('yargs') .alias('u', 'unicode') .string('flags') + // --flags sets all the flags, so the options that change a default flag can not go with it + .conflicts('flags', ['I', 'G', 'M', 's', 'u']) .describe( 'flags', - 'Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which also needs `v` or `u`.' + 'Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which uses `v`, or `u` when it is given.' ) .default('e', 'utf8') diff --git a/src/engine.ts b/src/engine.ts index 99c63454..a7fa3259 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -179,6 +179,21 @@ function unpackConfig(config) { config.hiddenCaptures = hiddenCaptures; // With the flag n of Regex+ only named groups capture, so --js gets no $1 and on config.namedOnly = namedOnly; + // The replacement text still reads $1 as the first named group, which can be on purpose + if ( + namedOnly && + 'string' === typeof config.replacement && + !config.replacementJs && + !config.outputMatch && + !config.literalReplacement && + [...config.replacement.matchAll(re.replacementToken)].some( + ([, , first, second]) => 0 < Number(first + second) + ) + ) { + warn( + 'With the flag n the groups have no numbers, but $1 in the replacement is the first named group. Use $.' + ); + } // The groups Regex+ adds for atomic groups and possessive quantifiers are not counted in the numbers of the replacement if ( @@ -728,15 +743,6 @@ const engineName = name => engineNames[name] ?? name ?? 'rr'; // regex+ also has v, free spacing (x) and only named groups capture (n) by default, and v makes -u needless. function regexFlags(config, regexPlus) { if (null != config.flags) { - if ( - config.voidGlobal || - config.voidIgnoreCase || - config.voidMultiline || - config.dotAll || - config.unicode - ) { - panic('--flags can not be used with -G, -I, -M, -s or -u, as it sets all the flags'); - } return String(config.flags); } return ( @@ -765,12 +771,9 @@ function compileRegex(pattern, config) { if (!regexPlus && re.regexPlusOnlyFlags.test(flags)) { panic('The flags x and n need -E regex+', flags); } - if (regexPlus && !flags.includes('v') && !flags.includes('u')) { - panic('-E regex+ needs the flag v or u', flags); - } - // Regex+ takes v, u, x and n as options. It uses v, or u on a runtime without v. + // Regex+ takes v, u, x and n as options. It uses v, unless --flags has u, or the runtime has no v. const nativeFlags = regexPlus ? flags.replace(re.regexPlusFlags, '') : flags; - const unicode = flags.includes('v') && supportsFlagV ? 'v' : 'u'; + const unicode = !flags.includes('u') && supportsFlagV ? 'v' : 'u'; try { if (config.literal) { // The text as it is, without the free spacing of Regex+ @@ -781,7 +784,7 @@ function compileRegex(pattern, config) { }; } if (regexPlus) { - const disable = {v: !flags.includes('v'), x: !flags.includes('x'), n: !flags.includes('n')}; + const disable = {v: flags.includes('u'), x: !flags.includes('x'), n: !flags.includes('n')}; const result = rewrite(pattern, {flags: nativeFlags, disable}); return { regex: new RegExp(result.pattern, result.flags), diff --git a/test/fix/854.test.js b/test/fix/854.test.js index 249f90d2..f49e3276 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -3,7 +3,8 @@ // regex+ is Regex+ with the flags v, x and n, and --flags sets all the flags of any engine. import {describe, test, expect} from 'bun:test'; import fs from 'fs'; -import {rr, rrStatus} from '../helper.js'; +import {$} from 'bun'; +import {rr, rrStatus, sh, cli} from '../helper.js'; // What the regex engine was asked to do: the source and the flags of the regex async function compiled(pattern, config = {}) { @@ -155,6 +156,16 @@ describe('#854 --flags', () => { expect(await out('o', 'x', 'oOo', {flags: 'm'})).toBe('xOo'); }); + test('gives regex+ the flag v when it has neither v nor u', async () => { + expect((await compiled('a', {...plus, flags: 'gx'})).flags).toContain('v'); + }); + + test('can not go with -I, -G, -M, -s or -u in the CLI', async () => { + fs.writeFileSync('a.txt', 'a'); + expect((await sh($`bun ${cli} a b a.txt --flags g -I`)).exit).toBe(3); + expect(fs.readFileSync('a.txt', 'utf8')).toBe('a'); + }); + test('gives regex+ the flag u in place of v', async () => { const {flags} = await compiled('a', {...plus, flags: 'gu'}); expect(flags).toContain('u'); @@ -162,10 +173,8 @@ describe('#854 --flags', () => { }); test.each([ - ['-I', {voidIgnoreCase: true, flags: 'g'}], ['x without regex+', {flags: 'gx'}], ['n without regex+', {flags: 'gn'}], - ['regex+ without v or u', {...plus, flags: 'gx'}], ['a flag that is not a flag', {flags: 'gq'}], ])('stops the run with %s', async (name, config) => { expect((await rr('a', 'b', {pipedData: 'a', ...config})).exit).toBe(3); @@ -182,6 +191,12 @@ describe('#854 --js with the flag n of regex+', () => { ); }); + test('warns when the replacement text uses a group number, as it is the first named group', async () => { + const result = await rr('(?\\d)', '[$1]', {pipedData: '1', engine: 'regex+'}); + expect(result.out).toBe('[1]'); + expect(result.err).toContain('With the flag n the groups have no numbers'); + }); + test('has the group numbers again without n', async () => { expect(await out('(\\d)(\\d)', '$2 + $1', '12', {...config, flags: 'gimvx'})).toBe('21'); }); From 3668a4a497d5066acd69b2553b5f77710e9616be Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sun, 25 Oct 2026 09:36:08 +1100 Subject: [PATCH 05/13] Name the unknown flags of --flags, and check the messages of the stopped runs in the tests --- src/engine.ts | 7 +++++++ test/fix/854.test.js | 25 ++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index a7fa3259..5868e1da 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -28,6 +28,8 @@ const re = { subroutineSyntax: /\\g<|\(\?\(DEFINE\)/, // The flags only Regex+ knows: free spacing (x) and only named groups capture (n) regexPlusOnlyFlags: /[xn]/, + // Every flag of a regex, and x and n of Regex+ + regexFlagNames: /[dgimsuvyxn]/g, // The flags Regex+ takes as options, and not as flags regexPlusFlags: /[uvxn]/g, // A $ in a replacement: $$, $&, $`, $', $, or a group number of one or two digits @@ -743,6 +745,11 @@ const engineName = name => engineNames[name] ?? name ?? 'rr'; // regex+ also has v, free spacing (x) and only named groups capture (n) by default, and v makes -u needless. function regexFlags(config, regexPlus) { if (null != config.flags) { + // The runtime does not always say which flag it does not know, so it is told here + const unknown = String(config.flags).replace(re.regexFlagNames, ''); + if (unknown) { + panic('Unknown flags in --flags', unknown); + } return String(config.flags); } return ( diff --git a/test/fix/854.test.js b/test/fix/854.test.js index f49e3276..1531f5e4 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -123,7 +123,11 @@ describe('#854 the other engines', () => { test.each([['regex+n'], ['N'], ['regex+x'], ['X'], ['regex+u'], ['U'], ['regex+v']])( '%s is not an engine', async engine => { - expect((await rr('a', 'b', {pipedData: 'a', engine})).exit).toBe(3); + expect(await rr('a', 'b', {pipedData: 'a', engine})).toEqual({ + out: '', + err: ` ❌ Engine ${engine} not supported\n No further processing.\n\n`, + exit: 3, + }); } ); @@ -131,6 +135,7 @@ describe('#854 the other engines', () => { fs.writeFileSync('a.txt', 'a"b'); const result = await rr('\\"', 'x', {includeGlob: ['a.txt'], engine: 'regex+'}); expect(result.exit).toBe(3); + expect(result.err).toStartWith(' ❌ Invalid regular expression'); expect(fs.readFileSync('a.txt', 'utf8')).toBe('a"b'); }); @@ -162,7 +167,9 @@ describe('#854 --flags', () => { test('can not go with -I, -G, -M, -s or -u in the CLI', async () => { fs.writeFileSync('a.txt', 'a'); - expect((await sh($`bun ${cli} a b a.txt --flags g -I`)).exit).toBe(3); + const result = await sh($`bun ${cli} a b a.txt --flags g -I`); + expect(result.exit).toBe(3); + expect(result.err).toEndWith('Arguments flags and I are mutually exclusive\n'); expect(fs.readFileSync('a.txt', 'utf8')).toBe('a'); }); @@ -173,11 +180,15 @@ describe('#854 --flags', () => { }); test.each([ - ['x without regex+', {flags: 'gx'}], - ['n without regex+', {flags: 'gn'}], - ['a flag that is not a flag', {flags: 'gq'}], - ])('stops the run with %s', async (name, config) => { - expect((await rr('a', 'b', {pipedData: 'a', ...config})).exit).toBe(3); + ['x without regex+', {flags: 'gx'}, ' ❌ The flags x and n need -E regex+\n gx'], + ['n without regex+', {flags: 'gn'}, ' ❌ The flags x and n need -E regex+\n gn'], + ['a flag that is not a flag', {flags: 'gqz'}, ' ❌ Unknown flags in --flags\n qz'], + ])('stops the run with %s, and says so', async (name, config, message) => { + expect(await rr('a', 'b', {pipedData: 'a', ...config})).toEqual({ + out: '', + err: message + '\n No further processing.\n\n', + exit: 3, + }); }); }); From 458cb9b4391deb6297220df186df0eac9c48fae8 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Mon, 26 Oct 2026 09:28:44 +1100 Subject: [PATCH 06/13] Take regex from npm and copy only its subroutines, drop V8, JSC, V and the $1 warning, and fix [(?>]\1 --- README.md | 8 +- package.json | 6 +- src/cli.ts | 9 +- src/engine.ts | 98 ++++---- src/libs/README.md | 30 +-- src/libs/regex-utilities/LICENSE | 21 -- src/libs/regex-utilities/index.js | 170 ------------- src/libs/regex/atomic.js | 224 ----------------- src/libs/regex/backcompat.js | 64 ----- src/libs/regex/flag-n.js | 49 ---- src/libs/regex/flag-x.js | 198 --------------- src/libs/regex/internals.js | 4 - src/libs/regex/pattern.js | 45 ---- src/libs/regex/regex.js | 390 ------------------------------ src/libs/regex/subclass.js | 98 -------- src/libs/regex/subroutines.js | 2 +- src/libs/regex/utils.js | 384 +---------------------------- test/cli/run.sh | 6 +- test/engine/engine.test.js | 21 +- test/fix/854.test.js | 26 +- yarn.lock | 12 + 21 files changed, 94 insertions(+), 1771 deletions(-) delete mode 100644 src/libs/regex-utilities/LICENSE delete mode 100644 src/libs/regex-utilities/index.js delete mode 100644 src/libs/regex/atomic.js delete mode 100644 src/libs/regex/backcompat.js delete mode 100644 src/libs/regex/flag-n.js delete mode 100644 src/libs/regex/flag-x.js delete mode 100644 src/libs/regex/internals.js delete mode 100644 src/libs/regex/pattern.js delete mode 100644 src/libs/regex/regex.js delete mode 100644 src/libs/regex/subclass.js diff --git a/README.md b/README.md index 48116406..4d6d2e2d 100644 --- a/README.md +++ b/README.md @@ -323,9 +323,9 @@ If planned renames clash, the process is halted and no change is applied to any | `-s` | **`--dot-all`** Have `.` also match newline. [boolean] | | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | -| | **`--flags`** Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which uses `v`, or `u` when it is given. [string] | +| | **`--flags`** The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. [string] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | -| `-E` | **`--engine`** What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. Use --flags to leave out x or n. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime. [choices: "rr", "default", "regex+", "V", "runtime", "V8", "JSC"] [default: "rr"] | +| `-E` | **`--engine`** The regex engine. `rr` (also `default`) reads patterns like the native engine and adds the extended syntax of Regex+: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g`. `regex+` is Regex+ with the flags v, x and n. `runtime` is the native engine of the runtime: V8 in Node and JavaScriptCore in Bun. [choices: "rr", "default", "regex+", "runtime"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | @@ -368,7 +368,7 @@ If planned renames clash, the process is halted and no change is applied to any | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | | `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so there is no `$1`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | -| `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | +| `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. It uses the same engine and flags as the pattern. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | @@ -390,7 +390,7 @@ Files identified for potential processing are identified by: - Patterns are described as javascript notation regex - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern -- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same. The one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ is copied into the code, so it is no extra dependency. +- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, so they do not work inside a lookbehind. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. - Supports backreference to matching groups in the replacement - Data can be piped in, both as the text to search and as the replacement - The names of the changed files are printed to stdout, so they can be piped on (to stderr with `--output` and `--output-match`) diff --git a/package.json b/package.json index e7c25b65..0736f929 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,8 @@ "build": "yarn bundle-node --outfile=bin/rexreplace.cli.node.js && yarn bundle-bun --outfile=bin/rexreplace.cli.bun.js", "build-minify": "yarn build && yarn minify", "minify": "yarn bundle-node --minify --outfile=bin/rexreplace.cli.node.min.js && yarn bundle-bun --minify --outfile=bin/rexreplace.cli.bun.min.js", - "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --packages=external", - "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --packages=external", + "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --external fast-glob --external yargs", + "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --external fast-glob --external yargs", "prebuild": "rm -fr bin && yarn format", "sync-version": "bun npm/sync-version.js", "test-js": "AGENT=1 bun test ./test/engine ./test/cli ./test/fix", @@ -72,6 +72,8 @@ }, "dependencies": { "fast-glob": "^3.3.3", + "regex": "6.1.0", + "regex-utilities": "2.3.0", "yargs": "16" } } diff --git a/src/cli.ts b/src/cli.ts index d4b8cefe..a76924af 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -119,11 +119,10 @@ const yargs = require('yargs') .alias('u', 'unicode') .string('flags') - // --flags sets all the flags, so the options that change a default flag can not go with it .conflicts('flags', ['I', 'G', 'M', 's', 'u']) .describe( 'flags', - 'Set all the flags of the regex, like `gimsv`. Without it the flags are `gim` (`gimvxn` with `-E regex+`), changed by -G, -I, -M, -s and -u, which can not be used with --flags. `x` (free spacing) and `n` (only named groups capture) need `-E regex+`, which uses `v`, or `u` when it is given.' + 'The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`.' ) .default('e', 'utf8') @@ -135,7 +134,7 @@ const yargs = require('yargs') .string('x') .describe( 'x', - 'Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times.' + 'Files with a path that matches this regular expression will be excluded from processing. It uses the same engine and flags as the pattern. Can be provided multiple times.' ) .alias('x', 'exclude-re') @@ -149,9 +148,9 @@ const yargs = require('yargs') .alias('E', 'engine') .describe( 'E', - 'What regex engine to use. `rr` (also `default`) is Regex+ wrapped to read patterns like the native engine, with the extended syntax of Regex+ on top: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g` with `(?(DEFINE)…)`. `regex+` (also `V`) is the Regex+ engine with flag v, free spacing (x), only named groups capture (n), atomic groups, possessive quantifiers and subroutines. On a runtime without v, like Node 18, it uses u. Use --flags to leave out x or n. `runtime` is the native engine of the runtime, and `V8` (Node) and `JSC` (JavaScriptCore, Bun) are the same, but stop the run on the other runtime.' + 'The regex engine. `rr` (also `default`) reads patterns like the native engine and adds the extended syntax of Regex+: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\\g`. `regex+` is Regex+ with the flags v, x and n. `runtime` is the native engine of the runtime: V8 in Node and JavaScriptCore in Bun.' ) - .choices('E', ['rr', 'default', 'regex+', 'V', 'runtime', 'V8', 'JSC']) + .choices('E', ['rr', 'default', 'regex+', 'runtime']) .default('E', 'rr') .boolean('q') diff --git a/src/engine.ts b/src/engine.ts index 5868e1da..1eec2f2f 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -24,15 +24,13 @@ const re = { // With a name the location is in parentheses at the end, like `at fn (/path/file.js:2:15)`. stackLineWithoutName: /^\s*at (?:file:\/\/)?((?:\/|[A-Za-z]:[\\/]).*?)(:\d+:\d+)\s*$/, stackLineWithName: /^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/, - // Subroutines of Regex+, which the rr engine only looks for when the pattern has them subroutineSyntax: /\\g<|\(\?\(DEFINE\)/, - // The flags only Regex+ knows: free spacing (x) and only named groups capture (n) + atomicSyntax: /\(\?>/, + // v is a native flag too, but only Regex+ knows x and n regexPlusOnlyFlags: /[xn]/, - // Every flag of a regex, and x and n of Regex+ regexFlagNames: /[dgimsuvyxn]/g, // The flags Regex+ takes as options, and not as flags regexPlusFlags: /[uvxn]/g, - // A $ in a replacement: $$, $&, $`, $', $, or a group number of one or two digits replacementToken: /\$(\$|&|`|'|<[^>]*>|(\d)(\d?))/g, // Only the chars with a meaning outside a character class. Unicode mode does not allow to escape the others (#686). regexSpecialChars: /[\[\]{}()*+?.\/\\^$|]/g, @@ -50,9 +48,11 @@ const re = { // Only the version ends up in the bundle import {version} from '../package.json'; -// Regex+ by Steven Levithan, copied into src/libs until it exports the subroutines (see src/libs/README.md) -import {atomic, possessive, subroutines} from './libs/regex/internals.js'; -import {rewrite} from './libs/regex/regex.js'; +// Regex+ by Steven Levithan. Its subroutines are copied into src/libs until it exports them. +import {rewrite} from 'regex'; +import {atomic, possessive} from 'regex/internals'; +import {Context, hasUnescaped} from 'regex-utilities'; +import {subroutines} from './libs/regex/subroutines.js'; export {version}; // The regex engine comes with the runtime: Node has V8 and Bun has JSC (JavaScriptCore) @@ -181,23 +181,8 @@ function unpackConfig(config) { config.hiddenCaptures = hiddenCaptures; // With the flag n of Regex+ only named groups capture, so --js gets no $1 and on config.namedOnly = namedOnly; - // The replacement text still reads $1 as the first named group, which can be on purpose - if ( - namedOnly && - 'string' === typeof config.replacement && - !config.replacementJs && - !config.outputMatch && - !config.literalReplacement && - [...config.replacement.matchAll(re.replacementToken)].some( - ([, , first, second]) => 0 < Number(first + second) - ) - ) { - warn( - 'With the flag n the groups have no numbers, but $1 in the replacement is the first named group. Use $.' - ); - } - // The groups Regex+ adds for atomic groups and possessive quantifiers are not counted in the numbers of the replacement + // The user does not know the groups Regex+ adds for atomic groups and possessive quantifiers if ( hiddenCaptures.length && 'string' === typeof config.replacement && @@ -738,11 +723,9 @@ function codeReplacement(file, config, text, numbers) { } // -E: default is another name for rr, and V for regex+ -const engineNames = {default: 'rr', V: 'regex+'}; -const engineName = name => engineNames[name] ?? name ?? 'rr'; +const engineName = name => ('default' === name ? 'rr' : (name ?? 'rr')); -// The flags of the regex: --flags as they are, or the default flags with -G, -I, -M, -s and -u. -// regex+ also has v, free spacing (x) and only named groups capture (n) by default, and v makes -u needless. +// -u is needless with regex+, as it has v function regexFlags(config, regexPlus) { if (null != config.flags) { // The runtime does not always say which flag it does not know, so it is told here @@ -750,7 +733,8 @@ function regexFlags(config, regexPlus) { if (unknown) { panic('Unknown flags in --flags', unknown); } - return String(config.flags); + // The runtime does not take a flag twice, and there is no harm in it + return [...new Set(String(config.flags))].join(''); } return ( (config.voidGlobal ? '' : 'g') + @@ -761,24 +745,17 @@ function regexFlags(config, regexPlus) { ); } -// The regex, the numbers of the groups the engine added to it, which the user does not know about, -// and if only named groups capture function compileRegex(pattern, config) { const engine = engineName(config.engine); - if (!['rr', 'regex+', 'V8', 'JSC', 'runtime'].includes(engine)) { + if (!['rr', 'regex+', 'runtime'].includes(engine)) { panic(`Engine ${config.engine} not supported`); } - if (('V8' === engine || 'JSC' === engine) && engine !== runtimeEngine) { - panic( - `When running via the ${runtime} runtime the native engine is ${runtimeEngine}, not ${engine}` - ); - } const regexPlus = 'regex+' === engine; const flags = regexFlags(config, regexPlus); if (!regexPlus && re.regexPlusOnlyFlags.test(flags)) { panic('The flags x and n need -E regex+', flags); } - // Regex+ takes v, u, x and n as options. It uses v, unless --flags has u, or the runtime has no v. + // Regex+ takes v, u, x and n as options, and gives an error when they come as flags const nativeFlags = regexPlus ? flags.replace(re.regexPlusFlags, '') : flags; const unicode = !flags.includes('u') && supportsFlagV ? 'v' : 'u'; try { @@ -791,16 +768,29 @@ function compileRegex(pattern, config) { }; } if (regexPlus) { - const disable = {v: flags.includes('u'), x: !flags.includes('x'), n: !flags.includes('n')}; + // The subroutines and the atomic groups are left to extendedSyntax, as rewrite() does not tell which groups they add + const disable = { + v: flags.includes('u'), + x: !flags.includes('x'), + n: !flags.includes('n'), + atomic: true, + subroutines: true, + }; const result = rewrite(pattern, {flags: nativeFlags, disable}); + const extended = extendedSyntax(result.pattern, result.flags); return { - regex: new RegExp(result.pattern, result.flags), - hiddenCaptures: result.hiddenCaptures, + regex: new RegExp(extended.pattern, result.flags), + hiddenCaptures: extended.hiddenCaptures, namedOnly: flags.includes('n'), }; } if ('rr' === engine) { - return {...compileRr(pattern, flags), namedOnly: false}; + const extended = extendedSyntax(pattern, flags); + return { + regex: new RegExp(extended.pattern, flags), + hiddenCaptures: extended.hiddenCaptures, + namedOnly: false, + }; } return {regex: new RegExp(pattern, flags), hiddenCaptures: [], namedOnly: false}; } catch (e) { @@ -808,27 +798,25 @@ function compileRegex(pattern, config) { } } -// The rr engine: the native engine with the flags of today, and the extended syntax of Regex+ on top. // Only the transforms of Regex+ run, in its own order, and not its flags, so a pattern that works natively gives the same regex. -function compileRr(pattern, flags) { +// A transform only runs when its syntax is outside a character class, as Regex+ does not always look, like for [(?>]\1 (#858). +function extendedSyntax(pattern, flags) { let hiddenCaptures = []; - const transform = plugin => { + const transform = (plugin, syntax = null) => { + if (syntax && !hasUnescaped(pattern, syntax.source, Context.DEFAULT)) { + return; + } const result = plugin(pattern, {flags, hiddenCaptures}); pattern = result.pattern; hiddenCaptures = result.hiddenCaptures ?? hiddenCaptures; }; - // The subroutines do not return early by themselves when the pattern has none - if (re.subroutineSyntax.test(pattern)) { - transform(subroutines); - } + transform(subroutines, re.subroutineSyntax); transform(possessive); - transform(atomic); - return {regex: new RegExp(pattern, flags), hiddenCaptures}; + transform(atomic, re.atomicSyntax); + return {pattern, hiddenCaptures}; } -// The group numbers in a replacement, like $1, count only the groups of the user, so they are moved past the hidden ones. -// It follows the rules of javascript for $n and $nn with the visible groups, and always gives two digits, -// so a digit after it is not read as part of the number. A number that is no visible group stays as text. +// Always two digits, so a digit after it is not read as part of the number. A number that is no group of the user stays text, as in javascript. function visibleGroupNumbers(replacement, hiddenCaptures, total) { const real = [0]; for (let group = 1; group <= total; group++) { @@ -880,7 +868,6 @@ function countFile(config, text, result) { function codeUses(code, regex) { const named = name => new RegExp('\\$' + name + '(?![A-Za-z])').test(code); const groups = namedGroups(regex); - // A named group is used as $year, or as $year_ with a space at the end const usesGroup = name => new RegExp( '(^|[^\\p{ID_Continue}$])\\$' + name.replace(/\$/g, '\\$') + '_?(?![\\p{ID_Continue}$])', @@ -1157,8 +1144,7 @@ function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; } }); - // A named group is $ and the name, like $year for (?…), and $year_ has a space at the end. - // It comes after the numbers, so a group named like a number, like n, wins over $n. + // After the numbers, so a group named like a number, like n, wins over $n for (const [name, value] of Object.entries(named)) { variables += `var $${name} = ${JSON.stringify(value)}, $${name}_ = $${name} + ' '; `; } diff --git a/src/libs/README.md b/src/libs/README.md index 56e21a11..33bbc619 100644 --- a/src/libs/README.md +++ b/src/libs/README.md @@ -1,28 +1,12 @@ -# Copied libraries +# Copied code -These are copies of other packages, bundled into RexReplace, so they are not runtime dependencies. -Keep them as they are upstream, so they are easy to compare. Biome leaves this folder alone. +RexReplace uses [Regex+](https://github.com/slevithan/regex) by Steven Levithan, from the npm packages [`regex`](https://www.npmjs.com/package/regex) and [`regex-utilities`](https://www.npmjs.com/package/regex-utilities). Both are dependencies, and the build bundles them, as they are ES modules only. -| Folder | Package | Upstream | -| --- | --- | --- | -| `regex/` | [`regex`](https://www.npmjs.com/package/regex) 6.1.0, the `src/` folder | https://github.com/slevithan/regex | -| `regex-utilities/` | [`regex-utilities`](https://www.npmjs.com/package/regex-utilities) 2.3.0, `src/index.js` | https://github.com/slevithan/regex-utilities | +One part of `regex` 6.1.0 is not exported: the subroutines. So it is copied here, as it is upstream, until upstream exports it: -Both are MIT, and their `LICENSE` files are next to the code. +- `regex/subroutines.js` and `regex/utils-internals.js`, unchanged. +- `regex/utils.js`, only the three parts of it that `subroutines.js` uses. -## Changes from upstream +The license of `regex` (MIT) is in `regex/LICENSE`. Biome leaves this folder alone, so it stays easy to compare with upstream. -Changes 2 and 3 are marked with a `// rexreplace patch:` comment. - -1. `regex/*.js` import `regex-utilities` as `../regex-utilities/index.js`, as the build leaves bare imports out of the bundle. -2. `regex/internals.js` also exports `subroutines`. Asked for upstream. -3. `rewrite()` in `regex/regex.js` also gives `hiddenCaptures`, the numbers of the groups Regex+ added, so RexReplace can skip them without `RegExpSubclass`, which makes `.replace()` 3 to 5 times slower. - -## Removal plan - -When upstream exports `subroutines`: - -1. Add `regex` as a dependency. -2. Import from `regex/internals` and `regex` in place of `./libs/regex/internals.js` and `./libs/regex/regex.js`. -3. If `rewrite()` does not give `hiddenCaptures` upstream, ask for it first, as change 3 is still needed. -4. Delete `src/libs/` and its line in `biome.json`. The tests must pass without changes. +When `regex` exports `subroutines`, import it from `regex/internals`, and delete this folder and its line in `biome.json`. diff --git a/src/libs/regex-utilities/LICENSE b/src/libs/regex-utilities/LICENSE deleted file mode 100644 index 2790a22c..00000000 --- a/src/libs/regex-utilities/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Steven Levithan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/src/libs/regex-utilities/index.js b/src/libs/regex-utilities/index.js deleted file mode 100644 index 5cf3c4b5..00000000 --- a/src/libs/regex-utilities/index.js +++ /dev/null @@ -1,170 +0,0 @@ -// Constant properties for tracking regex syntax context -export const Context = Object.freeze({ - DEFAULT: 'DEFAULT', - CHAR_CLASS: 'CHAR_CLASS', -}); - -/** -Replaces all unescaped instances of a regex pattern in the given context, using a replacement -string or callback. - -Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used -with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. -@param {string} expression Search target -@param {string} needle Search as a regex pattern, with flags `su` applied -@param {string | (match: RegExpExecArray, details: { - context: 'DEFAULT' | 'CHAR_CLASS'; - negated: boolean; -}) => string} replacement -@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified -@returns {string} Updated expression -@example -const str = '.\\.\\\\.[[\\.].].'; -replaceUnescaped(str, '\\.', '@'); -// → '@\\.\\\\@[[\\.]@]@' -replaceUnescaped(str, '\\.', '@', Context.DEFAULT); -// → '@\\.\\\\@[[\\.].]@' -replaceUnescaped(str, '\\.', '@', Context.CHAR_CLASS); -// → '.\\.\\\\.[[\\.]@].' -*/ -export function replaceUnescaped(expression, needle, replacement, context) { - const re = new RegExp(String.raw`${needle}|(?<$skip>\[\^?|\\?.)`, 'gsu'); - const negated = [false]; - let numCharClassesOpen = 0; - let result = ''; - for (const match of expression.matchAll(re)) { - const {0: m, groups: {$skip}} = match; - if (!$skip && (!context || (context === Context.DEFAULT) === !numCharClassesOpen)) { - if (replacement instanceof Function) { - result += replacement(match, { - context: numCharClassesOpen ? Context.CHAR_CLASS : Context.DEFAULT, - negated: negated[negated.length - 1], - }); - } else { - result += replacement; - } - continue; - } - if (m[0] === '[') { - numCharClassesOpen++; - negated.push(m[1] === '^'); - } else if (m === ']' && numCharClassesOpen) { - numCharClassesOpen--; - negated.pop(); - } - result += m; - } - return result; -} - -/** -Runs a callback for each unescaped instance of a regex pattern in the given context. - -Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used -with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. -@param {string} expression Search target -@param {string} needle Search as a regex pattern, with flags `su` applied -@param {(match: RegExpExecArray, details: { - context: 'DEFAULT' | 'CHAR_CLASS'; - negated: boolean; -}) => void} callback -@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified -*/ -export function forEachUnescaped(expression, needle, callback, context) { - // Do this the easy way - replaceUnescaped(expression, needle, callback, context); -} - -/** -Returns a match object for the first unescaped instance of a regex pattern in the given context, or -`null`. - -Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used -with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. -@param {string} expression Search target -@param {string} needle Search as a regex pattern, with flags `su` applied -@param {number} [pos] Offset to start the search -@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified -@returns {RegExpExecArray | null} -*/ -export function execUnescaped(expression, needle, pos = 0, context) { - // Quick partial test; avoid the loop if not needed - if (!(new RegExp(needle, 'su').test(expression))) { - return null; - } - const re = new RegExp(`${needle}|(?<$skip>\\\\?.)`, 'gsu'); - re.lastIndex = pos; - let numCharClassesOpen = 0; - let match; - while (match = re.exec(expression)) { - const {0: m, groups: {$skip}} = match; - if (!$skip && (!context || (context === Context.DEFAULT) === !numCharClassesOpen)) { - return match; - } - if (m === '[') { - numCharClassesOpen++; - } else if (m === ']' && numCharClassesOpen) { - numCharClassesOpen--; - } - // Avoid an infinite loop on zero-length matches - if (re.lastIndex == match.index) { - re.lastIndex++; - } - } - return null; -} - -/** -Checks whether an unescaped instance of a regex pattern appears in the given context. - -Doesn't skip over complete multicharacter tokens (only `\` plus its folowing char) so must be used -with knowledge of what's safe to do given regex syntax. Assumes UnicodeSets-mode syntax. -@param {string} expression Search target -@param {string} needle Search as a regex pattern, with flags `su` applied -@param {'DEFAULT' | 'CHAR_CLASS'} [context] All contexts if not specified -@returns {boolean} Whether the pattern was found -*/ -export function hasUnescaped(expression, needle, context) { - // Do this the easy way - return !!execUnescaped(expression, needle, 0, context); -} - -/** -Extracts the full contents of a group (subpattern) from the given expression, accounting for -escaped characters, nested groups, and character classes. The group is identified by the position -where its contents start (the string index just after the group's opening delimiter). Returns the -rest of the string if the group is unclosed. - -Assumes UnicodeSets-mode syntax. -@param {string} expression Search target -@param {number} contentsStartPos -@returns {string} -*/ -export function getGroupContents(expression, contentsStartPos) { - const token = /\\?./gsu; - token.lastIndex = contentsStartPos; - let contentsEndPos = expression.length; - let numCharClassesOpen = 0; - // Starting search within an open group, after the group's opening - let numGroupsOpen = 1; - let match; - while (match = token.exec(expression)) { - const [m] = match; - if (m === '[') { - numCharClassesOpen++; - } else if (!numCharClassesOpen) { - if (m === '(') { - numGroupsOpen++; - } else if (m === ')') { - numGroupsOpen--; - if (!numGroupsOpen) { - contentsEndPos = match.index; - break; - } - } - } else if (m === ']') { - numCharClassesOpen--; - } - } - return expression.slice(contentsStartPos, contentsEndPos); -} diff --git a/src/libs/regex/atomic.js b/src/libs/regex/atomic.js deleted file mode 100644 index 1254003d..00000000 --- a/src/libs/regex/atomic.js +++ /dev/null @@ -1,224 +0,0 @@ -import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js'; -import {Context, replaceUnescaped} from '../regex-utilities/index.js'; -/** -@import {PluginData, PluginResult} from './regex.js'; -*/ - -const atomicPluginToken = new RegExp(String.raw`(?${noncapturingDelim})|(?\((?:\?<[^>]+>)?)|\\?.`, 'gsu'); - -/** -Apply transformations for atomic groups: `(?>…)`. -@param {string} expression -@param {PluginData} [data] -@returns {Required} -*/ -function atomic(expression, data) { - const hiddenCaptures = data?.hiddenCaptures ?? []; - // Capture transfer is used by - let captureTransfers = data?.captureTransfers ?? new Map(); - if (!/\(\?>/.test(expression)) { - return { - pattern: expression, - captureTransfers, - hiddenCaptures, - }; - } - - const aGDelim = '(?>'; - const emulatedAGDelim = '(?:(?=('; - const captureNumMap = [0]; - const addedHiddenCaptures = []; - let numCapturesBeforeAG = 0; - let numAGs = 0; - let aGPos = NaN; - let hasProcessedAG; - do { - hasProcessedAG = false; - let numCharClassesOpen = 0; - let numGroupsOpenInAG = 0; - let inAG = false; - let match; - atomicPluginToken.lastIndex = Number.isNaN(aGPos) ? 0 : aGPos + emulatedAGDelim.length; - while (match = atomicPluginToken.exec(expression)) { - const {0: m, index, groups: {capturingStart, noncapturingStart}} = match; - if (m === '[') { - numCharClassesOpen++; - } else if (!numCharClassesOpen) { - - if (m === aGDelim && !inAG) { - aGPos = index; - inAG = true; - } else if (inAG && noncapturingStart) { - numGroupsOpenInAG++; - } else if (capturingStart) { - if (inAG) { - numGroupsOpenInAG++; - } else { - numCapturesBeforeAG++; - captureNumMap.push(numCapturesBeforeAG + numAGs); - } - } else if (m === ')' && inAG) { - if (!numGroupsOpenInAG) { - numAGs++; - const addedCaptureNum = numCapturesBeforeAG + numAGs; - // Replace `expression` and use `<$$N>` as a temporary wrapper for the backref so it - // can avoid backref renumbering afterward. Wrap the whole substitution (including the - // lookahead and following backref) in a noncapturing group to handle following - // quantifiers and literal digits - expression = `${expression.slice(0, aGPos)}${emulatedAGDelim}${ - expression.slice(aGPos + aGDelim.length, index) - }))<$$${addedCaptureNum}>)${expression.slice(index + 1)}`; - hasProcessedAG = true; - addedHiddenCaptures.push(addedCaptureNum); - incrementIfAtLeast(hiddenCaptures, addedCaptureNum); - if (captureTransfers.size) { - const newCaptureTransfers = new Map(); - captureTransfers.forEach((from, to) => { - newCaptureTransfers.set( - to >= addedCaptureNum ? to + 1 : to, - from.map(f => f >= addedCaptureNum ? f + 1 : f) - ); - }); - captureTransfers = newCaptureTransfers; - } - break; - } - numGroupsOpenInAG--; - } - - } else if (m === ']') { - numCharClassesOpen--; - } - } - // Start over from the beginning of the atomic group's contents, in case the processed group - // contains additional atomic groups - } while (hasProcessedAG); - - hiddenCaptures.push(...addedHiddenCaptures); - - // Second pass to adjust numbered backrefs - expression = replaceUnescaped( - expression, - String.raw`\\(?[1-9]\d*)|<\$\$(?\d+)>`, - ({0: m, groups: {backrefNum, wrappedBackrefNum}}) => { - if (backrefNum) { - const bNum = +backrefNum; - if (bNum > captureNumMap.length - 1) { - throw new Error(`Backref "${m}" greater than number of captures`); - } - return `\\${captureNumMap[bNum]}`; - } - return `\\${wrappedBackrefNum}`; - }, - Context.DEFAULT - ); - - return { - pattern: expression, - captureTransfers, - hiddenCaptures, - }; -} - -const baseQuantifier = String.raw`(?:[?*+]|\{\d+(?:,\d*)?\})`; -// Complete tokenizer for base syntax; doesn't (need to) know about character-class-only syntax -const possessivePluginToken = new RegExp(String.raw` -\\(?: \d+ - | c[A-Za-z] - | [gk]<[^>]+> - | [pPu]\{[^\}]+\} - | u[A-Fa-f\d]{4} - | x[A-Fa-f\d]{2} - ) -| \((?: \? (?: [:=!>] - | <(?:[=!]|[^>]+>) - | [A-Za-z\-]+: - | \(DEFINE\) - ))? -| (?${baseQuantifier})(?[?+]?)(?[?*+\{]?) -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -Transform posessive quantifiers into atomic groups. The posessessive quantifiers are: -`?+`, `*+`, `++`, `{N}+`, `{N,}+`, `{N,N}+`. -This follows Java, PCRE, Perl, and Python. -Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`. -@param {string} expression -@returns {PluginResult} -*/ -function possessive(expression) { - if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) { - return { - pattern: expression, - }; - } - - const openGroupIndices = []; - let lastGroupIndex = null; - let lastCharClassIndex = null; - let lastToken = ''; - let numCharClassesOpen = 0; - let match; - possessivePluginToken.lastIndex = 0; - while (match = possessivePluginToken.exec(expression)) { - const {0: m, index, groups: {qBase, qMod, invalidQ}} = match; - if (m === '[') { - if (!numCharClassesOpen) { - lastCharClassIndex = index; - } - numCharClassesOpen++; - } else if (m === ']') { - if (numCharClassesOpen) { - numCharClassesOpen--; - // Unmatched `]` - } else { - lastCharClassIndex = null; - } - } else if (!numCharClassesOpen) { - - if (qMod === '+' && lastToken && !lastToken.startsWith('(')) { - // Invalid following quantifier would become valid via the wrapping group - if (invalidQ) { - throw new Error(`Invalid quantifier "${m}"`); - } - let charsAdded = -1; // -1 for removed trailing `+` - // Possessivizing fixed repetition quantifiers like `{2}` does't change their behavior, so - // avoid doing so (convert them to greedy) - if (/^\{\d+\}$/.test(qBase)) { - expression = spliceStr(expression, index + qBase.length, qMod, ''); - } else { - if (lastToken === ')' || lastToken === ']') { - const nodeIndex = lastToken === ')' ? lastGroupIndex : lastCharClassIndex; - // Unmatched `)` would break out of the wrapping group and mess with handling. - // Unmatched `]` wouldn't be a problem, but it's unnecessary to have dedicated support - // for unescaped `]++` since this won't work with flag u or v anyway - if (nodeIndex === null) { - throw new Error(`Invalid unmatched "${lastToken}"`); - } - expression = `${expression.slice(0, nodeIndex)}(?>${expression.slice(nodeIndex, index)}${qBase})${expression.slice(index + m.length)}`; - } else { - expression = `${expression.slice(0, index - lastToken.length)}(?>${lastToken}${qBase})${expression.slice(index + m.length)}`; - } - charsAdded += 4; // `(?>)` - } - possessivePluginToken.lastIndex += charsAdded; - } else if (m[0] === '(') { - openGroupIndices.push(index); - } else if (m === ')') { - lastGroupIndex = openGroupIndices.length ? openGroupIndices.pop() : null; - } - - } - lastToken = m; - } - - return { - pattern: expression, - }; -} - -export { - atomic, - possessive, -}; diff --git a/src/libs/regex/backcompat.js b/src/libs/regex/backcompat.js deleted file mode 100644 index 2a5a0d45..00000000 --- a/src/libs/regex/backcompat.js +++ /dev/null @@ -1,64 +0,0 @@ -import {doublePunctuatorChars} from './utils.js'; -/** -@import {PluginResult} from './regex.js'; -*/ - -const incompatibleEscapeChars = '&!#%,:;<=>@`~'; -const token = new RegExp(String.raw` -\[\^?-? -| --?\] -| (?[${doublePunctuatorChars}])\k -| -- -| \\(?[${incompatibleEscapeChars}]) -| \\[pPu]\{[^}]+\} -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -Applies flag v rules when using flag u, for forward compatibility. -Assumes flag u and doesn't worry about syntax errors that are caught by it. -@param {string} expression -@returns {PluginResult} -*/ -function backcompatPlugin(expression) { - const unescapedLiteralHyphenMsg = 'Invalid unescaped "-" in character class'; - let inCharClass = false; - let result = ''; - for (const {0: m, groups: {dp, vOnlyEscape}} of expression.matchAll(token)) { - if (m[0] === '[') { - if (inCharClass) { - throw new Error('Invalid nested character class when flag v not supported; possibly from interpolation'); - } - if (m.endsWith('-')) { - throw new Error(unescapedLiteralHyphenMsg); - } - inCharClass = true; - } else if (m.endsWith(']')) { - if (m[0] === '-') { - throw new Error(unescapedLiteralHyphenMsg); - } - inCharClass = false; - } else if (inCharClass) { - if (m === '&&' || m === '--') { - throw new Error(`Invalid set operator "${m}" when flag v not supported`); - } else if (dp) { - throw new Error(`Invalid double punctuator "${m}", reserved by flag v`); - } else if ('(){}/|'.includes(m)) { - throw new Error(`Invalid unescaped "${m}" in character class`); - } else if (vOnlyEscape) { - // Remove the escaping backslash to emulate flag v rules, since this character is allowed - // to be escaped within character classes with flag v but not with flag u - result += vOnlyEscape; - continue; - } - } - result += m; - } - return { - pattern: result, - }; -} - -export { - backcompatPlugin, -}; diff --git a/src/libs/regex/flag-n.js b/src/libs/regex/flag-n.js deleted file mode 100644 index 288fc747..00000000 --- a/src/libs/regex/flag-n.js +++ /dev/null @@ -1,49 +0,0 @@ -import {getEndContextForIncompleteExpression, RegexContext} from './utils.js'; -import {noncapturingDelim} from './utils-internals.js'; -/** -@import {Preprocessor} from './utils.js'; -*/ - -const token = new RegExp(String.raw` -${noncapturingDelim} -| \(\?< -| (?\\[1-9]\d*) -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -Apply transformations for flag n (named capture only). - -Preprocessors are applied to the outer regex and interpolated patterns, but not interpolated -regexes or strings. -@type {Preprocessor} -*/ -function flagNPreprocessor(value, runningContext) { - value = String(value); - let expression = ''; - let transformed = ''; - for (const {0: m, groups: {backrefNum}} of value.matchAll(token)) { - expression += m; - runningContext = getEndContextForIncompleteExpression(expression, runningContext); - const {regexContext} = runningContext; - if (regexContext === RegexContext.DEFAULT) { - if (m === '(') { - transformed += '(?:'; - } else if (backrefNum) { - throw new Error(`Invalid decimal escape "${m}" with implicit flag n; replace with named backreference`); - } else { - transformed += m; - } - } else { - transformed += m; - } - } - return { - transformed, - runningContext, - }; -} - -export { - flagNPreprocessor, -}; diff --git a/src/libs/regex/flag-x.js b/src/libs/regex/flag-x.js deleted file mode 100644 index 043595b4..00000000 --- a/src/libs/regex/flag-x.js +++ /dev/null @@ -1,198 +0,0 @@ -import {CharClassContext, doublePunctuatorChars, getEndContextForIncompleteExpression, RegexContext, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js'; -import {noncapturingDelim} from './utils-internals.js'; -import {Context, replaceUnescaped} from '../regex-utilities/index.js'; -/** -@import {PluginResult} from './regex.js'; -@import {Preprocessor} from './utils.js'; -*/ - -const ws = /^\s$/; -const escapedWsOrHash = /^\\[\s#]$/; -const charClassWs = /^[ \t]$/; -const escapedCharClassWs = /^\\[ \t]$/; -const token = new RegExp(String.raw` -\\(?: [gk]< - | [pPu]\{ - | c[A-Za-z] - | u[A-Fa-f\d]{4} - | x[A-Fa-f\d]{2} - | 0\d+ -) -| \[\^ -| ${noncapturingDelim} -| \(\?< -| (?[${doublePunctuatorChars}])\k -| -- -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -Apply transformations for flag x (insignificant whitespace and line comments). - -Preprocessors are applied to the outer regex and interpolated patterns, but not interpolated -regexes or strings. -@type {Preprocessor} -*/ -function flagXPreprocessor(value, runningContext, options) { - value = String(value); - let ignoringWs = false; - let ignoringCharClassWs = false; - let ignoringComment = false; - let expression = ''; - let transformed = ''; - let lastSignificantToken = ''; - let lastSignificantCharClassContext = ''; - let separatorNeeded = false; - const update = (str, options) => { - const opts = { - prefix: true, - postfix: false, - ...options, - }; - str = (separatorNeeded && opts.prefix ? '(?:)' : '') + str + (opts.postfix ? '(?:)' : ''); - separatorNeeded = false; - return str; - }; - for (const {0: m, index} of value.matchAll(token)) { - if (ignoringComment) { - if (m === '\n') { - ignoringComment = false; - separatorNeeded = true; - } - continue; - } - if (ignoringWs) { - if (ws.test(m)) { - continue; - } - ignoringWs = false; - separatorNeeded = true; - } else if (ignoringCharClassWs) { - if (charClassWs.test(m)) { - continue; - } - ignoringCharClassWs = false; - } - - expression += m; - runningContext = getEndContextForIncompleteExpression(expression, runningContext); - const {regexContext, charClassContext} = runningContext; - if ( - // `--` is matched in one step, so boundary chars aren't `-` unless separated by whitespace - m === '-' && - regexContext === RegexContext.CHAR_CLASS && - lastSignificantCharClassContext === CharClassContext.RANGE && - (options.flags.includes('v') || options.unicodeSetsPlugin) - ) { - // Need to handle this here since the main regex-parsing code would think the hyphen forms - // part of a subtraction operator since we've removed preceding whitespace - throw new Error('Invalid unescaped hyphen as the end value for a range'); - } - if ( - // `??` is matched in one step by the double punctuator token - (regexContext === RegexContext.DEFAULT && /^(?:[?*+]|\?\?)$/.test(m)) || - (regexContext === RegexContext.INTERVAL_QUANTIFIER && m === '{') - ) { - // Skip the separator prefix and connect the quantifier to the previous token. This also - // allows whitespace between a quantifier and the `?` that makes it lazy. Add a postfix - // separator if `m` is `?` and we're following token `(`, to sandbox the `?` from following - // tokens (since `?` can be a group-type marker). Ex: `( ?:)` becomes `(?(?:):)` and throws. - // The loop we're in matches valid group openings in one step, so we won't arrive here if - // matching e.g. `(?:`. Flag n could prevent the need for the postfix since bare `(` is - // converted to `(?:`, but flag x handling always comes first and flag n can be turned off - transformed += update(m, {prefix: false, postfix: lastSignificantToken === '(' && m === '?'}); - } else if (regexContext === RegexContext.DEFAULT) { - if (ws.test(m)) { - ignoringWs = true; - } else if (m.startsWith('#')) { - ignoringComment = true; - } else if (escapedWsOrHash.test(m)) { - transformed += update(m[1], {prefix: false}); - } else { - transformed += update(m); - } - } else if (regexContext === RegexContext.CHAR_CLASS && m !== '[' && m !== '[^') { - if ( - charClassWs.test(m) && - ( charClassContext === CharClassContext.DEFAULT || - charClassContext === CharClassContext.ENCLOSED_Q || - charClassContext === CharClassContext.RANGE - ) - ) { - ignoringCharClassWs = true; - } else if (charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN) { - // Need to handle this here since the main regex-parsing code wouldn't know where the token - // ends if we removed whitespace after an incomplete token that is followed by something - // that completes the token - throw new Error(`Invalid incomplete token in character class: "${m}"`); - } else if ( - escapedCharClassWs.test(m) && - (charClassContext === CharClassContext.DEFAULT || charClassContext === CharClassContext.ENCLOSED_Q) - ) { - transformed += update(m[1], {prefix: false}); - } else if (charClassContext === CharClassContext.DEFAULT) { - const nextChar = value[index + 1] ?? ''; - let updated = sandboxUnsafeNulls(m); - // Avoid escaping lone double punctuators unless required, since some of them are not - // allowed to be escaped with flag u (the `unicodeSetsPlugin` already unescapes them when - // using flag u, but it can be set to `null` via an option) - if (charClassWs.test(nextChar) || m === '^') { - updated = sandboxLoneDoublePunctuatorChar(updated); - } - transformed += update(updated); - } else { - transformed += update(m); - } - } else { - transformed += update(m); - } - if (!(ignoringWs || ignoringCharClassWs || ignoringComment)) { - lastSignificantToken = m; - lastSignificantCharClassContext = charClassContext; - } - } - return { - transformed, - runningContext, - }; -} - -/** -Remove `(?:)` token separators (most likely added by flag x) in cases where it's safe to do so. -@param {string} expression -@returns {PluginResult} -*/ -function clean(expression) { - const sep = String.raw`\(\?:\)`; - // No need for repeated separators - expression = replaceUnescaped(expression, `(?:${sep}){2,}`, '(?:)', Context.DEFAULT); - // No need for separators at: - // - The beginning, if not followed by a quantifier. - // - The end. - // - Outside of character classes: - // - If followed by one of `)|.[$\\`, or `(` if that's not followed by `DEFINE)`. - // - Technically we shouldn't remove `(?:)` if preceded by `(?(DEFINE` and followed by `)`, - // but in this case flag x injects a sandboxing `(?:)` after the preceding invalid `(?`, - // so we already get an error from that. - // - If preceded by one of `()|.]^>`, `\\[bBdDfnrsStvwW]`, `(?:`, or a lookaround opening. - // - So long as the separator is not followed by a quantifier. - // Examples of things that are not safe to remove `(?:)` at the boundaries of: - // - Anywhere: Letters, numbers, or any of `-=_,`. - // - If preceded by any of `\\[cgkpPux]`. - // - Anything inside character classes. - expression = replaceUnescaped( - expression, - String.raw`${sep}(?=[)|.[$\\]|\((?!DEFINE)|$)|(?<=[()|.\]^>]|\\[bBdDfnrsStvwW]|\(\?(?:[:=!]|<[=!])|^)${sep}(?![?*+{])`, - '', - Context.DEFAULT - ); - return { - pattern: expression, - }; -} - -export { - clean, - flagXPreprocessor, -}; diff --git a/src/libs/regex/internals.js b/src/libs/regex/internals.js deleted file mode 100644 index 27f51bba..00000000 --- a/src/libs/regex/internals.js +++ /dev/null @@ -1,4 +0,0 @@ -export {atomic, possessive} from './atomic.js'; -export {RegExpSubclass} from './subclass.js'; -// rexreplace patch: export subroutines (requested upstream) -export {subroutines} from './subroutines.js'; diff --git a/src/libs/regex/pattern.js b/src/libs/regex/pattern.js deleted file mode 100644 index 19cb677d..00000000 --- a/src/libs/regex/pattern.js +++ /dev/null @@ -1,45 +0,0 @@ -class Pattern { - #value; - /** @param {string} value */ - constructor(value) { - this.#value = value; - } - /** @returns {string} */ - toString() { - return String(this.#value); - } -} - -/** -Returns a value that can be interpolated into a `regex` template string without having its special -characters escaped. - -Can be called as a function or template tag: -- `pattern(value)` - String or value coerced to string. -- `` pattern`…` `` - Same as ``pattern(String.raw`…`)``. - -@overload -@param {string | number} value -@returns {Pattern} - -@overload -@param {TemplateStringsArray} template -@param {...string} substitutions -@returns {Pattern} -*/ -function pattern(first, ...substitutions) { - if (Array.isArray(first?.raw)) { - return new Pattern( - // Intersperse raw template strings and substitutions - first.raw.flatMap((raw, i) => i < first.raw.length - 1 ? [raw, substitutions[i]] : raw).join('') - ); - } else if (!substitutions.length) { - return new Pattern(first === undefined ? '' : first); - } - throw new Error(`Unexpected arguments: ${JSON.stringify([first, ...substitutions])}`); -} - -export { - Pattern, - pattern, -}; diff --git a/src/libs/regex/regex.js b/src/libs/regex/regex.js deleted file mode 100644 index cfa5841c..00000000 --- a/src/libs/regex/regex.js +++ /dev/null @@ -1,390 +0,0 @@ -import {atomic, possessive} from './atomic.js'; -import {backcompatPlugin} from './backcompat.js'; -import {flagNPreprocessor} from './flag-n.js'; -import {clean, flagXPreprocessor} from './flag-x.js'; -import {Pattern, pattern} from './pattern.js'; -import {RegExpSubclass} from './subclass.js'; -import {subroutines} from './subroutines.js'; -import {adjustNumberedBackrefs, CharClassContext, containsCharClassUnion, countCaptures, enclosedTokenCharClassContexts, enclosedTokenRegexContexts, envSupportsFlagGroups, envSupportsFlagV, escapeV, getBreakoutChar, getEndContextForIncompleteExpression, preprocess, RegexContext, sandboxLoneCharClassCaret, sandboxLoneDoublePunctuatorChar, sandboxUnsafeNulls} from './utils.js'; -import {Context, hasUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; - -/** -@typedef {string | RegExp | Pattern | number} InterpolatedValue -@typedef {{ - flags?: string; - captureTransfers?: Map>; - hiddenCaptures?: Array; -}} PluginData -@typedef {{ - pattern: string; - captureTransfers?: Map>; - hiddenCaptures?: Array; -}} PluginResult -@typedef {TemplateStringsArray | {raw: Array}} RawTemplate -@typedef {{ - flags?: string; - subclass?: boolean; - plugins?: Array<(expression: string, data: PluginData) => PluginResult>; - unicodeSetsPlugin?: ((expression: string, data: PluginData) => PluginResult) | null; - disable?: { - x?: boolean; - n?: boolean; - v?: boolean; - atomic?: boolean; - subroutines?: boolean; - }; - force?: { - v?: boolean; - }; -}} RegexTagOptions -*/ -/** -@template T -@typedef RegexTag -@type {{ - (template: RawTemplate, ...substitutions: ReadonlyArray): T; - (flags?: string): RegexTag; - (options: RegexTagOptions & {subclass?: false}): RegexTag; - (options: RegexTagOptions & {subclass: true}): RegexTag; -}} -*/ -/** -Template tag for constructing a regex with extended syntax and context-aware interpolation of -regexes, strings, and patterns. - -Can be called in several ways: -1. `` regex`…` `` - Regex pattern as a raw string. -2. `` regex('gi')`…` `` - To specify flags. -3. `` regex({flags: 'gi'})`…` `` - With options. -@type {RegexTag} -*/ -const regex = (first, ...substitutions) => { - // Given a template - if (Array.isArray(first?.raw)) { - return regexFromTemplate({}, first, ...substitutions); - // Given flags - } else if ((typeof first === 'string' || first === undefined) && !substitutions.length) { - return regexFromTemplate.bind(null, {flags: first ?? ''}); - // Given an options object - } else if ({}.toString.call(first) === '[object Object]' && !substitutions.length) { - return regexFromTemplate.bind(null, first); - } - throw new Error(`Unexpected arguments: ${JSON.stringify([first, ...substitutions])}`); -}; - -/** -@template T -@typedef RegexFromTemplate -@type {{ - (options: RegexTagOptions, template: RawTemplate, ...substitutions: ReadonlyArray) : T; -}} -*/ -/** -Returns a RegExp from a template and substitutions to fill the template holes. -@type {RegexFromTemplate} -*/ -const regexFromTemplate = (options, template, ...substitutions) => { - const opts = getOptions(options); - const prepped = runPreprocessors(template, substitutions, opts); - - let precedingCaptures = 0; - let expression = ''; - let runningContext; - // Intersperse raw template strings and substitutions - prepped.template.raw.forEach((raw, i) => { - const hasNonEmptyRawBoundary = !!(prepped.template.raw[i] || prepped.template.raw[i + 1]); - // Even with flag n enabled, we might have named captures - precedingCaptures += countCaptures(raw); - // Sandbox `\0` in character classes. Not needed outside character classes because in other - // cases a following interpolated value would always be atomized - expression += sandboxUnsafeNulls(raw, Context.CHAR_CLASS); - runningContext = getEndContextForIncompleteExpression(expression, runningContext); - const {regexContext, charClassContext} = runningContext; - if (i < prepped.template.raw.length - 1) { - const substitution = prepped.substitutions[i]; - expression += interpolate( - substitution, - opts.flags, - regexContext, - charClassContext, - hasNonEmptyRawBoundary, - precedingCaptures - ); - if (substitution instanceof RegExp) { - precedingCaptures += countCaptures(substitution.source); - } else if (substitution instanceof Pattern) { - precedingCaptures += countCaptures(String(substitution)); - } - } - }); - - const plugged = runPlugins(expression, opts); - expression = plugged.pattern; - try { - return opts.subclass ? - new RegExpSubclass(expression, opts.flags, {hiddenCaptures: plugged.hiddenCaptures}) : - new RegExp(expression, opts.flags); - } catch (err) { - // Improve DX by always including the generated source in the error message. Some browsers - // include it automatically, but not Firefox or Safari - const stripped = err.message.replace(/ \/.+\/[a-z]*:/, ''); - err.message = `${stripped}: /${expression}/${opts.flags}`; - throw err; - } -}; - -/** -Returns the processed expression and flags as strings. -@param {string} expression -@param {RegexTagOptions} [options] -@returns {{ - pattern: string; - flags: string; -}} -*/ -function rewrite(expression = '', options) { - const opts = getOptions(options); - if (opts.subclass) { - throw new Error('Cannot use option subclass'); - } - return { - // NOTE: Since `pattern` is a Regex+ export with special meaning, the term `expression` is used - // in code to refer to regex source strings, except in the public API - // rexreplace patch: also return hiddenCaptures - ...runPlugins( - runPreprocessors({raw: [expression]}, [], opts).template.raw[0], - opts - ), - flags: opts.flags, - }; -} - -/** -Returns a complete set of options, with default values set for options that weren't provided, and -some options augmented for use. -@param {RegexTagOptions} [options] -@returns {Required} -*/ -function getOptions(options) { - const opts = { - flags: '', - subclass: false, - plugins: [], - unicodeSetsPlugin: backcompatPlugin, - disable: {/* n, v, x, atomic, subroutines */}, - force: {/* v */}, - ...options, - }; - if (/[nuvx]/.test(opts.flags)) { - throw new Error('Implicit flags v/u/x/n cannot be explicitly added'); - } - const useFlagV = opts.force.v || (opts.disable.v ? false : envSupportsFlagV); - opts.flags += useFlagV ? 'v' : 'u'; - if (useFlagV) { - opts.unicodeSetsPlugin = null; - } - return opts; -} - -/** -@param {RawTemplate} template -@param {ReadonlyArray} substitutions -@param {Required} options -@returns {{ - template: RawTemplate; - substitutions: ReadonlyArray; -}} -*/ -function runPreprocessors(template, substitutions, options) { - const preprocessors = []; - // Implicit flag x is handled first because otherwise some regex syntax (if unescaped) within - // comments could cause problems when parsing - if (!options.disable.x) { - preprocessors.push(flagXPreprocessor); - } - // Implicit flag n is a preprocessor because capturing groups affect backreference rewriting in - // both interpolation and plugins - if (!options.disable.n) { - preprocessors.push(flagNPreprocessor); - } - for (const pp of preprocessors) { - ({template, substitutions} = preprocess(template, substitutions, pp, options)); - } - return { - template, - substitutions, - }; -} - -/** -@param {string} expression -@param {Required} options -@returns {Required} -*/ -function runPlugins(expression, {flags, plugins, unicodeSetsPlugin, disable}) { - let hiddenCaptures = []; - [ ...plugins, // Run first, so provided plugins can output extended syntax - ...(disable.subroutines ? [] : [subroutines]), - ...(disable.atomic ? [] : [possessive, atomic]), - ...(disable.x ? [] : [clean]), - // Run last, so it doesn't have to worry about parsing extended syntax - ...(!unicodeSetsPlugin ? [] : [unicodeSetsPlugin]), - ].forEach(plugin => { - const result = plugin(expression, {flags, hiddenCaptures}); - if (typeof result?.pattern !== 'string') { - throw new Error('Plugin must return an object with a string property "pattern"'); - } - expression = result.pattern; - if (result.hiddenCaptures) { - hiddenCaptures = result.hiddenCaptures; - } - }); - return { - pattern: expression, - hiddenCaptures, - }; -} - -/** -@param {InterpolatedValue} value -@param {string} flags -@param {string} regexContext -@param {string} charClassContext -@param {boolean} hasNonEmptyRawBoundary -@param {number} precedingCaptures -@returns {string} -*/ -function interpolate(value, flags, regexContext, charClassContext, hasNonEmptyRawBoundary, precedingCaptures) { - if (value instanceof RegExp) { - if (regexContext !== RegexContext.DEFAULT) { - throw new Error('Cannot interpolate a RegExp at this position because the syntax context does not match'); - } - const transformed = transformForLocalFlags(value, flags); - const backrefsAdjusted = adjustNumberedBackrefs(transformed, precedingCaptures); - // Sandbox and atomize - return wrapIfUnsafe(backrefsAdjusted); - } - - if ( - regexContext === RegexContext.INVALID_INCOMPLETE_TOKEN || - charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN - ) { - // Throw in all cases, but only *need* to handle a preceding unescaped backslash (which would - // break sandboxing) since other errors would be handled by the invalid generated regex syntax - throw new Error('Interpolation preceded by invalid incomplete token'); - } - - if ( - typeof value === 'number' && - (regexContext === RegexContext.ENCLOSED_U || charClassContext === CharClassContext.ENCLOSED_U) - ) { - return value.toString(16); - } - - const isPattern = value instanceof Pattern; - value = String(value); - const escapedValue = isPattern ? null : escapeV( - value, - regexContext === RegexContext.CHAR_CLASS ? Context.CHAR_CLASS : Context.DEFAULT - ); - // Checks `escapedValue` in addition to patterns since potential breakout char `>` isn't escaped - const breakoutChar = getBreakoutChar(escapedValue || value, regexContext, charClassContext); - if (breakoutChar) { - throw new Error(`Unescaped stray "${breakoutChar}" in the interpolated value would have side effects outside it`); - } - - if ( - regexContext === RegexContext.INTERVAL_QUANTIFIER || - regexContext === RegexContext.GROUP_NAME || - enclosedTokenRegexContexts.has(regexContext) || - enclosedTokenCharClassContexts.has(charClassContext) - ) { - return isPattern ? value : escapedValue; - } - if (regexContext === RegexContext.CHAR_CLASS) { - if (isPattern) { - if (hasUnescaped(value, '^-|^&&|-$|&&$')) { - // Sandboxing so we don't change the chars outside the pattern into being part of an - // operation they didn't initiate. Same problem as starting a pattern with a quantifier - throw new Error('Cannot use range or set operator at boundary of interpolated pattern; move the operation into the pattern or the operator outside of it'); - } - const sandboxedValue = sandboxLoneCharClassCaret(sandboxLoneDoublePunctuatorChar(value)); - // Atomize via nested character class `[…]` if it contains implicit or explicit union (check - // the unadjusted value) - return containsCharClassUnion(value) ? `[${sandboxedValue}]` : sandboxUnsafeNulls(sandboxedValue); - } - // Atomize via nested character class `[…]` if more than one node - return containsCharClassUnion(escapedValue) ? `[${escapedValue}]` : escapedValue; - } - if (isPattern) { - // Sandbox and atomize - return wrapIfUnsafe(value); - } - // Sandbox and atomize, but avoid noise from noncapturing group if unnecessary (safe to avoid - // because interpolated patterns and regexes are always wrapped) - return hasNonEmptyRawBoundary ? `(?:${escapedValue})` : escapedValue; -} - -/** -@param {string} str -@returns {string} -*/ -function wrapIfUnsafe(str) { - return (str.startsWith('(') && str.endsWith(')')) ? str : `(?:${str})`; -} - -/** -@param {RegExp} re -@param {string} outerFlags -@returns {string} -*/ -function transformForLocalFlags(re, outerFlags) { - /** @type {{i: boolean | null; m: boolean | null; s: boolean | null;}} */ - const modFlagsObj = { - i: null, - m: null, - s: null, - }; - const newlines = '\\n\\r\\u2028\\u2029'; - let value = re.source; - if (re.ignoreCase !== outerFlags.includes('i')) { - if (envSupportsFlagGroups) { - modFlagsObj.i = re.ignoreCase; - } else { - throw new Error('Flag groups not supported, so flag i on the outer and interpolated regex must match'); - } - } - if (re.dotAll !== outerFlags.includes('s')) { - if (envSupportsFlagGroups) { - modFlagsObj.s = re.dotAll; - } else { - value = replaceUnescaped(value, '\\.', (re.dotAll ? '[^]' : `[^${newlines}]`), Context.DEFAULT); - } - } - if (re.multiline !== outerFlags.includes('m')) { - if (envSupportsFlagGroups) { - modFlagsObj.m = re.multiline; - } else { - value = replaceUnescaped(value, '\\^', (re.multiline ? `(?<=^|[${newlines}])` : '(? modFlagsObj[k] === true).join(''); - const modOff = keys.filter(k => modFlagsObj[k] === false).join(''); - if (modOff) { - modifier += `-${modOff}`; - } - if (modifier) { - return `(?${modifier}:${value})`; - } - } - return value; -} - -export { - pattern, - regex, - rewrite, -}; diff --git a/src/libs/regex/subclass.js b/src/libs/regex/subclass.js deleted file mode 100644 index 46f17b4f..00000000 --- a/src/libs/regex/subclass.js +++ /dev/null @@ -1,98 +0,0 @@ -/** -Works the same as JavaScript's native `RegExp` constructor in all contexts, but automatically -adjusts subpattern matches and indices (with flag `d`) to account for captures added as part of -emulating extended syntax. -*/ -class RegExpSubclass extends RegExp { - // Avoid `#private` to allow for subclassing - /** - @private - @type {Map} - */ - _captureMap; - /** - @overload - @param {string} expression - @param {string} [flags] - @param {{ - hiddenCaptures?: Array; - }} [options] - */ - /** - @overload - @param {RegExpSubclass} expression - @param {string} [flags] - */ - constructor(expression, flags, options) { - // Argument `options` isn't provided when regexes are copied via `new RegExpSubclass(regexp)`, - // including as part of the internal handling of string methods `matchAll` and `split` - if (expression instanceof RegExp) { - if (options) { - throw new Error('Cannot provide options when copying a regexp'); - } - super(expression, flags); - if (expression instanceof RegExpSubclass) { - this._captureMap = expression._captureMap; - } else { - this._captureMap = new Map(); - } - } else { - super(expression, flags); - const hiddenCaptures = options?.hiddenCaptures ?? []; - this._captureMap = createCaptureMap(hiddenCaptures); - } - } - /** - Called internally by all String/RegExp methods that use regexes. - @override - @param {string} str - @returns {RegExpExecArray | null} - */ - exec(str) { - const match = super.exec(str); - if (!match || !this._captureMap.size) { - return match; - } - const matchCopy = [...match]; - // Empty all but the first value of the array while preserving its other properties - match.length = 1; - let indicesCopy; - if (this.hasIndices) { - indicesCopy = [...match.indices]; - match.indices.length = 1; - } - for (let i = 1; i < matchCopy.length; i++) { - if (!this._captureMap.get(i)?.hidden) { - match.push(matchCopy[i]); - if (this.hasIndices) { - match.indices.push(indicesCopy[i]); - } - } - } - return match; - } -} - -/** -Build the capturing group map, with hidden captures marked to indicate their submatches shouldn't -appear in match results. -@param {Array} hiddenCaptures -@returns {Map} -*/ -function createCaptureMap(hiddenCaptures) { - const captureMap = new Map(); - for (const num of hiddenCaptures) { - captureMap.set(num, { - hidden: true, - }); - } - return captureMap; -} - -export { - RegExpSubclass, -}; diff --git a/src/libs/regex/subroutines.js b/src/libs/regex/subroutines.js index 8af34bf8..6ad569a2 100644 --- a/src/libs/regex/subroutines.js +++ b/src/libs/regex/subroutines.js @@ -1,6 +1,6 @@ import {capturingDelim, countCaptures, namedCapturingDelim} from './utils.js'; import {incrementIfAtLeast, spliceStr} from './utils-internals.js'; -import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; +import {Context, execUnescaped, forEachUnescaped, getGroupContents, hasUnescaped, replaceUnescaped} from 'regex-utilities'; /** @import {PluginData, PluginResult} from './regex.js'; */ diff --git a/src/libs/regex/utils.js b/src/libs/regex/utils.js index cd88ce97..e5101b03 100644 --- a/src/libs/regex/utils.js +++ b/src/libs/regex/utils.js @@ -1,133 +1,9 @@ -import {Pattern, pattern} from './pattern.js'; -import {Context, forEachUnescaped, replaceUnescaped} from '../regex-utilities/index.js'; -/** -@import {InterpolatedValue, RawTemplate, RegexTagOptions} from './regex.js'; -*/ - -const RegexContext = { - DEFAULT: 'DEFAULT', - CHAR_CLASS: 'CHAR_CLASS', - ENCLOSED_P: 'ENCLOSED_P', - ENCLOSED_U: 'ENCLOSED_U', - GROUP_NAME: 'GROUP_NAME', - INTERVAL_QUANTIFIER: 'INTERVAL_QUANTIFIER', - INVALID_INCOMPLETE_TOKEN: 'INVALID_INCOMPLETE_TOKEN', -}; - -const CharClassContext = { - DEFAULT: 'DEFAULT', - ENCLOSED_P: 'ENCLOSED_P', - ENCLOSED_Q: 'ENCLOSED_Q', - ENCLOSED_U: 'ENCLOSED_U', - INVALID_INCOMPLETE_TOKEN: 'INVALID_INCOMPLETE_TOKEN', - RANGE: 'RANGE', -}; +// rexreplace: the parts of utils.js of regex 6.1.0 that subroutines.js uses, copied as they are +import {Context, forEachUnescaped} from 'regex-utilities'; -const enclosedTokenRegexContexts = new Set([ - RegexContext.ENCLOSED_P, - RegexContext.ENCLOSED_U, -]); - -const enclosedTokenCharClassContexts = new Set([ - CharClassContext.ENCLOSED_P, - CharClassContext.ENCLOSED_Q, - CharClassContext.ENCLOSED_U, -]); - -const envSupportsFlagGroups = (() => { - try { - new RegExp('(?i:)'); - } catch { - return false; - } - return true; -})(); - -const envSupportsFlagV = (() => { - try { - new RegExp('', 'v'); - } catch { - return false; - } - return true; -})(); - -const doublePunctuatorChars = '&!#$%*+,.:;<=>?@^`~'; const namedCapturingDelim = String.raw`\(\?<(?![=!])(?[^>]+)>`; const capturingDelim = String.raw`\((?!\?)(?!(?<=\(\?\()DEFINE\))|${namedCapturingDelim}`; -/** -@param {string} expression -@param {number} precedingCaptures -@returns {string} -*/ -function adjustNumberedBackrefs(expression, precedingCaptures) { - return replaceUnescaped( - expression, - String.raw`\\(?[1-9]\d*)`, - ({groups: {num}}) => `\\${+num + precedingCaptures}`, - Context.DEFAULT - ); -} - -// Properties of strings as of ES2024 -const stringPropertyNames = [ - 'Basic_Emoji', - 'Emoji_Keycap_Sequence', - 'RGI_Emoji_Modifier_Sequence', - 'RGI_Emoji_Flag_Sequence', - 'RGI_Emoji_Tag_Sequence', - 'RGI_Emoji_ZWJ_Sequence', - 'RGI_Emoji', -].join('|'); -const charClassUnionToken = new RegExp(String.raw` -\\(?: c[A-Za-z] - | p\{(?${stringPropertyNames})\} - | [pP]\{[^\}]+\} - | (?q) - | u(?:[A-Fa-f\d]{4}|\{[A-Fa-f\d]+\}) - | x[A-Fa-f\d]{2} - | . -) -| -- -| && -| . -`.replace(/\s+/g, ''), 'gsu'); - -// Assumes flag v and doesn't worry about syntax errors that are caught by it -function containsCharClassUnion(charClassPattern) { - // Return `true` if it contains: - // - `\p` (lowercase only) and the name is a property of strings (case sensitive). - // - `\q`. - // - Two single-char-matching tokens in sequence. - // - One single-char-matching token followed immediately by unescaped `[`. - // - One single-char-matching token preceded immediately by unescaped `]`. - // Else, `false`. - // Ranges with `-` create a single token. - // Subtraction and intersection with `--` and `&&` create a single token. - // Supports any number of nested classes - let hasFirst = false; - let lastM; - for (const {0: m, groups} of charClassPattern.matchAll(charClassUnionToken)) { - if (groups.pStrProp || groups.qStrProp) { - return true; - } - if (m === '[' && hasFirst) { - return true; - } - if (['-', '--', '&&'].includes(m)) { - hasFirst = false; - } else if (m !== '[' && m !== ']') { - if (hasFirst || lastM === ']') { - return true; - } - hasFirst = true; - } - lastM = m; - } - return false; -} - /** @param {string} expression @returns {number} @@ -138,264 +14,8 @@ function countCaptures(expression) { return num; } -/** -Escape special characters for the given context, assuming flag v. -@param {string} str String to escape -@param {'DEFAULT' | 'CHAR_CLASS'} context `Context` option from lib `regex-utilities` -@returns {string} Escaped string -*/ -function escapeV(str, context) { - if (context === Context.CHAR_CLASS) { - // Escape all double punctuators (including ^, which is special on its own in the first - // position) in case they're bordered by the same character in or outside of the escaped string - return str.replace(new RegExp(String.raw`[()\[\]{}|\\/\-${doublePunctuatorChars}]`, 'g'), '\\$&'); - } - return str.replace(/[()\[\]{}|\\^$*+?.]/g, '\\$&'); -} - -// Look for characters that would change the meaning of subsequent tokens outside an interpolated value -function getBreakoutChar(expression, regexContext, charClassContext) { - const escapesRemoved = expression.replace(/\\./gsu, ''); - // Trailing unescaped `\`; checking `.includes('\\')` would also work - if (escapesRemoved.endsWith('\\')) { - return '\\'; - } - if (regexContext === RegexContext.DEFAULT) { - // Unbalanced `[` or `]` are also errors but don't breakout; they're caught by the wrapper - return getUnbalancedChar(escapesRemoved, '(', ')'); - } else if ( - regexContext === RegexContext.CHAR_CLASS && - !enclosedTokenCharClassContexts.has(charClassContext) - ) { - return getUnbalancedChar(escapesRemoved, '[', ']'); - } else if ( - regexContext === RegexContext.INTERVAL_QUANTIFIER || - enclosedTokenRegexContexts.has(regexContext) || - enclosedTokenCharClassContexts.has(charClassContext) - ) { - if (escapesRemoved.includes('}')) { - return '}'; - } - } else if (regexContext === RegexContext.GROUP_NAME) { - if (escapesRemoved.includes('>')) { - return '>'; - } - } - return ''; -} - -const contextToken = new RegExp(String.raw` -(?\(\?<(?![=!])|\\[gk]<) -| (?\\[pPu]\{) -| (?\\q\{) -| (?\{) -| (?\\(?: $ - | c(?![A-Za-z]) - | u(?![A-Fa-f\d]{4})[A-Fa-f\d]{0,3} - | x(?![A-Fa-f\d]{2})[A-Fa-f\d]? - ) -) -| -- -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -@typedef {{ - regexContext: string; - charClassContext: string; - charClassDepth: number; - lastPos: number; -}} RunningContext -*/ -/** -Accepts and returns its full state so it doesn't have to reprocess parts that have already been -seen. Assumes flag v and doesn't worry about syntax errors that are caught by it. -@param {string} incompleteExpression -@param {Partial} [runningContext] -@returns {RunningContext} -*/ -function getEndContextForIncompleteExpression(incompleteExpression, runningContext) { - let {regexContext, charClassContext, charClassDepth, lastPos} = { - regexContext: RegexContext.DEFAULT, - charClassContext: CharClassContext.DEFAULT, - charClassDepth: 0, - lastPos: 0, - ...runningContext, - }; - contextToken.lastIndex = lastPos; - let match; - while (match = contextToken.exec(incompleteExpression)) { - const {0: m, groups: {groupN, enclosedPU, enclosedQ, intervalQ, incompleteT}} = match; - if (m === '[') { - charClassDepth++; - regexContext = RegexContext.CHAR_CLASS; - charClassContext = CharClassContext.DEFAULT; - } else if (m === ']' && regexContext === RegexContext.CHAR_CLASS) { - if (charClassDepth) { - charClassDepth--; - } - if (!charClassDepth) { - regexContext = RegexContext.DEFAULT; - } - charClassContext = CharClassContext.DEFAULT; - } else if (regexContext === RegexContext.CHAR_CLASS) { - if (incompleteT) { - charClassContext = CharClassContext.INVALID_INCOMPLETE_TOKEN; - } else if (m === '-') { - charClassContext = CharClassContext.RANGE; - } else if (enclosedPU) { - charClassContext = m[1] === 'u' ? CharClassContext.ENCLOSED_U : CharClassContext.ENCLOSED_P; - } else if (enclosedQ) { - charClassContext = CharClassContext.ENCLOSED_Q; - } else if ( - (m === '}' && enclosedTokenCharClassContexts.has(charClassContext)) || - // Don't continue in these contexts since we've advanced another token - charClassContext === CharClassContext.INVALID_INCOMPLETE_TOKEN || - charClassContext === CharClassContext.RANGE - ) { - charClassContext = CharClassContext.DEFAULT; - } - } else { - if (incompleteT) { - regexContext = RegexContext.INVALID_INCOMPLETE_TOKEN; - } else if (groupN) { - regexContext = RegexContext.GROUP_NAME; - } else if (enclosedPU) { - regexContext = m[1] === 'u' ? RegexContext.ENCLOSED_U : RegexContext.ENCLOSED_P; - } else if (intervalQ) { - regexContext = RegexContext.INTERVAL_QUANTIFIER; - } else if ( - (m === '>' && regexContext === RegexContext.GROUP_NAME) || - (m === '}' && (regexContext === RegexContext.INTERVAL_QUANTIFIER || enclosedTokenRegexContexts.has(regexContext))) || - // Don't continue in this context since we've advanced another token - regexContext === RegexContext.INVALID_INCOMPLETE_TOKEN - ) { - regexContext = RegexContext.DEFAULT; - } - } - } - return { - regexContext, - charClassContext, - charClassDepth, - lastPos: incompleteExpression.length, - }; -} - -// No special handling for escaped versions of the characters -function getUnbalancedChar(expression, leftChar, rightChar) { - let numOpen = 0; - for (const [m] of expression.matchAll(new RegExp(`[${escapeV(leftChar + rightChar, Context.CHAR_CLASS)}]`, 'g'))) { - numOpen += m === leftChar ? 1 : -1; - if (numOpen < 0) { - return rightChar; - } - } - if (numOpen > 0) { - return leftChar; - } - return ''; -} - -/** -@typedef {InterpolatedValue} InterpolatedValue -@typedef {RawTemplate} RawTemplate -@typedef {RegexTagOptions} RegexTagOptions -@typedef {( - value: InterpolatedValue, - runningContext: RunningContext, - options: Required -) => { - transformed: string; - runningContext: RunningContext; -}} Preprocessor -*/ -/** -Returns transformed versions of a template and substitutions, using the given preprocessor. Only -processes substitutions that are instanceof `Pattern`. -@param {RawTemplate} template -@param {ReadonlyArray} substitutions -@param {Preprocessor} preprocessor -@param {Required} options -@returns {{template: RawTemplate; substitutions: ReadonlyArray;}} -*/ -function preprocess(template, substitutions, preprocessor, options) { - let /** @type {RawTemplate} */ newTemplate = {raw: []}; - let newSubstitutions = []; - let runningContext; - template.raw.forEach((raw, i) => { - const result = preprocessor(raw, {...runningContext, lastPos: 0}, options); - newTemplate.raw.push(result.transformed); - runningContext = result.runningContext; - if (i < template.raw.length - 1) { - const substitution = substitutions[i]; - if (substitution instanceof Pattern) { - const result = preprocessor(substitution, {...runningContext, lastPos: 0}, options); - newSubstitutions.push(pattern(result.transformed)); - runningContext = result.runningContext; - } else { - newSubstitutions.push(substitution); - } - } - }); - return { - template: newTemplate, - substitutions: newSubstitutions, - }; -} - -// Sandbox `^` if relevant, done so it can't change the meaning of the surrounding character class -// if we happen to be at the first position. See `sandboxLoneDoublePunctuatorChar` for more details -function sandboxLoneCharClassCaret(str) { - return str.replace(/^\^/, '\\^^'); -} - -// Sandbox without escaping by repeating the character and escaping only the first one. The second -// one is so that, if followed by the same symbol, the resulting double punctuator will still throw -// as expected. Details: -// - Only need to check the first position because, if it's part of an implicit union, -// interpolation handling will wrap it in nested `[…]`. -// - Can't just wrap in nested `[…]` here, since the value might be used in a range. -// - Can't add a second unescaped symbol if a lone symbol is the entire string because it might be -// followed by the same unescaped symbol outside an interpolation, and since it won't be wrapped, -// the second symbol wouldn't be sandboxed from the one following it. -function sandboxLoneDoublePunctuatorChar(str) { - return str.replace(new RegExp(`^([${doublePunctuatorChars}])(?!\\1)`), (m, _, pos) => { - return `\\${m}${pos + 1 === str.length ? '' : m}`; - }); -} - -/** -Converts `\0` tokens to `\x00` in the given context. -@param {string} str -@param {'DEFAULT' | 'CHAR_CLASS'} [context] `Context` option from lib `regex-utilities` -@returns {string} -*/ -function sandboxUnsafeNulls(str, context) { - // regex`[\0${0}]` and regex`[${pattern`\0`}0]` can't be guarded against via nested `[…]` - // sandboxing in character classes if the interpolated value doesn't contain union (since it - // might be placed on a range boundary). So escape `\0` in character classes as `\x00` - return replaceUnescaped(str, String.raw`\\0(?!\d)`, '\\x00', context); -} - export { - adjustNumberedBackrefs, capturingDelim, - CharClassContext, - containsCharClassUnion, countCaptures, - doublePunctuatorChars, - enclosedTokenCharClassContexts, - enclosedTokenRegexContexts, - envSupportsFlagGroups, - envSupportsFlagV, - escapeV, - getBreakoutChar, - getEndContextForIncompleteExpression, namedCapturingDelim, - preprocess, - RegexContext, - sandboxLoneCharClassCaret, - sandboxLoneDoublePunctuatorChar, - sandboxUnsafeNulls, }; diff --git a/test/cli/run.sh b/test/cli/run.sh index c2cbfd9e..85b3ce6c 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -80,11 +80,7 @@ reset assert "rexreplace o x my.file --output --engine default" "fxxbar" assert "rexreplace '(?>o+)(b)' '[\$1]' my.file --output" "f[b]ar" assert "rexreplace o x my.file --output --engine runtime" "fxxbar" -if [[ "$(rexreplace --version)" == *+bun ]]; then - assert_failure "rexreplace o x my.file --output --engine V8" -else - assert_failure "rexreplace o x my.file --output --engine JSC" -fi +assert_failure "rexreplace o x my.file --output --engine V8" # --encoding diff --git a/test/engine/engine.test.js b/test/engine/engine.test.js index 96016ec0..7e2c2f5b 100644 --- a/test/engine/engine.test.js +++ b/test/engine/engine.test.js @@ -7,19 +7,8 @@ describe('--engine', () => { expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'default'})).out).toBe('fxxbar'); }); - // The tests are run by Bun, and Bun has JSC - test('JSC does the job with Bun as the runtime', async () => { - expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'JSC'})).out).toBe('fxxbar'); - }); - - test('stops with an error if the engine is the one of another runtime', async () => { - fs.writeFileSync('my.file', 'foobar'); - const result = await rr('o', 'x', {includeGlob: ['my.file'], engine: 'V8'}); - expect(result.exit).toBe(3); - expect(result.err).toContain( - 'When running via the Bun runtime the native engine is JSC, not V8' - ); - expect(fs.readFileSync('my.file', 'utf8')).toBe('foobar'); + test('runtime is the native engine', async () => { + expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'runtime'})).out).toBe('fxxbar'); }); test('stops with an error if the engine is unknown', async () => { @@ -31,14 +20,14 @@ describe('--engine', () => { }); test('is case sensitive', async () => { - expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'jsc'})).exit).toBe(3); + expect((await rr('o', 'x', {pipedData: 'foobar', engine: 'Runtime'})).exit).toBe(3); }); describe('combined', () => { - test('JSC with all the regex flags', async () => { + test('runtime with all the regex flags', async () => { const config = { pipedData: 'Foo\nfoo 🦖', - engine: 'JSC', + engine: 'runtime', voidIgnoreCase: true, voidGlobal: true, voidMultiline: true, diff --git a/test/fix/854.test.js b/test/fix/854.test.js index 1531f5e4..da8eb1b8 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -33,6 +33,9 @@ describe('#854 the rr engine', () => { ['[+]+'], ['\\{2}+'], ['[(?>]'], + // #858 + ['[(?>]\\1'], + ['[\\g]'], ['[a++]'], ['\\(?>'], ['(?a)\\k'], @@ -104,14 +107,11 @@ describe('#854 the groups Regex+ adds', () => { describe('#854 the other engines', () => { // The tests are run by Bun, so the native engine is JSC - test.each([['JSC'], ['runtime']])( - '%s is the native engine, without the extended syntax', - async engine => { - expect(await out('(?a)\\g', 'X', 'ag', {engine})).toBe('X'); - } - ); + test.each([['runtime']])('%s is the native engine, without the extended syntax', async engine => { + expect(await out('(?a)\\g', 'X', 'ag', {engine})).toBe('X'); + }); - test.each([['regex+'], ['V']])( + test.each([['regex+']])( '%s has free spacing, only named groups capture, and the v flag', async engine => { expect(await out('foo bar', 'X', 'foobar', {engine})).toBe('X'); @@ -120,7 +120,7 @@ describe('#854 the other engines', () => { } ); - test.each([['regex+n'], ['N'], ['regex+x'], ['X'], ['regex+u'], ['U'], ['regex+v']])( + test.each([['V'], ['V8'], ['JSC'], ['regex+n'], ['regex+x'], ['regex+u'], ['regex+v']])( '%s is not an engine', async engine => { expect(await rr('a', 'b', {pipedData: 'a', engine})).toEqual({ @@ -161,6 +161,10 @@ describe('#854 --flags', () => { expect(await out('o', 'x', 'oOo', {flags: 'm'})).toBe('xOo'); }); + test('takes a flag that is given twice once', async () => { + expect(await out('o', 'x', 'oO', {flags: 'gg'})).toBe('xO'); + }); + test('gives regex+ the flag v when it has neither v nor u', async () => { expect((await compiled('a', {...plus, flags: 'gx'})).flags).toContain('v'); }); @@ -202,12 +206,6 @@ describe('#854 --js with the flag n of regex+', () => { ); }); - test('warns when the replacement text uses a group number, as it is the first named group', async () => { - const result = await rr('(?\\d)', '[$1]', {pipedData: '1', engine: 'regex+'}); - expect(result.out).toBe('[1]'); - expect(result.err).toContain('With the flag n the groups have no numbers'); - }); - test('has the group numbers again without n', async () => { expect(await out('(\\d)(\\d)', '$2 + $1', '12', {...config, flags: 'gimvx'})).toBe('21'); }); diff --git a/yarn.lock b/yarn.lock index dce61abe..20f409b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -461,6 +461,18 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +regex-utilities@2.3.0, regex-utilities@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/regex-utilities/-/regex-utilities-2.3.0.tgz#87163512a15dce2908cf079c8960d5158ff43280" + integrity sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng== + +regex@6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/regex/-/regex-6.1.0.tgz#d7ce98f8ee32da7497c13f6601fca2bc4a6a7803" + integrity sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg== + dependencies: + regex-utilities "^2.3.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From cde436c740abfd9b4d4cf56a31c9a025a89e39fc Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Tue, 27 Oct 2026 13:04:09 +1100 Subject: [PATCH 07/13] Load regex as a dependency when RexReplace runs, and copy atomic.js too, as regex/internals is an ES module only --- package.json | 4 +- src/engine.ts | 4 +- src/libs/README.md | 8 +- src/libs/regex/atomic.js | 224 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 src/libs/regex/atomic.js diff --git a/package.json b/package.json index 0736f929..fce85605 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,8 @@ "build": "yarn bundle-node --outfile=bin/rexreplace.cli.node.js && yarn bundle-bun --outfile=bin/rexreplace.cli.bun.js", "build-minify": "yarn build && yarn minify", "minify": "yarn bundle-node --minify --outfile=bin/rexreplace.cli.node.min.js && yarn bundle-bun --minify --outfile=bin/rexreplace.cli.bun.min.js", - "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --external fast-glob --external yargs", - "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --external fast-glob --external yargs", + "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --external fast-glob --external yargs --external regex", + "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --external fast-glob --external yargs --external regex", "prebuild": "rm -fr bin && yarn format", "sync-version": "bun npm/sync-version.js", "test-js": "AGENT=1 bun test ./test/engine ./test/cli ./test/fix", diff --git a/src/engine.ts b/src/engine.ts index 1eec2f2f..9f6d1e91 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -48,9 +48,9 @@ const re = { // Only the version ends up in the bundle import {version} from '../package.json'; -// Regex+ by Steven Levithan. Its subroutines are copied into src/libs until it exports them. +// Regex+ by Steven Levithan. regex/internals is an ES module only, so what is used from it is copied into src/libs. import {rewrite} from 'regex'; -import {atomic, possessive} from 'regex/internals'; +import {atomic, possessive} from './libs/regex/atomic.js'; import {Context, hasUnescaped} from 'regex-utilities'; import {subroutines} from './libs/regex/subroutines.js'; export {version}; diff --git a/src/libs/README.md b/src/libs/README.md index 33bbc619..9329fb15 100644 --- a/src/libs/README.md +++ b/src/libs/README.md @@ -1,12 +1,12 @@ # Copied code -RexReplace uses [Regex+](https://github.com/slevithan/regex) by Steven Levithan, from the npm packages [`regex`](https://www.npmjs.com/package/regex) and [`regex-utilities`](https://www.npmjs.com/package/regex-utilities). Both are dependencies, and the build bundles them, as they are ES modules only. +RexReplace uses [Regex+](https://github.com/slevithan/regex) by Steven Levithan, from the npm packages [`regex`](https://www.npmjs.com/package/regex) and [`regex-utilities`](https://www.npmjs.com/package/regex-utilities). `regex` is a dependency, loaded when RexReplace runs. `regex-utilities` is an ES module only, so the build bundles it. -One part of `regex` 6.1.0 is not exported: the subroutines. So it is copied here, as it is upstream, until upstream exports it: +RexReplace runs the atomic groups, the possessive quantifiers and the subroutines of `regex` 6.1.0 itself, to know which groups they add. The first two are only in `regex/internals`, an ES module only, which the CommonJS build of RexReplace can not load, and the subroutines are not exported at all. So they are copied here, as they are upstream: -- `regex/subroutines.js` and `regex/utils-internals.js`, unchanged. +- `regex/atomic.js`, `regex/subroutines.js` and `regex/utils-internals.js`, unchanged. - `regex/utils.js`, only the three parts of it that `subroutines.js` uses. The license of `regex` (MIT) is in `regex/LICENSE`. Biome leaves this folder alone, so it stays easy to compare with upstream. -When `regex` exports `subroutines`, import it from `regex/internals`, and delete this folder and its line in `biome.json`. +When `regex` exports these from a build that `require()` can load, import them from there, and delete this folder and its line in `biome.json`. diff --git a/src/libs/regex/atomic.js b/src/libs/regex/atomic.js new file mode 100644 index 00000000..1a24fa0d --- /dev/null +++ b/src/libs/regex/atomic.js @@ -0,0 +1,224 @@ +import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js'; +import {Context, replaceUnescaped} from 'regex-utilities'; +/** +@import {PluginData, PluginResult} from './regex.js'; +*/ + +const atomicPluginToken = new RegExp(String.raw`(?${noncapturingDelim})|(?\((?:\?<[^>]+>)?)|\\?.`, 'gsu'); + +/** +Apply transformations for atomic groups: `(?>…)`. +@param {string} expression +@param {PluginData} [data] +@returns {Required} +*/ +function atomic(expression, data) { + const hiddenCaptures = data?.hiddenCaptures ?? []; + // Capture transfer is used by + let captureTransfers = data?.captureTransfers ?? new Map(); + if (!/\(\?>/.test(expression)) { + return { + pattern: expression, + captureTransfers, + hiddenCaptures, + }; + } + + const aGDelim = '(?>'; + const emulatedAGDelim = '(?:(?=('; + const captureNumMap = [0]; + const addedHiddenCaptures = []; + let numCapturesBeforeAG = 0; + let numAGs = 0; + let aGPos = NaN; + let hasProcessedAG; + do { + hasProcessedAG = false; + let numCharClassesOpen = 0; + let numGroupsOpenInAG = 0; + let inAG = false; + let match; + atomicPluginToken.lastIndex = Number.isNaN(aGPos) ? 0 : aGPos + emulatedAGDelim.length; + while (match = atomicPluginToken.exec(expression)) { + const {0: m, index, groups: {capturingStart, noncapturingStart}} = match; + if (m === '[') { + numCharClassesOpen++; + } else if (!numCharClassesOpen) { + + if (m === aGDelim && !inAG) { + aGPos = index; + inAG = true; + } else if (inAG && noncapturingStart) { + numGroupsOpenInAG++; + } else if (capturingStart) { + if (inAG) { + numGroupsOpenInAG++; + } else { + numCapturesBeforeAG++; + captureNumMap.push(numCapturesBeforeAG + numAGs); + } + } else if (m === ')' && inAG) { + if (!numGroupsOpenInAG) { + numAGs++; + const addedCaptureNum = numCapturesBeforeAG + numAGs; + // Replace `expression` and use `<$$N>` as a temporary wrapper for the backref so it + // can avoid backref renumbering afterward. Wrap the whole substitution (including the + // lookahead and following backref) in a noncapturing group to handle following + // quantifiers and literal digits + expression = `${expression.slice(0, aGPos)}${emulatedAGDelim}${ + expression.slice(aGPos + aGDelim.length, index) + }))<$$${addedCaptureNum}>)${expression.slice(index + 1)}`; + hasProcessedAG = true; + addedHiddenCaptures.push(addedCaptureNum); + incrementIfAtLeast(hiddenCaptures, addedCaptureNum); + if (captureTransfers.size) { + const newCaptureTransfers = new Map(); + captureTransfers.forEach((from, to) => { + newCaptureTransfers.set( + to >= addedCaptureNum ? to + 1 : to, + from.map(f => f >= addedCaptureNum ? f + 1 : f) + ); + }); + captureTransfers = newCaptureTransfers; + } + break; + } + numGroupsOpenInAG--; + } + + } else if (m === ']') { + numCharClassesOpen--; + } + } + // Start over from the beginning of the atomic group's contents, in case the processed group + // contains additional atomic groups + } while (hasProcessedAG); + + hiddenCaptures.push(...addedHiddenCaptures); + + // Second pass to adjust numbered backrefs + expression = replaceUnescaped( + expression, + String.raw`\\(?[1-9]\d*)|<\$\$(?\d+)>`, + ({0: m, groups: {backrefNum, wrappedBackrefNum}}) => { + if (backrefNum) { + const bNum = +backrefNum; + if (bNum > captureNumMap.length - 1) { + throw new Error(`Backref "${m}" greater than number of captures`); + } + return `\\${captureNumMap[bNum]}`; + } + return `\\${wrappedBackrefNum}`; + }, + Context.DEFAULT + ); + + return { + pattern: expression, + captureTransfers, + hiddenCaptures, + }; +} + +const baseQuantifier = String.raw`(?:[?*+]|\{\d+(?:,\d*)?\})`; +// Complete tokenizer for base syntax; doesn't (need to) know about character-class-only syntax +const possessivePluginToken = new RegExp(String.raw` +\\(?: \d+ + | c[A-Za-z] + | [gk]<[^>]+> + | [pPu]\{[^\}]+\} + | u[A-Fa-f\d]{4} + | x[A-Fa-f\d]{2} + ) +| \((?: \? (?: [:=!>] + | <(?:[=!]|[^>]+>) + | [A-Za-z\-]+: + | \(DEFINE\) + ))? +| (?${baseQuantifier})(?[?+]?)(?[?*+\{]?) +| \\?. +`.replace(/\s+/g, ''), 'gsu'); + +/** +Transform posessive quantifiers into atomic groups. The posessessive quantifiers are: +`?+`, `*+`, `++`, `{N}+`, `{N,}+`, `{N,N}+`. +This follows Java, PCRE, Perl, and Python. +Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`. +@param {string} expression +@returns {PluginResult} +*/ +function possessive(expression) { + if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) { + return { + pattern: expression, + }; + } + + const openGroupIndices = []; + let lastGroupIndex = null; + let lastCharClassIndex = null; + let lastToken = ''; + let numCharClassesOpen = 0; + let match; + possessivePluginToken.lastIndex = 0; + while (match = possessivePluginToken.exec(expression)) { + const {0: m, index, groups: {qBase, qMod, invalidQ}} = match; + if (m === '[') { + if (!numCharClassesOpen) { + lastCharClassIndex = index; + } + numCharClassesOpen++; + } else if (m === ']') { + if (numCharClassesOpen) { + numCharClassesOpen--; + // Unmatched `]` + } else { + lastCharClassIndex = null; + } + } else if (!numCharClassesOpen) { + + if (qMod === '+' && lastToken && !lastToken.startsWith('(')) { + // Invalid following quantifier would become valid via the wrapping group + if (invalidQ) { + throw new Error(`Invalid quantifier "${m}"`); + } + let charsAdded = -1; // -1 for removed trailing `+` + // Possessivizing fixed repetition quantifiers like `{2}` does't change their behavior, so + // avoid doing so (convert them to greedy) + if (/^\{\d+\}$/.test(qBase)) { + expression = spliceStr(expression, index + qBase.length, qMod, ''); + } else { + if (lastToken === ')' || lastToken === ']') { + const nodeIndex = lastToken === ')' ? lastGroupIndex : lastCharClassIndex; + // Unmatched `)` would break out of the wrapping group and mess with handling. + // Unmatched `]` wouldn't be a problem, but it's unnecessary to have dedicated support + // for unescaped `]++` since this won't work with flag u or v anyway + if (nodeIndex === null) { + throw new Error(`Invalid unmatched "${lastToken}"`); + } + expression = `${expression.slice(0, nodeIndex)}(?>${expression.slice(nodeIndex, index)}${qBase})${expression.slice(index + m.length)}`; + } else { + expression = `${expression.slice(0, index - lastToken.length)}(?>${lastToken}${qBase})${expression.slice(index + m.length)}`; + } + charsAdded += 4; // `(?>)` + } + possessivePluginToken.lastIndex += charsAdded; + } else if (m[0] === '(') { + openGroupIndices.push(index); + } else if (m === ')') { + lastGroupIndex = openGroupIndices.length ? openGroupIndices.pop() : null; + } + + } + lastToken = m; + } + + return { + pattern: expression, + }; +} + +export { + atomic, + possessive, +}; From a6ea45bd4d6a5ce91cb5e6fd96f02fbcf5b38a3b Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Tue, 27 Oct 2026 21:35:43 +1100 Subject: [PATCH 08/13] Bundle regex again, as the build of #862 will load it as an ES module --- package.json | 4 +- src/engine.ts | 4 +- src/libs/README.md | 8 +- src/libs/regex/atomic.js | 224 --------------------------------------- 4 files changed, 8 insertions(+), 232 deletions(-) delete mode 100644 src/libs/regex/atomic.js diff --git a/package.json b/package.json index fce85605..0736f929 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,8 @@ "build": "yarn bundle-node --outfile=bin/rexreplace.cli.node.js && yarn bundle-bun --outfile=bin/rexreplace.cli.bun.js", "build-minify": "yarn build && yarn minify", "minify": "yarn bundle-node --minify --outfile=bin/rexreplace.cli.node.min.js && yarn bundle-bun --minify --outfile=bin/rexreplace.cli.bun.min.js", - "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --external fast-glob --external yargs --external regex", - "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --external fast-glob --external yargs --external regex", + "bundle-node": "bun build src/cli.node.ts --target=node --format=cjs --external fast-glob --external yargs", + "bundle-bun": "bun build src/cli.bun.ts --target=bun --format=cjs --external fast-glob --external yargs", "prebuild": "rm -fr bin && yarn format", "sync-version": "bun npm/sync-version.js", "test-js": "AGENT=1 bun test ./test/engine ./test/cli ./test/fix", diff --git a/src/engine.ts b/src/engine.ts index 9f6d1e91..1eec2f2f 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -48,9 +48,9 @@ const re = { // Only the version ends up in the bundle import {version} from '../package.json'; -// Regex+ by Steven Levithan. regex/internals is an ES module only, so what is used from it is copied into src/libs. +// Regex+ by Steven Levithan. Its subroutines are copied into src/libs until it exports them. import {rewrite} from 'regex'; -import {atomic, possessive} from './libs/regex/atomic.js'; +import {atomic, possessive} from 'regex/internals'; import {Context, hasUnescaped} from 'regex-utilities'; import {subroutines} from './libs/regex/subroutines.js'; export {version}; diff --git a/src/libs/README.md b/src/libs/README.md index 9329fb15..33bbc619 100644 --- a/src/libs/README.md +++ b/src/libs/README.md @@ -1,12 +1,12 @@ # Copied code -RexReplace uses [Regex+](https://github.com/slevithan/regex) by Steven Levithan, from the npm packages [`regex`](https://www.npmjs.com/package/regex) and [`regex-utilities`](https://www.npmjs.com/package/regex-utilities). `regex` is a dependency, loaded when RexReplace runs. `regex-utilities` is an ES module only, so the build bundles it. +RexReplace uses [Regex+](https://github.com/slevithan/regex) by Steven Levithan, from the npm packages [`regex`](https://www.npmjs.com/package/regex) and [`regex-utilities`](https://www.npmjs.com/package/regex-utilities). Both are dependencies, and the build bundles them, as they are ES modules only. -RexReplace runs the atomic groups, the possessive quantifiers and the subroutines of `regex` 6.1.0 itself, to know which groups they add. The first two are only in `regex/internals`, an ES module only, which the CommonJS build of RexReplace can not load, and the subroutines are not exported at all. So they are copied here, as they are upstream: +One part of `regex` 6.1.0 is not exported: the subroutines. So it is copied here, as it is upstream, until upstream exports it: -- `regex/atomic.js`, `regex/subroutines.js` and `regex/utils-internals.js`, unchanged. +- `regex/subroutines.js` and `regex/utils-internals.js`, unchanged. - `regex/utils.js`, only the three parts of it that `subroutines.js` uses. The license of `regex` (MIT) is in `regex/LICENSE`. Biome leaves this folder alone, so it stays easy to compare with upstream. -When `regex` exports these from a build that `require()` can load, import them from there, and delete this folder and its line in `biome.json`. +When `regex` exports `subroutines`, import it from `regex/internals`, and delete this folder and its line in `biome.json`. diff --git a/src/libs/regex/atomic.js b/src/libs/regex/atomic.js deleted file mode 100644 index 1a24fa0d..00000000 --- a/src/libs/regex/atomic.js +++ /dev/null @@ -1,224 +0,0 @@ -import {incrementIfAtLeast, noncapturingDelim, spliceStr} from './utils-internals.js'; -import {Context, replaceUnescaped} from 'regex-utilities'; -/** -@import {PluginData, PluginResult} from './regex.js'; -*/ - -const atomicPluginToken = new RegExp(String.raw`(?${noncapturingDelim})|(?\((?:\?<[^>]+>)?)|\\?.`, 'gsu'); - -/** -Apply transformations for atomic groups: `(?>…)`. -@param {string} expression -@param {PluginData} [data] -@returns {Required} -*/ -function atomic(expression, data) { - const hiddenCaptures = data?.hiddenCaptures ?? []; - // Capture transfer is used by - let captureTransfers = data?.captureTransfers ?? new Map(); - if (!/\(\?>/.test(expression)) { - return { - pattern: expression, - captureTransfers, - hiddenCaptures, - }; - } - - const aGDelim = '(?>'; - const emulatedAGDelim = '(?:(?=('; - const captureNumMap = [0]; - const addedHiddenCaptures = []; - let numCapturesBeforeAG = 0; - let numAGs = 0; - let aGPos = NaN; - let hasProcessedAG; - do { - hasProcessedAG = false; - let numCharClassesOpen = 0; - let numGroupsOpenInAG = 0; - let inAG = false; - let match; - atomicPluginToken.lastIndex = Number.isNaN(aGPos) ? 0 : aGPos + emulatedAGDelim.length; - while (match = atomicPluginToken.exec(expression)) { - const {0: m, index, groups: {capturingStart, noncapturingStart}} = match; - if (m === '[') { - numCharClassesOpen++; - } else if (!numCharClassesOpen) { - - if (m === aGDelim && !inAG) { - aGPos = index; - inAG = true; - } else if (inAG && noncapturingStart) { - numGroupsOpenInAG++; - } else if (capturingStart) { - if (inAG) { - numGroupsOpenInAG++; - } else { - numCapturesBeforeAG++; - captureNumMap.push(numCapturesBeforeAG + numAGs); - } - } else if (m === ')' && inAG) { - if (!numGroupsOpenInAG) { - numAGs++; - const addedCaptureNum = numCapturesBeforeAG + numAGs; - // Replace `expression` and use `<$$N>` as a temporary wrapper for the backref so it - // can avoid backref renumbering afterward. Wrap the whole substitution (including the - // lookahead and following backref) in a noncapturing group to handle following - // quantifiers and literal digits - expression = `${expression.slice(0, aGPos)}${emulatedAGDelim}${ - expression.slice(aGPos + aGDelim.length, index) - }))<$$${addedCaptureNum}>)${expression.slice(index + 1)}`; - hasProcessedAG = true; - addedHiddenCaptures.push(addedCaptureNum); - incrementIfAtLeast(hiddenCaptures, addedCaptureNum); - if (captureTransfers.size) { - const newCaptureTransfers = new Map(); - captureTransfers.forEach((from, to) => { - newCaptureTransfers.set( - to >= addedCaptureNum ? to + 1 : to, - from.map(f => f >= addedCaptureNum ? f + 1 : f) - ); - }); - captureTransfers = newCaptureTransfers; - } - break; - } - numGroupsOpenInAG--; - } - - } else if (m === ']') { - numCharClassesOpen--; - } - } - // Start over from the beginning of the atomic group's contents, in case the processed group - // contains additional atomic groups - } while (hasProcessedAG); - - hiddenCaptures.push(...addedHiddenCaptures); - - // Second pass to adjust numbered backrefs - expression = replaceUnescaped( - expression, - String.raw`\\(?[1-9]\d*)|<\$\$(?\d+)>`, - ({0: m, groups: {backrefNum, wrappedBackrefNum}}) => { - if (backrefNum) { - const bNum = +backrefNum; - if (bNum > captureNumMap.length - 1) { - throw new Error(`Backref "${m}" greater than number of captures`); - } - return `\\${captureNumMap[bNum]}`; - } - return `\\${wrappedBackrefNum}`; - }, - Context.DEFAULT - ); - - return { - pattern: expression, - captureTransfers, - hiddenCaptures, - }; -} - -const baseQuantifier = String.raw`(?:[?*+]|\{\d+(?:,\d*)?\})`; -// Complete tokenizer for base syntax; doesn't (need to) know about character-class-only syntax -const possessivePluginToken = new RegExp(String.raw` -\\(?: \d+ - | c[A-Za-z] - | [gk]<[^>]+> - | [pPu]\{[^\}]+\} - | u[A-Fa-f\d]{4} - | x[A-Fa-f\d]{2} - ) -| \((?: \? (?: [:=!>] - | <(?:[=!]|[^>]+>) - | [A-Za-z\-]+: - | \(DEFINE\) - ))? -| (?${baseQuantifier})(?[?+]?)(?[?*+\{]?) -| \\?. -`.replace(/\s+/g, ''), 'gsu'); - -/** -Transform posessive quantifiers into atomic groups. The posessessive quantifiers are: -`?+`, `*+`, `++`, `{N}+`, `{N,}+`, `{N,N}+`. -This follows Java, PCRE, Perl, and Python. -Possessive quantifiers in Oniguruma and Onigmo are only: `?+`, `*+`, `++`. -@param {string} expression -@returns {PluginResult} -*/ -function possessive(expression) { - if (!(new RegExp(`${baseQuantifier}\\+`).test(expression))) { - return { - pattern: expression, - }; - } - - const openGroupIndices = []; - let lastGroupIndex = null; - let lastCharClassIndex = null; - let lastToken = ''; - let numCharClassesOpen = 0; - let match; - possessivePluginToken.lastIndex = 0; - while (match = possessivePluginToken.exec(expression)) { - const {0: m, index, groups: {qBase, qMod, invalidQ}} = match; - if (m === '[') { - if (!numCharClassesOpen) { - lastCharClassIndex = index; - } - numCharClassesOpen++; - } else if (m === ']') { - if (numCharClassesOpen) { - numCharClassesOpen--; - // Unmatched `]` - } else { - lastCharClassIndex = null; - } - } else if (!numCharClassesOpen) { - - if (qMod === '+' && lastToken && !lastToken.startsWith('(')) { - // Invalid following quantifier would become valid via the wrapping group - if (invalidQ) { - throw new Error(`Invalid quantifier "${m}"`); - } - let charsAdded = -1; // -1 for removed trailing `+` - // Possessivizing fixed repetition quantifiers like `{2}` does't change their behavior, so - // avoid doing so (convert them to greedy) - if (/^\{\d+\}$/.test(qBase)) { - expression = spliceStr(expression, index + qBase.length, qMod, ''); - } else { - if (lastToken === ')' || lastToken === ']') { - const nodeIndex = lastToken === ')' ? lastGroupIndex : lastCharClassIndex; - // Unmatched `)` would break out of the wrapping group and mess with handling. - // Unmatched `]` wouldn't be a problem, but it's unnecessary to have dedicated support - // for unescaped `]++` since this won't work with flag u or v anyway - if (nodeIndex === null) { - throw new Error(`Invalid unmatched "${lastToken}"`); - } - expression = `${expression.slice(0, nodeIndex)}(?>${expression.slice(nodeIndex, index)}${qBase})${expression.slice(index + m.length)}`; - } else { - expression = `${expression.slice(0, index - lastToken.length)}(?>${lastToken}${qBase})${expression.slice(index + m.length)}`; - } - charsAdded += 4; // `(?>)` - } - possessivePluginToken.lastIndex += charsAdded; - } else if (m[0] === '(') { - openGroupIndices.push(index); - } else if (m === ')') { - lastGroupIndex = openGroupIndices.length ? openGroupIndices.pop() : null; - } - - } - lastToken = m; - } - - return { - pattern: expression, - }; -} - -export { - atomic, - possessive, -}; From a7ada55665385641f31b3eaa8bf0ca8feea6993b Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Tue, 27 Oct 2026 23:20:01 +1100 Subject: [PATCH 09/13] Take the last --flags, and tell in the README about the groups a subroutine copies, the lookbehind and -u with regex+ --- README.md | 4 ++-- src/cli.ts | 4 +++- src/engine.ts | 1 - test/fix/854.test.js | 9 +++++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4d6d2e2d..84597a40 100644 --- a/README.md +++ b/README.md @@ -323,7 +323,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-s` | **`--dot-all`** Have `.` also match newline. [boolean] | | `-M` | **`--void-multiline`** Void multiline search pattern. Makes ^ and $ match start/end of whole content rather than each line. [boolean] | | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | -| | **`--flags`** The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. [string] | +| | **`--flags`** The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. Not with -G, -I, -M, -s or -u. [string] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | | `-E` | **`--engine`** The regex engine. `rr` (also `default`) reads patterns like the native engine and adds the extended syntax of Regex+: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g`. `regex+` is Regex+ with the flags v, x and n. `runtime` is the native engine of the runtime: V8 in Node and JavaScriptCore in Bun. [choices: "rr", "default", "regex+", "runtime"] [default: "rr"] | | `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | @@ -390,7 +390,7 @@ Files identified for potential processing are identified by: - Patterns are described as javascript notation regex - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern -- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, so they do not work inside a lookbehind. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. +- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, so inside a lookbehind they match wrongly, with no error. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. A subroutine copies the groups inside the group it calls, and an unnamed one there gets a number of its own: in `(?

(a)(b))\g

`, `$4` and `$5` are the copies of `$2` and `$3`, and `--output-match` prints them too. Name the groups inside, or leave them out, to avoid it. `-E regex+` has `v`, so `-u` is not used with it, and `--flags` with `u` gives `u`. - Supports backreference to matching groups in the replacement - Data can be piped in, both as the text to search and as the replacement - The names of the changed files are printed to stdout, so they can be piped on (to stderr with `--output` and `--output-match`) diff --git a/src/cli.ts b/src/cli.ts index a76924af..d9beb2f4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -120,9 +120,11 @@ const yargs = require('yargs') .string('flags') .conflicts('flags', ['I', 'G', 'M', 's', 'u']) + // Like the other options, the last one wins when it is given twice + .coerce('flags', value => [].concat(value).pop()) .describe( 'flags', - 'The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`.' + 'The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. Not with -G, -I, -M, -s or -u.' ) .default('e', 'utf8') diff --git a/src/engine.ts b/src/engine.ts index 1eec2f2f..ebcfba1b 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -722,7 +722,6 @@ function codeReplacement(file, config, text, numbers) { }; } -// -E: default is another name for rr, and V for regex+ const engineName = name => ('default' === name ? 'rr' : (name ?? 'rr')); // -u is needless with regex+, as it has v diff --git a/test/fix/854.test.js b/test/fix/854.test.js index da8eb1b8..d1d5d121 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -100,6 +100,10 @@ describe('#854 the groups Regex+ adds', () => { expect(await out('(?>a+)(b)', 'x', 'aab', {outputMatch: true})).toBe('b\n'); }); + test('are not hidden for the unnamed groups a subroutine copies, as the README says', async () => { + expect(await out('(?

(a)(b))\\g

', '[$4$5]', 'abab')).toBe('[ab]'); + }); + test('are told in the status report', async () => { expect(await compiled('(?>a+)(b)')).toMatchObject({engine: 'rr', hiddenCaptures: [1]}); }); @@ -169,6 +173,11 @@ describe('#854 --flags', () => { expect((await compiled('a', {...plus, flags: 'gx'})).flags).toContain('v'); }); + test('takes the last one when it is given twice in the CLI', async () => { + fs.writeFileSync('a.txt', 'Aa'); + expect((await sh($`bun ${cli} a _ a.txt --flags g --flags i -o`)).out).toBe('_a\n'); + }); + test('can not go with -I, -G, -M, -s or -u in the CLI', async () => { fs.writeFileSync('a.txt', 'a'); const result = await sh($`bun ${cli} a b a.txt --flags g -I`); From 8f58db8d54f753f92dfb2658260784ec4c9d45c5 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Wed, 28 Oct 2026 00:37:13 +1100 Subject: [PATCH 10/13] Let -u give regex+ the flag u, stop on u with v, and explain the lookbehind with an example --- README.md | 2 +- src/engine.ts | 8 ++++++-- test/fix/854.test.js | 7 +++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84597a40..6870fc2a 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ Files identified for potential processing are identified by: - Patterns are described as javascript notation regex - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern -- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, so inside a lookbehind they match wrongly, with no error. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. A subroutine copies the groups inside the group it calls, and an unnamed one there gets a number of its own: in `(?

(a)(b))\g

`, `$4` and `$5` are the copies of `$2` and `$3`, and `--output-match` prints them too. Name the groups inside, or leave them out, to avoid it. `-E regex+` has `v`, so `-u` is not used with it, and `--flags` with `u` gives `u`. +- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, and a lookbehind reads from right to left, so inside a lookbehind they look forward in place of back, and match in the wrong place with no error: `(?<=(?>a))` finds the places in front of an `a`, not after it. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. A subroutine copies the groups inside the group it calls, and an unnamed one there gets a number of its own: in `(?

(a)(b))\g

`, `$4` and `$5` are the copies of `$2` and `$3`, and `--output-match` prints them too. Name the groups inside, or leave them out, to avoid it. With `-E regex+`, `-u` gives `u` in place of `v`. The flags `u` and `v` can not go together. - Supports backreference to matching groups in the replacement - Data can be piped in, both as the text to search and as the replacement - The names of the changed files are printed to stdout, so they can be piped on (to stderr with `--output` and `--output-match`) diff --git a/src/engine.ts b/src/engine.ts index ebcfba1b..7285a410 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -724,7 +724,7 @@ function codeReplacement(file, config, text, numbers) { const engineName = name => ('default' === name ? 'rr' : (name ?? 'rr')); -// -u is needless with regex+, as it has v +// regex+ has v, or u with -u function regexFlags(config, regexPlus) { if (null != config.flags) { // The runtime does not always say which flag it does not know, so it is told here @@ -740,7 +740,7 @@ function regexFlags(config, regexPlus) { (config.voidIgnoreCase ? '' : 'i') + (config.voidMultiline ? '' : 'm') + (config.dotAll ? 's' : '') + - (regexPlus ? 'vxn' : config.unicode ? 'u' : '') + (regexPlus ? (config.unicode ? 'uxn' : 'vxn') : config.unicode ? 'u' : '') ); } @@ -754,6 +754,10 @@ function compileRegex(pattern, config) { if (!regexPlus && re.regexPlusOnlyFlags.test(flags)) { panic('The flags x and n need -E regex+', flags); } + // The runtime does not say why it does not take them + if (flags.includes('u') && flags.includes('v')) { + panic('The flags u and v can not go together', flags); + } // Regex+ takes v, u, x and n as options, and gives an error when they come as flags const nativeFlags = regexPlus ? flags.replace(re.regexPlusFlags, '') : flags; const unicode = !flags.includes('u') && supportsFlagV ? 'v' : 'u'; diff --git a/test/fix/854.test.js b/test/fix/854.test.js index d1d5d121..fae82bba 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -186,6 +186,12 @@ describe('#854 --flags', () => { expect(fs.readFileSync('a.txt', 'utf8')).toBe('a'); }); + test('gives regex+ the flag u in place of v with -u too', async () => { + const {flags} = await compiled('a', {engine: 'regex+', unicode: true}); + expect(flags).toContain('u'); + expect(flags).not.toContain('v'); + }); + test('gives regex+ the flag u in place of v', async () => { const {flags} = await compiled('a', {...plus, flags: 'gu'}); expect(flags).toContain('u'); @@ -196,6 +202,7 @@ describe('#854 --flags', () => { ['x without regex+', {flags: 'gx'}, ' ❌ The flags x and n need -E regex+\n gx'], ['n without regex+', {flags: 'gn'}, ' ❌ The flags x and n need -E regex+\n gn'], ['a flag that is not a flag', {flags: 'gqz'}, ' ❌ Unknown flags in --flags\n qz'], + ['u and v', {flags: 'guv'}, ' ❌ The flags u and v can not go together\n guv'], ])('stops the run with %s, and says so', async (name, config, message) => { expect(await rr('a', 'b', {pipedData: 'a', ...config})).toEqual({ out: '', From f7a87cd2ee73f1c9ca3c08a149421981251a4c99 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Wed, 28 Oct 2026 01:08:42 +1100 Subject: [PATCH 11/13] Start each file, name and path from the beginning with the flag y --- src/engine.ts | 8 +++++++- test/fix/854.test.js | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/engine.ts b/src/engine.ts index 7285a410..6a5c8ce5 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -331,6 +331,8 @@ function openFile(file, config, number, total) { function doReplacement(file: string, config: any, data: string, number: number, total: number) { verbose('Work on content from: ' + file); + // A regex with the flag y and not g starts where the last file ended, so each file starts from the beginning + config.regex.lastIndex = 0; // Main regexp of the whole thing. With --output-match the replacement prints the matches, so the code of --replacement-js is not run. const result = data.replace( config.regex, @@ -456,6 +458,7 @@ function planRenames(files, config) { config.replacementJs && !config.outputMatch ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) : config.replacement; + config.regex.lastIndex = 0; const result = text.replace(config.regex, replacement); countFile(config, text, result); if (result === text) { @@ -1187,7 +1190,10 @@ function getFilePaths(conf) { excludeRe .map(pattern => compileRegex(regexPattern(pattern, conf), conf).regex) .forEach(exclude => { - files = files.filter(file => !file.match(exclude)); + files = files.filter(file => { + exclude.lastIndex = 0; + return !file.match(exclude); + }); }); files = sortFiles(files, sort); diff --git a/test/fix/854.test.js b/test/fix/854.test.js index fae82bba..3b4d2064 100644 --- a/test/fix/854.test.js +++ b/test/fix/854.test.js @@ -165,6 +165,31 @@ describe('#854 --flags', () => { expect(await out('o', 'x', 'oOo', {flags: 'm'})).toBe('xOo'); }); + test('with y, starts each file from the beginning', async () => { + const names = ['a1.txt', 'a2.txt', 'a3.txt']; + names.forEach(name => fs.writeFileSync(name, 'axx')); + await rr('a', '_', {includeGlob: names, flags: 'y'}); + expect(names.map(name => fs.readFileSync(name, 'utf8'))).toEqual(['_xx', '_xx', '_xx']); + }); + + test('with y, checks each path of --exclude-re from the beginning', async () => { + const names = ['a1.txt', 'a2.txt', 'b.txt']; + names.forEach(name => fs.writeFileSync(name, 'o')); + const result = await rr('o', 'x', { + includeGlob: names, + excludeRe: ['a'], + flags: 'y', + simulate: true, + }); + expect(result.out).toBe('b.txt\n'); + }); + + test('with y, starts each name of --rename from the beginning', async () => { + ['a1.txt', 'a2.txt'].forEach(name => fs.writeFileSync(name, 'o')); + await rr('a', 'b', {includeGlob: ['a1.txt', 'a2.txt'], flags: 'y', rename: true}); + expect(fs.readdirSync('.').sort()).toEqual(['b1.txt', 'b2.txt']); + }); + test('takes a flag that is given twice once', async () => { expect(await out('o', 'x', 'oO', {flags: 'gg'})).toBe('xO'); }); From 4d13d1f34406205c880ff0b39228fc5347503961 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Wed, 28 Oct 2026 01:40:39 +1100 Subject: [PATCH 12/13] Let a group named like a cousin win in any order, and tell the limits of [[] and of 99 groups --- README.md | 4 ++-- src/cli.ts | 2 +- src/engine.ts | 8 ++++++-- test/fix/825.test.js | 4 ++++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6870fc2a..be37368c 100644 --- a/README.md +++ b/README.md @@ -367,7 +367,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so there is no `$1`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so the code has no `$1`, while a replacement text still reads `$1` as the first named group, as in javascript. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. It uses the same engine and flags as the pattern. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | @@ -390,7 +390,7 @@ Files identified for potential processing are identified by: - Patterns are described as javascript notation regex - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern -- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, and a lookbehind reads from right to left, so inside a lookbehind they look forward in place of back, and match in the wrong place with no error: `(?<=(?>a))` finds the places in front of an `a`, not after it. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. A subroutine copies the groups inside the group it calls, and an unnamed one there gets a number of its own: in `(?

(a)(b))\g

`, `$4` and `$5` are the copies of `$2` and `$3`, and `--output-match` prints them too. Name the groups inside, or leave them out, to avoid it. With `-E regex+`, `-u` gives `u` in place of `v`. The flags `u` and `v` can not go together. +- Supports the extended syntax of [Regex+](https://github.com/slevithan/regex) in the pattern: atomic groups `(?>…)`, possessive quantifiers like `a++`, and subroutines `\g` with `(?(DEFINE)…)`. A pattern that works in the native engine of the runtime works the same, with one difference: `\g` calls the group, where the native engine reads it as the text `g` (use `-E runtime` for that). Regex+ emulates atomic groups and possessive quantifiers with a lookahead and a backreference, and a lookbehind reads from right to left, so inside a lookbehind they look forward in place of back, and match in the wrong place with no error: `(?<=(?>a))` finds the places in front of an `a`, not after it. A subroutine needs a named group that exists, as Regex+ has no numbered ones like `\g<1>`. A subroutine copies the groups inside the group it calls, and an unnamed one there gets a number of its own: in `(?

(a)(b))\g

`, `$4` and `$5` are the copies of `$2` and `$3`, and `--output-match` prints them too. Name the groups inside, or leave them out, to avoid it. Without `-u`, the native engine reads `[[]` as a class with a `[` in it, but Regex+ sees the class as still open, so the extended syntax after it is not read: `[[](?>a)` stops the run, and `[\[](?>a)` works. Counting the groups Regex+ adds, a replacement text can only reach the first 99 groups, as in javascript. With `-E regex+`, `-u` gives `u` in place of `v`. The flags `u` and `v` can not go together. - Supports backreference to matching groups in the replacement - Data can be piped in, both as the text to search and as the replacement - The names of the changed files are printed to stdout, so they can be piped on (to stderr with `--output` and `--output-match`) diff --git a/src/cli.ts b/src/cli.ts index d9beb2f4..2e83a2ec 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -423,7 +423,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). For speed, the code runs once per file unless it names the match, a group, \`$m\` or \`$M\`. -Access a named group as \`$\` and the name: \`(?...)\` gives you \`$year\`. A group named like a number, like \`n\`, wins over the number. With the flag \`n\` of \`-E regex+\` only named groups capture, so there is no \`$1\`. +Access a named group as \`$\` and the name: \`(?...)\` gives you \`$year\`. A group named like a number, like \`n\`, wins over the number. With the flag \`n\` of \`-E regex+\` only named groups capture, so the code has no \`$1\`, while a replacement text still reads \`$1\` as the first named group, as in javascript. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 6a5c8ce5..96ae73be 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1150,9 +1150,13 @@ function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; } }); - // After the numbers, so a group named like a number, like n, wins over $n + // After the numbers, so a group named like a number, like n, wins over $n. + // The cousins come first, so a group named like a cousin, like a_, wins over the cousin of a. for (const [name, value] of Object.entries(named)) { - variables += `var $${name} = ${JSON.stringify(value)}, $${name}_ = $${name} + ' '; `; + variables += `var $${name}_ = ${JSON.stringify(value + ' ')}; `; + } + for (const [name, value] of Object.entries(named)) { + variables += `var $${name} = ${JSON.stringify(value)}; `; } return runCode({m: match, M: config.numbers.matches}, variables); }; diff --git a/test/fix/825.test.js b/test/fix/825.test.js index cd668437..95ea0782 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -44,6 +44,10 @@ describe('#825 a named group with the name of another variable', () => { ); }); + test('can end with _, and then wins over the cousin with a space of the group without it, in either order', async () => { + expect((await rr('(?\\w)(?\\d)', '$a + $a_', {...js, pipedData: 'x1'})).out).toBe('1x'); + }); + test('can end with _, and then wins over the cousin with a space of the group without it', async () => { expect((await rr('(?\\d)(?\\w)', '$a + $a_', {...js, pipedData: '1x'})).out).toBe('1x'); }); From b315499f2a2c532025c8f151b386f46c3c3c813e Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Wed, 28 Oct 2026 01:43:10 +1100 Subject: [PATCH 13/13] Leave the order of the cousins and the wording of $1 to the end of the train, as the PRs above change the same lines --- README.md | 2 +- src/cli.ts | 2 +- src/engine.ts | 8 ++------ test/fix/825.test.js | 4 ---- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index be37368c..48474cf5 100644 --- a/README.md +++ b/README.md @@ -367,7 +367,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so the code has no `$1`, while a replacement text still reads `$1` as the first named group, as in javascript. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, the code runs once per file unless it names the match, a group, `$m` or `$M`. Access a named group as `$` and the name: `(?...)` gives you `$year`. A group named like a number, like `n`, wins over the number. With the flag `n` of `-E regex+` only named groups capture, so there is no `$1`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. It uses the same engine and flags as the pattern. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 2e83a2ec..d9beb2f4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -423,7 +423,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). For speed, the code runs once per file unless it names the match, a group, \`$m\` or \`$M\`. -Access a named group as \`$\` and the name: \`(?...)\` gives you \`$year\`. A group named like a number, like \`n\`, wins over the number. With the flag \`n\` of \`-E regex+\` only named groups capture, so the code has no \`$1\`, while a replacement text still reads \`$1\` as the first named group, as in javascript. +Access a named group as \`$\` and the name: \`(?...)\` gives you \`$year\`. A group named like a number, like \`n\`, wins over the number. With the flag \`n\` of \`-E regex+\` only named groups capture, so there is no \`$1\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 96ae73be..6a5c8ce5 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1150,13 +1150,9 @@ function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; } }); - // After the numbers, so a group named like a number, like n, wins over $n. - // The cousins come first, so a group named like a cousin, like a_, wins over the cousin of a. + // After the numbers, so a group named like a number, like n, wins over $n for (const [name, value] of Object.entries(named)) { - variables += `var $${name}_ = ${JSON.stringify(value + ' ')}; `; - } - for (const [name, value] of Object.entries(named)) { - variables += `var $${name} = ${JSON.stringify(value)}; `; + variables += `var $${name} = ${JSON.stringify(value)}, $${name}_ = $${name} + ' '; `; } return runCode({m: match, M: config.numbers.matches}, variables); }; diff --git a/test/fix/825.test.js b/test/fix/825.test.js index 95ea0782..cd668437 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -44,10 +44,6 @@ describe('#825 a named group with the name of another variable', () => { ); }); - test('can end with _, and then wins over the cousin with a space of the group without it, in either order', async () => { - expect((await rr('(?\\w)(?\\d)', '$a + $a_', {...js, pipedData: 'x1'})).out).toBe('1x'); - }); - test('can end with _, and then wins over the cousin with a space of the group without it', async () => { expect((await rr('(?\\d)(?\\w)', '$a + $a_', {...js, pipedData: '1x'})).out).toBe('1x'); });