Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions README.md

Large diffs are not rendered by default.

16 changes: 3 additions & 13 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,26 +118,16 @@ const yargs = createRequire(import.meta.url)('yargs')
.alias('M', 'void-multiline')

.boolean('u')
.describe(
'u',
'Treat patterns as a sequence of unicode code points. For the default engine it is on by default. For the regex+ engine (`-E regex+`) it swaps from v mode to u mode.'
)
.describe('u', 'Treat pattern as a sequence of unicode code points.')
.alias('u', 'unicode')

.boolean('U')
.describe(
'U',
'Void unicode. For the default engine it means reading the pattern as UTF-16 parts (escapes of chars that need no escaping are allowed). For the regex+ engine it has no impact, as it defaults to v mode.'
)
.alias('U', 'void-unicode')

.string('flags')
.conflicts('flags', ['I', 'G', 'M', 's', 'u', 'U'])
.conflicts('flags', ['I', 'G', 'M', 's', 'u'])
// Like the other options, the last one wins when it is given twice
.coerce('flags', value => [].concat(value).pop())
.describe(
'flags',
'The flags of the regex, like `gimsv`, in place of the defaults: `gimu`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. Not with -G, -I, -M, -s, -u or -U.'
'The flags of the regex, like `gimsv`, in place of the defaults: `gim`, or `gimvxn` with `-E regex+`. `x` and `n` need `-E regex+`. Not with -G, -I, -M, -s or -u.'
)

.default('e', 'utf8')
Expand Down
17 changes: 1 addition & 16 deletions src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,7 @@ function regexFlags(config, regexPlus) {
(config.voidIgnoreCase ? '' : 'i') +
(config.voidMultiline ? '' : 'm') +
(config.dotAll ? 's' : '') +
(regexPlus ? (config.unicode ? 'uxn' : 'vxn') : config.voidUnicode ? '' : 'u')
(regexPlus ? (config.unicode ? 'uxn' : 'vxn') : config.unicode ? 'u' : '')
);
}

Expand Down Expand Up @@ -805,25 +805,10 @@ function compileRegex(pattern, config) {
}
return {regex: new RegExp(pattern, flags), hiddenCaptures: [], namedOnly: false};
} catch (e) {
// The flag u is on by default, and it is strict about the syntax, so a pattern that worked without it can fail on it
if (!regexPlus && null == config.flags && worksWithoutUnicode(pattern, flags, engine)) {
return panic(e.message, 'It works without the flag u, which -U (--void-unicode) turns off.');
}
return panic(e.message);
}
}

function worksWithoutUnicode(pattern, flags, engine) {
const legacy = flags.replace('u', '');
try {
return Boolean(
new RegExp('rr' === engine ? extendedSyntax(pattern, legacy).pattern : pattern, legacy)
);
} catch (e) {
return false;
}
}

// Only the transforms of Regex+ run, in its own order, and not its flags, so a pattern that works natively gives the same regex.
// A transform only runs when its syntax is outside a character class, as Regex+ does not always look, like for [(?>]\1 (#858).
function extendedSyntax(pattern, flags) {
Expand Down
2 changes: 1 addition & 1 deletion test/cli/debug.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ describe('--debug', () => {
engine: 'rr',
hiddenCaptures: [],
search: 'o',
flags: 'gimu',
flags: 'gim',
replacement: 'x',
replacementSrc: null,
});
Expand Down
2 changes: 1 addition & 1 deletion test/cli/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ assert "rexreplace '.€' '€' my.file -o --void-euro" 'foobar'
reset
echo foo[bar] > my.file
assert "rexreplace '[§]]' '[' my.file -o" 'foo[bar['
assert "rexreplace '[§]]' '[' my.file -o --void-section -U" 'foo[bar]'
assert "rexreplace '[§]]' '[' my.file -o --void-section" 'foo[bar]'
reset
rexreplace 'b' '§n' my.file
assert "cat my.file" "foo\nar"
Expand Down
4 changes: 2 additions & 2 deletions test/engine/crossover.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import fs from 'fs';
import {rr} from '../helper.js';

describe('regex flags', () => {
const flags = ['voidIgnoreCase', 'voidGlobal', 'voidMultiline', 'dotAll', 'voidUnicode'];
const flags = ['voidIgnoreCase', 'voidGlobal', 'voidMultiline', 'dotAll', 'unicode'];
const text = 'Foo\nfoo 🦖\r\nFOO bar\n';
const patterns = ['foo', '^.', '.$', 'o.f', '^f.*o$', '\\S+$', 'O\\s', '.'];

Expand All @@ -22,7 +22,7 @@ describe('regex flags', () => {
(config.voidIgnoreCase ? '' : 'i') +
(config.voidMultiline ? '' : 'm') +
(config.dotAll ? 's' : '') +
(config.voidUnicode ? '' : 'u');
(config.unicode ? 'u' : '');

for (const pattern of patterns) {
expect((await rr(pattern, '<$&>', {pipedData: text, ...config})).out, pattern).toBe(
Expand Down
19 changes: 5 additions & 14 deletions test/engine/unicode.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ describe('--unicode', () => {
expect((await rr('^.$', 'x', {pipedData: '🦖', unicode: true})).out).toBe('x');
});

test('is the default, and --void-unicode turns it off', async () => {
expect((await rr('^.$', 'x', {pipedData: '🦖'})).out).toBe('x');
expect((await rr('^.$', 'x', {pipedData: '🦖', voidUnicode: true})).out).toBe('🦖');
test('is not the default', async () => {
expect((await rr('^.$', 'x', {pipedData: '🦖'})).out).toBe('🦖');
});

test('understands code point escapes', async () => {
expect((await rr('\\u{1F996}', 'x', {pipedData: 'a🦖b', unicode: true})).out).toBe('axb');
expect((await rr('\\u{1F996}', 'x', {pipedData: 'a🦖b', voidUnicode: true})).out).toBe('a🦖b');
expect((await rr('\\u{1F996}', 'x', {pipedData: 'a🦖b'})).out).toBe('a🦖b');
});

test('understands unicode property escapes', async () => {
Expand All @@ -37,21 +36,13 @@ describe('--unicode', () => {
});

test('is strict about escapes that have no meaning', async () => {
expect((await rr('\\-', 'x', {pipedData: 'a-b', voidUnicode: true})).out).toBe('axb');
const result = await rr('\\-', 'x', {pipedData: 'a-b'});
expect((await rr('\\-', 'x', {pipedData: 'a-b'})).out).toBe('axb');
const result = await rr('\\-', 'x', {pipedData: 'a-b', unicode: true});
expect(result.exit).toBe(3);
expect(result.err).toContain('Invalid regular expression');
expect(result.out).toBe('');
});

test('tells that a pattern from before works without it', async () => {
const result = await rr('\\-', 'x', {pipedData: 'a-b'});
expect(result.err).toContain(
'\n It works without the flag u, which -U (--void-unicode) turns off.\n'
);
expect((await rr('(', 'x', {pipedData: 'a'})).err).not.toContain('-U');
});

describe('combined', () => {
test('with files', async () => {
fs.writeFileSync('my.file', 'a🦖b');
Expand Down
8 changes: 4 additions & 4 deletions test/engine/utf8.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ describe('#735 UTF-8 in the pattern', () => {
expect(asUtf8((await rr('🦖', 'x', {pipedData: text})).out)).toBe('æ 中 x 👩‍💻\r\n');
});

test('. takes half an emoji with --void-unicode, which then shows as �', async () => {
const out = (await rr('.', 'x', {pipedData: '🦖a', voidGlobal: true, voidUnicode: true})).out;
test('. takes half an emoji without --unicode, which then shows as �', async () => {
const out = (await rr('.', 'x', {pipedData: '🦖a', voidGlobal: true})).out;
expect(asUtf8(out)).toBe('x�a');
});

test('. takes the whole emoji, as --unicode is the default', async () => {
const config = {pipedData: '🦖a', voidGlobal: true};
test('. takes the whole emoji with --unicode', async () => {
const config = {pipedData: '🦖a', voidGlobal: true, unicode: true};
expect(asUtf8((await rr('.', 'x', config)).out)).toBe('xa');
});

Expand Down
6 changes: 3 additions & 3 deletions test/engine/void-section.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ describe('--void-section', () => {

test('makes § a plain char in the pattern', async () => {
expect((await rr('§d', '#', {pipedData: 'a1 §d', voidSection: true})).out).toBe('a1 #');
expect(
(await rr('[§]]', '[', {pipedData: 'foo[bar]', voidSection: true, voidUnicode: true})).out
).toBe('foo[bar]');
expect((await rr('[§]]', '[', {pipedData: 'foo[bar]', voidSection: true})).out).toBe(
'foo[bar]'
);
});

test('makes § a plain char in the replacement', async () => {
Expand Down
7 changes: 3 additions & 4 deletions test/fix/683.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,9 @@ describe('#683 --exclude-re follows the regex flags of the main search', () => {
test('--unicode lets \\u{...} match in the path', async () => {
fs.writeFileSync('a.file', 'foo');
fs.writeFileSync('b.file', 'foo');
expect(
(await rr('foo', 'x', {includeGlob: ['*.file'], excludeRe: ['\\u{61}'], voidUnicode: true}))
.out
).toBe('a.file\nb.file\n');
expect((await rr('foo', 'x', {includeGlob: ['*.file'], excludeRe: ['\\u{61}']})).out).toBe(
'a.file\nb.file\n'
);
fs.writeFileSync('a.file', 'foo');
fs.writeFileSync('b.file', 'foo');
expect(
Expand Down
2 changes: 1 addition & 1 deletion test/fix/742.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ describe('#742 the engine part of the status', () => {
engine: 'rr',
hiddenCaptures: [],
search: 'o+',
flags: 'giu',
flags: 'gi',
replacement,
replacementSrc: src,
});
Expand Down
14 changes: 5 additions & 9 deletions test/fix/854.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,13 @@ describe('#854 the rr engine', () => {
['(?<a>x)\\1'],
['$'],
['[[]'],
])('gives the same regex as the native engine for %s, without the flag u', async pattern => {
])('gives the same regex as the native engine for %s', async pattern => {
const native = new RegExp(pattern, 'gim');
expect(await compiled(pattern, {voidUnicode: true})).toMatchObject({
search: native.source,
flags: 'gim',
});
expect(await compiled(pattern)).toMatchObject({search: native.source, flags: 'gim'});
});

test('has the flag u by default, and not with -U', async () => {
expect((await compiled('a')).flags).toBe('gimu');
expect((await compiled('a', {voidUnicode: true})).flags).toBe('gim');
test('only uses the u flag with -u', async () => {
expect((await compiled('a', {unicode: true})).flags).toBe('gimu');
});

test.each([
Expand Down Expand Up @@ -116,7 +112,7 @@ describe('#854 the groups Regex+ adds', () => {
describe('#854 the other engines', () => {
// The tests are run by Bun, so the native engine is JSC
test.each([['runtime']])('%s is the native engine, without the extended syntax', async engine => {
expect(await out('(?<d>a)\\g<d>', 'X', 'ag<d>', {engine, voidUnicode: true})).toBe('X');
expect(await out('(?<d>a)\\g<d>', 'X', 'ag<d>', {engine})).toBe('X');
});

test.each([['regex+']])(
Expand Down
10 changes: 9 additions & 1 deletion test/fuzzy/basics.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,27 @@ import {describe, test, expect} from 'bun:test';
import fs from 'fs';
import {rr, random, ended} from '../helper.js';

// rr reads a quantifier followed by + as possessive, which String.replace has no way to give
const re = {possessive: /[*+?]\+/};

describe('basics', () => {
test('gives the same as String.replace with the flags gim', async () => {
const rnd = random(1);
for (let i = 0; i < 500; i++) {
const pattern = rnd.text('abc.*+?|^$', 1, 4);
const replacement = rnd.text('xyz &1', 0, 3);
const text = rnd.text('abcABC \n', 0, 40);
if (re.possessive.test(pattern)) {
continue;
}

let regex;
try {
regex = new RegExp(pattern, 'gim');
} catch (e) {
expect((await rr(pattern, replacement, {pipedData: text})).exit, pattern).toBe(1);
expect((await rr(pattern, replacement, {pipedData: text})).exit, pattern).toBeGreaterThan(
0
);
continue;
}

Expand Down
24 changes: 24 additions & 0 deletions test/performance/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Performance tests

These tests measure how fast RexReplace is. They are not part of `yarn test`, because the times depend on the machine and on what else it is doing. Run them by hand, and compare numbers from the same machine only.

Each test has its own folder with a `run.sh`. It builds rr from the working tree into a temp folder, runs [hyperfine](https://github.com/sharkdp/hyperfine), and ends with two tables. The first gives each time relative to the fastest in its row, and the second the seconds for one run. Both use the median, as one slow run from other work on the machine moves the mean. The runs start from the repo root, a usual place to run rr.

`shared` has what the tests have in common, the build of rr (`build.sh`), the check that each tool gives the same file (`check.sh`) and the tables (`table.js`). The text to change is the book 1984 by George Orwell, in `tools/1984.html`.

Before the timing of each size, every command runs once, and the test stops if one fails or gives another file than the others. A tool that does other work, or no work, would else be timed for it. The file to change is in the ignored `tmp` folder of the repo and is given by a path from the repo, as rr is most often run on files under the working folder.

## engines

Compares the regex engines of rr, and what the flags u and v cost. rr from the working tree runs on Node and on Bun, on files from 1 KB to 100 MB.

```bash
bash test/performance/engines/run.sh
bash test/performance/engines/run.sh 1KB 10MB
```

The engines are `rr`, the default, `runtime`, the native engine of the runtime, and `regex+`, which uses the flag v. `rr` and `runtime` each run with the default flags, with the flag u (`-u`), and with the flag v. Each row is one runtime, so the 1.00x is the fastest engine on that runtime.

Each size runs with two patterns. One removes the HTML tags, so it matches often. The other looks for a tag that ends in `~>`, which is not in the book, so it never matches and the file stays the same. Both try at each `<` and read the tag, so they do the same work, apart from the replacing.

`results-nov-2026.md` was measured while the flag u was on by default, so its `rr` is with u and its `rr -U` is without.
50 changes: 50 additions & 0 deletions test/performance/engines/results-nov-2026.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
Time relative to the fastest in the row

| | rr | rr -U | rr v | runtime | runtime -U | runtime v | regex+ |
|---|---:|---:|---:|---:|---:|---:|---:|
| 1KB matches on Node | 1.03x | 1.03x | 1.02x | 1.00x | 1.00x | 1.01x | 1.06x |
| 1KB matches on Bun | 1.03x | 1.02x | 1.00x | 1.00x | 1.02x | 1.00x | 1.08x |
| 1KB no-matches on Node | 1.00x | 1.01x | 1.01x | 1.02x | 1.00x | 1.01x | 1.02x |
| 1KB no-matches on Bun | 1.02x | 1.01x | 1.01x | 1.00x | 1.00x | 1.01x | 1.03x |
| 100KB matches on Node | 1.00x | 1.02x | 1.01x | 1.01x | 1.02x | 1.00x | 1.02x |
| 100KB matches on Bun | 1.02x | 1.00x | 1.01x | 1.01x | 1.00x | 1.06x | 1.06x |
| 100KB no-matches on Node | 1.00x | 1.00x | 1.01x | 1.02x | 1.03x | 1.01x | 1.03x |
| 100KB no-matches on Bun | 1.01x | 1.05x | 1.04x | 1.01x | 1.00x | 1.01x | 1.03x |
| 1MB matches on Node | 1.02x | 1.00x | 1.01x | 1.01x | 1.03x | 1.00x | 1.04x |
| 1MB matches on Bun | 1.05x | 1.03x | 1.06x | 1.00x | 1.00x | 1.02x | 1.04x |
| 1MB no-matches on Node | 1.01x | 1.00x | 1.02x | 1.02x | 1.04x | 1.02x | 1.07x |
| 1MB no-matches on Bun | 1.02x | 1.00x | 1.03x | 1.02x | 1.00x | 1.02x | 1.05x |
| 10MB matches on Node | 1.06x | 1.00x | 1.05x | 1.04x | 1.01x | 1.02x | 1.07x |
| 10MB matches on Bun | 1.05x | 1.02x | 1.04x | 1.04x | 1.00x | 1.02x | 1.04x |
| 10MB no-matches on Node | 1.12x | 1.00x | 1.13x | 1.10x | 1.01x | 1.12x | 1.12x |
| 10MB no-matches on Bun | 1.12x | 1.02x | 1.14x | 1.22x | 1.00x | 1.13x | 1.14x |
| 100MB matches on Node | 1.09x | 1.00x | 1.13x | 1.08x | 1.01x | 1.11x | 1.09x |
| 100MB matches on Bun | 1.19x | 1.00x | 1.15x | 1.13x | 1.01x | 1.20x | 1.10x |
| 100MB no-matches on Node | 1.30x | 1.00x | 1.30x | 1.30x | 1.01x | 1.30x | 1.32x |
| 100MB no-matches on Bun | 1.33x | 1.00x | 1.39x | 1.32x | 1.01x | 1.33x | 1.40x |

Seconds for one run

| | rr | rr -U | rr v | runtime | runtime -U | runtime v | regex+ |
|---|---:|---:|---:|---:|---:|---:|---:|
| 1KB matches on Node | 0.073 | 0.073 | 0.073 | 0.071 | 0.072 | 0.072 | 0.075 |
| 1KB matches on Bun | 0.037 | 0.037 | 0.037 | 0.037 | 0.037 | 0.036 | 0.040 |
| 1KB no-matches on Node | 0.070 | 0.071 | 0.071 | 0.072 | 0.070 | 0.071 | 0.072 |
| 1KB no-matches on Bun | 0.037 | 0.037 | 0.036 | 0.036 | 0.036 | 0.037 | 0.037 |
| 100KB matches on Node | 0.073 | 0.074 | 0.074 | 0.074 | 0.074 | 0.073 | 0.074 |
| 100KB matches on Bun | 0.037 | 0.037 | 0.037 | 0.037 | 0.037 | 0.039 | 0.039 |
| 100KB no-matches on Node | 0.071 | 0.071 | 0.072 | 0.073 | 0.073 | 0.072 | 0.073 |
| 100KB no-matches on Bun | 0.037 | 0.038 | 0.038 | 0.037 | 0.036 | 0.037 | 0.038 |
| 1MB matches on Node | 0.079 | 0.078 | 0.079 | 0.079 | 0.081 | 0.078 | 0.081 |
| 1MB matches on Bun | 0.040 | 0.039 | 0.040 | 0.038 | 0.038 | 0.039 | 0.040 |
| 1MB no-matches on Node | 0.074 | 0.073 | 0.075 | 0.075 | 0.076 | 0.075 | 0.078 |
| 1MB no-matches on Bun | 0.038 | 0.037 | 0.038 | 0.038 | 0.037 | 0.038 | 0.039 |
| 10MB matches on Node | 0.133 | 0.125 | 0.131 | 0.130 | 0.127 | 0.128 | 0.135 |
| 10MB matches on Bun | 0.063 | 0.061 | 0.062 | 0.062 | 0.060 | 0.061 | 0.062 |
| 10MB no-matches on Node | 0.115 | 0.102 | 0.115 | 0.113 | 0.103 | 0.114 | 0.115 |
| 10MB no-matches on Bun | 0.059 | 0.054 | 0.060 | 0.064 | 0.052 | 0.059 | 0.060 |
| 100MB matches on Node | 0.649 | 0.598 | 0.676 | 0.644 | 0.607 | 0.663 | 0.650 |
| 100MB matches on Bun | 0.292 | 0.246 | 0.284 | 0.278 | 0.249 | 0.296 | 0.271 |
| 100MB no-matches on Node | 0.506 | 0.389 | 0.505 | 0.505 | 0.392 | 0.506 | 0.514 |
| 100MB no-matches on Bun | 0.256 | 0.192 | 0.267 | 0.254 | 0.194 | 0.256 | 0.268 |

Loading
Loading