From e6307078531263c7d67f051af8a8cda1946e00e9 Mon Sep 17 00:00:00 2001 From: SimY4 Date: Thu, 12 Nov 2026 14:44:10 +1100 Subject: [PATCH] fix(core): align char classes and escapes with java.util.regex. - `.` and negated classes (`[^a]`, `\D`, `\W`, `\S`, `\H`, `\V`) cover all BMP chars except surrogates; negation is no longer affected by DOTALL/UNIX_LINES, so `[^a]` matches line terminators as in Pattern. - `\s`/`\S` include `\x0B`, `\p{Space}` is `[ \t\n\x0B\f\r]`, `\p{all}` is any char, `\p{javaLowerCase}`, `\p{javaUpperCase}` and `\p{javaWhitespace}` use the respective Character predicates. - Support `\h`, `\H`, `\v`, `\V`, `\f`, `\a`, `\e` and `\cX`; previously parsed as literal letters, generating strings the pattern rejects. - Case-insensitive matching folds US-ASCII only unless UNICODE_CASE (or UNICODE_CHARACTER_CLASS) is set, as in Pattern. - Drop Lazy in favour of plain constants. --- .../github/simy4/coregex/core/Coregex.java | 6 +- .../simy4/coregex/core/CoregexParser.java | 64 +++++++-- .../com/github/simy4/coregex/core/Lazy.java | 55 -------- .../com/github/simy4/coregex/core/Set.java | 124 ++++++++++-------- .../coregex/core/CoregexParserSuite.scala | 55 ++++++++ .../github/simy4/coregex/core/SetSuite.scala | 18 ++- 6 files changed, 196 insertions(+), 126 deletions(-) delete mode 100644 core/src/main/java/com/github/simy4/coregex/core/Lazy.java diff --git a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java index 89afe32..561db87 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java @@ -54,8 +54,6 @@ public abstract class Coregex implements Serializable { private static final long serialVersionUID = 1L; - private static final Lazy EMPTY = - new Lazy<>(() -> Set.builder().build().quantify(0, 0, Quantified.Type.GREEDY)); /** * Constructs {@link Coregex} from provided {@link Pattern} instance. @@ -89,7 +87,7 @@ public static Coregex any(int flags) { * @return predefined constructor for empty regex. */ public static Coregex empty() { - return EMPTY.get(); + return Quantified.EMPTY; } /** @@ -601,6 +599,8 @@ public static final class Quantified extends Coregex { private static final long serialVersionUID = 1L; + static final Quantified EMPTY = new Quantified(Set.EMPTY, 0, 0, Type.GREEDY); + private final Coregex quantified; private final int min; private final int max; diff --git a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java index c3a3789..44f1cf1 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java +++ b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java @@ -350,7 +350,7 @@ private void setItem(Set.Builder set, Context ctx) { } else { Set metachar = metachar(ctx); if (metachar.single()) { - rangeOrSingle(set, (char) metachar.sample(0L).orElse(0), ctx); + rangeOrSingle(set, (char) metachar.sample(0L), ctx); } else { set.union(metachar); } @@ -510,7 +510,8 @@ private int flags(Context ctx) { /* *
{@code
-   * metachar ::= '\', 't' | 'r' | 'n' | 'd' | 'D' | 'w' | 'W' | 's' | 'p', '{', ? posix ?, '}' | 'S' | single
+   * metachar ::= '\', 't' | 'r' | 'n' | 'f' | 'a' | 'e' | 'c', ? char ? | 'd' | 'D' | 'w' | 'W'
+   *              | 's' | 'S' | 'h' | 'H' | 'v' | 'V' | 'p', '{', ? posix ?, '}' | single
    * }
*/ @SuppressWarnings("fallthrough") @@ -549,11 +550,50 @@ private Set metachar(Context ctx) { break; case 's': ctx.match('s'); - metachar.set('\r', '\n', '\t', '\f', ' '); + metachar.set(' ', '\t', '\n', '\u000B', '\f', '\r'); break; case 'S': ctx.match('S'); - metachar.set('\r', '\n', '\t', '\f', ' ').negate(); + metachar.set(' ', '\t', '\n', '\u000B', '\f', '\r').negate(); + break; + case 'h': + ctx.match('h'); + metachar + .set(' ', '\t', '\u00A0', '\u1680', '\u180E', '\u202F', '\u205F', '\u3000') + .range('\u2000', '\u200A'); + break; + case 'H': + ctx.match('H'); + metachar + .set(' ', '\t', '\u00A0', '\u1680', '\u180E', '\u202F', '\u205F', '\u3000') + .range('\u2000', '\u200A') + .negate(); + break; + case 'v': + ctx.match('v'); + metachar.set('\n', '\u000B', '\f', '\r', '\u0085', '\u2028', '\u2029'); + break; + case 'V': + ctx.match('V'); + metachar.set('\n', '\u000B', '\f', '\r', '\u0085', '\u2028', '\u2029').negate(); + break; + case 'f': + ctx.match('f'); + metachar.single('\f'); + break; + case 'a': + ctx.match('a'); + metachar.single('\u0007'); + break; + case 'e': + ctx.match('e'); + metachar.single('\u001B'); + break; + case 'c': + ctx.match('c'); + char control = ctx.peek(); + ctx.match(control); + metachar.single((char) (control ^ 0x40)); break; // handling predefined predicates from java.util.regex.CharPredicates case 'p': @@ -562,16 +602,20 @@ private Set metachar(Context ctx) { String posix = ctx.span('}'); switch (posix) { case "all": - metachar.range(Character.MIN_VALUE, Character.MIN_SURROGATE); + metachar.matching(any -> true, "\\p{all}"); break; case "Lower": - case "javaLowerCase": metachar.range('a', 'z'); break; + case "javaLowerCase": + metachar.matching(Character::isLowerCase, "\\p{javaLowerCase}"); + break; case "Upper": - case "javaUpperCase": metachar.range('A', 'Z'); break; + case "javaUpperCase": + metachar.matching(Character::isUpperCase, "\\p{javaUpperCase}"); + break; case "ASCII": metachar.range(Character.MIN_VALUE, (char) 0x7F); break; @@ -602,9 +646,13 @@ private Set metachar(Context ctx) { break; case "Blank": + metachar.set(' ', '\t'); + break; case "Space": + metachar.set(' ', '\t', '\n', '\u000B', '\f', '\r'); + break; case "javaWhitespace": - metachar.set(' ', '\t'); + metachar.matching(Character::isWhitespace, "\\p{javaWhitespace}"); break; default: throw ctx.unsupported("posix char class \\" + posix); diff --git a/core/src/main/java/com/github/simy4/coregex/core/Lazy.java b/core/src/main/java/com/github/simy4/coregex/core/Lazy.java deleted file mode 100644 index 772d791..0000000 --- a/core/src/main/java/com/github/simy4/coregex/core/Lazy.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2021-2026 Alex Simkin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.github.simy4.coregex.core; - -import static java.util.Objects.requireNonNull; - -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.util.function.Supplier; - -@SuppressWarnings("serial") -final class Lazy implements Supplier, Serializable { - private static final long serialVersionUID = 1L; - - private transient volatile Supplier supplier; - private T value; - - public Lazy(Supplier supplier) { - this.supplier = requireNonNull(supplier, "supplier"); - } - - @Override - public T get() { - return (supplier == null) ? value : doGet(); - } - - private synchronized T doGet() { - Supplier supplier = this.supplier; - if (supplier != null) { - value = supplier.get(); - this.supplier = null; - } - return value; - } - - private void writeObject(ObjectOutputStream oos) throws IOException { - get(); // resolve value - oos.defaultWriteObject(); - } -} diff --git a/core/src/main/java/com/github/simy4/coregex/core/Set.java b/core/src/main/java/com/github/simy4/coregex/core/Set.java index f437ccb..f8be9d7 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Set.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Set.java @@ -20,7 +20,7 @@ import java.io.Serializable; import java.util.BitSet; -import java.util.OptionalInt; +import java.util.function.IntPredicate; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -36,41 +36,39 @@ public final class Set extends Coregex implements Serializable { private static final long serialVersionUID = 1L; - static final Lazy ALL = - new Lazy<>( - () -> { - BitSet chars = new BitSet(Character.MIN_SURROGATE); - chars.set(Character.MIN_VALUE, Character.MIN_SURROGATE); - chars.clear('\r'); - chars.clear('\n'); - chars.clear('\u0085'); - chars.clear('\u2028'); - chars.clear('\u2029'); - return new Set(chars, "."); - }); - static final Lazy UNIX_LINES = - new Lazy<>( - () -> { - BitSet chars = new BitSet(Character.MIN_SURROGATE); - chars.set(Character.MIN_VALUE, Character.MIN_SURROGATE); - chars.clear('\n'); - return new Set(chars, "."); - }); - static final Lazy DOTALL = - new Lazy<>( - () -> { - BitSet chars = new BitSet(Character.MIN_SURROGATE); - chars.set(Character.MIN_VALUE, Character.MIN_SURROGATE); - return new Set(chars, "."); - }); + static final Set EMPTY = new Set(new BitSet(0), ""); + + /* + * All chars except surrogates, which only exist in pairs: supplementary code points are out of + * scope. Every set, including negated ones, is a subset of this one. + */ + static final Set DOTALL; + static final Set UNIX_LINES; + static final Set ALL; + + static { + BitSet dotAll = new BitSet(Character.MAX_VALUE + 1); + dotAll.set(Character.MIN_VALUE, Character.MAX_VALUE + 1); + dotAll.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1); + DOTALL = new Set(dotAll, "."); + BitSet unixLines = (BitSet) dotAll.clone(); + unixLines.clear('\n'); + UNIX_LINES = new Set(unixLines, "."); + BitSet all = (BitSet) unixLines.clone(); + all.clear('\r'); + all.clear('\u0085'); + all.clear('\u2028'); + all.clear('\u2029'); + ALL = new Set(all, "."); + } public static Set any(int flags) { if (0 != (flags & Pattern.DOTALL)) { - return DOTALL.get(); + return DOTALL; } else if (0 != (flags & Pattern.UNIX_LINES)) { - return UNIX_LINES.get(); + return UNIX_LINES; } else { - return ALL.get(); + return ALL; } } @@ -134,9 +132,9 @@ private Set(BitSet chars, String description) { @Override void generate(Context ctx) { - OptionalInt sample = sample(ctx.rng.nextLong()); - if (sample.isPresent()) { - ctx.append((char) sample.getAsInt()); + int sample = sample(ctx.rng.nextLong()); + if (0 <= sample) { + ctx.append((char) sample); } } @@ -170,16 +168,19 @@ public Stream shrink() { return builder.build(); } - OptionalInt sample(long seed) { + /** + * @return sampled char or -1 if this set is empty. + */ + int sample(long seed) { if (chars.isEmpty()) { - return OptionalInt.empty(); + return -1; } long skip = Math.abs(seed % chars.cardinality()); int sample = chars.nextSetBit(0); while (skip-- > 0) { sample = chars.nextSetBit(sample + 1); } - return OptionalInt.of(sample); + return sample; } @Override @@ -235,13 +236,8 @@ public Builder range(char start, char end) { } chars.set(start, end + 1); if (0 != (flags & Pattern.CASE_INSENSITIVE)) { - for (char ch = start; ch <= end; ch++) { - if (Character.isLowerCase(ch)) { - chars.set(Character.toUpperCase(ch)); - } - if (Character.isUpperCase(ch)) { - chars.set(Character.toLowerCase(ch)); - } + for (int ch = start; ch <= end; ch++) { + foldCase((char) ch); } } description.append(start).append('-').append(end); @@ -295,26 +291,48 @@ public Builder intersect(Set set) { */ public Builder single(char ch) { chars.set(ch); - if (0 != (flags & Pattern.CASE_INSENSITIVE)) { - if (Character.isLowerCase(ch)) { - chars.set(Character.toUpperCase(ch)); - } - if (Character.isUpperCase(ch)) { - chars.set(Character.toLowerCase(ch)); + foldCase(ch); + description.append(ch); + return this; + } + + /* + * Adds all chars matching given predicate, i.e. Character::isLowerCase. + */ + Builder matching(IntPredicate predicate, String description) { + for (int ch = DOTALL.chars.nextSetBit(0); ch >= 0; ch = DOTALL.chars.nextSetBit(ch + 1)) { + if (predicate.test(ch)) { + chars.set(ch); + foldCase((char) ch); } } - description.append(ch); + this.description.append(description); return this; } + /* + * Same as Pattern: CASE_INSENSITIVE alone only folds US-ASCII, UNICODE_CASE (implied by + * UNICODE_CHARACTER_CLASS) folds the rest. + */ + private void foldCase(char ch) { + if (0 == (flags & Pattern.CASE_INSENSITIVE) + || (ch > 0x7F + && 0 == (flags & (Pattern.UNICODE_CASE | Pattern.UNICODE_CHARACTER_CLASS)))) { + return; + } + chars.set(Character.toUpperCase(ch)); + chars.set(Character.toLowerCase(ch)); + } + /** * Negates this set. * * @return this builder instance */ public Builder negate() { - chars.flip(Character.MIN_VALUE, Character.MIN_SURROGATE); - chars.and(any(flags).chars); + // negation is not affected by DOTALL or UNIX_LINES: [^a] matches line terminators + chars.flip(Character.MIN_VALUE, Character.MAX_VALUE + 1); + chars.and(DOTALL.chars); description.insert(0, '^'); return this; } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala index ee1573b..3e10fa9 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala @@ -73,6 +73,7 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { Pattern.compile("^(a+)+$"), Pattern.compile("^([a-zA-Z]+)*$"), Pattern.compile("[\\.-z] [\\w-] [\\w.-]"), + Pattern.compile("(?U)\\w+ [\\d\\s]+ \\p{Alpha}"), Pattern.compile("^(a|aa)+$"), Pattern.compile("^(a|a?)+$"), Pattern.compile("^(?:||)$"), @@ -124,6 +125,60 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { } } + List( + ".", + "(?s).", + "(?d).", + "[^a]", + "[^\\n]", + "\\d", + "\\D", + "\\w", + "\\W", + "\\s", + "\\S", + "\\h", + "\\H", + "\\v", + "\\V", + "\\t", + "\\f", + "\\a", + "\\e", + "\\cA", + "\\p{all}", + "\\p{Lower}", + "\\p{Upper}", + "\\p{ASCII}", + "\\p{Digit}", + "\\p{XDigit}", + "\\p{Alnum}", + "\\p{Alpha}", + "\\p{Punct}", + "\\p{Graph}", + "\\p{Print}", + "\\p{Blank}", + "\\p{Space}", + "\\p{javaWhitespace}", + "\\p{javaLowerCase}", + "\\p{javaUpperCase}", + "(?i)[a-z]", + "(?i)\u00E9", + "(?iu)\u00E9", + "(?iU)\u00E9", + "[\\w&&[^\\d]]" + ).foreach { regex => + test(s"$regex should match the same chars as java.util.regex") { + val pattern = Pattern.compile(regex) + val coregex = Coregex.from(pattern) + val wrong = (Character.MIN_VALUE to Character.MAX_VALUE) + .filterNot(Character.isSurrogate) + .map(_.toString) + .filter(ch => coregex.matches(ch, null) != pattern.matcher(ch).matches()) + assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...") + } + } + test("should produce a well-formatted error message for unsupported constructs") { val err = intercept[UnsupportedOperationException](Coregex.from(Pattern.compile("abc \\G foo"))) assert( diff --git a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala index 4181568..1bca01d 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala @@ -53,7 +53,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { if (start == end) (end + 1).asInstanceOf[Char] else end } val range = Set.builder().range(start, end).build() - val generated = range.sample(seed).orElse(-1) + val generated = range.sample(seed) (start.toInt <= generated && generated <= end.toInt) :| s"$start <= $generated <= $end" } } @@ -61,7 +61,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { property("sampled should be in set") { forAll { (first: Char, rest: String, seed: Long) => val set = Set.builder().set(first, rest.toCharArray: _*).build() - val generated = set.sample(seed).orElse(-1) + val generated = set.sample(seed) rest.map(_.toInt =? generated).foldLeft(first.toInt =? generated)(_ || _) :| s"$generated in $set" } } @@ -76,13 +76,19 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { property("sampled should not be in intersection") { forAll { (left: Set, right: Set, seed1: Long, seed2: Long) => - val leftWithCommon = Set.builder().union(left).single(right.sample(seed1).orElse(-1).toChar).build() + val leftWithCommon = Set.builder().union(left).single(right.sample(seed1).toChar).build() val intersection = Set.builder().union(leftWithCommon).intersect(right).build() val generated = intersection.generate(seed2) (leftWithCommon.matches(generated, null) && right.matches(generated, null)) :| s"$generated in $intersection" } } + property("empty set should generate empty") { + forAll { (seed: Long) => + (Set.builder().build().sample(seed) ?= -1) && (Set.builder().build().generate(seed) ?= "") + } + } + test("empty set should not shrink") { !Set.builder().build().shrink().iterator().hasNext :| "empty set should not shrink" } @@ -128,11 +134,10 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (seed: Long) => val result = Set .builder(Pattern.DOTALL) - .union(Set.ALL.get()) + .union(Set.ALL) .negate() .build() .sample(seed) - .orElse(-1) (result ?= '\r') || (result ?= '\n') || (result ?= '\u0085') || (result ?= '\u2028') || (result ?= '\u2029'); } @@ -142,11 +147,10 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (seed: Long) => val result = Set .builder(Pattern.DOTALL) - .union(Set.UNIX_LINES.get()) + .union(Set.UNIX_LINES) .negate() .build() .sample(seed) - .orElse(-1) result ?= '\n' }