From 6a1aaf88fb11eba45909f18294fc6bedc97aa1de Mon Sep 17 00:00:00 2001 From: SimY4 Date: Thu, 12 Nov 2026 14:44:44 +1100 Subject: [PATCH 1/2] feat(core): language inclusion for regular Coregexes. New exact language operations on Coregex, decided on a Thompson NFA built from the regex: - `isSubsetOf(Coregex)`: whether every string fully matched by this regex is matched by the other. - `difference(Coregex)`: the shortest string matched by this regex but not by the other. - `intersection(Coregex)`: the shortest string matched by both. The search walks the product of this NFA with the lazily determinized other NFA, pruning pairs with antichains, so subset checks stay fast even with nested repetitions. Non-regular constructs (back references, atomic groups, possessive quantifiers, lookarounds, word boundaries) and ASCII approximations of UNICODE_CHARACTER_CLASS char classes are rejected with UnsupportedOperationException. Anchors (`^`, `\A`, `$`, `\z`, `\Z`) are supported at the input boundary, where they are no-ops under Matcher#matches(). --- .../github/simy4/coregex/core/Coregex.java | 354 +++++++++++++++- .../simy4/coregex/core/CoregexParser.java | 19 +- .../com/github/simy4/coregex/core/NFA.java | 385 ++++++++++++++++++ .../com/github/simy4/coregex/core/Set.java | 44 +- .../coregex/core/CoregexParserSuite.scala | 10 + .../github/simy4/coregex/core/NFASuite.scala | 360 ++++++++++++++++ version.sbt | 2 +- 7 files changed, 1153 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/com/github/simy4/coregex/core/NFA.java create mode 100644 core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala diff --git a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java index 561db87..8cd277a 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java @@ -44,6 +44,7 @@ * * @see Coregex.Concat * @see Coregex.Group + * @see Coregex.Labeled * @see Coregex.Quantified * @see Coregex.Ref * @see Set @@ -110,17 +111,60 @@ public static Coregex literal(String literal, int flags) { return new Concat(first, rest); } + /** + * @return predefined constructor for {@code ^} boundary matcher. + */ + public static Labeled lineStart() { + return Labeled.LINE_START; + } + + /** + * @return predefined constructor for {@code $} boundary matcher. + */ + public static Labeled lineEnd() { + return Labeled.LINE_END; + } + + /** + * @return predefined constructor for {@code \A} boundary matcher. + */ + public static Labeled inputStart() { + return Labeled.INPUT_START; + } + + /** + * @return predefined constructor for {@code \z} boundary matcher. + */ + public static Labeled inputEnd() { + return Labeled.INPUT_END; + } + + /** + * @return predefined constructor for {@code \Z} boundary matcher. + * @since 1.5.0 + */ + public static Labeled inputEndBeforeTerminator() { + return Labeled.INPUT_END_BEFORE_TERMINATOR; + } + + /** + * @param flags regex flags + * @param positive {@code true} for {@code \b}, {@code false} for {@code \B} + * @return predefined constructor for word boundary matcher. + */ public static Coregex wordBoundary(int flags, boolean positive) { Set word = Set.builder(flags).range('a', 'z').range('A', 'Z').range('0', '9').single('_').build(); Set notWord = Set.builder(flags).union(word).negate().build(); Group.Type lookbehind = positive ? Group.Type.LOOKBEHIND : Group.Type.NEGATIVE_LOOKBEHIND; Group.Type lookahead = positive ? Group.Type.LOOKAHEAD : Group.Type.NEGATIVE_LOOKAHEAD; - return new Union( - new Concat(new Group(lookbehind, empty()), new Group(lookahead, word)), - new Concat(new Group(lookbehind, word), new Group(lookahead, empty())), - new Concat(new Group(lookbehind, word), new Group(lookahead, notWord)), - new Concat(new Group(lookbehind, notWord), new Group(lookahead, word))); + return new Labeled( + positive ? "\\b" : "\\B", + new Union( + new Concat(new Group(lookbehind, empty()), new Group(lookahead, word)), + new Concat(new Group(lookbehind, word), new Group(lookahead, empty())), + new Concat(new Group(lookbehind, word), new Group(lookahead, notWord)), + new Concat(new Group(lookbehind, notWord), new Group(lookahead, word)))); } Coregex() {} @@ -136,6 +180,95 @@ public static Coregex wordBoundary(int flags, boolean positive) { */ public abstract Stream shrink(); + /* + * Thompson's construction: wires this regex between `from` and `to` such that paths from `from` + * to `to` spell exactly the language of this regex. Boundary anchors are stripped beforehand, + * see #compileAtBoundary(NFA, int, int). + */ + abstract void compile(NFA nfa, int from, int to); + + /* + * Same as #compile(NFA, int, int), for a regex spanning the whole input. Under Matcher#matches() + * semantics, anchors at the input boundary are no-ops (regardless of MULTILINE), so they are + * stripped from the very start and end of the regex, looking through alternatives and transparent + * groups. Anchors anywhere else are rejected by Labeled#compile. + * + * A single regex is at both boundaries at once. + */ + void compileAtBoundary(NFA nfa, int from, int to) { + if (isNoOpAt(true) || isNoOpAt(false)) { + nfa.epsilon(from, to); + } else { + compile(nfa, from, to); + } + } + + /* + * Whether this regex can be dropped at the start (or end) of the input, see + * #compileAtBoundary(NFA, int, int). + */ + boolean isNoOpAt(boolean start) { + return false; + } + + /** + * Tests if every string fully matched by this regex is also fully matched by the other one, i.e. + * if the other regex fully covers this one. + * + *

Unlike generation, language operations are exact. Hence, both regexes may only use regular + * constructs. Anchors ({@code ^}, {@code \A} and {@code $}, {@code \z}, {@code \Z}) are only + * supported as leading and trailing elements of the regex, of its top level alternatives or of + * groups that are the only remaining element, i.e. {@code ^a$|^(?:^b$)$}, where, under {@link + * java.util.regex.Matcher#matches()} semantics, they are no-ops regardless of {@link + * Pattern#MULTILINE}. Anchors nested anywhere else, i.e. {@code (^a)b}, are not supported. + * + *

Only characters of the Basic Multilingual Plane are considered: strings containing + * surrogates, i.e. supplementary code points, are out of scope. Case-insensitive matching folds + * case the same way {@link Pattern#CASE_INSENSITIVE} and {@link Pattern#UNICODE_CASE} do, except + * for the few Unicode characters with more than one case variant. + * + * @param other regex to compare with + * @return {@code true} if language of this regex is a subset of the other's language + * @throws UnsupportedOperationException if either regex uses constructs that are not regular: + * back references, atomic groups, possessive quantifiers, lookarounds or word boundaries. + * @see #difference(Coregex) + * @since 1.5.0 + */ + public final boolean isSubsetOf(Coregex other) { + return !difference(other).isPresent(); + } + + /** + * Finds the shortest string fully matched by this regex but not by the other one. Useful as a + * counterexample when {@link #isSubsetOf(Coregex)} fails. + * + *

See {@link #isSubsetOf(Coregex)} for supported constructs. + * + * @param other regex to subtract + * @return the shortest string matched by this but not the other regex or nothing if there is none + * @throws UnsupportedOperationException if either regex uses constructs that are not regular. + * @see #isSubsetOf(Coregex) + * @since 1.5.0 + */ + public final Optional difference(Coregex other) { + return NFA.of(this).difference(NFA.of(requireNonNull(other, "other"))); + } + + /** + * Finds the shortest string fully matched by both this and the other regex. + * + *

See {@link #isSubsetOf(Coregex)} for supported constructs. + * + * @param other regex to intersect with + * @return the shortest string matched by both regexes or nothing if they are disjoint + * @throws UnsupportedOperationException if either regex uses constructs that are not regular. + * @see #isSubsetOf(Coregex) + * @since 1.5.0 + */ + public final Optional intersection(Coregex other) { + return NFA.of(this).intersection(NFA.of(requireNonNull(other, "other"))); + } + /** * Samples one random string that matches this regex. * @@ -241,6 +374,29 @@ public Stream shrink() { }))); } + @Override + void compile(NFA nfa, int from, int to) { + nfa.chain(concat(), from, to); + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + List concat = concat(); + int first = 0; + int last = concat.size(); + while (first < last && concat.get(first).isNoOpAt(true)) { + first++; + } + while (first < last && concat.get(last - 1).isNoOpAt(false)) { + last--; + } + if (1 == last - first) { + concat.get(first).compileAtBoundary(nfa, from, to); + } else { + nfa.chain(concat.subList(first, last), from, to); + } + } + /** * @return underlying regexes in order of concatenation. */ @@ -503,6 +659,32 @@ public Stream shrink() { } } + @Override + void compile(NFA nfa, int from, int to) { + switch (type) { + case CAPTURING: + case NAMED: + case NON_CAPTURING: + group.compile(nfa, from, to); + break; + default: + throw new UnsupportedOperationException("group is not supported: " + this); + } + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + switch (type) { + case CAPTURING: + case NAMED: + case NON_CAPTURING: + group.compileAtBoundary(nfa, from, to); + break; + default: + compile(nfa, from, to); + } + } + /** * @return group type */ @@ -594,6 +776,111 @@ public enum Type { } } + /** + * Regex with a label, i.e. a boundary matcher such as {@code ^} or {@code $}. Behaves exactly as + * the labeled regex, but is represented by its label. + * + * @since 1.5.0 + */ + public static final class Labeled extends Coregex { + private static final long serialVersionUID = 1L; + + static final Labeled LINE_START = new Labeled("^", new Group(Group.Type.LOOKBEHIND, empty())); + static final Labeled LINE_END = new Labeled("$", new Group(Group.Type.LOOKAHEAD, empty())); + static final Labeled INPUT_START = + new Labeled("\\A", new Group(Group.Type.LOOKBEHIND, empty())); + static final Labeled INPUT_END = new Labeled("\\z", new Group(Group.Type.LOOKAHEAD, empty())); + static final Labeled INPUT_END_BEFORE_TERMINATOR = + new Labeled("\\Z", new Group(Group.Type.LOOKAHEAD, new Union(empty(), Set.single('\n')))); + + private final String label; + private final Coregex labeled; + + Labeled(String label, Coregex labeled) { + this.label = requireNonNull(label, "label"); + this.labeled = requireNonNull(labeled, "labeled"); + } + + @Override + void generate(Context ctx) throws RewindException { + labeled.generate(ctx); + } + + @Override + String[] match(String input, Context ctx) { + return labeled.match(input, ctx); + } + + /** {@inheritDoc} */ + @Override + public Stream shrink() { + return Stream.empty(); + } + + @Override + void compile(NFA nfa, int from, int to) { + // all labeled regexes are boundary matchers, which are lookarounds: boundary anchors are + // stripped by #compileAtBoundary(NFA, int, int), anything else is not regular + throw new UnsupportedOperationException(label + " is not supported"); + } + + /* + * Boundary anchors: `^` or `\A` at the start, `$`, `\z` or `\Z` at the end. Under + * Matcher#matches() semantics, `\Z` at the very end is a no-op as nothing may follow it. + */ + @Override + boolean isNoOpAt(boolean start) { + // labels are canonical: Labeled can only be created by the parser and predefined constructors + switch (label) { + case "^": + case "\\A": + return start; + case "$": + case "\\z": + case "\\Z": + return !start; + default: + return false; + } + } + + /** + * @return label + */ + public String label() { + return label; + } + + /** + * @return labeled regex + */ + public Coregex labeled() { + return labeled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Labeled that = (Labeled) o; + return label.equals(that.label) && labeled.equals(that.labeled); + } + + @Override + public int hashCode() { + return 31 * label.hashCode() + labeled.hashCode(); + } + + @Override + public String toString() { + return label; + } + } + /** Quantified regex. */ public static final class Quantified extends Coregex { @@ -700,6 +987,42 @@ public Stream shrink() { quantified.shrink().map(quantified -> new Quantified(quantified, min, max, type))); } + @Override + void compile(NFA nfa, int from, int to) { + if (Type.POSSESSIVE == type) { + throw new UnsupportedOperationException("possessive quantifier is not supported: " + this); + } + int current = from; + for (int i = 0; i < min; i++) { + int next = nfa.state(); + quantified.compile(nfa, current, next); + current = next; + } + if (-1 == max) { + // fresh loop state, as `from` may be shared and must not get incoming transitions + int loop = nfa.state(); + nfa.epsilon(current, loop); + quantified.compile(nfa, loop, loop); + nfa.epsilon(loop, to); + } else { + for (int i = min; i < max; i++) { + int next = nfa.state(); + nfa.epsilon(current, to); + quantified.compile(nfa, current, next); + current = next; + } + nfa.epsilon(current, to); + } + } + + /* + * Empty regex, i.e. inline flags like `(?m)` that may precede anchors. + */ + @Override + boolean isNoOpAt(boolean start) { + return EMPTY.equals(this); + } + /** * @return quantified regex */ @@ -838,6 +1161,11 @@ public Stream shrink() { return Stream.empty(); } + @Override + void compile(NFA nfa, int from, int to) { + throw new UnsupportedOperationException("back reference is not supported: " + this); + } + public Serializable ref() { return ref; } @@ -927,6 +1255,22 @@ public Stream shrink() { })))); } + @Override + void compile(NFA nfa, int from, int to) { + first.compile(nfa, from, to); + for (Coregex coregex : rest) { + coregex.compile(nfa, from, to); + } + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + first.compileAtBoundary(nfa, from, to); + for (Coregex coregex : rest) { + coregex.compileAtBoundary(nfa, from, to); + } + } + /** * @return underlying regexes forming this unification. */ diff --git a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java index 44f1cf1..d887241 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java +++ b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java @@ -184,11 +184,11 @@ private Coregex elementaryRE(Context ctx) { break; case '^': ctx.match('^'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty()); + elementaryRE = Coregex.lineStart(); break; case '$': ctx.match('$'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty()); + elementaryRE = Coregex.lineEnd(); break; case '\\': switch (ctx.peek(2)) { @@ -198,7 +198,7 @@ private Coregex elementaryRE(Context ctx) { case 'A': ctx.match('\\'); ctx.match('A'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty()); + elementaryRE = Coregex.inputStart(); break; case 'b': ctx.match('\\'); @@ -223,15 +223,12 @@ private Coregex elementaryRE(Context ctx) { case 'z': ctx.match('\\'); ctx.match('z'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty()); + elementaryRE = Coregex.inputEnd(); break; case 'Z': ctx.match('\\'); ctx.match('Z'); - elementaryRE = - new Coregex.Group( - Coregex.Group.Type.LOOKAHEAD, - new Coregex.Union(Coregex.empty(), Set.single('\n'))); + elementaryRE = Coregex.inputEndBeforeTerminator(); break; case 'G': throw ctx.unsupported("metacharacter \\G"); @@ -519,6 +516,8 @@ private Set metachar(Context ctx) { ctx.match('\\'); char ch = ctx.peek(); Set.Builder metachar = Set.builder(ctx.flags); + // char classes that are ASCII-only here, but Unicode with UNICODE_CHARACTER_CLASS + boolean asciiOnly = 0 <= "dDwWsS".indexOf(ch); switch (ch) { case 't': ctx.match('t'); @@ -600,6 +599,7 @@ private Set metachar(Context ctx) { ctx.match('p'); ctx.match('{'); String posix = ctx.span('}'); + asciiOnly = !"all".equals(posix) && !"ASCII".equals(posix) && !posix.startsWith("java"); switch (posix) { case "all": metachar.matching(any -> true, "\\p{all}"); @@ -665,6 +665,9 @@ private Set metachar(Context ctx) { metachar.single(ch); break; } + if (asciiOnly && 0 != (ctx.flags & Pattern.UNICODE_CHARACTER_CLASS)) { + metachar.approximated(); + } return metachar.build(); } diff --git a/core/src/main/java/com/github/simy4/coregex/core/NFA.java b/core/src/main/java/com/github/simy4/coregex/core/NFA.java new file mode 100644 index 0000000..d95b3b5 --- /dev/null +++ b/core/src/main/java/com/github/simy4/coregex/core/NFA.java @@ -0,0 +1,385 @@ +/* + * Copyright 2021-2026 Alex Simkin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.simy4.coregex.core; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; + +/* + * Non-deterministic finite automaton built from Coregex using Thompson's construction. Every + * Coregex wires itself between given states, see Coregex#compile(NFA, int, int). + * Fragments never add edges into `from` or out of `to`, so fragments can safely share endpoints + * (union) without leaking paths between each other. + * + * Unlike Coregex itself, which is free to approximate a regex as long as everything it generates + * matches, an NFA has to accept exactly the language of the regex. Therefore, constructs that are + * not regular are rejected with UnsupportedOperationException. + * + * The alphabet is UTF-16 chars except surrogates, see Set#DOTALL. + * + * Language operations are decided exactly by a lazy breadth-first walk of the product automaton, + * so whenever a witness exists, the shortest one is returned. + */ +final class NFA { + + private static final int START = 0; + private static final int ACCEPT = 1; + + static NFA of(Coregex coregex) { + NFA nfa = new NFA(); + coregex.compileAtBoundary(nfa, START, ACCEPT); + return nfa; + } + + private final List> epsilons = new ArrayList<>(); + private final List> edges = new ArrayList<>(); + + private NFA() { + state(); // START + state(); // ACCEPT + } + + // region construction + int state() { + epsilons.add(new ArrayList<>()); + edges.add(new ArrayList<>()); + return epsilons.size() - 1; + } + + void epsilon(int from, int to) { + if (from != to) { + epsilons.get(from).add(to); + } + } + + /* + * Label is shared, not copied: automaton never mutates it. + */ + void edge(int from, BitSet label, int to) { + edges.get(from).add(new Edge(label, to)); + } + + void chain(List chunks, int from, int to) { + int current = from; + for (int i = 0; i < chunks.size(); i++) { + int next = chunks.size() - 1 == i ? to : state(); + chunks.get(i).compile(this, current, next); + current = next; + } + if (chunks.isEmpty()) { + epsilon(from, to); + } + } + + // endregion + + boolean matches(CharSequence input) { + BitSet current = initial(); + for (int i = 0; i < input.length() && !current.isEmpty(); i++) { + current = step(current, input.charAt(i)); + } + return current.get(ACCEPT); + } + + /* + * Shortest string accepted by this but not the other automaton. + */ + Optional difference(NFA other) { + return search(other, false); + } + + /* + * Shortest string accepted by both this and the other automaton. + */ + Optional intersection(NFA other) { + return search(other, true); + } + + /* + * Breadth-first search over pairs of (single state of this NFA, set of states of other NFA). + * Only the other side is (lazily) determinized, as only it needs to be complemented. + * + * otherAccepts: whether other has to accept (intersection) or reject (difference) the witness. + * + * Pairs are pruned with antichains: for difference, (state, others) is subsumed by an already + * visited (state, subset of others), as every word rejected from the bigger set is rejected from + * the smaller one too. Symmetrically for intersection with supersets. Being breadth-first, the + * subsuming pair is never deeper, so the first witness found is still the shortest one. + */ + private Optional search(NFA other, boolean otherAccepts) { + char[] alphabet = alphabet(other); + Frontier frontier = new Frontier(edges.size(), otherAccepts); + frontier.offer(initial(), other.initial(), null, '\0'); + for (Node node = frontier.poll(); null != node; node = frontier.poll()) { + if (ACCEPT == node.state && otherAccepts == node.otherStates.get(ACCEPT)) { + return Optional.of(node.witness()); + } + for (char ch : alphabet) { + BitSet next = step(singleton(node.state), ch); + if (next.isEmpty()) { + continue; + } + BitSet nextOther = other.step(node.otherStates, ch); + if (otherAccepts && nextOther.isEmpty()) { + continue; // other can never accept from here on + } + frontier.offer(next, nextOther, node, ch); + } + } + return Optional.empty(); + } + + /* + * Splits all characters on edge labels of both automatons into equivalence classes (minterms): + * characters that are indistinguishable by every edge. Each class is then represented by its + * simplest character, so that witnesses are readable. + */ + private char[] alphabet(NFA other) { + java.util.Set labels = new LinkedHashSet<>(); + collectLabels(labels); + other.collectLabels(labels); + BitSet universe = new BitSet(); + for (BitSet label : labels) { + universe.or(label); + } + List blocks = new ArrayList<>(); + if (!universe.isEmpty()) { + blocks.add(universe); + } + for (BitSet label : labels) { + List refined = new ArrayList<>(blocks.size() * 2); + for (BitSet block : blocks) { + BitSet in = (BitSet) block.clone(); + in.and(label); + if (in.isEmpty() || in.equals(block)) { + refined.add(block); + } else { + BitSet out = (BitSet) block.clone(); + out.andNot(label); + refined.add(in); + refined.add(out); + } + } + blocks = refined; + } + char[] alphabet = new char[blocks.size()]; + for (int i = 0; i < alphabet.length; i++) { + alphabet[i] = simplest(blocks.get(i)); + } + Arrays.sort(alphabet); + return alphabet; + } + + /* + * Simplest of given chars, in the same order of preference as Set#shrink(): digits, lowercase and + * uppercase letters, then printable ASCII, then anything else. + */ + private static char simplest(BitSet chars) { + for (char[] range : new char[][] {{'0', '9'}, {'a', 'z'}, {'A', 'Z'}, {' ', '~'}}) { + int ch = chars.nextSetBit(range[0]); + if (0 <= ch && ch <= range[1]) { + return (char) ch; + } + } + return (char) chars.nextSetBit(0); + } + + private void collectLabels(java.util.Set labels) { + for (List stateEdges : edges) { + for (Edge edge : stateEdges) { + labels.add(edge.label); + } + } + } + + private BitSet initial() { + return closure(singleton(START)); + } + + private BitSet step(BitSet states, char ch) { + BitSet next = new BitSet(edges.size()); + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + for (Edge edge : edges.get(state)) { + if (edge.label.get(ch)) { + next.set(edge.target); + } + } + } + return closure(next); + } + + private BitSet closure(BitSet states) { + Deque stack = new ArrayDeque<>(); + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + stack.push(state); + } + while (!stack.isEmpty()) { + for (int target : epsilons.get(stack.pop())) { + if (!states.get(target)) { + states.set(target); + stack.push(target); + } + } + } + return states; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int state = 0; state < edges.size(); state++) { + sb.append(state); + if (START == state) { + sb.append(" (start)"); + } else if (ACCEPT == state) { + sb.append(" (accept)"); + } + for (Edge edge : edges.get(state)) { + sb.append(" --"); + appendLabel(sb, edge.label); + sb.append("--> ").append(edge.target); + } + for (int target : epsilons.get(state)) { + sb.append(" --ε--> ").append(target); + } + sb.append('\n'); + } + return sb.toString(); + } + + private static void appendLabel(StringBuilder sb, BitSet label) { + BitSet negated = new BitSet(); + negated.set(Character.MIN_VALUE, Character.MAX_VALUE + 1); + negated.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1); + negated.andNot(label); + boolean negate = ranges(negated) < ranges(label); + BitSet chars = negate ? negated : label; + sb.append(negate ? "[^" : "["); + for (int start = chars.nextSetBit(0); start >= 0; ) { + int end = chars.nextClearBit(start) - 1; + sb.append((char) start); + if (start < end) { + sb.append(start + 1 < end ? "-" : ""); + sb.append((char) end); + } + start = chars.nextSetBit(end + 1); + } + sb.append(']'); + } + + private static int ranges(BitSet chars) { + int ranges = 0; + for (int ch = chars.nextSetBit(0); ch >= 0; ch = chars.nextSetBit(chars.nextClearBit(ch))) { + ranges++; + } + return ranges; + } + + private static BitSet singleton(int state) { + BitSet set = new BitSet(state + 1); + set.set(state); + return set; + } + + private static final class Edge { + private final BitSet label; + private final int target; + + private Edge(BitSet label, int target) { + this.label = label; + this.target = target; + } + } + + /* + * Breadth-first queue of pairs, deduplicated with an antichain: per state of this NFA, a set of + * pairwise incomparable sets of states of the other NFA, see #search(NFA, boolean). + */ + private static final class Frontier { + private final Deque queue = new ArrayDeque<>(); + private final List> visited; + private final boolean supersets; + + private Frontier(int states, boolean supersets) { + this.visited = new ArrayList<>(states); + for (int i = 0; i < states; i++) { + visited.add(new ArrayList<>()); + } + this.supersets = supersets; + } + + /* + * Enqueues (state, others) for every given state, unless subsumed by an already visited pair. + */ + private void offer(BitSet states, BitSet others, Node parent, char ch) { + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + List antichain = visited.get(state); + if (antichain.stream().noneMatch(seen -> subsumes(seen, others))) { + antichain.removeIf(seen -> subsumes(others, seen)); + antichain.add(others); + queue.add(new Node(state, others, parent, ch)); + } + } + } + + private Node poll() { + return queue.poll(); + } + + /* + * Difference needs the other NFA to reject, so smaller sets subsume. Intersection needs it to + * accept, so bigger sets do. + */ + private boolean subsumes(BitSet subsuming, BitSet subsumed) { + return supersets ? isSubset(subsumed, subsuming) : isSubset(subsuming, subsumed); + } + + private static boolean isSubset(BitSet sub, BitSet sup) { + BitSet difference = (BitSet) sub.clone(); + difference.andNot(sup); + return difference.isEmpty(); + } + } + + private static final class Node { + private final int state; + private final BitSet otherStates; + private final Node parent; + private final char ch; + + private Node(int state, BitSet otherStates, Node parent, char ch) { + this.state = state; + this.otherStates = otherStates; + this.parent = parent; + this.ch = ch; + } + + private String witness() { + StringBuilder sb = new StringBuilder(); + for (Node node = this; null != node.parent; node = node.parent) { + sb.append(node.ch); + } + return sb.reverse().toString(); + } + } +} diff --git a/core/src/main/java/com/github/simy4/coregex/core/Set.java b/core/src/main/java/com/github/simy4/coregex/core/Set.java index f8be9d7..cfbeef8 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Set.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Set.java @@ -36,7 +36,7 @@ public final class Set extends Coregex implements Serializable { private static final long serialVersionUID = 1L; - static final Set EMPTY = new Set(new BitSet(0), ""); + static final Set EMPTY = new Set(new BitSet(0), "", false); /* * All chars except surrogates, which only exist in pairs: supplementary code points are out of @@ -50,16 +50,16 @@ public final class Set extends Coregex implements Serializable { BitSet dotAll = new BitSet(Character.MAX_VALUE + 1); dotAll.set(Character.MIN_VALUE, Character.MAX_VALUE + 1); dotAll.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1); - DOTALL = new Set(dotAll, "."); + DOTALL = new Set(dotAll, ".", false); BitSet unixLines = (BitSet) dotAll.clone(); unixLines.clear('\n'); - UNIX_LINES = new Set(unixLines, "."); + UNIX_LINES = new Set(unixLines, ".", false); BitSet all = (BitSet) unixLines.clone(); all.clear('\r'); all.clear('\u0085'); all.clear('\u2028'); all.clear('\u2029'); - ALL = new Set(all, "."); + ALL = new Set(all, ".", false); } public static Set any(int flags) { @@ -125,9 +125,17 @@ public static Set single(char ch, int flags) { private final BitSet chars; private final String description; - private Set(BitSet chars, String description) { + /* + * Whether chars are only an approximation of the actual char class, i.e. ASCII-only \w for + * UNICODE_CHARACTER_CLASS. Good enough for generation, as all generated chars still match, but not + * for exact language operations. + */ + private final boolean approximated; + + private Set(BitSet chars, String description, boolean approximated) { this.chars = chars; this.description = description; + this.approximated = approximated; } @Override @@ -168,7 +176,18 @@ public Stream shrink() { return builder.build(); } - /** + @Override + void compile(NFA nfa, int from, int to) { + if (approximated) { + throw new UnsupportedOperationException( + "char class with UNICODE_CHARACTER_CLASS is not supported: " + this); + } + if (!chars.isEmpty()) { + nfa.edge(from, chars, to); + } + } + + /* * @return sampled char or -1 if this set is empty. */ int sample(long seed) { @@ -217,6 +236,7 @@ public static final class Builder { private final int flags; private final BitSet chars; private final StringBuilder description = new StringBuilder(); + private boolean approximated; private Builder(int flags, int size) { this.flags = flags; @@ -267,6 +287,7 @@ public Builder set(char first, char... rest) { */ public Builder union(Set set) { chars.or(requireNonNull(set, "set").chars); + approximated |= set.approximated; description.append(set); return this; } @@ -279,6 +300,7 @@ public Builder union(Set set) { */ public Builder intersect(Set set) { chars.and(requireNonNull(set, "set").chars); + approximated |= set.approximated; description.append("&&").append(set); return this; } @@ -296,6 +318,14 @@ public Builder single(char ch) { return this; } + /* + * Marks this set as only an approximation of the actual char class, see Set#approximated. + */ + Builder approximated() { + approximated = true; + return this; + } + /* * Adds all chars matching given predicate, i.e. Character::isLowerCase. */ @@ -341,7 +371,7 @@ public Builder negate() { * @return compiled set */ public Set build() { - return new Set(chars, description.toString()); + return new Set(chars, description.toString(), approximated); } } } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala index 3e10fa9..5bf0db6 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala @@ -125,6 +125,16 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { } } + test("boundary matchers should be represented by their labels") { + val regex = "^\\Aa\\b\\B\\Z\\z$" + assertEquals(Coregex.from(Pattern.compile(regex)).toString, regex) + assertEquals(Coregex.from(Pattern.compile("^")), Coregex.lineStart()) + assertEquals(Coregex.from(Pattern.compile("$")), Coregex.lineEnd()) + assertEquals(Coregex.from(Pattern.compile("\\A")), Coregex.inputStart()) + assertEquals(Coregex.from(Pattern.compile("\\z")), Coregex.inputEnd()) + assertEquals(Coregex.from(Pattern.compile("\\Z")), Coregex.inputEndBeforeTerminator()) + } + List( ".", "(?s).", diff --git a/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala new file mode 100644 index 0000000..9fb6c3a --- /dev/null +++ b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala @@ -0,0 +1,360 @@ +/* + * Copyright 2021-2026 Alex Simkin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.simy4.coregex.core + +import munit.ScalaCheckSuite +import org.scalacheck.{ Arbitrary, Gen } +import org.scalacheck.Prop._ + +import java.util.Optional +import java.util.regex.Pattern + +class NFASuite extends ScalaCheckSuite with CoregexArbitraries { + + def genPatternPairs(): Gen[(Pattern, Pattern)] = Gen.oneOf( + List( + Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}") -> + Pattern.compile("[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}"), + Pattern.compile("((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])") -> + Pattern.compile("(?:\\d{1,3}\\.){3}\\d{1,3}"), + Pattern.compile("#([0-9a-fA-F]{3}){1,2}") -> Pattern.compile("#[0-9a-fA-F]{3,6}"), + Pattern.compile("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d([+-][0-2]\\d:[0-5]\\d|Z)") -> + Pattern.compile("[\\dTZ:+-]+"), + Pattern.compile("4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}") -> Pattern.compile("\\d{13,16}"), + Pattern.compile( + "^arn:(?\\w+):(?\\w+):(?[\\w-]+):(?\\d{12}):(?(?[\\w-]+)[:\\/])?(?[\\w.-]+)$" + ) -> Pattern.compile("arn:[\\w.:/-]+"), + Pattern.compile( + "^(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)(?:-(?(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + ) -> Pattern.compile("[0-9a-zA-Z.+-]+"), + Pattern.compile("\\Qhttps://\\E(?foo|bar|baz)\\Q.com/\\E(?[\\w-]+)") -> + Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"), + // `\w*` alone allows empty, but concatenated with `\w` it does not. + Pattern.compile("(?\\w*)(?\\w)") -> Pattern.compile("[\\w-]+"), + Pattern.compile("^(a+)+$") -> Pattern.compile("a+"), + Pattern.compile("^([a-zA-Z]+)*$") -> Pattern.compile("[a-zA-Z]*"), + Pattern.compile("^(a|aa)+$") -> Pattern.compile("a+"), + Pattern.compile("^(a|a?)+$") -> Pattern.compile("a*"), + Pattern.compile("^(?:||)$") -> Pattern.compile("") + ) + ) + + implicit val arbPatternPairs: Arbitrary[(Pattern, Pattern)] = Arbitrary(genPatternPairs()) + + private def re(regex: String): Coregex = Coregex.from(Pattern.compile(regex)) + + private def nfa(regex: String): NFA = NFA.of(re(regex)) + + property("should cover example regex") { + forAll { (pair: (Pattern, Pattern), seed: Long) => + val (sub, sup) = pair + val subset = Coregex.from(sub) + val superset = Coregex.from(sup) + val generated = subset.generate(seed) + val witness = superset.difference(subset) + (subset.isSubsetOf(superset) :| s"${sub.pattern()} should be a subset of ${sup.pattern()}") && + (sup.matcher(generated).matches() :| s"${sup.pattern()} should match generated: $generated") && + (subset.intersection(superset).isPresent :| s"${sub.pattern()} should intersect ${sup.pattern()}") && + ((!witness.isPresent || sup.matcher(witness.get).matches() && !sub.matcher(witness.get).matches()) :| + s"${sup.pattern()} but not ${sub.pattern()} should match witness: $witness") + } + } + + // region language operations + test("possibly empty fragment is not a subset of non empty one") { + assertEquals(re("(?\\w*)").difference(re("[\\w-]+")), Optional.of("")) + } + + test("dot fragment is not a subset of word characters") { + val witness = re("(?.+)").difference(re("[\\w-]+")) + assert(witness.isPresent && 1 == witness.get.length, s"$witness should be a single char") + assert(!Pattern.matches("[\\w-]+", witness.get), s"${witness.get} should not match [\\w-]+") + } + + test("intersection of disjoint languages is empty") { + assertEquals(re("[a-z]+").intersection(re("\\d+")), Optional.empty[String]()) + } + + test("intersection returns the shortest common string") { + assertEquals(re("[a-z]+").intersection(re("x.*")), Optional.of("x")) + assertEquals(re("a{3}b?").intersection(re("a{3}b?")), Optional.of("aaa")) + assertEquals(re("[a&&b]").intersection(re(".*")), Optional.empty[String]()) + } + + test("toString shows states with transitions") { + assertEquals( + nfa("a|b*").toString, + """0 (start) --[a]--> 1 --ε--> 2 + |1 (accept) + |2 --[b]--> 2 --ε--> 1 + |""".stripMargin + ) + assertEquals(nfa(".[^ab]").toString.linesIterator.next(), "0 (start) --[^\\n\\r\\u0085\\u2028\\u2029]--> 2") + } + + test("witnesses use the simplest chars, same as shrinking") { + assertEquals(re("#[0-9a-fA-F]{3,6}").difference(re("#(?:[0-9a-fA-F]{3}){1,2}")), Optional.of("#0000")) + assertEquals(re("[a-zA-Z]+").difference(re("[a-z]+")), Optional.of("A")) + assertEquals(re("[\\s\\u00E9]").intersection(re("\\W")), Optional.of(" ")) + assertEquals(re("\\W").difference(re("[ -~]")), Optional.of("\u0000")) + } + + test("empty regex matches empty string only") { + assertEquals(re("(?:)").intersection(re("")), Optional.of("")) + assertEquals(re("a(?:)b").intersection(re("ab")), Optional.of("ab")) + assertEquals(re("(?:a|)").difference(re("a")), Optional.of("")) + } + + // (sub, sup, input matched by sub only if sub is not a subset of sup) + List( + ("[^a]", "[^a\\n]", Some("\n")), + ("\\n", "\\D", None), + ("\\n", "[^\\w]", None), + ("\\uFF01", ".", None), + ("(?s).", "[^\\uD800-\\uDFFF]", None), + ("\u00C9", "(?i)\u00E9", Some("\u00C9")), + ("\u00C9", "(?iu)\u00E9", None), + ("K", "(?i)k", None), + ("a\\Z", "a", None), + ("a", "a\\Z", None) + ).foreach { case (sub, sup, witness) => + test(s"$sub is ${if (witness.isEmpty) "" else "not "}a subset of $sup like in java.util.regex") { + witness.foreach { w => + assert(Pattern.matches(sub, w) && !Pattern.matches(sup, w), s"$w should only match $sub") + } + assertEquals(re(sub).difference(re(sup)), witness.fold(Optional.empty[String]())(Optional.of(_))) + } + } + + List( + ".", + "(?s).", + "(?d).", + "[^a]", + "[^\\n]", + "\\d", + "\\D", + "\\w", + "\\W", + "\\s", + "\\S", + "\\h", + "\\H", + "\\v", + "\\V", + "\\t", + "\\f", + "\\a", + "\\e", + "\\cA", + "\\p{all}", + "\\p{Lower}", + "\\p{Upper}", + "\\p{ASCII}", + "\\p{Digit}", + "\\p{XDigit}", + "\\p{Alnum}", + "\\p{Alpha}", + "\\p{Punct}", + "\\p{Graph}", + "\\p{Print}", + "\\p{Blank}", + "\\p{Space}", + "\\p{javaWhitespace}", + "\\p{javaLowerCase}", + "\\p{javaUpperCase}", + "(?i)[a-z]", + "(?i)\u00E9", + "(?iu)\u00E9", + "(?iU)\u00E9", + "(?U)\\p{ASCII}", + "[\\w&&[^\\d]]" + ).foreach { regex => + test(s"$regex should accept the same chars as java.util.regex") { + val pattern = Pattern.compile(regex) + val nfa = NFA.of(Coregex.from(pattern)) + val wrong = (Character.MIN_VALUE to Character.MAX_VALUE) + .filterNot(Character.isSurrogate) + .map(_.toString) + .filter(ch => nfa.matches(ch) != pattern.matcher(ch).matches()) + assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...") + } + } + + test("unicode character class only affects predefined char classes") { + assert(re("(?U)abc").isSubsetOf(re("abc"))) + assert(re("(?U)[a-c]+").isSubsetOf(re("\\w+"))) + } + + test("reluctant and greedy quantifiers have the same language") { + assert(re("a+?b*?").isSubsetOf(re("a+b*"))) + assert(re("a+b*").isSubsetOf(re("a+?b*?"))) + } + + test("nested stars") { + val nested = nfa("(?:a*b*)*") + assert(nested.matches("")) + assert(nested.matches("abba")) + assert(!nested.matches("abc")) + assert(re("(?:a*b*)*").isSubsetOf(re("[ab]*"))) + assert(re("[ab]*").isSubsetOf(re("(?:a*b*)*"))) + } + + test("star inside union does not leak") { + val union = nfa("a*|b") + assert(!union.matches("bb")) + assert(!union.matches("ab")) + } + + List( + "^abc$", + "\\Aabc\\z", + "(?m)^abc$", + "(^abc$)", + "^(?:abc)$", + "^^abc$$", + "^a$|^bc$|(?:^ab)", + "(?:^a|^bc)$", + "^abc\\Z", + "^(?:^abc$)$", + "^(?:a|^bc)$", + "(?m)(?i)^abc$" + ).foreach { regex => + test(s"boundary anchors in $regex are no-ops") { + val pattern = Pattern.compile(regex) + val nfa = NFA.of(Coregex.from(pattern)) + List("", "a", "ab", "abc", "bc", "abcd", "\n", "abc\n").foreach { input => + assertEquals(nfa.matches(input), pattern.matcher(input).matches(), s"$input in $regex") + } + } + } + // endregion + + // region unsupported constructs + List( + "(a)\\1", + "(?a)\\k", + "(?>a*)", + "a*+", + "(?=a)a", + "(?!a)b", + "(?<=a)b", + "(? + test(s"$regex is not supported") { + val _ = intercept[UnsupportedOperationException](re(regex).isSubsetOf(re("a"))) + val _ = intercept[UnsupportedOperationException](re("a").isSubsetOf(re(regex))) + } + } + // endregion + + // region properties + private val supportedRange: Gen[QuantifyRange] = + for { + min <- Gen.choose(0, 5) + add <- Gen.choose(0, 5) + max <- Gen.oneOf( + Gen.const(-1), + Gen.choose(min, min + add) + ) + } yield QuantifyRange(min, max, Coregex.Quantified.Type.GREEDY) + private val genSupported: Gen[Coregex] = Gen.resize(20, genCoregex(supportedRange)) + + private def mutate(s: String): Gen[String] = + if (s.isEmpty) Gen.alphaNumChar.map(_.toString) + else + for { + i <- Gen.choose(0, s.length - 1) + ch <- Gen.alphaNumChar + mutated <- Gen.oneOf( + s.patch(i, Nil, 1), + s.patch(i, ch.toString, 0), + s.patch(i, ch.toString, 1), + s + s.charAt(i) + ) + } yield mutated + + property("nfa should accept generated strings") { + forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) => + val generated = coregex.generate(seed) + NFA.of(coregex).matches(generated) :| s"$generated in $coregex" + } + } + + property("nfa should agree with coregex matcher") { + forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) => + forAll(mutate(coregex.generate(seed))) { input => + (NFA.of(coregex).matches(input) ?= coregex.matches(input, null)) :| s"$input in $coregex" + } + } + } + + property("every coregex is a subset of itself and of its union") { + forAll(genSupported, genSupported) { (left, right) => + (left.isSubsetOf(left) :| "reflexive") && + (left.isSubsetOf(new Coregex.Union(left, right)) :| "union") + } + } + + property("difference witness should be accepted by left and rejected by right") { + forAll(genSupported, genSupported) { (left, right) => + val witness = left.difference(right) + witness.isPresent ==> { + (left.matches(witness.get, null) :| s"${witness.get} in $left") && + (!right.matches(witness.get, null) :| s"${witness.get} not in $right") + } + } + } + + property("difference should find the shortest witness if any exists") { + forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, right, seed) => + val generated = left.generate(seed) + !right.matches(generated, null) ==> { + val witness = left.difference(right) + (witness.isPresent :| s"$generated in $left but not in $right") && + (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" + } + } + } + + property("intersection should find the shortest witness if any exists") { + forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, other, seed) => + val right = new Coregex.Union(other, left) + val generated = left.generate(seed) + val witness = left.intersection(right) + (witness.isPresent :| s"$generated in both $left and $right") && { + (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" && + (left.matches(witness.get, null) && right.matches(witness.get, null)) :| s"${witness.get} in both" + } + } + } + // endregion +} diff --git a/version.sbt b/version.sbt index edb9358..c919701 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "1.4.6-SNAPSHOT" +ThisBuild / version := "1.5.0-SNAPSHOT" From c64298eb23f65d9664a964ea4225466df30fb486 Mon Sep 17 00:00:00 2001 From: SimY4 Date: Thu, 12 Nov 2026 18:19:20 +1100 Subject: [PATCH 2/2] fix(core): exact lookaround generation via NFA. Lookarounds used to be checked by Coregex#match() against the whole input generated before (lookbehind) or after (lookahead), rather than a suffix or prefix of it. Negative lookarounds could generate non-matching strings, e.g. `ba(? match = Arrays.asList(first.match(input, ctx)); - for (int i = 0; i < rest.length && !match.isEmpty(); i++) { - Collection nextMatch = new HashSet<>(match.size()); - Coregex coregex = rest[i]; - for (String tail : match) { - Collections.addAll(nextMatch, coregex.match(tail, ctx)); - } - match = nextMatch; - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -539,6 +510,7 @@ public static final class Group extends Coregex { private final int index; private final String name; private final Coregex group; + private transient volatile NFA lookaround; /** * Unnamed capturing group. @@ -603,7 +575,7 @@ void generate(Context ctx) throws RewindException { positive = false; // fall through case LOOKBEHIND: - if (positive == group.matches(ctx.toString(true), ctx)) { + if (positive == lookaround().matches(ctx.toString(true))) { break; } throw new RewindException(); @@ -612,13 +584,10 @@ void generate(Context ctx) throws RewindException { // fall through case LOOKAHEAD: int length = ctx.length(true); - Coregex fullLookbehind = - 0 == length - ? group - : new Concat(any().quantify(length, length, Quantified.Type.GREEDY), group); - final boolean pos = positive; + NFA lookahead = lookaround(); + boolean pos = positive; ctx.finalizeWithLookbehind( - root -> pos == fullLookbehind.matches(root.toString(true), root)); + root -> pos == lookahead.matches(root.toString(true).substring(length))); break; default: try (Context childCtx = new Context(ctx, index, name)) { @@ -627,24 +596,6 @@ void generate(Context ctx) throws RewindException { } } - @Override - String[] match(String input, Context ctx) { - switch (type) { - case NON_CAPTURING: - case ATOMIC: - return group.match(input, ctx); - case LOOKAHEAD: - case LOOKBEHIND: - case NEGATIVE_LOOKAHEAD: - case NEGATIVE_LOOKBEHIND: - return new String[] {input}; - default: - try (Context childCtx = new Context(ctx, index, name)) { - return group.match(input, childCtx); - } - } - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -659,6 +610,24 @@ public Stream shrink() { } } + /* + * Lookbehind body has to match a suffix of the input generated so far, lookahead body a prefix + * of the input generated after: `[\s\S]*body` and `body[\s\S]*` respectively. Compiled as is, + * since the body is not at the input boundary and anchors in it are not no-ops. + */ + private NFA lookaround() { + NFA nfa = lookaround; + if (null == nfa) { + boolean behind = Type.LOOKBEHIND == type || Type.NEGATIVE_LOOKBEHIND == type; + Coregex anything = Set.DOTALL.quantify(0, -1, Quantified.Type.GREEDY); + nfa = new NFA(); + (behind ? new Concat(anything, group) : new Concat(group, anything)) + .compile(nfa, NFA.START, NFA.ACCEPT); + lookaround = nfa; + } + return nfa; + } + @Override void compile(NFA nfa, int from, int to) { switch (type) { @@ -785,13 +754,23 @@ public enum Type { public static final class Labeled extends Coregex { private static final long serialVersionUID = 1L; - static final Labeled LINE_START = new Labeled("^", new Group(Group.Type.LOOKBEHIND, empty())); - static final Labeled LINE_END = new Labeled("$", new Group(Group.Type.LOOKAHEAD, empty())); - static final Labeled INPUT_START = - new Labeled("\\A", new Group(Group.Type.LOOKBEHIND, empty())); - static final Labeled INPUT_END = new Labeled("\\z", new Group(Group.Type.LOOKAHEAD, empty())); + // (? shrink() { @@ -952,29 +926,6 @@ void generate(Context ctx) throws RewindException { } } - @Override - String[] match(String input, Context ctx) { - int quantifier = 0, min = this.min, max = this.max; - java.util.Set match = new HashSet<>(Collections.singleton(input)); - for (; quantifier < min && !match.isEmpty(); quantifier++) { - java.util.Set nextMatch = new HashSet<>(match.size()); - for (String tail : match) { - Collections.addAll(nextMatch, quantified.match(tail, ctx)); - } - match = nextMatch; - } - for (; (-1 == max || quantifier < max) && !match.isEmpty(); quantifier++) { - java.util.Set nextMatch = new HashSet<>(match.size()); - for (String tail : match) { - Collections.addAll(nextMatch, quantified.match(tail, ctx)); - } - if (!match.addAll(nextMatch)) { - break; - } - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -1149,12 +1100,6 @@ void generate(Context ctx) { ctx.append(ctx.ref(ref)); } - @Override - String[] match(String input, Context ctx) { - String ref = ctx.ref(this.ref); - return input.startsWith(ref) ? new String[] {input.substring(ref.length())} : new String[0]; - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -1216,16 +1161,6 @@ void generate(Context ctx) throws RewindException { (index < rest.length ? rest[index] : first).generate(ctx); } - @Override - String[] match(String input, Context ctx) { - Collection match = new HashSet<>(rest.length + 1); - Collections.addAll(match, first.match(input, ctx)); - for (Coregex coregex : rest) { - Collections.addAll(match, coregex.match(input, ctx)); - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { diff --git a/core/src/main/java/com/github/simy4/coregex/core/NFA.java b/core/src/main/java/com/github/simy4/coregex/core/NFA.java index d95b3b5..42a8624 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/NFA.java +++ b/core/src/main/java/com/github/simy4/coregex/core/NFA.java @@ -42,8 +42,8 @@ */ final class NFA { - private static final int START = 0; - private static final int ACCEPT = 1; + static final int START = 0; + static final int ACCEPT = 1; static NFA of(Coregex coregex) { NFA nfa = new NFA(); @@ -54,7 +54,7 @@ static NFA of(Coregex coregex) { private final List> epsilons = new ArrayList<>(); private final List> edges = new ArrayList<>(); - private NFA() { + NFA() { state(); // START state(); // ACCEPT } diff --git a/core/src/main/java/com/github/simy4/coregex/core/Set.java b/core/src/main/java/com/github/simy4/coregex/core/Set.java index cfbeef8..f058587 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Set.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Set.java @@ -146,13 +146,6 @@ void generate(Context ctx) { } } - @Override - String[] match(String input, Context ctx) { - return !input.isEmpty() && chars.get(input.charAt(0)) - ? new String[] {input.substring(1)} - : new String[0]; - } - boolean single() { return 1 == chars.cardinality(); } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala index 5bf0db6..ba57b1e 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala @@ -79,6 +79,14 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { Pattern.compile("^(?:||)$"), Pattern.compile("(\\d+) (\\w+) == (\\1) \\2 \\3"), Pattern.compile("(?!=.{10,}).+"), + Pattern.compile("(?!a)[ab]b"), + Pattern.compile("(?=a)[ab]b"), + Pattern.compile("b[ab](?<=a)"), + Pattern.compile("[ab]{2}(? coregex.matches(ch, null) != pattern.matcher(ch).matches()) + .filter(ch => NFA.of(coregex).matches(ch) != pattern.matcher(ch).matches()) assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...") } } @@ -199,12 +207,20 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { ) } + test("anchors inside lookarounds are not at the input boundary") { + List("a(?=^)b", "a(?<=$)b", "a(?<=\\z)b", "a(?<=\\Z)b", "(?<=^)a", "a(?=$)").foreach { regex => + val _ = intercept[UnsupportedOperationException](Coregex.from(Pattern.compile(regex)).generate(0L)) + } + } + property("should fail to generate strings") { forAll( Gen.oneOf( Pattern.compile("(?=(abc))"), Pattern.compile("\\d+(?= dollars)"), - Pattern.compile("(?<=[a-z0-9])(?=[A-Z])") + Pattern.compile("(?<=[a-z0-9])(?=[A-Z])"), + Pattern.compile("ba(? diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala index 96e87b0..530908f 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala @@ -44,7 +44,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { } test("empty should match empty") { - assert(Coregex.empty().matches("", null)) + assert(NFA.of(Coregex.empty()).matches("")) } property("literal don't shrink") { @@ -55,7 +55,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { property("literal should match self") { forAll { (literal: String, flags: Int) => - Coregex.literal(literal, flags).matches(literal, null) :| s"$literal match self" + NFA.of(Coregex.literal(literal, flags)).matches(literal) :| s"$literal match self" } } @@ -89,7 +89,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (head: Simple, tail: Seq[Simple], seed: Long) => val concat = new Concat(head.value, tail.map(_.value): _*) val generated = concat.generate(seed) - concat.matches(generated, null) :| s"$generated in $concat" + NFA.of(concat).matches(generated) :| s"$generated in $concat" } } // endregion @@ -134,7 +134,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (head: Simple, tail: Seq[Simple], seed: Long) => val union = new Union(head.value, tail.map(_.value): _*) val generated = union.generate(seed) - union.matches(generated, null) :| s"$generated in ($union)" + NFA.of(union).matches(generated) :| s"$generated in ($union)" } } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala index 9fb6c3a..047260d 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala @@ -103,7 +103,7 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { |2 --[b]--> 2 --ε--> 1 |""".stripMargin ) - assertEquals(nfa(".[^ab]").toString.linesIterator.next(), "0 (start) --[^\\n\\r\\u0085\\u2028\\u2029]--> 2") + assert(nfa(".[^ab]").toString.startsWith("0 (start) --[^\n\r\u0085\u2028\u2029]--> 2\n")) } test("witnesses use the simplest chars, same as shrinking") { @@ -288,20 +288,6 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { } yield QuantifyRange(min, max, Coregex.Quantified.Type.GREEDY) private val genSupported: Gen[Coregex] = Gen.resize(20, genCoregex(supportedRange)) - private def mutate(s: String): Gen[String] = - if (s.isEmpty) Gen.alphaNumChar.map(_.toString) - else - for { - i <- Gen.choose(0, s.length - 1) - ch <- Gen.alphaNumChar - mutated <- Gen.oneOf( - s.patch(i, Nil, 1), - s.patch(i, ch.toString, 0), - s.patch(i, ch.toString, 1), - s + s.charAt(i) - ) - } yield mutated - property("nfa should accept generated strings") { forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) => val generated = coregex.generate(seed) @@ -309,14 +295,6 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { } } - property("nfa should agree with coregex matcher") { - forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) => - forAll(mutate(coregex.generate(seed))) { input => - (NFA.of(coregex).matches(input) ?= coregex.matches(input, null)) :| s"$input in $coregex" - } - } - } - property("every coregex is a subset of itself and of its union") { forAll(genSupported, genSupported) { (left, right) => (left.isSubsetOf(left) :| "reflexive") && @@ -328,8 +306,8 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { forAll(genSupported, genSupported) { (left, right) => val witness = left.difference(right) witness.isPresent ==> { - (left.matches(witness.get, null) :| s"${witness.get} in $left") && - (!right.matches(witness.get, null) :| s"${witness.get} not in $right") + (NFA.of(left).matches(witness.get) :| s"${witness.get} in $left") && + (!NFA.of(right).matches(witness.get) :| s"${witness.get} not in $right") } } } @@ -337,7 +315,7 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { property("difference should find the shortest witness if any exists") { forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, right, seed) => val generated = left.generate(seed) - !right.matches(generated, null) ==> { + !NFA.of(right).matches(generated) ==> { val witness = left.difference(right) (witness.isPresent :| s"$generated in $left but not in $right") && (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" @@ -352,7 +330,7 @@ class NFASuite extends ScalaCheckSuite with CoregexArbitraries { val witness = left.intersection(right) (witness.isPresent :| s"$generated in both $left and $right") && { (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" && - (left.matches(witness.get, null) && right.matches(witness.get, null)) :| s"${witness.get} in both" + (NFA.of(left).matches(witness.get) && NFA.of(right).matches(witness.get)) :| s"${witness.get} in both" } } } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala index 668bead..483484e 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala @@ -31,7 +31,9 @@ object QuantifyRange { Gen.const(-1), Gen.choose(min, min + add) ) - tpe <- Gen.oneOf(Coregex.Quantified.Type.values.toIndexedSeq) + // possessive generates the same as greedy, but is not regular, so generated strings could not be + // verified with NFA + tpe <- Gen.oneOf(Coregex.Quantified.Type.GREEDY, Coregex.Quantified.Type.RELUCTANT) } yield QuantifyRange(min, max, tpe) } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala index 1bca01d..66538dd 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala @@ -29,7 +29,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (set: Set, seed: Long) => val generated = set.generate(seed) - val inSetCheck = set.matches(generated, null) :| s"$generated in $set" + val inSetCheck = NFA.of(set).matches(generated) :| s"$generated in $set" val lengthCheck = (generated.length ?= 1) :| s"$generated.length == 1" inSetCheck && lengthCheck @@ -37,11 +37,11 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { } property("quantified generated should be in set") { - forAll { (set: Set, range: QuantifyRange, `type`: Coregex.Quantified.Type, seed: Long) => - val quantified = set.quantify(range.min, range.max, `type`) + forAll { (set: Set, range: QuantifyRange, seed: Long) => + val quantified = set.quantify(range.min, range.max, range.`type`) val generated = quantified.generate(seed) - quantified.matches(generated, null) :| s"$generated in $set" + NFA.of(quantified).matches(generated) :| s"$generated in $set" } } @@ -70,7 +70,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (left: Set, right: Set, seed: Long) => val union = Set.builder().union(left).union(right).build() val generated = union.generate(seed) - (left.matches(generated, null) || right.matches(generated, null)) :| s"$generated in $union" + (NFA.of(left).matches(generated) || NFA.of(right).matches(generated)) :| s"$generated in $union" } } @@ -79,7 +79,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { val leftWithCommon = Set.builder().union(left).single(right.sample(seed1).toChar).build() val intersection = Set.builder().union(leftWithCommon).intersect(right).build() val generated = intersection.generate(seed2) - (leftWithCommon.matches(generated, null) && right.matches(generated, null)) :| s"$generated in $intersection" + (NFA.of(leftWithCommon).matches(generated) && NFA.of(right).matches(generated)) :| s"$generated in $intersection" } } @@ -126,7 +126,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { .build() .generate(seed) - set.matches(result.toLowerCase(Locale.ENGLISH), null) || set.matches(result.toUpperCase(Locale.ENGLISH), null) + NFA.of(set).matches(result.toLowerCase(Locale.ENGLISH)) || NFA.of(set).matches(result.toUpperCase(Locale.ENGLISH)) } }