diff --git a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java index 561db87..0e5f956 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java @@ -21,10 +21,8 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -44,6 +42,7 @@ * * @see Coregex.Concat * @see Coregex.Group + * @see Coregex.Labeled * @see Coregex.Quantified * @see Coregex.Ref * @see Set @@ -110,25 +109,65 @@ public static Coregex literal(String literal, int flags) { return new Concat(first, rest); } + /** + * @return predefined constructor for {@code ^} boundary matcher. + */ + public static Labeled lineStart() { + return Labeled.LINE_START; + } + + /** + * @return predefined constructor for {@code $} boundary matcher. + */ + public static Labeled lineEnd() { + return Labeled.LINE_END; + } + + /** + * @return predefined constructor for {@code \A} boundary matcher. + */ + public static Labeled inputStart() { + return Labeled.INPUT_START; + } + + /** + * @return predefined constructor for {@code \z} boundary matcher. + */ + public static Labeled inputEnd() { + return Labeled.INPUT_END; + } + + /** + * @return predefined constructor for {@code \Z} boundary matcher. + * @since 1.5.0 + */ + public static Labeled inputEndBeforeTerminator() { + return Labeled.INPUT_END_BEFORE_TERMINATOR; + } + + /** + * @param flags regex flags + * @param positive {@code true} for {@code \b}, {@code false} for {@code \B} + * @return predefined constructor for word boundary matcher. + */ public static Coregex wordBoundary(int flags, boolean positive) { Set word = Set.builder(flags).range('a', 'z').range('A', 'Z').range('0', '9').single('_').build(); - Set notWord = Set.builder(flags).union(word).negate().build(); - Group.Type lookbehind = positive ? Group.Type.LOOKBEHIND : Group.Type.NEGATIVE_LOOKBEHIND; - Group.Type lookahead = positive ? Group.Type.LOOKAHEAD : Group.Type.NEGATIVE_LOOKAHEAD; - return new Union( - new Concat(new Group(lookbehind, empty()), new Group(lookahead, word)), - new Concat(new Group(lookbehind, word), new Group(lookahead, empty())), - new Concat(new Group(lookbehind, word), new Group(lookahead, notWord)), - new Concat(new Group(lookbehind, notWord), new Group(lookahead, word))); + // \b: (?<=\w)(?!\w)|(? shrink(); + /* + * Thompson's construction: wires this regex between `from` and `to` such that paths from `from` + * to `to` spell exactly the language of this regex. Boundary anchors are stripped beforehand, + * see #compileAtBoundary(NFA, int, int). + */ + abstract void compile(NFA nfa, int from, int to); + + /* + * Same as #compile(NFA, int, int), for a regex spanning the whole input. Under Matcher#matches() + * semantics, anchors at the input boundary are no-ops (regardless of MULTILINE), so they are + * stripped from the very start and end of the regex, looking through alternatives and transparent + * groups. Anchors anywhere else are rejected by Labeled#compile. + * + * A single regex is at both boundaries at once. + */ + void compileAtBoundary(NFA nfa, int from, int to) { + if (isNoOpAt(true) || isNoOpAt(false)) { + nfa.epsilon(from, to); + } else { + compile(nfa, from, to); + } + } + + /* + * Whether this regex can be dropped at the start (or end) of the input, see + * #compileAtBoundary(NFA, int, int). + */ + boolean isNoOpAt(boolean start) { + return false; + } + + /** + * Tests if every string fully matched by this regex is also fully matched by the other one, i.e. + * if the other regex fully covers this one. + * + *

Unlike generation, language operations are exact. Hence, both regexes may only use regular + * constructs. Anchors ({@code ^}, {@code \A} and {@code $}, {@code \z}, {@code \Z}) are only + * supported as leading and trailing elements of the regex, of its top level alternatives or of + * groups that are the only remaining element, i.e. {@code ^a$|^(?:^b$)$}, where, under {@link + * java.util.regex.Matcher#matches()} semantics, they are no-ops regardless of {@link + * Pattern#MULTILINE}. Anchors nested anywhere else, i.e. {@code (^a)b}, are not supported. + * + *

Only characters of the Basic Multilingual Plane are considered: strings containing + * surrogates, i.e. supplementary code points, are out of scope. Case-insensitive matching folds + * case the same way {@link Pattern#CASE_INSENSITIVE} and {@link Pattern#UNICODE_CASE} do, except + * for the few Unicode characters with more than one case variant. + * + * @param other regex to compare with + * @return {@code true} if language of this regex is a subset of the other's language + * @throws UnsupportedOperationException if either regex uses constructs that are not regular: + * back references, atomic groups, possessive quantifiers, lookarounds or word boundaries. + * @see #difference(Coregex) + * @since 1.5.0 + */ + public final boolean isSubsetOf(Coregex other) { + return !difference(other).isPresent(); + } + + /** + * Finds the shortest string fully matched by this regex but not by the other one. Useful as a + * counterexample when {@link #isSubsetOf(Coregex)} fails. + * + *

See {@link #isSubsetOf(Coregex)} for supported constructs. + * + * @param other regex to subtract + * @return the shortest string matched by this but not the other regex or nothing if there is none + * @throws UnsupportedOperationException if either regex uses constructs that are not regular. + * @see #isSubsetOf(Coregex) + * @since 1.5.0 + */ + public final Optional difference(Coregex other) { + return NFA.of(this).difference(NFA.of(requireNonNull(other, "other"))); + } + + /** + * Finds the shortest string fully matched by both this and the other regex. + * + *

See {@link #isSubsetOf(Coregex)} for supported constructs. + * + * @param other regex to intersect with + * @return the shortest string matched by both regexes or nothing if they are disjoint + * @throws UnsupportedOperationException if either regex uses constructs that are not regular. + * @see #isSubsetOf(Coregex) + * @since 1.5.0 + */ + public final Optional intersection(Coregex other) { + return NFA.of(this).intersection(NFA.of(requireNonNull(other, "other"))); + } + /** * Samples one random string that matches this regex. * @@ -156,16 +284,6 @@ public final String generate(long seed) { "Cannot generate string for " + this + " after " + Context.maxAttempts + " attempts"); } - final boolean matches(String input, Context ctx) { - String[] matches = match(input, ctx); - for (String match : matches) { - if (match.isEmpty()) { - return true; - } - } - return false; - } - /** * Quantify this regex. * @@ -207,20 +325,6 @@ void generate(Context ctx) { } } - @Override - String[] match(String input, Context ctx) { - Collection match = Arrays.asList(first.match(input, ctx)); - for (int i = 0; i < rest.length && !match.isEmpty(); i++) { - Collection nextMatch = new HashSet<>(match.size()); - Coregex coregex = rest[i]; - for (String tail : match) { - Collections.addAll(nextMatch, coregex.match(tail, ctx)); - } - match = nextMatch; - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -241,6 +345,29 @@ public Stream shrink() { }))); } + @Override + void compile(NFA nfa, int from, int to) { + nfa.chain(concat(), from, to); + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + List concat = concat(); + int first = 0; + int last = concat.size(); + while (first < last && concat.get(first).isNoOpAt(true)) { + first++; + } + while (first < last && concat.get(last - 1).isNoOpAt(false)) { + last--; + } + if (1 == last - first) { + concat.get(first).compileAtBoundary(nfa, from, to); + } else { + nfa.chain(concat.subList(first, last), from, to); + } + } + /** * @return underlying regexes in order of concatenation. */ @@ -383,6 +510,7 @@ public static final class Group extends Coregex { private final int index; private final String name; private final Coregex group; + private transient volatile NFA lookaround; /** * Unnamed capturing group. @@ -447,7 +575,7 @@ void generate(Context ctx) throws RewindException { positive = false; // fall through case LOOKBEHIND: - if (positive == group.matches(ctx.toString(true), ctx)) { + if (positive == lookaround().matches(ctx.toString(true))) { break; } throw new RewindException(); @@ -456,13 +584,10 @@ void generate(Context ctx) throws RewindException { // fall through case LOOKAHEAD: int length = ctx.length(true); - Coregex fullLookbehind = - 0 == length - ? group - : new Concat(any().quantify(length, length, Quantified.Type.GREEDY), group); - final boolean pos = positive; + NFA lookahead = lookaround(); + boolean pos = positive; ctx.finalizeWithLookbehind( - root -> pos == fullLookbehind.matches(root.toString(true), root)); + root -> pos == lookahead.matches(root.toString(true).substring(length))); break; default: try (Context childCtx = new Context(ctx, index, name)) { @@ -471,35 +596,61 @@ void generate(Context ctx) throws RewindException { } } + /** {@inheritDoc} */ @Override - String[] match(String input, Context ctx) { + public Stream shrink() { switch (type) { - case NON_CAPTURING: - case ATOMIC: - return group.match(input, ctx); case LOOKAHEAD: case LOOKBEHIND: case NEGATIVE_LOOKAHEAD: case NEGATIVE_LOOKBEHIND: - return new String[] {input}; + return Stream.empty(); default: - try (Context childCtx = new Context(ctx, index, name)) { - return group.match(input, childCtx); - } + return group.shrink().map(group -> new Group(type, index, name, group)); } } - /** {@inheritDoc} */ + /* + * Lookbehind body has to match a suffix of the input generated so far, lookahead body a prefix + * of the input generated after: `[\s\S]*body` and `body[\s\S]*` respectively. Compiled as is, + * since the body is not at the input boundary and anchors in it are not no-ops. + */ + private NFA lookaround() { + NFA nfa = lookaround; + if (null == nfa) { + boolean behind = Type.LOOKBEHIND == type || Type.NEGATIVE_LOOKBEHIND == type; + Coregex anything = Set.DOTALL.quantify(0, -1, Quantified.Type.GREEDY); + nfa = new NFA(); + (behind ? new Concat(anything, group) : new Concat(group, anything)) + .compile(nfa, NFA.START, NFA.ACCEPT); + lookaround = nfa; + } + return nfa; + } + @Override - public Stream shrink() { + void compile(NFA nfa, int from, int to) { switch (type) { - case LOOKAHEAD: - case LOOKBEHIND: - case NEGATIVE_LOOKAHEAD: - case NEGATIVE_LOOKBEHIND: - return Stream.empty(); + case CAPTURING: + case NAMED: + case NON_CAPTURING: + group.compile(nfa, from, to); + break; default: - return group.shrink().map(group -> new Group(type, index, name, group)); + throw new UnsupportedOperationException("group is not supported: " + this); + } + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + switch (type) { + case CAPTURING: + case NAMED: + case NON_CAPTURING: + group.compileAtBoundary(nfa, from, to); + break; + default: + compile(nfa, from, to); } } @@ -594,6 +745,116 @@ public enum Type { } } + /** + * Regex with a label, i.e. a boundary matcher such as {@code ^} or {@code $}. Behaves exactly as + * the labeled regex, but is represented by its label. + * + * @since 1.5.0 + */ + public static final class Labeled extends Coregex { + private static final long serialVersionUID = 1L; + + // (? shrink() { + return Stream.empty(); + } + + @Override + void compile(NFA nfa, int from, int to) { + // all labeled regexes are boundary matchers, which are lookarounds: boundary anchors are + // stripped by #compileAtBoundary(NFA, int, int), anything else is not regular + throw new UnsupportedOperationException(label + " is not supported"); + } + + /* + * Boundary anchors: `^` or `\A` at the start, `$`, `\z` or `\Z` at the end. Under + * Matcher#matches() semantics, `\Z` at the very end is a no-op as nothing may follow it. + */ + @Override + boolean isNoOpAt(boolean start) { + // labels are canonical: Labeled can only be created by the parser and predefined constructors + switch (label) { + case "^": + case "\\A": + return start; + case "$": + case "\\z": + case "\\Z": + return !start; + default: + return false; + } + } + + /** + * @return label + */ + public String label() { + return label; + } + + /** + * @return labeled regex + */ + public Coregex labeled() { + return labeled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Labeled that = (Labeled) o; + return label.equals(that.label) && labeled.equals(that.labeled); + } + + @Override + public int hashCode() { + return 31 * label.hashCode() + labeled.hashCode(); + } + + @Override + public String toString() { + return label; + } + } + /** Quantified regex. */ public static final class Quantified extends Coregex { @@ -665,29 +926,6 @@ void generate(Context ctx) throws RewindException { } } - @Override - String[] match(String input, Context ctx) { - int quantifier = 0, min = this.min, max = this.max; - java.util.Set match = new HashSet<>(Collections.singleton(input)); - for (; quantifier < min && !match.isEmpty(); quantifier++) { - java.util.Set nextMatch = new HashSet<>(match.size()); - for (String tail : match) { - Collections.addAll(nextMatch, quantified.match(tail, ctx)); - } - match = nextMatch; - } - for (; (-1 == max || quantifier < max) && !match.isEmpty(); quantifier++) { - java.util.Set nextMatch = new HashSet<>(match.size()); - for (String tail : match) { - Collections.addAll(nextMatch, quantified.match(tail, ctx)); - } - if (!match.addAll(nextMatch)) { - break; - } - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -700,6 +938,42 @@ public Stream shrink() { quantified.shrink().map(quantified -> new Quantified(quantified, min, max, type))); } + @Override + void compile(NFA nfa, int from, int to) { + if (Type.POSSESSIVE == type) { + throw new UnsupportedOperationException("possessive quantifier is not supported: " + this); + } + int current = from; + for (int i = 0; i < min; i++) { + int next = nfa.state(); + quantified.compile(nfa, current, next); + current = next; + } + if (-1 == max) { + // fresh loop state, as `from` may be shared and must not get incoming transitions + int loop = nfa.state(); + nfa.epsilon(current, loop); + quantified.compile(nfa, loop, loop); + nfa.epsilon(loop, to); + } else { + for (int i = min; i < max; i++) { + int next = nfa.state(); + nfa.epsilon(current, to); + quantified.compile(nfa, current, next); + current = next; + } + nfa.epsilon(current, to); + } + } + + /* + * Empty regex, i.e. inline flags like `(?m)` that may precede anchors. + */ + @Override + boolean isNoOpAt(boolean start) { + return EMPTY.equals(this); + } + /** * @return quantified regex */ @@ -826,18 +1100,17 @@ void generate(Context ctx) { ctx.append(ctx.ref(ref)); } - @Override - String[] match(String input, Context ctx) { - String ref = ctx.ref(this.ref); - return input.startsWith(ref) ? new String[] {input.substring(ref.length())} : new String[0]; - } - /** {@inheritDoc} */ @Override public Stream shrink() { return Stream.empty(); } + @Override + void compile(NFA nfa, int from, int to) { + throw new UnsupportedOperationException("back reference is not supported: " + this); + } + public Serializable ref() { return ref; } @@ -888,16 +1161,6 @@ void generate(Context ctx) throws RewindException { (index < rest.length ? rest[index] : first).generate(ctx); } - @Override - String[] match(String input, Context ctx) { - Collection match = new HashSet<>(rest.length + 1); - Collections.addAll(match, first.match(input, ctx)); - for (Coregex coregex : rest) { - Collections.addAll(match, coregex.match(input, ctx)); - } - return match.toArray(new String[0]); - } - /** {@inheritDoc} */ @Override public Stream shrink() { @@ -927,6 +1190,22 @@ public Stream shrink() { })))); } + @Override + void compile(NFA nfa, int from, int to) { + first.compile(nfa, from, to); + for (Coregex coregex : rest) { + coregex.compile(nfa, from, to); + } + } + + @Override + void compileAtBoundary(NFA nfa, int from, int to) { + first.compileAtBoundary(nfa, from, to); + for (Coregex coregex : rest) { + coregex.compileAtBoundary(nfa, from, to); + } + } + /** * @return underlying regexes forming this unification. */ diff --git a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java index 44f1cf1..d887241 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java +++ b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java @@ -184,11 +184,11 @@ private Coregex elementaryRE(Context ctx) { break; case '^': ctx.match('^'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty()); + elementaryRE = Coregex.lineStart(); break; case '$': ctx.match('$'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty()); + elementaryRE = Coregex.lineEnd(); break; case '\\': switch (ctx.peek(2)) { @@ -198,7 +198,7 @@ private Coregex elementaryRE(Context ctx) { case 'A': ctx.match('\\'); ctx.match('A'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty()); + elementaryRE = Coregex.inputStart(); break; case 'b': ctx.match('\\'); @@ -223,15 +223,12 @@ private Coregex elementaryRE(Context ctx) { case 'z': ctx.match('\\'); ctx.match('z'); - elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty()); + elementaryRE = Coregex.inputEnd(); break; case 'Z': ctx.match('\\'); ctx.match('Z'); - elementaryRE = - new Coregex.Group( - Coregex.Group.Type.LOOKAHEAD, - new Coregex.Union(Coregex.empty(), Set.single('\n'))); + elementaryRE = Coregex.inputEndBeforeTerminator(); break; case 'G': throw ctx.unsupported("metacharacter \\G"); @@ -519,6 +516,8 @@ private Set metachar(Context ctx) { ctx.match('\\'); char ch = ctx.peek(); Set.Builder metachar = Set.builder(ctx.flags); + // char classes that are ASCII-only here, but Unicode with UNICODE_CHARACTER_CLASS + boolean asciiOnly = 0 <= "dDwWsS".indexOf(ch); switch (ch) { case 't': ctx.match('t'); @@ -600,6 +599,7 @@ private Set metachar(Context ctx) { ctx.match('p'); ctx.match('{'); String posix = ctx.span('}'); + asciiOnly = !"all".equals(posix) && !"ASCII".equals(posix) && !posix.startsWith("java"); switch (posix) { case "all": metachar.matching(any -> true, "\\p{all}"); @@ -665,6 +665,9 @@ private Set metachar(Context ctx) { metachar.single(ch); break; } + if (asciiOnly && 0 != (ctx.flags & Pattern.UNICODE_CHARACTER_CLASS)) { + metachar.approximated(); + } return metachar.build(); } diff --git a/core/src/main/java/com/github/simy4/coregex/core/NFA.java b/core/src/main/java/com/github/simy4/coregex/core/NFA.java new file mode 100644 index 0000000..42a8624 --- /dev/null +++ b/core/src/main/java/com/github/simy4/coregex/core/NFA.java @@ -0,0 +1,385 @@ +/* + * Copyright 2021-2026 Alex Simkin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.simy4.coregex.core; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; + +/* + * Non-deterministic finite automaton built from Coregex using Thompson's construction. Every + * Coregex wires itself between given states, see Coregex#compile(NFA, int, int). + * Fragments never add edges into `from` or out of `to`, so fragments can safely share endpoints + * (union) without leaking paths between each other. + * + * Unlike Coregex itself, which is free to approximate a regex as long as everything it generates + * matches, an NFA has to accept exactly the language of the regex. Therefore, constructs that are + * not regular are rejected with UnsupportedOperationException. + * + * The alphabet is UTF-16 chars except surrogates, see Set#DOTALL. + * + * Language operations are decided exactly by a lazy breadth-first walk of the product automaton, + * so whenever a witness exists, the shortest one is returned. + */ +final class NFA { + + static final int START = 0; + static final int ACCEPT = 1; + + static NFA of(Coregex coregex) { + NFA nfa = new NFA(); + coregex.compileAtBoundary(nfa, START, ACCEPT); + return nfa; + } + + private final List> epsilons = new ArrayList<>(); + private final List> edges = new ArrayList<>(); + + NFA() { + state(); // START + state(); // ACCEPT + } + + // region construction + int state() { + epsilons.add(new ArrayList<>()); + edges.add(new ArrayList<>()); + return epsilons.size() - 1; + } + + void epsilon(int from, int to) { + if (from != to) { + epsilons.get(from).add(to); + } + } + + /* + * Label is shared, not copied: automaton never mutates it. + */ + void edge(int from, BitSet label, int to) { + edges.get(from).add(new Edge(label, to)); + } + + void chain(List chunks, int from, int to) { + int current = from; + for (int i = 0; i < chunks.size(); i++) { + int next = chunks.size() - 1 == i ? to : state(); + chunks.get(i).compile(this, current, next); + current = next; + } + if (chunks.isEmpty()) { + epsilon(from, to); + } + } + + // endregion + + boolean matches(CharSequence input) { + BitSet current = initial(); + for (int i = 0; i < input.length() && !current.isEmpty(); i++) { + current = step(current, input.charAt(i)); + } + return current.get(ACCEPT); + } + + /* + * Shortest string accepted by this but not the other automaton. + */ + Optional difference(NFA other) { + return search(other, false); + } + + /* + * Shortest string accepted by both this and the other automaton. + */ + Optional intersection(NFA other) { + return search(other, true); + } + + /* + * Breadth-first search over pairs of (single state of this NFA, set of states of other NFA). + * Only the other side is (lazily) determinized, as only it needs to be complemented. + * + * otherAccepts: whether other has to accept (intersection) or reject (difference) the witness. + * + * Pairs are pruned with antichains: for difference, (state, others) is subsumed by an already + * visited (state, subset of others), as every word rejected from the bigger set is rejected from + * the smaller one too. Symmetrically for intersection with supersets. Being breadth-first, the + * subsuming pair is never deeper, so the first witness found is still the shortest one. + */ + private Optional search(NFA other, boolean otherAccepts) { + char[] alphabet = alphabet(other); + Frontier frontier = new Frontier(edges.size(), otherAccepts); + frontier.offer(initial(), other.initial(), null, '\0'); + for (Node node = frontier.poll(); null != node; node = frontier.poll()) { + if (ACCEPT == node.state && otherAccepts == node.otherStates.get(ACCEPT)) { + return Optional.of(node.witness()); + } + for (char ch : alphabet) { + BitSet next = step(singleton(node.state), ch); + if (next.isEmpty()) { + continue; + } + BitSet nextOther = other.step(node.otherStates, ch); + if (otherAccepts && nextOther.isEmpty()) { + continue; // other can never accept from here on + } + frontier.offer(next, nextOther, node, ch); + } + } + return Optional.empty(); + } + + /* + * Splits all characters on edge labels of both automatons into equivalence classes (minterms): + * characters that are indistinguishable by every edge. Each class is then represented by its + * simplest character, so that witnesses are readable. + */ + private char[] alphabet(NFA other) { + java.util.Set labels = new LinkedHashSet<>(); + collectLabels(labels); + other.collectLabels(labels); + BitSet universe = new BitSet(); + for (BitSet label : labels) { + universe.or(label); + } + List blocks = new ArrayList<>(); + if (!universe.isEmpty()) { + blocks.add(universe); + } + for (BitSet label : labels) { + List refined = new ArrayList<>(blocks.size() * 2); + for (BitSet block : blocks) { + BitSet in = (BitSet) block.clone(); + in.and(label); + if (in.isEmpty() || in.equals(block)) { + refined.add(block); + } else { + BitSet out = (BitSet) block.clone(); + out.andNot(label); + refined.add(in); + refined.add(out); + } + } + blocks = refined; + } + char[] alphabet = new char[blocks.size()]; + for (int i = 0; i < alphabet.length; i++) { + alphabet[i] = simplest(blocks.get(i)); + } + Arrays.sort(alphabet); + return alphabet; + } + + /* + * Simplest of given chars, in the same order of preference as Set#shrink(): digits, lowercase and + * uppercase letters, then printable ASCII, then anything else. + */ + private static char simplest(BitSet chars) { + for (char[] range : new char[][] {{'0', '9'}, {'a', 'z'}, {'A', 'Z'}, {' ', '~'}}) { + int ch = chars.nextSetBit(range[0]); + if (0 <= ch && ch <= range[1]) { + return (char) ch; + } + } + return (char) chars.nextSetBit(0); + } + + private void collectLabels(java.util.Set labels) { + for (List stateEdges : edges) { + for (Edge edge : stateEdges) { + labels.add(edge.label); + } + } + } + + private BitSet initial() { + return closure(singleton(START)); + } + + private BitSet step(BitSet states, char ch) { + BitSet next = new BitSet(edges.size()); + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + for (Edge edge : edges.get(state)) { + if (edge.label.get(ch)) { + next.set(edge.target); + } + } + } + return closure(next); + } + + private BitSet closure(BitSet states) { + Deque stack = new ArrayDeque<>(); + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + stack.push(state); + } + while (!stack.isEmpty()) { + for (int target : epsilons.get(stack.pop())) { + if (!states.get(target)) { + states.set(target); + stack.push(target); + } + } + } + return states; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int state = 0; state < edges.size(); state++) { + sb.append(state); + if (START == state) { + sb.append(" (start)"); + } else if (ACCEPT == state) { + sb.append(" (accept)"); + } + for (Edge edge : edges.get(state)) { + sb.append(" --"); + appendLabel(sb, edge.label); + sb.append("--> ").append(edge.target); + } + for (int target : epsilons.get(state)) { + sb.append(" --ε--> ").append(target); + } + sb.append('\n'); + } + return sb.toString(); + } + + private static void appendLabel(StringBuilder sb, BitSet label) { + BitSet negated = new BitSet(); + negated.set(Character.MIN_VALUE, Character.MAX_VALUE + 1); + negated.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1); + negated.andNot(label); + boolean negate = ranges(negated) < ranges(label); + BitSet chars = negate ? negated : label; + sb.append(negate ? "[^" : "["); + for (int start = chars.nextSetBit(0); start >= 0; ) { + int end = chars.nextClearBit(start) - 1; + sb.append((char) start); + if (start < end) { + sb.append(start + 1 < end ? "-" : ""); + sb.append((char) end); + } + start = chars.nextSetBit(end + 1); + } + sb.append(']'); + } + + private static int ranges(BitSet chars) { + int ranges = 0; + for (int ch = chars.nextSetBit(0); ch >= 0; ch = chars.nextSetBit(chars.nextClearBit(ch))) { + ranges++; + } + return ranges; + } + + private static BitSet singleton(int state) { + BitSet set = new BitSet(state + 1); + set.set(state); + return set; + } + + private static final class Edge { + private final BitSet label; + private final int target; + + private Edge(BitSet label, int target) { + this.label = label; + this.target = target; + } + } + + /* + * Breadth-first queue of pairs, deduplicated with an antichain: per state of this NFA, a set of + * pairwise incomparable sets of states of the other NFA, see #search(NFA, boolean). + */ + private static final class Frontier { + private final Deque queue = new ArrayDeque<>(); + private final List> visited; + private final boolean supersets; + + private Frontier(int states, boolean supersets) { + this.visited = new ArrayList<>(states); + for (int i = 0; i < states; i++) { + visited.add(new ArrayList<>()); + } + this.supersets = supersets; + } + + /* + * Enqueues (state, others) for every given state, unless subsumed by an already visited pair. + */ + private void offer(BitSet states, BitSet others, Node parent, char ch) { + for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) { + List antichain = visited.get(state); + if (antichain.stream().noneMatch(seen -> subsumes(seen, others))) { + antichain.removeIf(seen -> subsumes(others, seen)); + antichain.add(others); + queue.add(new Node(state, others, parent, ch)); + } + } + } + + private Node poll() { + return queue.poll(); + } + + /* + * Difference needs the other NFA to reject, so smaller sets subsume. Intersection needs it to + * accept, so bigger sets do. + */ + private boolean subsumes(BitSet subsuming, BitSet subsumed) { + return supersets ? isSubset(subsumed, subsuming) : isSubset(subsuming, subsumed); + } + + private static boolean isSubset(BitSet sub, BitSet sup) { + BitSet difference = (BitSet) sub.clone(); + difference.andNot(sup); + return difference.isEmpty(); + } + } + + private static final class Node { + private final int state; + private final BitSet otherStates; + private final Node parent; + private final char ch; + + private Node(int state, BitSet otherStates, Node parent, char ch) { + this.state = state; + this.otherStates = otherStates; + this.parent = parent; + this.ch = ch; + } + + private String witness() { + StringBuilder sb = new StringBuilder(); + for (Node node = this; null != node.parent; node = node.parent) { + sb.append(node.ch); + } + return sb.reverse().toString(); + } + } +} diff --git a/core/src/main/java/com/github/simy4/coregex/core/Set.java b/core/src/main/java/com/github/simy4/coregex/core/Set.java index f8be9d7..f058587 100644 --- a/core/src/main/java/com/github/simy4/coregex/core/Set.java +++ b/core/src/main/java/com/github/simy4/coregex/core/Set.java @@ -36,7 +36,7 @@ public final class Set extends Coregex implements Serializable { private static final long serialVersionUID = 1L; - static final Set EMPTY = new Set(new BitSet(0), ""); + static final Set EMPTY = new Set(new BitSet(0), "", false); /* * All chars except surrogates, which only exist in pairs: supplementary code points are out of @@ -50,16 +50,16 @@ public final class Set extends Coregex implements Serializable { BitSet dotAll = new BitSet(Character.MAX_VALUE + 1); dotAll.set(Character.MIN_VALUE, Character.MAX_VALUE + 1); dotAll.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1); - DOTALL = new Set(dotAll, "."); + DOTALL = new Set(dotAll, ".", false); BitSet unixLines = (BitSet) dotAll.clone(); unixLines.clear('\n'); - UNIX_LINES = new Set(unixLines, "."); + UNIX_LINES = new Set(unixLines, ".", false); BitSet all = (BitSet) unixLines.clone(); all.clear('\r'); all.clear('\u0085'); all.clear('\u2028'); all.clear('\u2029'); - ALL = new Set(all, "."); + ALL = new Set(all, ".", false); } public static Set any(int flags) { @@ -125,9 +125,17 @@ public static Set single(char ch, int flags) { private final BitSet chars; private final String description; - private Set(BitSet chars, String description) { + /* + * Whether chars are only an approximation of the actual char class, i.e. ASCII-only \w for + * UNICODE_CHARACTER_CLASS. Good enough for generation, as all generated chars still match, but not + * for exact language operations. + */ + private final boolean approximated; + + private Set(BitSet chars, String description, boolean approximated) { this.chars = chars; this.description = description; + this.approximated = approximated; } @Override @@ -138,13 +146,6 @@ void generate(Context ctx) { } } - @Override - String[] match(String input, Context ctx) { - return !input.isEmpty() && chars.get(input.charAt(0)) - ? new String[] {input.substring(1)} - : new String[0]; - } - boolean single() { return 1 == chars.cardinality(); } @@ -168,7 +169,18 @@ public Stream shrink() { return builder.build(); } - /** + @Override + void compile(NFA nfa, int from, int to) { + if (approximated) { + throw new UnsupportedOperationException( + "char class with UNICODE_CHARACTER_CLASS is not supported: " + this); + } + if (!chars.isEmpty()) { + nfa.edge(from, chars, to); + } + } + + /* * @return sampled char or -1 if this set is empty. */ int sample(long seed) { @@ -217,6 +229,7 @@ public static final class Builder { private final int flags; private final BitSet chars; private final StringBuilder description = new StringBuilder(); + private boolean approximated; private Builder(int flags, int size) { this.flags = flags; @@ -267,6 +280,7 @@ public Builder set(char first, char... rest) { */ public Builder union(Set set) { chars.or(requireNonNull(set, "set").chars); + approximated |= set.approximated; description.append(set); return this; } @@ -279,6 +293,7 @@ public Builder union(Set set) { */ public Builder intersect(Set set) { chars.and(requireNonNull(set, "set").chars); + approximated |= set.approximated; description.append("&&").append(set); return this; } @@ -296,6 +311,14 @@ public Builder single(char ch) { return this; } + /* + * Marks this set as only an approximation of the actual char class, see Set#approximated. + */ + Builder approximated() { + approximated = true; + return this; + } + /* * Adds all chars matching given predicate, i.e. Character::isLowerCase. */ @@ -341,7 +364,7 @@ public Builder negate() { * @return compiled set */ public Set build() { - return new Set(chars, description.toString()); + return new Set(chars, description.toString(), approximated); } } } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala index 3e10fa9..ba57b1e 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala @@ -79,6 +79,14 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { Pattern.compile("^(?:||)$"), Pattern.compile("(\\d+) (\\w+) == (\\1) \\2 \\3"), Pattern.compile("(?!=.{10,}).+"), + Pattern.compile("(?!a)[ab]b"), + Pattern.compile("(?=a)[ab]b"), + Pattern.compile("b[ab](?<=a)"), + Pattern.compile("[ab]{2}(? coregex.matches(ch, null) != pattern.matcher(ch).matches()) + .filter(ch => NFA.of(coregex).matches(ch) != pattern.matcher(ch).matches()) assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...") } } @@ -189,12 +207,20 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries { ) } + test("anchors inside lookarounds are not at the input boundary") { + List("a(?=^)b", "a(?<=$)b", "a(?<=\\z)b", "a(?<=\\Z)b", "(?<=^)a", "a(?=$)").foreach { regex => + val _ = intercept[UnsupportedOperationException](Coregex.from(Pattern.compile(regex)).generate(0L)) + } + } + property("should fail to generate strings") { forAll( Gen.oneOf( Pattern.compile("(?=(abc))"), Pattern.compile("\\d+(?= dollars)"), - Pattern.compile("(?<=[a-z0-9])(?=[A-Z])") + Pattern.compile("(?<=[a-z0-9])(?=[A-Z])"), + Pattern.compile("ba(? diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala index 96e87b0..530908f 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala @@ -44,7 +44,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { } test("empty should match empty") { - assert(Coregex.empty().matches("", null)) + assert(NFA.of(Coregex.empty()).matches("")) } property("literal don't shrink") { @@ -55,7 +55,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { property("literal should match self") { forAll { (literal: String, flags: Int) => - Coregex.literal(literal, flags).matches(literal, null) :| s"$literal match self" + NFA.of(Coregex.literal(literal, flags)).matches(literal) :| s"$literal match self" } } @@ -89,7 +89,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (head: Simple, tail: Seq[Simple], seed: Long) => val concat = new Concat(head.value, tail.map(_.value): _*) val generated = concat.generate(seed) - concat.matches(generated, null) :| s"$generated in $concat" + NFA.of(concat).matches(generated) :| s"$generated in $concat" } } // endregion @@ -134,7 +134,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (head: Simple, tail: Seq[Simple], seed: Long) => val union = new Union(head.value, tail.map(_.value): _*) val generated = union.generate(seed) - union.matches(generated, null) :| s"$generated in ($union)" + NFA.of(union).matches(generated) :| s"$generated in ($union)" } } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala new file mode 100644 index 0000000..047260d --- /dev/null +++ b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala @@ -0,0 +1,338 @@ +/* + * Copyright 2021-2026 Alex Simkin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.simy4.coregex.core + +import munit.ScalaCheckSuite +import org.scalacheck.{ Arbitrary, Gen } +import org.scalacheck.Prop._ + +import java.util.Optional +import java.util.regex.Pattern + +class NFASuite extends ScalaCheckSuite with CoregexArbitraries { + + def genPatternPairs(): Gen[(Pattern, Pattern)] = Gen.oneOf( + List( + Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}") -> + Pattern.compile("[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}"), + Pattern.compile("((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])") -> + Pattern.compile("(?:\\d{1,3}\\.){3}\\d{1,3}"), + Pattern.compile("#([0-9a-fA-F]{3}){1,2}") -> Pattern.compile("#[0-9a-fA-F]{3,6}"), + Pattern.compile("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d([+-][0-2]\\d:[0-5]\\d|Z)") -> + Pattern.compile("[\\dTZ:+-]+"), + Pattern.compile("4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}") -> Pattern.compile("\\d{13,16}"), + Pattern.compile( + "^arn:(?\\w+):(?\\w+):(?[\\w-]+):(?\\d{12}):(?(?[\\w-]+)[:\\/])?(?[\\w.-]+)$" + ) -> Pattern.compile("arn:[\\w.:/-]+"), + Pattern.compile( + "^(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)(?:-(?(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + ) -> Pattern.compile("[0-9a-zA-Z.+-]+"), + Pattern.compile("\\Qhttps://\\E(?foo|bar|baz)\\Q.com/\\E(?[\\w-]+)") -> + Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"), + // `\w*` alone allows empty, but concatenated with `\w` it does not. + Pattern.compile("(?\\w*)(?\\w)") -> Pattern.compile("[\\w-]+"), + Pattern.compile("^(a+)+$") -> Pattern.compile("a+"), + Pattern.compile("^([a-zA-Z]+)*$") -> Pattern.compile("[a-zA-Z]*"), + Pattern.compile("^(a|aa)+$") -> Pattern.compile("a+"), + Pattern.compile("^(a|a?)+$") -> Pattern.compile("a*"), + Pattern.compile("^(?:||)$") -> Pattern.compile("") + ) + ) + + implicit val arbPatternPairs: Arbitrary[(Pattern, Pattern)] = Arbitrary(genPatternPairs()) + + private def re(regex: String): Coregex = Coregex.from(Pattern.compile(regex)) + + private def nfa(regex: String): NFA = NFA.of(re(regex)) + + property("should cover example regex") { + forAll { (pair: (Pattern, Pattern), seed: Long) => + val (sub, sup) = pair + val subset = Coregex.from(sub) + val superset = Coregex.from(sup) + val generated = subset.generate(seed) + val witness = superset.difference(subset) + (subset.isSubsetOf(superset) :| s"${sub.pattern()} should be a subset of ${sup.pattern()}") && + (sup.matcher(generated).matches() :| s"${sup.pattern()} should match generated: $generated") && + (subset.intersection(superset).isPresent :| s"${sub.pattern()} should intersect ${sup.pattern()}") && + ((!witness.isPresent || sup.matcher(witness.get).matches() && !sub.matcher(witness.get).matches()) :| + s"${sup.pattern()} but not ${sub.pattern()} should match witness: $witness") + } + } + + // region language operations + test("possibly empty fragment is not a subset of non empty one") { + assertEquals(re("(?\\w*)").difference(re("[\\w-]+")), Optional.of("")) + } + + test("dot fragment is not a subset of word characters") { + val witness = re("(?.+)").difference(re("[\\w-]+")) + assert(witness.isPresent && 1 == witness.get.length, s"$witness should be a single char") + assert(!Pattern.matches("[\\w-]+", witness.get), s"${witness.get} should not match [\\w-]+") + } + + test("intersection of disjoint languages is empty") { + assertEquals(re("[a-z]+").intersection(re("\\d+")), Optional.empty[String]()) + } + + test("intersection returns the shortest common string") { + assertEquals(re("[a-z]+").intersection(re("x.*")), Optional.of("x")) + assertEquals(re("a{3}b?").intersection(re("a{3}b?")), Optional.of("aaa")) + assertEquals(re("[a&&b]").intersection(re(".*")), Optional.empty[String]()) + } + + test("toString shows states with transitions") { + assertEquals( + nfa("a|b*").toString, + """0 (start) --[a]--> 1 --ε--> 2 + |1 (accept) + |2 --[b]--> 2 --ε--> 1 + |""".stripMargin + ) + assert(nfa(".[^ab]").toString.startsWith("0 (start) --[^\n\r\u0085\u2028\u2029]--> 2\n")) + } + + test("witnesses use the simplest chars, same as shrinking") { + assertEquals(re("#[0-9a-fA-F]{3,6}").difference(re("#(?:[0-9a-fA-F]{3}){1,2}")), Optional.of("#0000")) + assertEquals(re("[a-zA-Z]+").difference(re("[a-z]+")), Optional.of("A")) + assertEquals(re("[\\s\\u00E9]").intersection(re("\\W")), Optional.of(" ")) + assertEquals(re("\\W").difference(re("[ -~]")), Optional.of("\u0000")) + } + + test("empty regex matches empty string only") { + assertEquals(re("(?:)").intersection(re("")), Optional.of("")) + assertEquals(re("a(?:)b").intersection(re("ab")), Optional.of("ab")) + assertEquals(re("(?:a|)").difference(re("a")), Optional.of("")) + } + + // (sub, sup, input matched by sub only if sub is not a subset of sup) + List( + ("[^a]", "[^a\\n]", Some("\n")), + ("\\n", "\\D", None), + ("\\n", "[^\\w]", None), + ("\\uFF01", ".", None), + ("(?s).", "[^\\uD800-\\uDFFF]", None), + ("\u00C9", "(?i)\u00E9", Some("\u00C9")), + ("\u00C9", "(?iu)\u00E9", None), + ("K", "(?i)k", None), + ("a\\Z", "a", None), + ("a", "a\\Z", None) + ).foreach { case (sub, sup, witness) => + test(s"$sub is ${if (witness.isEmpty) "" else "not "}a subset of $sup like in java.util.regex") { + witness.foreach { w => + assert(Pattern.matches(sub, w) && !Pattern.matches(sup, w), s"$w should only match $sub") + } + assertEquals(re(sub).difference(re(sup)), witness.fold(Optional.empty[String]())(Optional.of(_))) + } + } + + List( + ".", + "(?s).", + "(?d).", + "[^a]", + "[^\\n]", + "\\d", + "\\D", + "\\w", + "\\W", + "\\s", + "\\S", + "\\h", + "\\H", + "\\v", + "\\V", + "\\t", + "\\f", + "\\a", + "\\e", + "\\cA", + "\\p{all}", + "\\p{Lower}", + "\\p{Upper}", + "\\p{ASCII}", + "\\p{Digit}", + "\\p{XDigit}", + "\\p{Alnum}", + "\\p{Alpha}", + "\\p{Punct}", + "\\p{Graph}", + "\\p{Print}", + "\\p{Blank}", + "\\p{Space}", + "\\p{javaWhitespace}", + "\\p{javaLowerCase}", + "\\p{javaUpperCase}", + "(?i)[a-z]", + "(?i)\u00E9", + "(?iu)\u00E9", + "(?iU)\u00E9", + "(?U)\\p{ASCII}", + "[\\w&&[^\\d]]" + ).foreach { regex => + test(s"$regex should accept the same chars as java.util.regex") { + val pattern = Pattern.compile(regex) + val nfa = NFA.of(Coregex.from(pattern)) + val wrong = (Character.MIN_VALUE to Character.MAX_VALUE) + .filterNot(Character.isSurrogate) + .map(_.toString) + .filter(ch => nfa.matches(ch) != pattern.matcher(ch).matches()) + assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...") + } + } + + test("unicode character class only affects predefined char classes") { + assert(re("(?U)abc").isSubsetOf(re("abc"))) + assert(re("(?U)[a-c]+").isSubsetOf(re("\\w+"))) + } + + test("reluctant and greedy quantifiers have the same language") { + assert(re("a+?b*?").isSubsetOf(re("a+b*"))) + assert(re("a+b*").isSubsetOf(re("a+?b*?"))) + } + + test("nested stars") { + val nested = nfa("(?:a*b*)*") + assert(nested.matches("")) + assert(nested.matches("abba")) + assert(!nested.matches("abc")) + assert(re("(?:a*b*)*").isSubsetOf(re("[ab]*"))) + assert(re("[ab]*").isSubsetOf(re("(?:a*b*)*"))) + } + + test("star inside union does not leak") { + val union = nfa("a*|b") + assert(!union.matches("bb")) + assert(!union.matches("ab")) + } + + List( + "^abc$", + "\\Aabc\\z", + "(?m)^abc$", + "(^abc$)", + "^(?:abc)$", + "^^abc$$", + "^a$|^bc$|(?:^ab)", + "(?:^a|^bc)$", + "^abc\\Z", + "^(?:^abc$)$", + "^(?:a|^bc)$", + "(?m)(?i)^abc$" + ).foreach { regex => + test(s"boundary anchors in $regex are no-ops") { + val pattern = Pattern.compile(regex) + val nfa = NFA.of(Coregex.from(pattern)) + List("", "a", "ab", "abc", "bc", "abcd", "\n", "abc\n").foreach { input => + assertEquals(nfa.matches(input), pattern.matcher(input).matches(), s"$input in $regex") + } + } + } + // endregion + + // region unsupported constructs + List( + "(a)\\1", + "(?a)\\k", + "(?>a*)", + "a*+", + "(?=a)a", + "(?!a)b", + "(?<=a)b", + "(? + test(s"$regex is not supported") { + val _ = intercept[UnsupportedOperationException](re(regex).isSubsetOf(re("a"))) + val _ = intercept[UnsupportedOperationException](re("a").isSubsetOf(re(regex))) + } + } + // endregion + + // region properties + private val supportedRange: Gen[QuantifyRange] = + for { + min <- Gen.choose(0, 5) + add <- Gen.choose(0, 5) + max <- Gen.oneOf( + Gen.const(-1), + Gen.choose(min, min + add) + ) + } yield QuantifyRange(min, max, Coregex.Quantified.Type.GREEDY) + private val genSupported: Gen[Coregex] = Gen.resize(20, genCoregex(supportedRange)) + + property("nfa should accept generated strings") { + forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) => + val generated = coregex.generate(seed) + NFA.of(coregex).matches(generated) :| s"$generated in $coregex" + } + } + + property("every coregex is a subset of itself and of its union") { + forAll(genSupported, genSupported) { (left, right) => + (left.isSubsetOf(left) :| "reflexive") && + (left.isSubsetOf(new Coregex.Union(left, right)) :| "union") + } + } + + property("difference witness should be accepted by left and rejected by right") { + forAll(genSupported, genSupported) { (left, right) => + val witness = left.difference(right) + witness.isPresent ==> { + (NFA.of(left).matches(witness.get) :| s"${witness.get} in $left") && + (!NFA.of(right).matches(witness.get) :| s"${witness.get} not in $right") + } + } + } + + property("difference should find the shortest witness if any exists") { + forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, right, seed) => + val generated = left.generate(seed) + !NFA.of(right).matches(generated) ==> { + val witness = left.difference(right) + (witness.isPresent :| s"$generated in $left but not in $right") && + (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" + } + } + } + + property("intersection should find the shortest witness if any exists") { + forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, other, seed) => + val right = new Coregex.Union(other, left) + val generated = left.generate(seed) + val witness = left.intersection(right) + (witness.isPresent :| s"$generated in both $left and $right") && { + (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" && + (NFA.of(left).matches(witness.get) && NFA.of(right).matches(witness.get)) :| s"${witness.get} in both" + } + } + } + // endregion +} diff --git a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala index 668bead..483484e 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala @@ -31,7 +31,9 @@ object QuantifyRange { Gen.const(-1), Gen.choose(min, min + add) ) - tpe <- Gen.oneOf(Coregex.Quantified.Type.values.toIndexedSeq) + // possessive generates the same as greedy, but is not regular, so generated strings could not be + // verified with NFA + tpe <- Gen.oneOf(Coregex.Quantified.Type.GREEDY, Coregex.Quantified.Type.RELUCTANT) } yield QuantifyRange(min, max, tpe) } diff --git a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala index 1bca01d..66538dd 100644 --- a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala +++ b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala @@ -29,7 +29,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (set: Set, seed: Long) => val generated = set.generate(seed) - val inSetCheck = set.matches(generated, null) :| s"$generated in $set" + val inSetCheck = NFA.of(set).matches(generated) :| s"$generated in $set" val lengthCheck = (generated.length ?= 1) :| s"$generated.length == 1" inSetCheck && lengthCheck @@ -37,11 +37,11 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { } property("quantified generated should be in set") { - forAll { (set: Set, range: QuantifyRange, `type`: Coregex.Quantified.Type, seed: Long) => - val quantified = set.quantify(range.min, range.max, `type`) + forAll { (set: Set, range: QuantifyRange, seed: Long) => + val quantified = set.quantify(range.min, range.max, range.`type`) val generated = quantified.generate(seed) - quantified.matches(generated, null) :| s"$generated in $set" + NFA.of(quantified).matches(generated) :| s"$generated in $set" } } @@ -70,7 +70,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { forAll { (left: Set, right: Set, seed: Long) => val union = Set.builder().union(left).union(right).build() val generated = union.generate(seed) - (left.matches(generated, null) || right.matches(generated, null)) :| s"$generated in $union" + (NFA.of(left).matches(generated) || NFA.of(right).matches(generated)) :| s"$generated in $union" } } @@ -79,7 +79,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { val leftWithCommon = Set.builder().union(left).single(right.sample(seed1).toChar).build() val intersection = Set.builder().union(leftWithCommon).intersect(right).build() val generated = intersection.generate(seed2) - (leftWithCommon.matches(generated, null) && right.matches(generated, null)) :| s"$generated in $intersection" + (NFA.of(leftWithCommon).matches(generated) && NFA.of(right).matches(generated)) :| s"$generated in $intersection" } } @@ -126,7 +126,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries { .build() .generate(seed) - set.matches(result.toLowerCase(Locale.ENGLISH), null) || set.matches(result.toUpperCase(Locale.ENGLISH), null) + NFA.of(set).matches(result.toLowerCase(Locale.ENGLISH)) || NFA.of(set).matches(result.toUpperCase(Locale.ENGLISH)) } } diff --git a/version.sbt b/version.sbt index edb9358..c919701 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "1.4.6-SNAPSHOT" +ThisBuild / version := "1.5.0-SNAPSHOT"