diff --git a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java
index 561db87..0e5f956 100644
--- a/core/src/main/java/com/github/simy4/coregex/core/Coregex.java
+++ b/core/src/main/java/com/github/simy4/coregex/core/Coregex.java
@@ -21,10 +21,8 @@
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -44,6 +42,7 @@
*
* @see Coregex.Concat
* @see Coregex.Group
+ * @see Coregex.Labeled
* @see Coregex.Quantified
* @see Coregex.Ref
* @see Set
@@ -110,25 +109,65 @@ public static Coregex literal(String literal, int flags) {
return new Concat(first, rest);
}
+ /**
+ * @return predefined constructor for {@code ^} boundary matcher.
+ */
+ public static Labeled lineStart() {
+ return Labeled.LINE_START;
+ }
+
+ /**
+ * @return predefined constructor for {@code $} boundary matcher.
+ */
+ public static Labeled lineEnd() {
+ return Labeled.LINE_END;
+ }
+
+ /**
+ * @return predefined constructor for {@code \A} boundary matcher.
+ */
+ public static Labeled inputStart() {
+ return Labeled.INPUT_START;
+ }
+
+ /**
+ * @return predefined constructor for {@code \z} boundary matcher.
+ */
+ public static Labeled inputEnd() {
+ return Labeled.INPUT_END;
+ }
+
+ /**
+ * @return predefined constructor for {@code \Z} boundary matcher.
+ * @since 1.5.0
+ */
+ public static Labeled inputEndBeforeTerminator() {
+ return Labeled.INPUT_END_BEFORE_TERMINATOR;
+ }
+
+ /**
+ * @param flags regex flags
+ * @param positive {@code true} for {@code \b}, {@code false} for {@code \B}
+ * @return predefined constructor for word boundary matcher.
+ */
public static Coregex wordBoundary(int flags, boolean positive) {
Set word =
Set.builder(flags).range('a', 'z').range('A', 'Z').range('0', '9').single('_').build();
- Set notWord = Set.builder(flags).union(word).negate().build();
- Group.Type lookbehind = positive ? Group.Type.LOOKBEHIND : Group.Type.NEGATIVE_LOOKBEHIND;
- Group.Type lookahead = positive ? Group.Type.LOOKAHEAD : Group.Type.NEGATIVE_LOOKAHEAD;
- return new Union(
- new Concat(new Group(lookbehind, empty()), new Group(lookahead, word)),
- new Concat(new Group(lookbehind, word), new Group(lookahead, empty())),
- new Concat(new Group(lookbehind, word), new Group(lookahead, notWord)),
- new Concat(new Group(lookbehind, notWord), new Group(lookahead, word)));
+ // \b: (?<=\w)(?!\w)|(? shrink();
+ /*
+ * Thompson's construction: wires this regex between `from` and `to` such that paths from `from`
+ * to `to` spell exactly the language of this regex. Boundary anchors are stripped beforehand,
+ * see #compileAtBoundary(NFA, int, int).
+ */
+ abstract void compile(NFA nfa, int from, int to);
+
+ /*
+ * Same as #compile(NFA, int, int), for a regex spanning the whole input. Under Matcher#matches()
+ * semantics, anchors at the input boundary are no-ops (regardless of MULTILINE), so they are
+ * stripped from the very start and end of the regex, looking through alternatives and transparent
+ * groups. Anchors anywhere else are rejected by Labeled#compile.
+ *
+ * A single regex is at both boundaries at once.
+ */
+ void compileAtBoundary(NFA nfa, int from, int to) {
+ if (isNoOpAt(true) || isNoOpAt(false)) {
+ nfa.epsilon(from, to);
+ } else {
+ compile(nfa, from, to);
+ }
+ }
+
+ /*
+ * Whether this regex can be dropped at the start (or end) of the input, see
+ * #compileAtBoundary(NFA, int, int).
+ */
+ boolean isNoOpAt(boolean start) {
+ return false;
+ }
+
+ /**
+ * Tests if every string fully matched by this regex is also fully matched by the other one, i.e.
+ * if the other regex fully covers this one.
+ *
+ *
Unlike generation, language operations are exact. Hence, both regexes may only use regular
+ * constructs. Anchors ({@code ^}, {@code \A} and {@code $}, {@code \z}, {@code \Z}) are only
+ * supported as leading and trailing elements of the regex, of its top level alternatives or of
+ * groups that are the only remaining element, i.e. {@code ^a$|^(?:^b$)$}, where, under {@link
+ * java.util.regex.Matcher#matches()} semantics, they are no-ops regardless of {@link
+ * Pattern#MULTILINE}. Anchors nested anywhere else, i.e. {@code (^a)b}, are not supported.
+ *
+ *
Only characters of the Basic Multilingual Plane are considered: strings containing
+ * surrogates, i.e. supplementary code points, are out of scope. Case-insensitive matching folds
+ * case the same way {@link Pattern#CASE_INSENSITIVE} and {@link Pattern#UNICODE_CASE} do, except
+ * for the few Unicode characters with more than one case variant.
+ *
+ * @param other regex to compare with
+ * @return {@code true} if language of this regex is a subset of the other's language
+ * @throws UnsupportedOperationException if either regex uses constructs that are not regular:
+ * back references, atomic groups, possessive quantifiers, lookarounds or word boundaries.
+ * @see #difference(Coregex)
+ * @since 1.5.0
+ */
+ public final boolean isSubsetOf(Coregex other) {
+ return !difference(other).isPresent();
+ }
+
+ /**
+ * Finds the shortest string fully matched by this regex but not by the other one. Useful as a
+ * counterexample when {@link #isSubsetOf(Coregex)} fails.
+ *
+ *
See {@link #isSubsetOf(Coregex)} for supported constructs.
+ *
+ * @param other regex to subtract
+ * @return the shortest string matched by this but not the other regex or nothing if there is none
+ * @throws UnsupportedOperationException if either regex uses constructs that are not regular.
+ * @see #isSubsetOf(Coregex)
+ * @since 1.5.0
+ */
+ public final Optional difference(Coregex other) {
+ return NFA.of(this).difference(NFA.of(requireNonNull(other, "other")));
+ }
+
+ /**
+ * Finds the shortest string fully matched by both this and the other regex.
+ *
+ * See {@link #isSubsetOf(Coregex)} for supported constructs.
+ *
+ * @param other regex to intersect with
+ * @return the shortest string matched by both regexes or nothing if they are disjoint
+ * @throws UnsupportedOperationException if either regex uses constructs that are not regular.
+ * @see #isSubsetOf(Coregex)
+ * @since 1.5.0
+ */
+ public final Optional intersection(Coregex other) {
+ return NFA.of(this).intersection(NFA.of(requireNonNull(other, "other")));
+ }
+
/**
* Samples one random string that matches this regex.
*
@@ -156,16 +284,6 @@ public final String generate(long seed) {
"Cannot generate string for " + this + " after " + Context.maxAttempts + " attempts");
}
- final boolean matches(String input, Context ctx) {
- String[] matches = match(input, ctx);
- for (String match : matches) {
- if (match.isEmpty()) {
- return true;
- }
- }
- return false;
- }
-
/**
* Quantify this regex.
*
@@ -207,20 +325,6 @@ void generate(Context ctx) {
}
}
- @Override
- String[] match(String input, Context ctx) {
- Collection match = Arrays.asList(first.match(input, ctx));
- for (int i = 0; i < rest.length && !match.isEmpty(); i++) {
- Collection nextMatch = new HashSet<>(match.size());
- Coregex coregex = rest[i];
- for (String tail : match) {
- Collections.addAll(nextMatch, coregex.match(tail, ctx));
- }
- match = nextMatch;
- }
- return match.toArray(new String[0]);
- }
-
/** {@inheritDoc} */
@Override
public Stream shrink() {
@@ -241,6 +345,29 @@ public Stream shrink() {
})));
}
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ nfa.chain(concat(), from, to);
+ }
+
+ @Override
+ void compileAtBoundary(NFA nfa, int from, int to) {
+ List concat = concat();
+ int first = 0;
+ int last = concat.size();
+ while (first < last && concat.get(first).isNoOpAt(true)) {
+ first++;
+ }
+ while (first < last && concat.get(last - 1).isNoOpAt(false)) {
+ last--;
+ }
+ if (1 == last - first) {
+ concat.get(first).compileAtBoundary(nfa, from, to);
+ } else {
+ nfa.chain(concat.subList(first, last), from, to);
+ }
+ }
+
/**
* @return underlying regexes in order of concatenation.
*/
@@ -383,6 +510,7 @@ public static final class Group extends Coregex {
private final int index;
private final String name;
private final Coregex group;
+ private transient volatile NFA lookaround;
/**
* Unnamed capturing group.
@@ -447,7 +575,7 @@ void generate(Context ctx) throws RewindException {
positive = false;
// fall through
case LOOKBEHIND:
- if (positive == group.matches(ctx.toString(true), ctx)) {
+ if (positive == lookaround().matches(ctx.toString(true))) {
break;
}
throw new RewindException();
@@ -456,13 +584,10 @@ void generate(Context ctx) throws RewindException {
// fall through
case LOOKAHEAD:
int length = ctx.length(true);
- Coregex fullLookbehind =
- 0 == length
- ? group
- : new Concat(any().quantify(length, length, Quantified.Type.GREEDY), group);
- final boolean pos = positive;
+ NFA lookahead = lookaround();
+ boolean pos = positive;
ctx.finalizeWithLookbehind(
- root -> pos == fullLookbehind.matches(root.toString(true), root));
+ root -> pos == lookahead.matches(root.toString(true).substring(length)));
break;
default:
try (Context childCtx = new Context(ctx, index, name)) {
@@ -471,35 +596,61 @@ void generate(Context ctx) throws RewindException {
}
}
+ /** {@inheritDoc} */
@Override
- String[] match(String input, Context ctx) {
+ public Stream shrink() {
switch (type) {
- case NON_CAPTURING:
- case ATOMIC:
- return group.match(input, ctx);
case LOOKAHEAD:
case LOOKBEHIND:
case NEGATIVE_LOOKAHEAD:
case NEGATIVE_LOOKBEHIND:
- return new String[] {input};
+ return Stream.empty();
default:
- try (Context childCtx = new Context(ctx, index, name)) {
- return group.match(input, childCtx);
- }
+ return group.shrink().map(group -> new Group(type, index, name, group));
}
}
- /** {@inheritDoc} */
+ /*
+ * Lookbehind body has to match a suffix of the input generated so far, lookahead body a prefix
+ * of the input generated after: `[\s\S]*body` and `body[\s\S]*` respectively. Compiled as is,
+ * since the body is not at the input boundary and anchors in it are not no-ops.
+ */
+ private NFA lookaround() {
+ NFA nfa = lookaround;
+ if (null == nfa) {
+ boolean behind = Type.LOOKBEHIND == type || Type.NEGATIVE_LOOKBEHIND == type;
+ Coregex anything = Set.DOTALL.quantify(0, -1, Quantified.Type.GREEDY);
+ nfa = new NFA();
+ (behind ? new Concat(anything, group) : new Concat(group, anything))
+ .compile(nfa, NFA.START, NFA.ACCEPT);
+ lookaround = nfa;
+ }
+ return nfa;
+ }
+
@Override
- public Stream shrink() {
+ void compile(NFA nfa, int from, int to) {
switch (type) {
- case LOOKAHEAD:
- case LOOKBEHIND:
- case NEGATIVE_LOOKAHEAD:
- case NEGATIVE_LOOKBEHIND:
- return Stream.empty();
+ case CAPTURING:
+ case NAMED:
+ case NON_CAPTURING:
+ group.compile(nfa, from, to);
+ break;
default:
- return group.shrink().map(group -> new Group(type, index, name, group));
+ throw new UnsupportedOperationException("group is not supported: " + this);
+ }
+ }
+
+ @Override
+ void compileAtBoundary(NFA nfa, int from, int to) {
+ switch (type) {
+ case CAPTURING:
+ case NAMED:
+ case NON_CAPTURING:
+ group.compileAtBoundary(nfa, from, to);
+ break;
+ default:
+ compile(nfa, from, to);
}
}
@@ -594,6 +745,116 @@ public enum Type {
}
}
+ /**
+ * Regex with a label, i.e. a boundary matcher such as {@code ^} or {@code $}. Behaves exactly as
+ * the labeled regex, but is represented by its label.
+ *
+ * @since 1.5.0
+ */
+ public static final class Labeled extends Coregex {
+ private static final long serialVersionUID = 1L;
+
+ // (? shrink() {
+ return Stream.empty();
+ }
+
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ // all labeled regexes are boundary matchers, which are lookarounds: boundary anchors are
+ // stripped by #compileAtBoundary(NFA, int, int), anything else is not regular
+ throw new UnsupportedOperationException(label + " is not supported");
+ }
+
+ /*
+ * Boundary anchors: `^` or `\A` at the start, `$`, `\z` or `\Z` at the end. Under
+ * Matcher#matches() semantics, `\Z` at the very end is a no-op as nothing may follow it.
+ */
+ @Override
+ boolean isNoOpAt(boolean start) {
+ // labels are canonical: Labeled can only be created by the parser and predefined constructors
+ switch (label) {
+ case "^":
+ case "\\A":
+ return start;
+ case "$":
+ case "\\z":
+ case "\\Z":
+ return !start;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * @return label
+ */
+ public String label() {
+ return label;
+ }
+
+ /**
+ * @return labeled regex
+ */
+ public Coregex labeled() {
+ return labeled;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Labeled that = (Labeled) o;
+ return label.equals(that.label) && labeled.equals(that.labeled);
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * label.hashCode() + labeled.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return label;
+ }
+ }
+
/** Quantified regex. */
public static final class Quantified extends Coregex {
@@ -665,29 +926,6 @@ void generate(Context ctx) throws RewindException {
}
}
- @Override
- String[] match(String input, Context ctx) {
- int quantifier = 0, min = this.min, max = this.max;
- java.util.Set match = new HashSet<>(Collections.singleton(input));
- for (; quantifier < min && !match.isEmpty(); quantifier++) {
- java.util.Set nextMatch = new HashSet<>(match.size());
- for (String tail : match) {
- Collections.addAll(nextMatch, quantified.match(tail, ctx));
- }
- match = nextMatch;
- }
- for (; (-1 == max || quantifier < max) && !match.isEmpty(); quantifier++) {
- java.util.Set nextMatch = new HashSet<>(match.size());
- for (String tail : match) {
- Collections.addAll(nextMatch, quantified.match(tail, ctx));
- }
- if (!match.addAll(nextMatch)) {
- break;
- }
- }
- return match.toArray(new String[0]);
- }
-
/** {@inheritDoc} */
@Override
public Stream shrink() {
@@ -700,6 +938,42 @@ public Stream shrink() {
quantified.shrink().map(quantified -> new Quantified(quantified, min, max, type)));
}
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ if (Type.POSSESSIVE == type) {
+ throw new UnsupportedOperationException("possessive quantifier is not supported: " + this);
+ }
+ int current = from;
+ for (int i = 0; i < min; i++) {
+ int next = nfa.state();
+ quantified.compile(nfa, current, next);
+ current = next;
+ }
+ if (-1 == max) {
+ // fresh loop state, as `from` may be shared and must not get incoming transitions
+ int loop = nfa.state();
+ nfa.epsilon(current, loop);
+ quantified.compile(nfa, loop, loop);
+ nfa.epsilon(loop, to);
+ } else {
+ for (int i = min; i < max; i++) {
+ int next = nfa.state();
+ nfa.epsilon(current, to);
+ quantified.compile(nfa, current, next);
+ current = next;
+ }
+ nfa.epsilon(current, to);
+ }
+ }
+
+ /*
+ * Empty regex, i.e. inline flags like `(?m)` that may precede anchors.
+ */
+ @Override
+ boolean isNoOpAt(boolean start) {
+ return EMPTY.equals(this);
+ }
+
/**
* @return quantified regex
*/
@@ -826,18 +1100,17 @@ void generate(Context ctx) {
ctx.append(ctx.ref(ref));
}
- @Override
- String[] match(String input, Context ctx) {
- String ref = ctx.ref(this.ref);
- return input.startsWith(ref) ? new String[] {input.substring(ref.length())} : new String[0];
- }
-
/** {@inheritDoc} */
@Override
public Stream[ shrink() {
return Stream.empty();
}
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ throw new UnsupportedOperationException("back reference is not supported: " + this);
+ }
+
public Serializable ref() {
return ref;
}
@@ -888,16 +1161,6 @@ void generate(Context ctx) throws RewindException {
(index < rest.length ? rest[index] : first).generate(ctx);
}
- @Override
- String[] match(String input, Context ctx) {
- Collection match = new HashSet<>(rest.length + 1);
- Collections.addAll(match, first.match(input, ctx));
- for (Coregex coregex : rest) {
- Collections.addAll(match, coregex.match(input, ctx));
- }
- return match.toArray(new String[0]);
- }
-
/** {@inheritDoc} */
@Override
public Stream shrink() {
@@ -927,6 +1190,22 @@ public Stream shrink() {
}))));
}
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ first.compile(nfa, from, to);
+ for (Coregex coregex : rest) {
+ coregex.compile(nfa, from, to);
+ }
+ }
+
+ @Override
+ void compileAtBoundary(NFA nfa, int from, int to) {
+ first.compileAtBoundary(nfa, from, to);
+ for (Coregex coregex : rest) {
+ coregex.compileAtBoundary(nfa, from, to);
+ }
+ }
+
/**
* @return underlying regexes forming this unification.
*/
diff --git a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java
index 44f1cf1..d887241 100644
--- a/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java
+++ b/core/src/main/java/com/github/simy4/coregex/core/CoregexParser.java
@@ -184,11 +184,11 @@ private Coregex elementaryRE(Context ctx) {
break;
case '^':
ctx.match('^');
- elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty());
+ elementaryRE = Coregex.lineStart();
break;
case '$':
ctx.match('$');
- elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty());
+ elementaryRE = Coregex.lineEnd();
break;
case '\\':
switch (ctx.peek(2)) {
@@ -198,7 +198,7 @@ private Coregex elementaryRE(Context ctx) {
case 'A':
ctx.match('\\');
ctx.match('A');
- elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKBEHIND, Coregex.empty());
+ elementaryRE = Coregex.inputStart();
break;
case 'b':
ctx.match('\\');
@@ -223,15 +223,12 @@ private Coregex elementaryRE(Context ctx) {
case 'z':
ctx.match('\\');
ctx.match('z');
- elementaryRE = new Coregex.Group(Coregex.Group.Type.LOOKAHEAD, Coregex.empty());
+ elementaryRE = Coregex.inputEnd();
break;
case 'Z':
ctx.match('\\');
ctx.match('Z');
- elementaryRE =
- new Coregex.Group(
- Coregex.Group.Type.LOOKAHEAD,
- new Coregex.Union(Coregex.empty(), Set.single('\n')));
+ elementaryRE = Coregex.inputEndBeforeTerminator();
break;
case 'G':
throw ctx.unsupported("metacharacter \\G");
@@ -519,6 +516,8 @@ private Set metachar(Context ctx) {
ctx.match('\\');
char ch = ctx.peek();
Set.Builder metachar = Set.builder(ctx.flags);
+ // char classes that are ASCII-only here, but Unicode with UNICODE_CHARACTER_CLASS
+ boolean asciiOnly = 0 <= "dDwWsS".indexOf(ch);
switch (ch) {
case 't':
ctx.match('t');
@@ -600,6 +599,7 @@ private Set metachar(Context ctx) {
ctx.match('p');
ctx.match('{');
String posix = ctx.span('}');
+ asciiOnly = !"all".equals(posix) && !"ASCII".equals(posix) && !posix.startsWith("java");
switch (posix) {
case "all":
metachar.matching(any -> true, "\\p{all}");
@@ -665,6 +665,9 @@ private Set metachar(Context ctx) {
metachar.single(ch);
break;
}
+ if (asciiOnly && 0 != (ctx.flags & Pattern.UNICODE_CHARACTER_CLASS)) {
+ metachar.approximated();
+ }
return metachar.build();
}
diff --git a/core/src/main/java/com/github/simy4/coregex/core/NFA.java b/core/src/main/java/com/github/simy4/coregex/core/NFA.java
new file mode 100644
index 0000000..42a8624
--- /dev/null
+++ b/core/src/main/java/com/github/simy4/coregex/core/NFA.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright 2021-2026 Alex Simkin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.github.simy4.coregex.core;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Deque;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Optional;
+
+/*
+ * Non-deterministic finite automaton built from Coregex using Thompson's construction. Every
+ * Coregex wires itself between given states, see Coregex#compile(NFA, int, int).
+ * Fragments never add edges into `from` or out of `to`, so fragments can safely share endpoints
+ * (union) without leaking paths between each other.
+ *
+ * Unlike Coregex itself, which is free to approximate a regex as long as everything it generates
+ * matches, an NFA has to accept exactly the language of the regex. Therefore, constructs that are
+ * not regular are rejected with UnsupportedOperationException.
+ *
+ * The alphabet is UTF-16 chars except surrogates, see Set#DOTALL.
+ *
+ * Language operations are decided exactly by a lazy breadth-first walk of the product automaton,
+ * so whenever a witness exists, the shortest one is returned.
+ */
+final class NFA {
+
+ static final int START = 0;
+ static final int ACCEPT = 1;
+
+ static NFA of(Coregex coregex) {
+ NFA nfa = new NFA();
+ coregex.compileAtBoundary(nfa, START, ACCEPT);
+ return nfa;
+ }
+
+ private final List]> epsilons = new ArrayList<>();
+ private final List> edges = new ArrayList<>();
+
+ NFA() {
+ state(); // START
+ state(); // ACCEPT
+ }
+
+ // region construction
+ int state() {
+ epsilons.add(new ArrayList<>());
+ edges.add(new ArrayList<>());
+ return epsilons.size() - 1;
+ }
+
+ void epsilon(int from, int to) {
+ if (from != to) {
+ epsilons.get(from).add(to);
+ }
+ }
+
+ /*
+ * Label is shared, not copied: automaton never mutates it.
+ */
+ void edge(int from, BitSet label, int to) {
+ edges.get(from).add(new Edge(label, to));
+ }
+
+ void chain(List chunks, int from, int to) {
+ int current = from;
+ for (int i = 0; i < chunks.size(); i++) {
+ int next = chunks.size() - 1 == i ? to : state();
+ chunks.get(i).compile(this, current, next);
+ current = next;
+ }
+ if (chunks.isEmpty()) {
+ epsilon(from, to);
+ }
+ }
+
+ // endregion
+
+ boolean matches(CharSequence input) {
+ BitSet current = initial();
+ for (int i = 0; i < input.length() && !current.isEmpty(); i++) {
+ current = step(current, input.charAt(i));
+ }
+ return current.get(ACCEPT);
+ }
+
+ /*
+ * Shortest string accepted by this but not the other automaton.
+ */
+ Optional difference(NFA other) {
+ return search(other, false);
+ }
+
+ /*
+ * Shortest string accepted by both this and the other automaton.
+ */
+ Optional intersection(NFA other) {
+ return search(other, true);
+ }
+
+ /*
+ * Breadth-first search over pairs of (single state of this NFA, set of states of other NFA).
+ * Only the other side is (lazily) determinized, as only it needs to be complemented.
+ *
+ * otherAccepts: whether other has to accept (intersection) or reject (difference) the witness.
+ *
+ * Pairs are pruned with antichains: for difference, (state, others) is subsumed by an already
+ * visited (state, subset of others), as every word rejected from the bigger set is rejected from
+ * the smaller one too. Symmetrically for intersection with supersets. Being breadth-first, the
+ * subsuming pair is never deeper, so the first witness found is still the shortest one.
+ */
+ private Optional search(NFA other, boolean otherAccepts) {
+ char[] alphabet = alphabet(other);
+ Frontier frontier = new Frontier(edges.size(), otherAccepts);
+ frontier.offer(initial(), other.initial(), null, '\0');
+ for (Node node = frontier.poll(); null != node; node = frontier.poll()) {
+ if (ACCEPT == node.state && otherAccepts == node.otherStates.get(ACCEPT)) {
+ return Optional.of(node.witness());
+ }
+ for (char ch : alphabet) {
+ BitSet next = step(singleton(node.state), ch);
+ if (next.isEmpty()) {
+ continue;
+ }
+ BitSet nextOther = other.step(node.otherStates, ch);
+ if (otherAccepts && nextOther.isEmpty()) {
+ continue; // other can never accept from here on
+ }
+ frontier.offer(next, nextOther, node, ch);
+ }
+ }
+ return Optional.empty();
+ }
+
+ /*
+ * Splits all characters on edge labels of both automatons into equivalence classes (minterms):
+ * characters that are indistinguishable by every edge. Each class is then represented by its
+ * simplest character, so that witnesses are readable.
+ */
+ private char[] alphabet(NFA other) {
+ java.util.Set labels = new LinkedHashSet<>();
+ collectLabels(labels);
+ other.collectLabels(labels);
+ BitSet universe = new BitSet();
+ for (BitSet label : labels) {
+ universe.or(label);
+ }
+ List blocks = new ArrayList<>();
+ if (!universe.isEmpty()) {
+ blocks.add(universe);
+ }
+ for (BitSet label : labels) {
+ List refined = new ArrayList<>(blocks.size() * 2);
+ for (BitSet block : blocks) {
+ BitSet in = (BitSet) block.clone();
+ in.and(label);
+ if (in.isEmpty() || in.equals(block)) {
+ refined.add(block);
+ } else {
+ BitSet out = (BitSet) block.clone();
+ out.andNot(label);
+ refined.add(in);
+ refined.add(out);
+ }
+ }
+ blocks = refined;
+ }
+ char[] alphabet = new char[blocks.size()];
+ for (int i = 0; i < alphabet.length; i++) {
+ alphabet[i] = simplest(blocks.get(i));
+ }
+ Arrays.sort(alphabet);
+ return alphabet;
+ }
+
+ /*
+ * Simplest of given chars, in the same order of preference as Set#shrink(): digits, lowercase and
+ * uppercase letters, then printable ASCII, then anything else.
+ */
+ private static char simplest(BitSet chars) {
+ for (char[] range : new char[][] {{'0', '9'}, {'a', 'z'}, {'A', 'Z'}, {' ', '~'}}) {
+ int ch = chars.nextSetBit(range[0]);
+ if (0 <= ch && ch <= range[1]) {
+ return (char) ch;
+ }
+ }
+ return (char) chars.nextSetBit(0);
+ }
+
+ private void collectLabels(java.util.Set labels) {
+ for (List stateEdges : edges) {
+ for (Edge edge : stateEdges) {
+ labels.add(edge.label);
+ }
+ }
+ }
+
+ private BitSet initial() {
+ return closure(singleton(START));
+ }
+
+ private BitSet step(BitSet states, char ch) {
+ BitSet next = new BitSet(edges.size());
+ for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) {
+ for (Edge edge : edges.get(state)) {
+ if (edge.label.get(ch)) {
+ next.set(edge.target);
+ }
+ }
+ }
+ return closure(next);
+ }
+
+ private BitSet closure(BitSet states) {
+ Deque stack = new ArrayDeque<>();
+ for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) {
+ stack.push(state);
+ }
+ while (!stack.isEmpty()) {
+ for (int target : epsilons.get(stack.pop())) {
+ if (!states.get(target)) {
+ states.set(target);
+ stack.push(target);
+ }
+ }
+ }
+ return states;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (int state = 0; state < edges.size(); state++) {
+ sb.append(state);
+ if (START == state) {
+ sb.append(" (start)");
+ } else if (ACCEPT == state) {
+ sb.append(" (accept)");
+ }
+ for (Edge edge : edges.get(state)) {
+ sb.append(" --");
+ appendLabel(sb, edge.label);
+ sb.append("--> ").append(edge.target);
+ }
+ for (int target : epsilons.get(state)) {
+ sb.append(" --ε--> ").append(target);
+ }
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ private static void appendLabel(StringBuilder sb, BitSet label) {
+ BitSet negated = new BitSet();
+ negated.set(Character.MIN_VALUE, Character.MAX_VALUE + 1);
+ negated.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1);
+ negated.andNot(label);
+ boolean negate = ranges(negated) < ranges(label);
+ BitSet chars = negate ? negated : label;
+ sb.append(negate ? "[^" : "[");
+ for (int start = chars.nextSetBit(0); start >= 0; ) {
+ int end = chars.nextClearBit(start) - 1;
+ sb.append((char) start);
+ if (start < end) {
+ sb.append(start + 1 < end ? "-" : "");
+ sb.append((char) end);
+ }
+ start = chars.nextSetBit(end + 1);
+ }
+ sb.append(']');
+ }
+
+ private static int ranges(BitSet chars) {
+ int ranges = 0;
+ for (int ch = chars.nextSetBit(0); ch >= 0; ch = chars.nextSetBit(chars.nextClearBit(ch))) {
+ ranges++;
+ }
+ return ranges;
+ }
+
+ private static BitSet singleton(int state) {
+ BitSet set = new BitSet(state + 1);
+ set.set(state);
+ return set;
+ }
+
+ private static final class Edge {
+ private final BitSet label;
+ private final int target;
+
+ private Edge(BitSet label, int target) {
+ this.label = label;
+ this.target = target;
+ }
+ }
+
+ /*
+ * Breadth-first queue of pairs, deduplicated with an antichain: per state of this NFA, a set of
+ * pairwise incomparable sets of states of the other NFA, see #search(NFA, boolean).
+ */
+ private static final class Frontier {
+ private final Deque queue = new ArrayDeque<>();
+ private final List> visited;
+ private final boolean supersets;
+
+ private Frontier(int states, boolean supersets) {
+ this.visited = new ArrayList<>(states);
+ for (int i = 0; i < states; i++) {
+ visited.add(new ArrayList<>());
+ }
+ this.supersets = supersets;
+ }
+
+ /*
+ * Enqueues (state, others) for every given state, unless subsumed by an already visited pair.
+ */
+ private void offer(BitSet states, BitSet others, Node parent, char ch) {
+ for (int state = states.nextSetBit(0); state >= 0; state = states.nextSetBit(state + 1)) {
+ List antichain = visited.get(state);
+ if (antichain.stream().noneMatch(seen -> subsumes(seen, others))) {
+ antichain.removeIf(seen -> subsumes(others, seen));
+ antichain.add(others);
+ queue.add(new Node(state, others, parent, ch));
+ }
+ }
+ }
+
+ private Node poll() {
+ return queue.poll();
+ }
+
+ /*
+ * Difference needs the other NFA to reject, so smaller sets subsume. Intersection needs it to
+ * accept, so bigger sets do.
+ */
+ private boolean subsumes(BitSet subsuming, BitSet subsumed) {
+ return supersets ? isSubset(subsumed, subsuming) : isSubset(subsuming, subsumed);
+ }
+
+ private static boolean isSubset(BitSet sub, BitSet sup) {
+ BitSet difference = (BitSet) sub.clone();
+ difference.andNot(sup);
+ return difference.isEmpty();
+ }
+ }
+
+ private static final class Node {
+ private final int state;
+ private final BitSet otherStates;
+ private final Node parent;
+ private final char ch;
+
+ private Node(int state, BitSet otherStates, Node parent, char ch) {
+ this.state = state;
+ this.otherStates = otherStates;
+ this.parent = parent;
+ this.ch = ch;
+ }
+
+ private String witness() {
+ StringBuilder sb = new StringBuilder();
+ for (Node node = this; null != node.parent; node = node.parent) {
+ sb.append(node.ch);
+ }
+ return sb.reverse().toString();
+ }
+ }
+}
diff --git a/core/src/main/java/com/github/simy4/coregex/core/Set.java b/core/src/main/java/com/github/simy4/coregex/core/Set.java
index f8be9d7..f058587 100644
--- a/core/src/main/java/com/github/simy4/coregex/core/Set.java
+++ b/core/src/main/java/com/github/simy4/coregex/core/Set.java
@@ -36,7 +36,7 @@ public final class Set extends Coregex implements Serializable {
private static final long serialVersionUID = 1L;
- static final Set EMPTY = new Set(new BitSet(0), "");
+ static final Set EMPTY = new Set(new BitSet(0), "", false);
/*
* All chars except surrogates, which only exist in pairs: supplementary code points are out of
@@ -50,16 +50,16 @@ public final class Set extends Coregex implements Serializable {
BitSet dotAll = new BitSet(Character.MAX_VALUE + 1);
dotAll.set(Character.MIN_VALUE, Character.MAX_VALUE + 1);
dotAll.clear(Character.MIN_SURROGATE, Character.MAX_SURROGATE + 1);
- DOTALL = new Set(dotAll, ".");
+ DOTALL = new Set(dotAll, ".", false);
BitSet unixLines = (BitSet) dotAll.clone();
unixLines.clear('\n');
- UNIX_LINES = new Set(unixLines, ".");
+ UNIX_LINES = new Set(unixLines, ".", false);
BitSet all = (BitSet) unixLines.clone();
all.clear('\r');
all.clear('\u0085');
all.clear('\u2028');
all.clear('\u2029');
- ALL = new Set(all, ".");
+ ALL = new Set(all, ".", false);
}
public static Set any(int flags) {
@@ -125,9 +125,17 @@ public static Set single(char ch, int flags) {
private final BitSet chars;
private final String description;
- private Set(BitSet chars, String description) {
+ /*
+ * Whether chars are only an approximation of the actual char class, i.e. ASCII-only \w for
+ * UNICODE_CHARACTER_CLASS. Good enough for generation, as all generated chars still match, but not
+ * for exact language operations.
+ */
+ private final boolean approximated;
+
+ private Set(BitSet chars, String description, boolean approximated) {
this.chars = chars;
this.description = description;
+ this.approximated = approximated;
}
@Override
@@ -138,13 +146,6 @@ void generate(Context ctx) {
}
}
- @Override
- String[] match(String input, Context ctx) {
- return !input.isEmpty() && chars.get(input.charAt(0))
- ? new String[] {input.substring(1)}
- : new String[0];
- }
-
boolean single() {
return 1 == chars.cardinality();
}
@@ -168,7 +169,18 @@ public Stream shrink() {
return builder.build();
}
- /**
+ @Override
+ void compile(NFA nfa, int from, int to) {
+ if (approximated) {
+ throw new UnsupportedOperationException(
+ "char class with UNICODE_CHARACTER_CLASS is not supported: " + this);
+ }
+ if (!chars.isEmpty()) {
+ nfa.edge(from, chars, to);
+ }
+ }
+
+ /*
* @return sampled char or -1 if this set is empty.
*/
int sample(long seed) {
@@ -217,6 +229,7 @@ public static final class Builder {
private final int flags;
private final BitSet chars;
private final StringBuilder description = new StringBuilder();
+ private boolean approximated;
private Builder(int flags, int size) {
this.flags = flags;
@@ -267,6 +280,7 @@ public Builder set(char first, char... rest) {
*/
public Builder union(Set set) {
chars.or(requireNonNull(set, "set").chars);
+ approximated |= set.approximated;
description.append(set);
return this;
}
@@ -279,6 +293,7 @@ public Builder union(Set set) {
*/
public Builder intersect(Set set) {
chars.and(requireNonNull(set, "set").chars);
+ approximated |= set.approximated;
description.append("&&").append(set);
return this;
}
@@ -296,6 +311,14 @@ public Builder single(char ch) {
return this;
}
+ /*
+ * Marks this set as only an approximation of the actual char class, see Set#approximated.
+ */
+ Builder approximated() {
+ approximated = true;
+ return this;
+ }
+
/*
* Adds all chars matching given predicate, i.e. Character::isLowerCase.
*/
@@ -341,7 +364,7 @@ public Builder negate() {
* @return compiled set
*/
public Set build() {
- return new Set(chars, description.toString());
+ return new Set(chars, description.toString(), approximated);
}
}
}
diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala
index 3e10fa9..ba57b1e 100644
--- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala
+++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexParserSuite.scala
@@ -79,6 +79,14 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries {
Pattern.compile("^(?:||)$"),
Pattern.compile("(\\d+) (\\w+) == (\\1) \\2 \\3"),
Pattern.compile("(?!=.{10,}).+"),
+ Pattern.compile("(?!a)[ab]b"),
+ Pattern.compile("(?=a)[ab]b"),
+ Pattern.compile("b[ab](?<=a)"),
+ Pattern.compile("[ab]{2}(? coregex.matches(ch, null) != pattern.matcher(ch).matches())
+ .filter(ch => NFA.of(coregex).matches(ch) != pattern.matcher(ch).matches())
assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...")
}
}
@@ -189,12 +207,20 @@ class CoregexParserSuite extends ScalaCheckSuite with CoregexArbitraries {
)
}
+ test("anchors inside lookarounds are not at the input boundary") {
+ List("a(?=^)b", "a(?<=$)b", "a(?<=\\z)b", "a(?<=\\Z)b", "(?<=^)a", "a(?=$)").foreach { regex =>
+ val _ = intercept[UnsupportedOperationException](Coregex.from(Pattern.compile(regex)).generate(0L))
+ }
+ }
+
property("should fail to generate strings") {
forAll(
Gen.oneOf(
Pattern.compile("(?=(abc))"),
Pattern.compile("\\d+(?= dollars)"),
- Pattern.compile("(?<=[a-z0-9])(?=[A-Z])")
+ Pattern.compile("(?<=[a-z0-9])(?=[A-Z])"),
+ Pattern.compile("ba(?
diff --git a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala
index 96e87b0..530908f 100644
--- a/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala
+++ b/core/src/test/scala/com/github/simy4/coregex/core/CoregexSuite.scala
@@ -44,7 +44,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries {
}
test("empty should match empty") {
- assert(Coregex.empty().matches("", null))
+ assert(NFA.of(Coregex.empty()).matches(""))
}
property("literal don't shrink") {
@@ -55,7 +55,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries {
property("literal should match self") {
forAll { (literal: String, flags: Int) =>
- Coregex.literal(literal, flags).matches(literal, null) :| s"$literal match self"
+ NFA.of(Coregex.literal(literal, flags)).matches(literal) :| s"$literal match self"
}
}
@@ -89,7 +89,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries {
forAll { (head: Simple, tail: Seq[Simple], seed: Long) =>
val concat = new Concat(head.value, tail.map(_.value): _*)
val generated = concat.generate(seed)
- concat.matches(generated, null) :| s"$generated in $concat"
+ NFA.of(concat).matches(generated) :| s"$generated in $concat"
}
}
// endregion
@@ -134,7 +134,7 @@ class CoregexSuite extends ScalaCheckSuite with CoregexArbitraries {
forAll { (head: Simple, tail: Seq[Simple], seed: Long) =>
val union = new Union(head.value, tail.map(_.value): _*)
val generated = union.generate(seed)
- union.matches(generated, null) :| s"$generated in ($union)"
+ NFA.of(union).matches(generated) :| s"$generated in ($union)"
}
}
diff --git a/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala
new file mode 100644
index 0000000..047260d
--- /dev/null
+++ b/core/src/test/scala/com/github/simy4/coregex/core/NFASuite.scala
@@ -0,0 +1,338 @@
+/*
+ * Copyright 2021-2026 Alex Simkin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.github.simy4.coregex.core
+
+import munit.ScalaCheckSuite
+import org.scalacheck.{ Arbitrary, Gen }
+import org.scalacheck.Prop._
+
+import java.util.Optional
+import java.util.regex.Pattern
+
+class NFASuite extends ScalaCheckSuite with CoregexArbitraries {
+
+ def genPatternPairs(): Gen[(Pattern, Pattern)] = Gen.oneOf(
+ List(
+ Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}") ->
+ Pattern.compile("[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}"),
+ Pattern.compile("((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])") ->
+ Pattern.compile("(?:\\d{1,3}\\.){3}\\d{1,3}"),
+ Pattern.compile("#([0-9a-fA-F]{3}){1,2}") -> Pattern.compile("#[0-9a-fA-F]{3,6}"),
+ Pattern.compile("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d([+-][0-2]\\d:[0-5]\\d|Z)") ->
+ Pattern.compile("[\\dTZ:+-]+"),
+ Pattern.compile("4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}") -> Pattern.compile("\\d{13,16}"),
+ Pattern.compile(
+ "^arn:(?\\w+):(?\\w+):(?[\\w-]+):(?\\d{12}):(?(?[\\w-]+)[:\\/])?(?[\\w.-]+)$"
+ ) -> Pattern.compile("arn:[\\w.:/-]+"),
+ Pattern.compile(
+ "^(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)\\.(?0|[1-9]\\d*)(?:-(?(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
+ ) -> Pattern.compile("[0-9a-zA-Z.+-]+"),
+ Pattern.compile("\\Qhttps://\\E(?foo|bar|baz)\\Q.com/\\E(?[\\w-]+)") ->
+ Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"),
+ // `\w*` alone allows empty, but concatenated with `\w` it does not.
+ Pattern.compile("(?\\w*)(?\\w)") -> Pattern.compile("[\\w-]+"),
+ Pattern.compile("^(a+)+$") -> Pattern.compile("a+"),
+ Pattern.compile("^([a-zA-Z]+)*$") -> Pattern.compile("[a-zA-Z]*"),
+ Pattern.compile("^(a|aa)+$") -> Pattern.compile("a+"),
+ Pattern.compile("^(a|a?)+$") -> Pattern.compile("a*"),
+ Pattern.compile("^(?:||)$") -> Pattern.compile("")
+ )
+ )
+
+ implicit val arbPatternPairs: Arbitrary[(Pattern, Pattern)] = Arbitrary(genPatternPairs())
+
+ private def re(regex: String): Coregex = Coregex.from(Pattern.compile(regex))
+
+ private def nfa(regex: String): NFA = NFA.of(re(regex))
+
+ property("should cover example regex") {
+ forAll { (pair: (Pattern, Pattern), seed: Long) =>
+ val (sub, sup) = pair
+ val subset = Coregex.from(sub)
+ val superset = Coregex.from(sup)
+ val generated = subset.generate(seed)
+ val witness = superset.difference(subset)
+ (subset.isSubsetOf(superset) :| s"${sub.pattern()} should be a subset of ${sup.pattern()}") &&
+ (sup.matcher(generated).matches() :| s"${sup.pattern()} should match generated: $generated") &&
+ (subset.intersection(superset).isPresent :| s"${sub.pattern()} should intersect ${sup.pattern()}") &&
+ ((!witness.isPresent || sup.matcher(witness.get).matches() && !sub.matcher(witness.get).matches()) :|
+ s"${sup.pattern()} but not ${sub.pattern()} should match witness: $witness")
+ }
+ }
+
+ // region language operations
+ test("possibly empty fragment is not a subset of non empty one") {
+ assertEquals(re("(?\\w*)").difference(re("[\\w-]+")), Optional.of(""))
+ }
+
+ test("dot fragment is not a subset of word characters") {
+ val witness = re("(?.+)").difference(re("[\\w-]+"))
+ assert(witness.isPresent && 1 == witness.get.length, s"$witness should be a single char")
+ assert(!Pattern.matches("[\\w-]+", witness.get), s"${witness.get} should not match [\\w-]+")
+ }
+
+ test("intersection of disjoint languages is empty") {
+ assertEquals(re("[a-z]+").intersection(re("\\d+")), Optional.empty[String]())
+ }
+
+ test("intersection returns the shortest common string") {
+ assertEquals(re("[a-z]+").intersection(re("x.*")), Optional.of("x"))
+ assertEquals(re("a{3}b?").intersection(re("a{3}b?")), Optional.of("aaa"))
+ assertEquals(re("[a&&b]").intersection(re(".*")), Optional.empty[String]())
+ }
+
+ test("toString shows states with transitions") {
+ assertEquals(
+ nfa("a|b*").toString,
+ """0 (start) --[a]--> 1 --ε--> 2
+ |1 (accept)
+ |2 --[b]--> 2 --ε--> 1
+ |""".stripMargin
+ )
+ assert(nfa(".[^ab]").toString.startsWith("0 (start) --[^\n\r\u0085\u2028\u2029]--> 2\n"))
+ }
+
+ test("witnesses use the simplest chars, same as shrinking") {
+ assertEquals(re("#[0-9a-fA-F]{3,6}").difference(re("#(?:[0-9a-fA-F]{3}){1,2}")), Optional.of("#0000"))
+ assertEquals(re("[a-zA-Z]+").difference(re("[a-z]+")), Optional.of("A"))
+ assertEquals(re("[\\s\\u00E9]").intersection(re("\\W")), Optional.of(" "))
+ assertEquals(re("\\W").difference(re("[ -~]")), Optional.of("\u0000"))
+ }
+
+ test("empty regex matches empty string only") {
+ assertEquals(re("(?:)").intersection(re("")), Optional.of(""))
+ assertEquals(re("a(?:)b").intersection(re("ab")), Optional.of("ab"))
+ assertEquals(re("(?:a|)").difference(re("a")), Optional.of(""))
+ }
+
+ // (sub, sup, input matched by sub only if sub is not a subset of sup)
+ List(
+ ("[^a]", "[^a\\n]", Some("\n")),
+ ("\\n", "\\D", None),
+ ("\\n", "[^\\w]", None),
+ ("\\uFF01", ".", None),
+ ("(?s).", "[^\\uD800-\\uDFFF]", None),
+ ("\u00C9", "(?i)\u00E9", Some("\u00C9")),
+ ("\u00C9", "(?iu)\u00E9", None),
+ ("K", "(?i)k", None),
+ ("a\\Z", "a", None),
+ ("a", "a\\Z", None)
+ ).foreach { case (sub, sup, witness) =>
+ test(s"$sub is ${if (witness.isEmpty) "" else "not "}a subset of $sup like in java.util.regex") {
+ witness.foreach { w =>
+ assert(Pattern.matches(sub, w) && !Pattern.matches(sup, w), s"$w should only match $sub")
+ }
+ assertEquals(re(sub).difference(re(sup)), witness.fold(Optional.empty[String]())(Optional.of(_)))
+ }
+ }
+
+ List(
+ ".",
+ "(?s).",
+ "(?d).",
+ "[^a]",
+ "[^\\n]",
+ "\\d",
+ "\\D",
+ "\\w",
+ "\\W",
+ "\\s",
+ "\\S",
+ "\\h",
+ "\\H",
+ "\\v",
+ "\\V",
+ "\\t",
+ "\\f",
+ "\\a",
+ "\\e",
+ "\\cA",
+ "\\p{all}",
+ "\\p{Lower}",
+ "\\p{Upper}",
+ "\\p{ASCII}",
+ "\\p{Digit}",
+ "\\p{XDigit}",
+ "\\p{Alnum}",
+ "\\p{Alpha}",
+ "\\p{Punct}",
+ "\\p{Graph}",
+ "\\p{Print}",
+ "\\p{Blank}",
+ "\\p{Space}",
+ "\\p{javaWhitespace}",
+ "\\p{javaLowerCase}",
+ "\\p{javaUpperCase}",
+ "(?i)[a-z]",
+ "(?i)\u00E9",
+ "(?iu)\u00E9",
+ "(?iU)\u00E9",
+ "(?U)\\p{ASCII}",
+ "[\\w&&[^\\d]]"
+ ).foreach { regex =>
+ test(s"$regex should accept the same chars as java.util.regex") {
+ val pattern = Pattern.compile(regex)
+ val nfa = NFA.of(Coregex.from(pattern))
+ val wrong = (Character.MIN_VALUE to Character.MAX_VALUE)
+ .filterNot(Character.isSurrogate)
+ .map(_.toString)
+ .filter(ch => nfa.matches(ch) != pattern.matcher(ch).matches())
+ assert(wrong.isEmpty, s"${wrong.take(5).map(ch => f"\\u${ch.charAt(0).toInt}%04X").mkString(", ")} ...")
+ }
+ }
+
+ test("unicode character class only affects predefined char classes") {
+ assert(re("(?U)abc").isSubsetOf(re("abc")))
+ assert(re("(?U)[a-c]+").isSubsetOf(re("\\w+")))
+ }
+
+ test("reluctant and greedy quantifiers have the same language") {
+ assert(re("a+?b*?").isSubsetOf(re("a+b*")))
+ assert(re("a+b*").isSubsetOf(re("a+?b*?")))
+ }
+
+ test("nested stars") {
+ val nested = nfa("(?:a*b*)*")
+ assert(nested.matches(""))
+ assert(nested.matches("abba"))
+ assert(!nested.matches("abc"))
+ assert(re("(?:a*b*)*").isSubsetOf(re("[ab]*")))
+ assert(re("[ab]*").isSubsetOf(re("(?:a*b*)*")))
+ }
+
+ test("star inside union does not leak") {
+ val union = nfa("a*|b")
+ assert(!union.matches("bb"))
+ assert(!union.matches("ab"))
+ }
+
+ List(
+ "^abc$",
+ "\\Aabc\\z",
+ "(?m)^abc$",
+ "(^abc$)",
+ "^(?:abc)$",
+ "^^abc$$",
+ "^a$|^bc$|(?:^ab)",
+ "(?:^a|^bc)$",
+ "^abc\\Z",
+ "^(?:^abc$)$",
+ "^(?:a|^bc)$",
+ "(?m)(?i)^abc$"
+ ).foreach { regex =>
+ test(s"boundary anchors in $regex are no-ops") {
+ val pattern = Pattern.compile(regex)
+ val nfa = NFA.of(Coregex.from(pattern))
+ List("", "a", "ab", "abc", "bc", "abcd", "\n", "abc\n").foreach { input =>
+ assertEquals(nfa.matches(input), pattern.matcher(input).matches(), s"$input in $regex")
+ }
+ }
+ }
+ // endregion
+
+ // region unsupported constructs
+ List(
+ "(a)\\1",
+ "(?a)\\k",
+ "(?>a*)",
+ "a*+",
+ "(?=a)a",
+ "(?!a)b",
+ "(?<=a)b",
+ "(?
+ test(s"$regex is not supported") {
+ val _ = intercept[UnsupportedOperationException](re(regex).isSubsetOf(re("a")))
+ val _ = intercept[UnsupportedOperationException](re("a").isSubsetOf(re(regex)))
+ }
+ }
+ // endregion
+
+ // region properties
+ private val supportedRange: Gen[QuantifyRange] =
+ for {
+ min <- Gen.choose(0, 5)
+ add <- Gen.choose(0, 5)
+ max <- Gen.oneOf(
+ Gen.const(-1),
+ Gen.choose(min, min + add)
+ )
+ } yield QuantifyRange(min, max, Coregex.Quantified.Type.GREEDY)
+ private val genSupported: Gen[Coregex] = Gen.resize(20, genCoregex(supportedRange))
+
+ property("nfa should accept generated strings") {
+ forAll(genSupported, Arbitrary.arbitrary[Long]) { (coregex, seed) =>
+ val generated = coregex.generate(seed)
+ NFA.of(coregex).matches(generated) :| s"$generated in $coregex"
+ }
+ }
+
+ property("every coregex is a subset of itself and of its union") {
+ forAll(genSupported, genSupported) { (left, right) =>
+ (left.isSubsetOf(left) :| "reflexive") &&
+ (left.isSubsetOf(new Coregex.Union(left, right)) :| "union")
+ }
+ }
+
+ property("difference witness should be accepted by left and rejected by right") {
+ forAll(genSupported, genSupported) { (left, right) =>
+ val witness = left.difference(right)
+ witness.isPresent ==> {
+ (NFA.of(left).matches(witness.get) :| s"${witness.get} in $left") &&
+ (!NFA.of(right).matches(witness.get) :| s"${witness.get} not in $right")
+ }
+ }
+ }
+
+ property("difference should find the shortest witness if any exists") {
+ forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, right, seed) =>
+ val generated = left.generate(seed)
+ !NFA.of(right).matches(generated) ==> {
+ val witness = left.difference(right)
+ (witness.isPresent :| s"$generated in $left but not in $right") &&
+ (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated"
+ }
+ }
+ }
+
+ property("intersection should find the shortest witness if any exists") {
+ forAll(genSupported, genSupported, Arbitrary.arbitrary[Long]) { (left, other, seed) =>
+ val right = new Coregex.Union(other, left)
+ val generated = left.generate(seed)
+ val witness = left.intersection(right)
+ (witness.isPresent :| s"$generated in both $left and $right") && {
+ (witness.get.length <= generated.length) :| s"${witness.get} is no longer than $generated" &&
+ (NFA.of(left).matches(witness.get) && NFA.of(right).matches(witness.get)) :| s"${witness.get} in both"
+ }
+ }
+ }
+ // endregion
+}
diff --git a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala
index 668bead..483484e 100644
--- a/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala
+++ b/core/src/test/scala/com/github/simy4/coregex/core/QuantifyRange.scala
@@ -31,7 +31,9 @@ object QuantifyRange {
Gen.const(-1),
Gen.choose(min, min + add)
)
- tpe <- Gen.oneOf(Coregex.Quantified.Type.values.toIndexedSeq)
+ // possessive generates the same as greedy, but is not regular, so generated strings could not be
+ // verified with NFA
+ tpe <- Gen.oneOf(Coregex.Quantified.Type.GREEDY, Coregex.Quantified.Type.RELUCTANT)
} yield QuantifyRange(min, max, tpe)
}
diff --git a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala
index 1bca01d..66538dd 100644
--- a/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala
+++ b/core/src/test/scala/com/github/simy4/coregex/core/SetSuite.scala
@@ -29,7 +29,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries {
forAll { (set: Set, seed: Long) =>
val generated = set.generate(seed)
- val inSetCheck = set.matches(generated, null) :| s"$generated in $set"
+ val inSetCheck = NFA.of(set).matches(generated) :| s"$generated in $set"
val lengthCheck = (generated.length ?= 1) :| s"$generated.length == 1"
inSetCheck && lengthCheck
@@ -37,11 +37,11 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries {
}
property("quantified generated should be in set") {
- forAll { (set: Set, range: QuantifyRange, `type`: Coregex.Quantified.Type, seed: Long) =>
- val quantified = set.quantify(range.min, range.max, `type`)
+ forAll { (set: Set, range: QuantifyRange, seed: Long) =>
+ val quantified = set.quantify(range.min, range.max, range.`type`)
val generated = quantified.generate(seed)
- quantified.matches(generated, null) :| s"$generated in $set"
+ NFA.of(quantified).matches(generated) :| s"$generated in $set"
}
}
@@ -70,7 +70,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries {
forAll { (left: Set, right: Set, seed: Long) =>
val union = Set.builder().union(left).union(right).build()
val generated = union.generate(seed)
- (left.matches(generated, null) || right.matches(generated, null)) :| s"$generated in $union"
+ (NFA.of(left).matches(generated) || NFA.of(right).matches(generated)) :| s"$generated in $union"
}
}
@@ -79,7 +79,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries {
val leftWithCommon = Set.builder().union(left).single(right.sample(seed1).toChar).build()
val intersection = Set.builder().union(leftWithCommon).intersect(right).build()
val generated = intersection.generate(seed2)
- (leftWithCommon.matches(generated, null) && right.matches(generated, null)) :| s"$generated in $intersection"
+ (NFA.of(leftWithCommon).matches(generated) && NFA.of(right).matches(generated)) :| s"$generated in $intersection"
}
}
@@ -126,7 +126,7 @@ class SetSuite extends ScalaCheckSuite with CoregexArbitraries {
.build()
.generate(seed)
- set.matches(result.toLowerCase(Locale.ENGLISH), null) || set.matches(result.toUpperCase(Locale.ENGLISH), null)
+ NFA.of(set).matches(result.toLowerCase(Locale.ENGLISH)) || NFA.of(set).matches(result.toUpperCase(Locale.ENGLISH))
}
}
diff --git a/version.sbt b/version.sbt
index edb9358..c919701 100644
--- a/version.sbt
+++ b/version.sbt
@@ -1 +1 @@
-ThisBuild / version := "1.4.6-SNAPSHOT"
+ThisBuild / version := "1.5.0-SNAPSHOT"