diff --git a/verifier/README.md b/verifier/README.md index 7d9d7f899..b16461ae6 100644 --- a/verifier/README.md +++ b/verifier/README.md @@ -408,6 +408,18 @@ by the Z3 solver during its search phase. If the solver times out, the verifier will throw a `CelVerificationException` which should be explicitly caught and handled by the caller. +### Validating Counterexamples with the CEL Runtime + +`enableCegar(true)` checks the inputs the solver finds with the CEL +runtime of the verifier's environment: a confirmed `INCONCLUSIVE` counterexample +(or satisfying input) becomes `VIOLATED` (or `VERIFIED`), and any refuted one +becomes `INCONCLUSIVE`. Only exact inputs are checked, when no unknown +identifiers are registered; when the runtime can't evaluate them (e.g. a +function has no binding), the message says why. Policy invariants aren't +checked yet. The runtime must follow the CEL spec (e.g. `CelOptions.DEFAULT`, which has no +iteration limit), and a function with bindings must have one for each overload, +which is synchronous, deterministic and doesn't throw. + ### Comprehensions and Bounded Model Checking Because CEL lists/maps can be dynamically sized, the verifier cannot diff --git a/verifier/src/main/java/dev/cel/verifier/BUILD.bazel b/verifier/src/main/java/dev/cel/verifier/BUILD.bazel index 2d2549456..b23b6f2c5 100644 --- a/verifier/src/main/java/dev/cel/verifier/BUILD.bazel +++ b/verifier/src/main/java/dev/cel/verifier/BUILD.bazel @@ -170,6 +170,7 @@ java_library( "CelZ3StructTranslator.java", "CelZ3TranslationContext.java", "CelZ3TypeConstraintGenerator.java", + "RuntimeOracle.java", "TranslatedValue.java", ], compatible_with = [], @@ -184,10 +185,12 @@ java_library( "//bundle:cel", "//common:cel_ast", "//common:compiler_common", + "//common:error_codes", "//common:operator", "//common/ast", "//common/ast:cel_block", "//common/internal:cel_descriptor_pools", + "//common/internal:map_keys", "//common/internal:proto_time_utils", "//common/internal:well_known_proto", "//common/types", @@ -198,6 +201,10 @@ java_library( "//optimizer", "//optimizer:optimization_exception", "//optimizer:optimizer_builder", + "//runtime", + "//runtime:function_resolver", + "//runtime:resolved_overload", + "//runtime:unknown_attributes", "//verifier/axioms", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", diff --git a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java index 25496cc8c..d150ec1c5 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java @@ -243,7 +243,7 @@ private TranslatedValue translateIdent(CelExpr celExpr, CelAbstractSyntaxTree as } String name = ident.name(); CelType type = ast.getTypeOrThrow(exprId); - if (type instanceof TypeType) { + if (type instanceof TypeType && !context.lookupSymbol(name).isPresent()) { TypeType typeType = (TypeType) type; if (typeType.type().kind() != CelKind.DYN) { return TranslatedValue.create( diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java b/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java index c49bb4343..e78f11a32 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java @@ -82,6 +82,23 @@ public interface CelVerifierBuilder { @CanIgnoreReturnValue CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit); + /** + * Checks the inputs the solver finds by evaluating them with the CEL runtime of the verifier's + * environment (off by default). For now this only validates them, without refining the solver + * (counterexample-guided abstraction refinement, CEGAR). An {@code INCONCLUSIVE} counterexample + * or satisfying input that the runtime confirms becomes {@code VIOLATED} (or {@code VERIFIED} for + * satisfiability), and any counterexample or satisfying input it disagrees with becomes {@code + * INCONCLUSIVE}. + * + *

Inputs are checked only when all their values are exact and no unknown identifiers are + * registered. When the runtime can't evaluate them, e.g. a function has no binding, the message + * says why. Policy invariants aren't checked yet. The runtime must follow the CEL spec (e.g. + * {@code CelOptions.DEFAULT}, which has no iteration limit), and a function with bindings must + * have one for each overload, which is synchronous, deterministic and doesn't throw. + */ + @CanIgnoreReturnValue + CelVerifierBuilder enableCegar(boolean value); + /** Builds the {@link CelVerifier} instance. */ CelVerifier build(); } diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java index 900821e92..48ffd534c 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; +import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -41,12 +42,15 @@ import dev.cel.optimizer.CelOptimizationException; import dev.cel.optimizer.CelOptimizer; import dev.cel.optimizer.CelOptimizerFactory; +import dev.cel.runtime.CelEvaluationException; import dev.cel.verifier.axioms.CelZ3FunctionAxiom; import dev.cel.verifier.axioms.CelZ3StandardAxioms; import java.time.Duration; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; /** Z3 implementation of the CelVerifier. */ @Immutable @@ -91,6 +95,7 @@ public Optional findType(String typeName) { private final ImmutableSet unknownIdentifiers; private final CelZ3FunctionRegistry functionRegistry; private final CelTypeProvider typeProvider; + private final boolean enableCegar; @SuppressWarnings("Immutable") // Cel environment is immutable, just not marked as such private final Cel cel; @@ -111,6 +116,7 @@ static final class Builder implements CelVerifierBuilder { private final ImmutableList.Builder functionAxioms; private final Cel cel; private CelTypeProvider typeProvider; + private boolean enableCegar; @Override @CanIgnoreReturnValue @@ -144,6 +150,13 @@ public CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit) { return this; } + @Override + @CanIgnoreReturnValue + public CelVerifierBuilder enableCegar(boolean value) { + this.enableCegar = value; + return this; + } + @CanIgnoreReturnValue Builder addFunctionAxioms(CelZ3FunctionAxiom... axioms) { return addFunctionAxioms(Arrays.asList(axioms)); @@ -170,6 +183,7 @@ public CelVerifier build() { unknownIdentifiers.build(), registry, typeProvider, + enableCegar, cel); } @@ -208,17 +222,19 @@ public CelVerificationResult verifyEquivalence( CanonicalizationOptimizer.newInstance( CanonicalizationOptimizer.CanonicalizationOptions.newBuilder().build())) .build(); + CelAbstractSyntaxTree canonicalA = astA; + CelAbstractSyntaxTree canonicalB = astB; try { - astA = optimizer.optimize(astA); - astB = optimizer.optimize(astB); + canonicalA = optimizer.optimize(astA); + canonicalB = optimizer.optimize(astB); } catch (CelOptimizationException e) { // Fall back to original ASTs if canonicalization or re-typechecking fails } try (Context ctx = new Context(ImmutableMap.of("model", "true"))) { CelAstToZ3Translator translator = newTranslator(ctx, /* parameterizeUnknowns= */ true); - TranslatedValue tvA = translator.translate(astA); - TranslatedValue tvB = translator.translate(astB); + TranslatedValue tvA = translator.translate(canonicalA); + TranslatedValue tvB = translator.translate(canonicalB); BoolExpr divergenceCondition = ctx.mkNot(ctx.mkEq(tvA.z3Expr(), tvB.z3Expr())); // Parameterized unknowns are keyed on the operation producing them, whereas CEL identifies @@ -243,7 +259,8 @@ public CelVerificationResult verifyEquivalence( /* unknownCondition= */ ctx.mkFalse(), /* isCounterexample= */ true, /* candidate= */ "a divergence", - /* exactMatchMessage= */ "Equivalence violation detected."); + /* exactMatchMessage= */ "Equivalence violation detected.", + runtimeCheck(results -> !results.get(0).equals(results.get(1)), astA, astB)); } } @@ -295,7 +312,8 @@ CelVerificationResult verifyImplication( unknownCondition, /* isCounterexample= */ true, /* candidate= */ "a counterexample", - /* exactMatchMessage= */ String.format("%s violation detected.", subjectName)); + /* exactMatchMessage= */ String.format("%s violation detected.", subjectName), + /* runtimeCheck= */ RuntimeChecker.UNCHECKED); } } @@ -316,7 +334,8 @@ private CelVerificationResult checkSatisfiability( /* unknownCondition= */ ctx.mkFalse(), /* isCounterexample= */ true, /* candidate= */ "a counterexample", - /* exactMatchMessage= */ "Condition is not always true."); + /* exactMatchMessage= */ "Condition is not always true.", + runtimeCheck(results -> !results.get(0).equals(true), ast)); } BoolExpr isUnmodeledUnknown = ctx.mkAnd(tv.isZ3Unknown(), tv.isApproximate()); return search( @@ -327,13 +346,16 @@ private CelVerificationResult checkSatisfiability( isUnmodeledUnknown, /* isCounterexample= */ false, /* candidate= */ "a satisfying input", - /* exactMatchMessage= */ "Condition is satisfiable."); + /* exactMatchMessage= */ "Condition is satisfiable.", + runtimeCheck(results -> results.get(0).equals(true), ast)); } } /** * Searches for a {@code condition} match: first an exact one, then an approximate one, then an - * unmodeled result satisfying {@code unknownCondition} that may stand in for one. + * unmodeled result satisfying {@code unknownCondition} that may stand in for one. {@code + * runtimeCheck} checks a witness with the runtime, noting why if it couldn't. Confirming an + * approximate match doesn't count internal runtime errors. */ private CelVerificationResult search( Context ctx, @@ -343,7 +365,8 @@ private CelVerificationResult search( BoolExpr unknownCondition, boolean isCounterexample, String candidate, - String exactMatchMessage) + String exactMatchMessage, + RuntimeChecker runtimeCheck) throws CelVerificationException { Solver solver = newSolver(ctx); solver.add(translator.getTypeConstraints().toArray(new BoolExpr[0])); @@ -353,9 +376,14 @@ private CelVerificationResult search( Status status = checkSat(solver, typeSystem, condition, ctx.mkNot(taint)); if (status == Status.SATISFIABLE) { CelWitness witness = extractWitness(ctx, translator, solver.getModel()); - return isCounterexample - ? CelVerificationResult.failed(exactMatchMessage, witness) - : CelVerificationResult.verified(exactMatchMessage, witness); + RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ true); + if (!check.isMatch().orElse(true)) { + return CelVerificationResult.inconclusive( + "Inconclusive: the CEL runtime disagrees with " + candidate + " the verifier found.", + witness, + isCounterexample); + } + return match(isCounterexample, exactMatchMessage + check.note(), witness); } else if (status == Status.UNKNOWN) { // Depends on solver internals; not deterministically testable. return solverGaveUp(solver); @@ -365,9 +393,19 @@ private CelVerificationResult search( status = checkSat(solver, typeSystem, condition); if (status == Status.SATISFIABLE) { Model model = solver.getModel(); + CelWitness witness = extractWitness(ctx, translator, model); + RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ false); + if (check.isMatch().orElse(false)) { + return match( + isCounterexample, + String.format( + "%s The CEL runtime confirmed %s the verifier approximated.", + exactMatchMessage, candidate), + witness); + } return CelVerificationResult.inconclusive( - inconclusiveReason(candidate, translator, model, taint), - extractWitness(ctx, translator, model), + inconclusiveReason(candidate, translator, model, taint) + check.note(), + witness, isCounterexample); } else if (status == Status.UNKNOWN) { // Depends on solver internals; not deterministically testable. @@ -389,6 +427,62 @@ private CelVerificationResult search( : CelVerificationResult.failed("Condition is not satisfiable."); } + private static CelVerificationResult match( + boolean isCounterexample, String reason, CelWitness witness) { + return isCounterexample + ? CelVerificationResult.failed(reason, witness) + : CelVerificationResult.verified(reason, witness); + } + + private interface RuntimeChecker { + RuntimeChecker UNCHECKED = (witness, countInternalErrors) -> RuntimeCheck.UNCHECKED; + + RuntimeCheck check(CelWitness witness, boolean countInternalErrors); + } + + // Whether the runtime agrees a witness is a match, if it checked it, and a note for the message. + @AutoValue + abstract static class RuntimeCheck { + static final RuntimeCheck UNCHECKED = + new AutoValue_CelVerifierZ3Impl_RuntimeCheck(Optional.empty(), ""); + + abstract Optional isMatch(); + + abstract String note(); + + static RuntimeCheck of(boolean isMatch) { + return new AutoValue_CelVerifierZ3Impl_RuntimeCheck( + Optional.of(isMatch), isMatch ? "" : " The CEL runtime disagrees with the one found."); + } + + static RuntimeCheck failed(String reason) { + return new AutoValue_CelVerifierZ3Impl_RuntimeCheck( + Optional.empty(), + " The CEL runtime couldn't check it: " + reason + (reason.endsWith(".") ? "" : ".")); + } + } + + // Checks the witness's inputs with the runtime, if refinement is on and they're exact. Unknowns + // aren't inputs the runtime can take. + private RuntimeChecker runtimeCheck( + Predicate> isMatch, CelAbstractSyntaxTree... asts) { + if (!enableCegar || !unknownIdentifiers.isEmpty()) { + return RuntimeChecker.UNCHECKED; + } + return (witness, countInternalErrors) -> { + Optional> inputs = RuntimeOracle.exactInputs(witness); + if (!inputs.isPresent()) { + return RuntimeCheck.UNCHECKED; + } + try { + return RuntimeCheck.of( + isMatch.test(RuntimeOracle.evaluate(cel, inputs.get(), countInternalErrors, asts))); + } catch (CelEvaluationException e) { + return RuntimeCheck.failed(e.getMessage()); + } + }; + } + // Throws on a timeout. Leaves the assertions in place unless they're unsatisfiable, so the model // or the reason the solver gave up can be read. private static Status checkSat(Solver solver, CelZ3TypeSystem typeSystem, BoolExpr... assertions) @@ -468,12 +562,14 @@ private CelVerifierZ3Impl( ImmutableSet unknownIdentifiers, CelZ3FunctionRegistry functionRegistry, CelTypeProvider typeProvider, + boolean enableCegar, Cel cel) { this.timeout = timeout; this.comprehensionUnrollLimit = comprehensionUnrollLimit; this.unknownIdentifiers = unknownIdentifiers; this.functionRegistry = functionRegistry; this.typeProvider = typeProvider; + this.enableCegar = enableCegar; this.cel = cel; } } diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java index 379fba20a..f4b4e1904 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java @@ -51,13 +51,6 @@ final class CelZ3ComprehensionTranslator { TranslatedValue translateComprehension(CelExpr celExpr, CelAbstractSyntaxTree ast) { CelComprehension comp = celExpr.comprehension(); CelExpr iterRangeExpr = comp.iterRange(); - if (iterRangeExpr.exprKind().getKind() == ExprKind.Kind.IDENT) { - iterRangeExpr = - context - .lookupSymbol(iterRangeExpr.ident().name()) - .flatMap(TranslatedValue::celExpr) - .orElse(iterRangeExpr); - } List iterationElements = new ArrayList<>(); List taints = new ArrayList<>(); List> allRangeElems = new ArrayList<>(); @@ -86,6 +79,7 @@ TranslatedValue translateComprehension(CelExpr celExpr, CelAbstractSyntaxTree as allRangeElems.add(key); allRangeElems.add(value); } + allRangeElems.add(translateExpr(iterRangeExpr, ast).z3Expr()); } else { return translateDynamicComprehension(celExpr, ast); } @@ -182,18 +176,36 @@ private TranslatedValue translateDynamicComprehension( isMap ? (ArrayExpr) typeSystem.getMapPresence(typeSystem.getMapRef(iterRange)) : null; BoolExpr isValidRange = isMap ? typeSystem.isMap(iterRange) : typeSystem.isList(iterRange); + int unrollLimit = unrollLimit(comp.iterRange()); BoolExpr isTruncated = context.recordUnrollLimitApproximation( - ctx.mkAnd(isValidRange, ctx.mkGt(lengthExpr, ctx.mkInt(comprehensionUnrollLimit)))); + ctx.mkAnd(isValidRange, ctx.mkGt(lengthExpr, ctx.mkInt(unrollLimit)))); context.addUnmodeledResult(isTruncated); if (isAllMacro(comp) || isExistsMacro(comp)) { return unrollAllAndExists( - celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv); + celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv, unrollLimit); } else { return unrollMapAndFilter( - celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv); + celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv, unrollLimit); + } + } + + /** The size of the literal bound to {@code iterRange}, else the unroll limit. */ + private int unrollLimit(CelExpr iterRange) { + if (iterRange.exprKind().getKind() != ExprKind.Kind.IDENT) { + return comprehensionUnrollLimit; } + return context + .lookupSymbol(iterRange.ident().name()) + .flatMap(TranslatedValue::celExpr) + .map(e -> + e.exprKind().getKind() == ExprKind.Kind.LIST + ? e.list().elements().size() + : e.exprKind().getKind() == ExprKind.Kind.MAP + ? e.map().entries().size() + : comprehensionUnrollLimit) + .orElse(comprehensionUnrollLimit); } private T withIterationScope( @@ -260,7 +272,8 @@ private TranslatedValue unrollAllAndExists( ArithExpr lengthExpr, ArrayExpr mapPresence, BoolExpr isTruncated, - TranslatedValue iterRangeTv) { + TranslatedValue iterRangeTv, + int unrollLimit) { CelComprehension comp = celExpr.comprehension(); List iterations = new ArrayList<>(); TranslatedValue accuInitTv = translateExpr(comp.accuInit(), ast); @@ -269,7 +282,7 @@ private TranslatedValue unrollAllAndExists( boolean isMap = mapPresence != null; boolean isTwoVar = !comp.iterVar2().isEmpty(); - for (int i = 0; i < comprehensionUnrollLimit; i++) { + for (int i = 0; i < unrollLimit; i++) { IntExpr idx = ctx.mkInt(i); BoolExpr inBounds = ctx.mkLt(idx, lengthExpr); Expr xVal = ctx.mkNth(seq, idx); @@ -301,7 +314,8 @@ private TranslatedValue unrollMapAndFilter( ArithExpr lengthExpr, ArrayExpr mapPresence, BoolExpr isTruncated, - TranslatedValue iterRangeTv) { + TranslatedValue iterRangeTv, + int unrollLimit) { CelComprehension comp = celExpr.comprehension(); // For macros like map and filter, we must sequentially thread the accumulator through the loop TranslatedValue chainedAccuTv = translateExpr(comp.accuInit(), ast); @@ -314,7 +328,7 @@ private TranslatedValue unrollMapAndFilter( taints.add(chainedAccuTv.isApproximate()); taints.add(iterRangeTv.isApproximate()); - for (int i = 0; i < comprehensionUnrollLimit; i++) { + for (int i = 0; i < unrollLimit; i++) { IntExpr idx = ctx.mkInt(i); BoolExpr inBounds = ctx.mkLt(idx, lengthExpr); Expr xVal = ctx.mkNth(seq, idx); diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java index c7aef6643..94bcef460 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java @@ -26,6 +26,7 @@ import com.microsoft.z3.FuncDecl; import com.microsoft.z3.IntNum; import com.microsoft.z3.Model; +import dev.cel.common.internal.MapKeys; import dev.cel.common.internal.ProtoTimeUtils; import dev.cel.common.values.CelByteString; import dev.cel.common.values.NullValue; @@ -265,6 +266,7 @@ private Value extractMap(Expr mapRef) { ImmutableMap.Builder values = ImmutableMap.builder(); List entries = new ArrayList<>(); Set> seenKeys = new HashSet<>(); + Set runtimeKeys = new HashSet<>(); ArrayExpr keySet = ctx.mkConstArray(typeSystem.celValueSort(), ctx.mkFalse()); for (long i = 0; i < limit; i++) { Expr key = evaluate(ctx.mkNth(typeSystem.getMapKeys(mapRef), ctx.mkInt(i))); @@ -276,7 +278,10 @@ private Value extractMap(Expr mapRef) { Value keyValue = extract(key); Value value = extract(evaluate(ctx.mkSelect((ArrayExpr) typeSystem.getMapValues(mapRef), key))); - if (keyValue.runtimeValue.isPresent() && value.runtimeValue.isPresent()) { + if (keyValue.runtimeValue.isPresent() + && value.runtimeValue.isPresent() + && MapKeys.isValidKeyType(keyValue.runtimeValue.get()) + && runtimeKeys.add(MapKeys.normalize(keyValue.runtimeValue.get()))) { values.put(keyValue.runtimeValue.get(), value.runtimeValue.get()); } else { isExact = false; diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java index c223dbd9b..6f1e17e43 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java @@ -65,6 +65,14 @@ final class CelZ3OperatorTranslator { Optional translateFunctionCall( String functionName, List args, long exprId, CelAbstractSyntaxTree ast) { + ImmutableList overloadIds = + ast.getReference(exprId).map(CelReference::overloadIds).orElse(ImmutableList.of()); + if (overloadIds.equals(ImmutableList.of("optional_or_optional"))) { + return Optional.of(translateOptionalOr(functionName, args, ast, /* unwrap= */ false)); + } + if (overloadIds.equals(ImmutableList.of("optional_orValue_value"))) { + return Optional.of(translateOptionalOr(functionName, args, ast, /* unwrap= */ true)); + } Optional opOpt = Operator.findReverse(functionName); ImmutableList> z3Args = args.stream().map(TranslatedValue::z3Expr).collect(toImmutableList()); @@ -238,6 +246,43 @@ private BoolExpr constrainInWitnessTypes( mayObserveTruncatedTypeConstraint(ast, lhsTv, rhsTv))); } + /** + * Translates {@code receiver.or(alternative)}, or {@code orValue} if {@code unwrap}, which only + * evaluates the alternative if the receiver is empty. + */ + private TranslatedValue translateOptionalOr( + String functionName, List args, CelAbstractSyntaxTree ast, boolean unwrap) { + TranslatedValue receiver = args.get(0); + TranslatedValue alternative = args.get(1); + Expr optRef = typeSystem.getOptionalRef(receiver.z3Expr()); + TranslatedValue hasValue = + TranslatedValue.propagateStrict( + ctx, + typeSystem, + functionName, + typeSystem.withRuntimeError( + typeSystem.wrapBool(typeSystem.optHasValue(optRef)), + ctx.mkNot(typeSystem.isOptional(receiver.z3Expr()))), + receiver); + TranslatedValue present = + unwrap + ? TranslatedValue.create(typeSystem.getOptionalValue(optRef), typeSystem, ctx.mkFalse()) + : receiver; + Expr alt = alternative.z3Expr(); + // `isUnknown` is untested: a model can't be forced to make an unknown input unknown. + TranslatedValue absent = + unwrap + ? alternative + : TranslatedValue.create( + ctx.mkITE( + ctx.mkOr(typeSystem.isOptional(alt), typeSystem.isUnknown(alt)), + alt, + typeSystem.mkError()), + typeSystem, + alternative.isApproximate()); + return translateConditional(ImmutableList.of(hasValue, present, absent), ast); + } + private BoolExpr mkTypeGuard(Expr arg, CelType expectedType) { switch (expectedType.kind()) { case LIST: @@ -864,9 +909,6 @@ private BoolExpr listMembership( // Whether lhs may equal an element of elemType: a number only across numeric types, and a list // only another list (anything else conservatively). private BoolExpr mayEqualElement(Expr lhs, CelType elemType) { - if (elemType instanceof NullableType) { - return ctx.mkTrue(); - } if (isNumericType(elemType)) { return isNumeric(lhs); } @@ -989,8 +1031,6 @@ private TranslatedValue translateEquality( structuralEq = (BoolExpr) ctx.mkITE(bothMaps, mapEquality, structuralEq); } else if (!isStaticallyKnown(type0) && !isStaticallyKnown(type1)) { // Performance only: a side with a known type isn't a list or map here. - // TODO: Unless the type checker narrowed it, e.g. `(int_list + - // dyn_var)[0]`. structuralEq = approximateEqualityWhere( context.recordApproximation( @@ -1027,13 +1067,11 @@ private static final class IndexAccess { private final BoolExpr valid; private final BoolExpr found; private final Expr value; - private final BoolExpr isApproximate; - private IndexAccess(BoolExpr valid, BoolExpr found, Expr value, BoolExpr isApproximate) { + private IndexAccess(BoolExpr valid, BoolExpr found, Expr value) { this.valid = valid; this.found = found; this.value = value; - this.isApproximate = isApproximate; } } @@ -1043,8 +1081,7 @@ private IndexAccess listAccess(Expr lhsTrans, Expr rhsTrans) { return new IndexAccess( ctx.mkAnd(typeSystem.isList(lhsTrans), typeSystem.isInt(rhsTrans)), ctx.mkAnd(ctx.mkGe(index, ctx.mkInt(0)), ctx.mkLt(index, ctx.mkLength(seq))), - ctx.mkNth(seq, index), - /* isApproximate= */ ctx.mkFalse()); + ctx.mkNth(seq, index)); } private IndexAccess mapAccess(Expr lhsTrans, Expr rhsTrans) { @@ -1052,45 +1089,32 @@ private IndexAccess mapAccess(Expr lhsTrans, Expr rhsTrans) { return new IndexAccess( typeSystem.isMap(lhsTrans), typeSystem.hasEqualKey(lhsTrans, equalKeys), - typeSystem.valueAtEqualKey(lhsTrans, equalKeys), - /* isApproximate= */ ctx.mkFalse()); + typeSystem.valueAtEqualKey(lhsTrans, equalKeys)); } /** - * Looks up {@code rhsTrans} in the list or map {@code lhsTrans}. The value is approximate if the - * static element type isn't guaranteed and it may not conform. + * Looks up {@code rhsTrans} in the list or map {@code lhsTrans} of type {@code containerType}. */ - private IndexAccess indexAccess( - Expr lhsTrans, Expr rhsTrans, CelType containerType, boolean isLhsTypeGuaranteed) { + private IndexAccess indexAccess(Expr lhsTrans, Expr rhsTrans, CelType containerType) { IndexAccess list = listAccess(lhsTrans, rhsTrans); IndexAccess map = mapAccess(lhsTrans, rhsTrans); IndexAccess access = new IndexAccess( ctx.mkOr(list.valid, map.valid), (BoolExpr) ctx.mkITE(list.valid, list.found, map.found), - ctx.mkITE(list.valid, list.value, map.value), - /* isApproximate= */ ctx.mkFalse()); + ctx.mkITE(list.valid, list.value, map.value)); BoolExpr isRead = ctx.mkAnd(access.valid, access.found); context.constrainValidValue(isRead, access.value); - if (containerType.kind() != CelKind.MAP && containerType.kind() != CelKind.LIST) { - return access; - } - CelType elemType = - containerType.kind() == CelKind.MAP - ? ((MapType) containerType).valueType() - : ((ListType) containerType).elemType(); - BoolExpr conforms = typeConstraintGenerator.createTypeConstraintForType(access.value, elemType); - if (isLhsTypeGuaranteed) { - context.addConstraint(ctx.mkImplies(isRead, conforms)); - return access; + if (containerType instanceof MapType || containerType instanceof ListType) { + CelType elemType = + containerType instanceof MapType + ? ((MapType) containerType).valueType() + : ((ListType) containerType).elemType(); + context.addConstraint( + ctx.mkImplies( + isRead, typeConstraintGenerator.createTypeConstraintForType(access.value, elemType))); } - // The type checker may narrow other static types (e.g. `(['a'] + dyn_var)[1]` is typed - // string), so the element may not conform. - return new IndexAccess( - access.valid, - access.found, - access.value, - CelZ3Exprs.mkAndFlattened(ctx, isRead, CelZ3Exprs.mkNotFlattened(ctx, conforms))); + return access; } /** @@ -1102,7 +1126,6 @@ private TranslatedValue translateIndex( TranslatedValue lhs = args.get(0); TranslatedValue rhs = args.get(1); CelType lhsType = extractAstTypeOrDefault(lhs, ast); - boolean isLhsTypeGuaranteed = lhsType.equals(guaranteedType(lhs, ast)); CelType containerType = lhsType instanceof OptionalType ? lhsType.parameters().get(0) : lhsType; Expr lhsTrans = lhs.z3Expr(); @@ -1112,7 +1135,7 @@ private TranslatedValue translateIndex( BoolExpr lhsHasValue = typeSystem.optHasValue(optRef); lhsTrans = ctx.mkITE(isOptionalLhs, typeSystem.getOptionalValue(optRef), lhsTrans); // An empty optional's value can't be read, so constraining it is harmless. - IndexAccess access = indexAccess(lhsTrans, rhsTrans, containerType, isLhsTypeGuaranteed); + IndexAccess access = indexAccess(lhsTrans, rhsTrans, containerType); Expr element = ctx.mkITE(ctx.mkAnd(access.valid, access.found), access.value, typeSystem.mkError()); Expr optionalElement = @@ -1139,8 +1162,7 @@ private TranslatedValue translateIndex( BoolExpr isMapIndexTruncated = context.isTruncatedMap(lhsTrans); return TranslatedValue.propagateStrict(ctx, typeSystem, op.getFunction(), actualValue, args) .withApproximation( - CelZ3Exprs.mkOrFlattened( - ctx, isListIndexTruncated, isMapIndexTruncated, access.isApproximate)); + CelZ3Exprs.mkOrFlattened(ctx, isListIndexTruncated, isMapIndexTruncated)); } private TranslatedValue translateConditional( @@ -1197,12 +1219,7 @@ private static CelType extractAstTypeOrDefault(TranslatedValue val, CelAbstractS return val.celExpr().map(node -> ast.getTypeOrThrow(node.id())).orElse(SimpleType.DYN); } - /** - * Returns the type that {@code val} is guaranteed to conform to at runtime: a constant's, the - * declared type of an input variable, or else dyn. Global constraints may only be derived from - * guaranteed types: the type checker may narrow static types (e.g. {@code ['a'] + dyn_var} is - * typed list(string)), so constraints derived from those could exclude valid inputs. - */ + /** Returns a constant's type, or an input variable's declared type, or else dyn. */ private CelType guaranteedType(TranslatedValue val, CelAbstractSyntaxTree ast) { return val.isLiteral(ExprKind.Kind.CONSTANT) ? extractAstTypeOrDefault(val, ast) @@ -1254,9 +1271,11 @@ private static boolean hasStructuralEquality(CelType type) { } } + /** Excludes wrappers, whose kind is their value's but which may also be null. */ private static boolean isNumericType(CelType type) { CelKind kind = type.kind(); - return kind == CelKind.INT || kind == CelKind.UINT || kind == CelKind.DOUBLE; + return !(type instanceof NullableType) + && (kind == CelKind.INT || kind == CelKind.UINT || kind == CelKind.DOUBLE); } /** diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java index 05e6e307a..ff5d80ab1 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java @@ -178,20 +178,24 @@ BoolExpr hasUnmodeledResult() { } void bindSymbol(String name, TranslatedValue value) { - localScope.put(name, value); + symbolTable.put(name, value); } Optional lookupSymbol(String name) { - return Optional.ofNullable(localScope.get(name)); + TranslatedValue local = localScope.get(name); + return Optional.ofNullable(local != null ? local : symbolTable.get(globalName(name))); } TranslatedValue computeSymbolIfAbsent( String rawName, Function mappingFunction) { - if (!rawName.startsWith(".") && localScope.containsKey(rawName)) { + if (localScope.containsKey(rawName)) { return localScope.get(rawName); } - return symbolTable.computeIfAbsent( - rawName.startsWith(".") ? rawName.substring(1) : rawName, mappingFunction); + return symbolTable.computeIfAbsent(globalName(rawName), mappingFunction); + } + + private static String globalName(String name) { + return name.startsWith(".") ? name.substring(1) : name; } /** Binds {@code varName} to {@code value} while running {@code action}, then restores it. */ diff --git a/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java b/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java new file mode 100644 index 000000000..00ac15ca3 --- /dev/null +++ b/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.verifier; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelErrorCode; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelFunctionResolver; +import dev.cel.runtime.CelResolvedOverload; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntime.Program; +import dev.cel.runtime.CelUnknownSet; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Evaluates expressions with the CEL runtime on a witness's inputs, to check the solver's + * candidates. + */ +final class RuntimeOracle { + + // Stands for every evaluation error, matching the verifier's single error value. + private enum EvaluationError { + ERROR + } + + /** The witness's inputs, if every value is exact. */ + static Optional> exactInputs(CelWitness witness) { + return witness.variables().values().stream().allMatch(v -> v.value().isPresent()) + ? Optional.of( + ImmutableMap.copyOf(Maps.transformValues(witness.variables(), v -> v.value().get()))) + : Optional.empty(); + } + + /** + * Returns the values of {@code asts} for {@code inputs}, with errors as one value. Throws if the + * runtime can't evaluate one, e.g. a function has no binding or its iteration limit is hit, or if + * it fails with an internal error and {@code countInternalErrors} is false. + */ + static ImmutableList evaluate( + CelRuntime runtime, + Map inputs, + boolean countInternalErrors, + CelAbstractSyntaxTree... asts) + throws CelEvaluationException { + ImmutableList.Builder results = ImmutableList.builder(); + for (CelAbstractSyntaxTree ast : asts) { + Program program = runtime.createProgram(ast); + // Notes calls to late-bound functions, which have no binding here. The legacy interpreter + // also consults them after any failed dispatch, so its dispatch errors go unchecked. + AtomicBoolean callsLateBound = new AtomicBoolean(); + CelFunctionResolver lateBoundFunctions = + new CelFunctionResolver() { + @Override + public Optional findOverloadMatchingArgs( + String functionName, Collection overloadIds, Object[] args) { + return findOverloadMatchingArgs(functionName, args); + } + + @Override + public Optional findOverloadMatchingArgs( + String functionName, Object[] args) { + callsLateBound.set(true); + return Optional.empty(); + } + }; + Object result; + try { + result = program.eval(inputs, lateBoundFunctions); + } catch (CelEvaluationException e) { + CelErrorCode code = e.getErrorCode(); + // The runtime reports some CEL errors as internal, but also misused bindings (e.g. async). + if (callsLateBound.get() + || code == CelErrorCode.ITERATION_BUDGET_EXCEEDED + || (!countInternalErrors && code == CelErrorCode.INTERNAL_ERROR)) { + throw e; + } + result = EvaluationError.ERROR; + } + // A binding may return null or unknowns. + if (result == null || result instanceof CelUnknownSet) { + throw new CelEvaluationException( + result == null ? "a function returned null" : "it evaluated to unknowns"); + } + results.add(result); + } + return results.build(); + } + + private RuntimeOracle() {} +} diff --git a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java index 748b6f6ca..956bb8a84 100644 --- a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java +++ b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java @@ -18,8 +18,6 @@ import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_NONE; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OF; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OF_NON_ZERO_VALUE; -import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR; -import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR_VALUE; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_VALUE; import com.google.common.base.Preconditions; @@ -27,7 +25,6 @@ import com.microsoft.z3.BoolExpr; import com.microsoft.z3.Context; import com.microsoft.z3.Expr; -import com.microsoft.z3.FPExpr; import com.microsoft.z3.SeqExpr; import dev.cel.common.CelFunctionDecl; import dev.cel.common.CelOverloadDecl; @@ -72,19 +69,6 @@ final class OptionalAxioms { Expr optRef = ts.getOptionalRef(val); return Optional.of( ctx.mkITE(ts.optHasValue(optRef), ts.getOptionalValue(optRef), ts.mkError())); - }), - createBinaryAxiom( - OPTIONAL_OR_VALUE, - (ctx, ts, sink, val, other) -> { - Expr optRef = ts.getOptionalRef(val); - return Optional.of( - ctx.mkITE(ts.optHasValue(optRef), ts.getOptionalValue(optRef), other)); - }), - createBinaryAxiom( - OPTIONAL_OR, - (ctx, ts, sink, val, other) -> { - Expr optRef = ts.getOptionalRef(val); - return Optional.of(ctx.mkITE(ts.optHasValue(optRef), val, other)); })); private static BoolExpr isZeroValue(Context ctx, CelZ3TypeSystem ts, Expr val) { @@ -93,7 +77,7 @@ private static BoolExpr isZeroValue(Context ctx, CelZ3TypeSystem ts, Expr val ctx.mkAnd(ts.isBool(val), ctx.mkEq(ts.unwrapBool(val), ctx.mkFalse())), ctx.mkAnd(ts.isInt(val), ctx.mkEq(ts.getInt(val), ctx.mkInt(0))), ctx.mkAnd(ts.isUint(val), ctx.mkEq(ts.getUint(val), ctx.mkInt(0))), - ctx.mkAnd(ts.isDouble(val), ctx.mkFPIsZero((FPExpr) ts.getDouble(val))), + ctx.mkAnd(ts.isDouble(val), ctx.mkFPIsZero(ts.getDouble(val))), ctx.mkAnd(ts.isTimestamp(val), ctx.mkEq(ts.getTimestamp(val), ctx.mkInt(0))), ctx.mkAnd(ts.isDuration(val), ctx.mkEq(ts.getDuration(val), ctx.mkInt(0))), ctx.mkAnd(ts.isString(val), ctx.mkEq(ts.getString(val), ctx.mkString(""))), @@ -113,25 +97,17 @@ private static BoolExpr isZeroValue(Context ctx, CelZ3TypeSystem ts, Expr val private static CelZ3FunctionAxiom createAxiom( CelFunctionDecl.Declarer declarer, CelZ3OverloadTranslator translator) { CelFunctionDecl functionDecl = declarer.functionDecl(); - CelZ3FunctionAxiom.Builder builder = CelZ3FunctionAxiom.newBuilder(functionDecl); - builder.addOverloadTranslator(getSingleOverloadOrThrow(functionDecl), translator); - return builder.build(); + return CelZ3FunctionAxiom.newBuilder(functionDecl) + .addOverloadTranslator(getSingleOverloadOrThrow(functionDecl), translator) + .build(); } private static CelZ3FunctionAxiom createUnaryAxiom( CelFunctionDecl.Declarer declarer, CelZ3FunctionAxiom.UnaryTranslator translator) { CelFunctionDecl functionDecl = declarer.functionDecl(); - CelZ3FunctionAxiom.Builder builder = CelZ3FunctionAxiom.newBuilder(functionDecl); - builder.addUnaryOverloadTranslator(getSingleOverloadOrThrow(functionDecl), translator); - return builder.build(); - } - - private static CelZ3FunctionAxiom createBinaryAxiom( - CelFunctionDecl.Declarer declarer, CelZ3FunctionAxiom.BinaryTranslator translator) { - CelFunctionDecl functionDecl = declarer.functionDecl(); - CelZ3FunctionAxiom.Builder builder = CelZ3FunctionAxiom.newBuilder(functionDecl); - builder.addBinaryOverloadTranslator(getSingleOverloadOrThrow(functionDecl), translator); - return builder.build(); + return CelZ3FunctionAxiom.newBuilder(functionDecl) + .addUnaryOverloadTranslator(getSingleOverloadOrThrow(functionDecl), translator) + .build(); } private static CelOverloadDecl getSingleOverloadOrThrow(CelFunctionDecl functionDecl) { diff --git a/verifier/src/test/java/dev/cel/verifier/BUILD.bazel b/verifier/src/test/java/dev/cel/verifier/BUILD.bazel index 91a24c23d..da198b15e 100644 --- a/verifier/src/test/java/dev/cel/verifier/BUILD.bazel +++ b/verifier/src/test/java/dev/cel/verifier/BUILD.bazel @@ -47,8 +47,8 @@ java_library( "//policy:parser_factory", "//policy:validation_exception", "//policy/testing:k8s_test_tag_handler", - "//runtime", - "//runtime:evaluation_exception", + "//runtime:function_binding", + "//runtime:unknown_attributes", "//testing/protos:single_file_java_proto", "@bazel_tools//tools/java/runfiles", "@maven//:com_google_errorprone_error_prone_annotations", diff --git a/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java index 8c18f4da9..cd16e7d50 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java @@ -450,6 +450,54 @@ public void verifyInvariants_restrictedDestinationsPolicy_verified() throws Exce "unrestricted_destination_allowed", CelVerificationResult.verified()); } + private enum VariableReadTestCase { + INSIDE_SHADOWING_COMPREHENSION( + "[x]", "[999].all(x, variables.v.all(e, e == 999))", VerificationStatus.VIOLATED), + PAST_SHADOWING_ITERATION_VARIABLE( + "x", "[2].all(variables, .variables.v == 1)", VerificationStatus.VERIFIED), + TYPE("[int][x]", "variables.v == int", VerificationStatus.VIOLATED), + TYPE_ELEMENT("[int]", "variables.v.all(e, e == int)", VerificationStatus.VERIFIED), + LITERAL_PAST_UNROLL_LIMIT( + "[1, 2, 3, 4, 5, 6]", "variables.v.all(e, e > 0)", VerificationStatus.VERIFIED), + REPEATED_KEY("{x: 1, 1u: 2}", "variables.v.all(k, true)", VerificationStatus.VIOLATED); + + private final String variable; + private final String assertion; + private final VerificationStatus expectedStatus; + + VariableReadTestCase(String variable, String assertion, VerificationStatus expectedStatus) { + this.variable = variable; + this.assertion = assertion; + this.expectedStatus = expectedStatus; + } + } + + @Test + public void verifyInvariants_variableRead(@TestParameter VariableReadTestCase testCase) + throws Exception { + CelPolicy policy = + PARSER.parse( + "name: variable_read_policy\n" // + + "rule:\n" // + + " match:\n" // + + " - output: 'true'\n" // + + "verification:\n" // + + " variables:\n" // + + " - v: '" + + testCase.variable + + "'\n" // + + " invariants:\n" // + + " - id: reads_variable\n" // + + " assume: x == 1\n" // + + " assert: '" + + testCase.assertion + + "'"); + + ImmutableMap results = VERIFIER.verifyInvariants(policy); + + assertThat(results.get("reads_variable").status()).isEqualTo(testCase.expectedStatus); + } + @Test public void verifyInvariants_invariantReferencesPolicyVariable_verified() throws Exception { String policySource = @@ -504,31 +552,6 @@ public void verifyInvariants_verificationVariablesAndMultiClauseLists_verified() assertThat(results).containsExactly("ergonomic_inv", CelVerificationResult.verified()); } - @Test - public void verifyInvariants_literalListVariableBeyondUnrollLimit_verified() throws Exception { - String policySource = - "name: literal_list_policy\n" // - + "rule:\n" // - + " variables:\n" // - + " - xs: '[1, 2, 3, 4, 5, 6]'\n" // - + " match:\n" // - + " - output: 'true'\n" // - + "verification:\n" // - + " invariants:\n" // - + " - id: all_positive\n" // - + " assert: variables.xs.all(x, x > 0)"; - CelPolicy policy = PARSER.parse(policySource); - CelPolicyVerifier verifier = - CelPolicyVerifierFactory.newVerifier( - POLICY_COMPILER, - CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(5).build()) - .build(); - - ImmutableMap results = verifier.verifyInvariants(policy); - - assertThat(results).containsExactly("all_positive", CelVerificationResult.verified()); - } - @Test public void verifyInvariants_workloadAdmissionFlawed_violationsDetected() throws Exception { Cel extendedCel = diff --git a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java index 0fd3ba2cb..325594b78 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.Immutable; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; // import com.google.testing.testsize.MediumTest; @@ -38,6 +39,7 @@ import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelContainer; import dev.cel.common.CelFunctionDecl; +import dev.cel.common.CelOptions; import dev.cel.common.CelOverloadDecl; import dev.cel.common.CelVarDecl; import dev.cel.common.Operator; @@ -58,8 +60,11 @@ import dev.cel.optimizer.CelOptimizer; import dev.cel.optimizer.CelOptimizerFactory; import dev.cel.optimizer.optimizers.ConstantFoldingOptimizer; +import dev.cel.optimizer.optimizers.SubexpressionOptimizer; import dev.cel.parser.CelMacro; import dev.cel.parser.CelStandardMacro; +import dev.cel.runtime.CelFunctionBinding; +import dev.cel.runtime.CelUnknownSet; import dev.cel.verifier.CelVerificationResult.VerificationStatus; import dev.cel.verifier.axioms.CelZ3FunctionAxiom; import dev.cel.verifier.axioms.CelZ3OverloadResult; @@ -69,7 +74,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; -import java.util.function.UnaryOperator; import java.util.stream.IntStream; import org.junit.Before; import org.junit.Test; @@ -141,15 +145,35 @@ public final class CelVerifierZ3ImplTest { TestAllTypes.getDescriptor(), TestAllTypes.NestedMessage.getDescriptor())) .build(); - private static final CelVerifier VERIFIER = withRuntimeOracle(CEL, b -> b); + private static final CelVerifier VERIFIER = checkedVerifier(CEL).build(); - // For ASTs with functions the runtime can't plan, and for verifyImplication. - private static final CelVerifier VERIFIER_WITHOUT_ORACLE = - CelVerifierFactory.newVerifier(CEL).build(); + // Doesn't check witnesses with the runtime: for approximate results, which it may confirm, for + // functions it has no bindings for, and for verifyImplication. + private static final CelVerifier UNCHECKED_VERIFIER = CelVerifierFactory.newVerifier(CEL).build(); - // Builds a verifier whose exact witnesses are checked against the runtime of the same `cel`. - private static CelVerifier withRuntimeOracle(Cel cel, UnaryOperator config) { - return RuntimeOracle.wrap(cel, config.apply(CelVerifierFactory.newVerifier(cel)).build()); + private static final CelFunctionDecl ZERO_DECL = + CelFunctionDecl.newFunctionDeclaration( + "zero", CelOverloadDecl.newGlobalOverload("zero_int", SimpleType.INT, SimpleType.INT)); + + private static final Cel UNBOUND_ZERO_CEL = + CEL.toCelBuilder().addFunctionDeclarations(ZERO_DECL).build(); + + // The verifier doesn't model `zero`, but the runtime has a binding for it. + private static final Cel ZERO_CEL = + UNBOUND_ZERO_CEL + .toCelBuilder() + .addFunctionBindings(CelFunctionBinding.from("zero_int", Long.class, x -> 0L)) + .build(); + + // A verifier builder that checks witnesses with the runtime of the same `cel`. + private static CelVerifierBuilder checkedVerifier(Cel cel) { + return CelVerifierFactory.newVerifier(cel).enableCegar(true); + } + + // An exact verdict the runtime didn't refute or fail to check. + private static void assertExact(CelVerificationResult result, VerificationStatus status) { + assertWithMessage(result.message()).that(result.status()).isEqualTo(status); + assertThat(result.message()).doesNotContain("CEL runtime"); } @Before @@ -159,6 +183,8 @@ public void setUp() { private enum IsSatisfiableTestCase { SATISFIABLE("x > 5", "x = (?:[6-9]|[1-9]\\d+)"), + OPTIONAL_OR_VALUE_SKIPS_APPROXIMATE_ALTERNATIVE( + "optional.of(x).orValue(int('123')) == 7", "x = 7"), DYNAMIC_ARITHMETIC("request == 1 && request + 2 == 3", "request = 1"), DYNAMIC_ARITHMETIC_UNARY("request == 1 && -request == -1", "request = 1"), GREATER_DOUBLE("d > 1.5", "d = "), @@ -212,8 +238,13 @@ private enum IsSatisfiableTestCase { INT_MAX_DOUBLE_EQUALITY( "dyn(request) == 9223372036854775808.0", "request = 9223372036854775[2-8]\\d+"), UINT_MAX_DOUBLE_EQUALITY("dyn(u) == 18446744073709551616.0", "u = 1844674407370955\\d+u"), - // The type checker narrows `int_list + dyn_var` to list(int), which must not constrain dyn_var. IN_NARROWED_LIST_CONCATENATION("dyn('a') in (int_list + dyn_var)", "dyn_var = "), + DYN_TYPE_PARAMETER_JOIN( + "!b && (b ? dyn_int_map : string_dyn_map)['a'] != (b ? dyn_int_map : string_dyn_map)['a']", + "string_dyn_map = "), + OR_VALUE_OF_DYN_NAN( + "dyn_var.orValue(1) != dyn_var.orValue(1)", + "dyn_var = optional\\.of\\(double\\(\"NaN\"\\)\\)"), // Indexing and conditionals return existing values, so they do not inherit truncated type // constraints from their operands. DYN_ELEMENT_OF_LONG_NESTED_LIST_EQUALITY( @@ -253,7 +284,7 @@ public void isSatisfiable_success(@TestParameter IsSatisfiableTestCase testCase) CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); for (String fragment : testCase.expectedFragments) { assertThat(result.message()).containsMatch(fragment); } @@ -265,7 +296,7 @@ public void isSatisfiable_withVariable_returnsSatisfyingModel() throws Exception CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).containsMatch("x = (?:[6-9]|[1-9]\\d+)"); @@ -333,6 +364,7 @@ public void isSatisfiable_witnessValueSatisfiesExpressionAndDisplayRoundTrips( CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); CelWitness.Variable variable = result.witness().get().variables().get(testCase.variableName); assertThat(variable.displayValue()).isEqualTo(testCase.expectedDisplayValue); ImmutableMap input = @@ -370,6 +402,7 @@ public void isSatisfiable_noExactRuntimeValue_witnessValueEmpty( CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.witness().get().variables().get(testCase.variableName).value()).isEmpty(); } @@ -433,6 +466,7 @@ public void isSatisfiable_listAtDisplayLimit_displaysAllElements() throws Except CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .endsWith("int_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]"); } @@ -440,12 +474,13 @@ public void isSatisfiable_listAtDisplayLimit_displaysAllElements() throws Except @Test public void isSatisfiable_listPastDisplayLimit_elidesRemainingElements() throws Exception { // Within the unroll limit the whole list is extracted, so the display must still elide it. - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(16)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(16).build(); CelAbstractSyntaxTree ast = CEL.compile("int_list == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]").getAst(); CelVerificationResult result = verifier.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .endsWith( "int_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ... (1 more" @@ -458,7 +493,7 @@ public void isSatisfiable_mapNoContainerError_returnsSatisfyingModel() throws Ex CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("string_int_map = {"); @@ -471,7 +506,7 @@ public void isSatisfiable_listNoContainerError_returnsSatisfyingModel() throws E CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("dyn_list = ["); @@ -484,7 +519,7 @@ public void isSatisfiable_dynMapNoContainerError_returnsSatisfyingModel() throws CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("dyn_map = {"); @@ -497,7 +532,7 @@ public void counterexample_nullValueFormattedAsNull() throws Exception { CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("request = null"); } @@ -508,7 +543,7 @@ public void counterexample_mapFormattedCorrectly() throws Exception { CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).containsMatch("string_int_map = \\{[^}]*,[^}]*\\}"); assertThat(result.message()).contains("\"a\": 1"); } @@ -535,7 +570,7 @@ public void isAlwaysTrue_counterexampleNeverContainsError( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).doesNotContain("Error"); } @@ -545,7 +580,7 @@ public void isSatisfiable_unconditional_returnsUnconditionalMessage() throws Exc CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .isEqualTo( "Condition is satisfiable. (The expression is satisfiable unconditionally, regardless" @@ -625,7 +660,7 @@ public void isSatisfiable_approximate_listsApproximationSources( @TestParameter ApproximationSourceTestCase testCase) throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.reason()) @@ -665,7 +700,7 @@ public void isSatisfiable_uninterpretedFunctions_listsApproximationSources( .build(); CelAbstractSyntaxTree ast = celCompiler.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.reason()) @@ -680,7 +715,7 @@ public void isSatisfiable_uninterpretedFunctions_listsApproximationSources( public void isSatisfiable_approximate_returnsPotentialSatisfyingInput() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("int('123') == 123 ? x > 5 : false").getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()).contains("Inconclusive: a satisfying input may exist"); @@ -689,6 +724,11 @@ public void isSatisfiable_approximate_returnsPotentialSatisfyingInput() throws E } private enum IsSatisfiableInconclusiveTestCase { + OPTIONAL_OR_VALUE_EMPTY_RECEIVER_APPROXIMATE_ALTERNATIVE( + "optional.none().orValue(int('123')) == 7"), + OPTIONAL_OR_VALUE_APPROXIMATE_RECEIVER("optional.of(int('123')).orValue(1) == 7"), + OPTIONAL_OR_EMPTY_RECEIVER_APPROXIMATE_ALTERNATIVE( + "optional.none().or(optional.of(int('123'))) == optional.of(7)"), // A list of nullable ints may hold null past the unroll limit. NULL_IN_NULLABLE_LIST_PAST_UNROLL_LIMIT( "size(nullable_int_list) == 6 && nullable_int_list[0] != null && nullable_int_list[1] !=" @@ -784,7 +824,7 @@ public void isSatisfiable_inconclusive(@TestParameter IsSatisfiableInconclusiveT throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -799,7 +839,8 @@ public void isSatisfiable_mapLiteralWithDynMapBeyondBmcLimit_inconclusive( throws Exception { CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); // A small unroll limit keeps the solver fast; the maps are still past it. - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(1)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(1).build(); CelVerificationResult result = verifier.isSatisfiable(ast); @@ -814,8 +855,10 @@ public void isSatisfiable_maskedByBmcNested_inconclusive() throws Exception { CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelVerifier customVerifier = - withRuntimeOracle( - CEL, b -> b.setComprehensionUnrollLimit(3).setTypeProvider(TYPE_PROVIDER)); + CelVerifierFactory.newVerifier(CEL) + .setComprehensionUnrollLimit(3) + .setTypeProvider(TYPE_PROVIDER) + .build(); CelVerificationResult result = customVerifier.isSatisfiable(ast); @@ -827,14 +870,17 @@ public void isSatisfiable_comprehensionZeroUnrollLimit_inconclusive() throws Exc String expr = "int_list == [1] ? int_list.exists(x, x == 1) : false"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } private enum IsUnsatisfiableTestCase { + OPTIONAL_OR_NON_OPTIONAL_ALTERNATIVE("optional.none().or(dyn(x)) == dyn(x)"), DYN_MESSAGE_PRESENT_UNDECLARED_FIELD("dyn(test_all_types).unknown_field == 42"), + NULLABLE_NULL_AND_ONE("nullable_int == null && nullable_int == 1"), REPEATED_FIELD_FROM_NON_LIST( "dyn_var == 42 && TestAllTypes{repeated_int64: dyn_var} == TestAllTypes{}"), UNSATISFIABLE("x > 5 && x < 3"), @@ -958,7 +1004,7 @@ public void isSatisfiable_failure(@TestParameter IsUnsatisfiableTestCase testCas CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).isEqualTo("Condition is not satisfiable."); } @@ -1225,7 +1271,9 @@ private enum IsAlwaysTrueTestCase { OPTIONAL_OF_EQ_OF_VAR("optional.of(x) == optional.of(x)"), OPTIONAL_OF_NEQ_OF_VAR("x != y ? optional.of(x) != optional.of(y) : true"), OPTIONAL_OR_VALUE_VAR("optional.of(x).orValue(y) == x"), - // Indexing follows the value, whatever type the checker narrowed it to. + OPTIONAL_OR_VALUE_SKIPS_FAILING_ALTERNATIVE("optional.of(x).orValue(1 / 0) == x"), + OPTIONAL_OR_SKIPS_FAILING_ALTERNATIVE( + "optional.of(x).or(optional.of(1 / 0)) == optional.of(x)"), NARROWED_MAP_INDEXED_AS_LIST( "dyn_var == optional.of([2]) ? dyn_var.orValue({0: 1})[0] == 2 : true"), NARROWED_LIST_INDEXED_AS_MAP( @@ -1249,8 +1297,6 @@ private enum IsAlwaysTrueTestCase { OPTIONAL_LIST_INDEXED_BY_STRING("TestAllTypes{}.?repeated_int64[dyn('a')] == optional.none()"), OPTIONAL_LIST_OPTIONALLY_INDEXED_BY_STRING( "TestAllTypes{}.?repeated_int64[?dyn('a')] == optional.none()"), - // The checker finds several overloads for an optional(dyn) lhs, so the runtime indexes a value - // typed optional that isn't one by what it is. NARROWED_OPTIONAL_DYN_INDEX( "dyn_var == optional.of([2]) ? dyn_var.orValue(optional.of(dyn([1])))[0] == 2 : true"), NARROWED_OPTIONAL_DYN_OPTIONAL_INDEX( @@ -1585,7 +1631,8 @@ public void isAlwaysTrue_withUnknownIdentifier_evaluatesToUnknown( throws Exception { CelAbstractSyntaxTree ast = CEL.compile(expression).getAst(); - CelVerifier verifierWithUnknown = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("x")); + CelVerifier verifierWithUnknown = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("x").build(); // 'x == x' is not a tautology if it can be unknown // i.e: CelUnknown == CelUnknown is unknown. @@ -1619,7 +1666,8 @@ private enum ApproximateBesideUnknownTestCase { public void isAlwaysTrue_approximateBesideUnknown_inconclusive( @TestParameter ApproximateBesideUnknownTestCase testCase) throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("unknown_var")); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("unknown_var").build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); @@ -1640,9 +1688,9 @@ public void isAlwaysTrue_dynamicComprehensionNonBoolYieldsError() throws Excepti + " dyn_list.all(x, x.not_a_bool) == false) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelVerificationResult result = - withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(1)).isAlwaysTrue(ast); + checkedVerifier(CEL).setComprehensionUnrollLimit(1).build().isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .matches( "Condition is not always true\\. Counterexample input:\n" @@ -1654,7 +1702,8 @@ public void isAlwaysTrue_comprehensionExceedsMaxIterations_returnsUnknown() thro String expr = "int_list == [1, 2, 3] ? int_list.all(x, x > 0) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(2)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(2).build(); CelVerificationResult verifiedValue = verifier.isAlwaysTrue(ast); // Truncated loops return Unknown, which negates to Unknown. @@ -1671,7 +1720,7 @@ public void isAlwaysTrue_comprehensionZeroUnrollLimit_emptyList() throws Excepti String expr = "int_list == [] ? int_list.all(x, x > 0) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult verifiedValue = verifier.isAlwaysTrue(ast); assertThat(verifiedValue.status()).isEqualTo(VerificationStatus.VERIFIED); @@ -1701,7 +1750,7 @@ public void isAlwaysTrue_strictCustomFunction_errorPropagates() throws Exception CelAbstractSyntaxTree ast = celWithCustomFunc.compile("custom_func(1 / 0) == 5 || !(custom_func(1 / 0) == 5)").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); // Should fail verification because the error propagates through custom_func, // so the whole expression evaluates to an error, not true. @@ -1766,18 +1815,30 @@ public void verifyEquivalence_withUnknownIdentifiers_inconclusive( CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); CelVerifier verifier = - withRuntimeOracle( - CEL, - b -> - b.addUnknownIdentifier("a") - .addUnknownIdentifier("x") - .addUnknownIdentifier("unknown_var")); + CelVerifierFactory.newVerifier(CEL) + .addUnknownIdentifier("a") + .addUnknownIdentifier("x") + .addUnknownIdentifier("unknown_var") + .build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } + @Test + public void verifyEquivalence_orValueOfNonOptionalWithUnknownAlternative_verified() + throws Exception { + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("unknown_var").build(); + CelAbstractSyntaxTree astA = CEL.compile("dyn(1).orValue(unknown_var)").getAst(); + CelAbstractSyntaxTree astB = CEL.compile("1 / 0").getAst(); + + CelVerificationResult result = verifier.verifyEquivalence(astA, astB); + + assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + } + @Test public void verifyEquivalence_freeVariableIndicesDeduplicated() throws Exception { CelAbstractSyntaxTree astA = @@ -1785,7 +1846,7 @@ public void verifyEquivalence_freeVariableIndicesDeduplicated() throws Exception CelAbstractSyntaxTree astB = CEL.compile("x == y && y == port ? dyn_list.all(e, y == port) : false").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -1805,7 +1866,7 @@ public void isSatisfiable_approximateIterRangeInMap_inconclusive() throws Except CelAbstractSyntaxTree ast = celWithCustomFunc.compile("approx_list(1).map(x, x * 2) == []").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -1825,7 +1886,7 @@ public void isSatisfiable_mapWithUnusedApproximateIteration_verified() throws Ex CelAbstractSyntaxTree ast = celWithCustomFunc.compile("y == [] && y.map(x, approx_func(1)) == y").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); } @@ -1874,7 +1935,7 @@ public void isAlwaysTrue_unconditionalError_failsVerification( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isEqualTo( "Condition is not always true. (The expression fails unconditionally, regardless of" @@ -1889,7 +1950,7 @@ public void verifyEquivalence_unconditionalError_failsVerification( CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isEqualTo( "Equivalence violation detected. (The expression fails unconditionally, regardless of" @@ -1904,7 +1965,7 @@ public void verifyEquivalence_dynamicMapEquality_enforcesExtensionalityOnPresent CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("string_int_map ="); assertThat(result.message()).doesNotContain("string_int_map = {'a': 1}"); } @@ -1978,7 +2039,7 @@ public void verifyEquivalence_infinityConstants_notEquivalent() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } private enum IsAlwaysTrueViolationTestCase { @@ -1988,23 +2049,31 @@ private enum IsAlwaysTrueViolationTestCase { "string_dyn_map == string_dyn_map", "Condition is not always true\\."), // `[NaN]` is found nowhere, not even by its term. IN_LIST_NAN_LIST("[d] in [[d]]", "d = double\\(\"NaN\"\\)"), - // The checker types the value string, but at runtime it's the int 1, which equals 1.0. + NULLABLE_CONDITIONAL( + "b || nullable_int != null || dyn({'a': b ? 1 : nullable_int}.a) != null", + "nullable_int = null"), + OR_VALUE_OF_DYN_MAP_LITERAL_EQUALITY( + "dyn_var == optional.of({'a': 1.0}) ? dyn_var.orValue(string_int_map) != {'a': 1} : true", + "Condition is not always true\\."), + FIELD_OF_MESSAGE_OR_NULL( + "b || has((b ? test_all_types : null).single_int64)" + + " || !has((b ? test_all_types : null).single_int64)", + "Condition is not always true\\."), NARROWED_TYPE_NUMERIC_EQUALITY( "dyn_var == optional.of(1) ? optional.ofNonZeroValue('').or(dyn_var).value() != dyn(1.0)" + " : true", "dyn_var = optional\\.of\\(1\\)"), - // Indexing 42 errors whether or not the overload is picked by its narrowed type. NARROWED_OPTIONAL_INDEX_NOT_CONTAINER( "dyn_var == optional.of(42) ? dyn_var.orValue(optional.of([1]))[0] in [42, optional.of(42)]" + " : true", "dyn_var = optional\\.of\\(42\\)"), - // Selecting from an optional follows its value, here 42 rather than the map its type says. NARROWED_OPTIONAL_MAP_SELECTED( "dyn_var == optional.of(optional.of(42))" + " ? (dyn_var.orValue(optional.of({'a': 1})).a.hasValue()" + " || !dyn_var.orValue(optional.of({'a': 1})).a.hasValue()) : true", "dyn_var = optional\\.of\\(optional\\.of\\(42\\)\\)"), // Map literal keys must be strings, ints, uints or bools, and can't repeat. + MAP_LITERAL_RANGE_REPEATED_KEY("x == 1 ? {x: 'a', 1u: 'b'}.all(k, true) : true", "x = 1"), MAP_LITERAL_DOUBLE_KEY("{1.0: 'a'}[dyn(1)] == 'a'", "Condition is not always true\\."), MAP_LITERAL_BYTES_KEY("size({b'a': 1}) == 1", "Condition is not always true\\."), MAP_LITERAL_REPEATED_KEY("size({1: 'a', 1: 'b'}) == 1", "Condition is not always true\\."), @@ -2293,8 +2362,6 @@ private enum IsAlwaysTrueViolationTestCase { "size(int_list) == 5 ? int_list[5] == 1 : true", "Condition is not always true\\.", "Counterexample input:"), - // The type checker narrows static types (e.g. `['guest'] + dyn_var` is typed list(string)), so - // those must not constrain inputs, even in subexpressions that are never evaluated. NARROWED_LIST_IN_DOES_NOT_CONSTRAIN_INPUTS( "dyn_var != [0] || (false && 'x' in (['guest'] + dyn_var))", "Condition is not always true\\.", @@ -2338,7 +2405,7 @@ public void isAlwaysTrue_violation_returnsFalse( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); for (String fragment : testCase.expectedFragments) { assertThat(result.message()).containsMatch(fragment); } @@ -2355,11 +2422,11 @@ public void isAlwaysTrue_dynValuesComparedWithIntConstant_verified( CelAbstractSyntaxTree ast = CEL.compile(expression).getAst(); // A tighter timeout than the default, so that removing the fast path for comparing doubles with // exact integer constants fails as a performance regression (~0.3 s with it, 19-45 s without). - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setTimeout(Duration.ofSeconds(5))); + CelVerifier verifier = checkedVerifier(CEL).setTimeout(Duration.ofSeconds(5)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -2374,11 +2441,11 @@ public void isAlwaysTrue_dynamicMapComprehensionNestedEquality_violated() throws // (longer on GitHub runners). // TODO: Restore 20 s once the slowdown since map literal key validation is // recovered. - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setTimeout(Duration.ofSeconds(60))); + CelVerifier verifier = checkedVerifier(CEL).setTimeout(Duration.ofSeconds(60)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("request = "); } @@ -2497,7 +2564,7 @@ public void isAlwaysTrue_inconclusive(@TestParameter IsInconclusiveTestCase test throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -2506,7 +2573,7 @@ public void isAlwaysTrue_inconclusive(@TestParameter IsInconclusiveTestCase test public void isAlwaysTrue_inconclusive_containsPotentialCounterexample() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("request.matches('^[a-z]+$') == true").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()) @@ -2514,6 +2581,9 @@ public void isAlwaysTrue_inconclusive_containsPotentialCounterexample() throws E } private enum EquivalenceInconclusiveTestCase { + OPTIONAL_OR_VALUE_OF_TRUNCATED_RECEIVER( + "size(int_list) > 5 ? int_list.map(n, optional.none())[0].orValue(1) : 0", + "size(int_list) > 5 ? int_list.map(n, optional.none())[0].orValue(2) : 0"), // A list or map typed dyn holding NaN (e.g. `[NaN]`) differs from itself, so these do diverge, // but lists and maps typed dyn on both sides compare approximately. DYN_NAN_INPUT_AND_LITERAL("dyn_var != dyn_var ? 0.0 / 0.0 : dyn_var", "dyn_var"), @@ -2607,7 +2677,7 @@ public void verifyEquivalence_dynComprehensionRange_listedAsApproximationSource( CelAbstractSyntaxTree astA = CEL.compile("dyn_var.all(e, e > 0)").getAst(); CelAbstractSyntaxTree astB = CEL.compile("dyn_var.all(e, e < 0)").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.reason()) .isEqualTo( @@ -2619,7 +2689,8 @@ public void verifyEquivalence_dynComprehensionRange_listedAsApproximationSource( public void verifyEquivalence_unknownResults_listedAsApproximationSource() throws Exception { CelAbstractSyntaxTree astA = CEL.compile("unknown_var * 1").getAst(); CelAbstractSyntaxTree astB = CEL.compile("unknown_var + 0").getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("unknown_var")); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("unknown_var").build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -2635,12 +2706,13 @@ public void verifyEquivalence_inconclusive( CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } private enum EquivalenceTestCase { + OPTIONAL_OR_VALUE_OF_NON_OPTIONAL("dyn(1).orValue(int('123'))", "1 / 0"), STRUCT_UNSET_TIMESTAMP_DEFAULT("TestAllTypes{}.single_timestamp", "timestamp(0)"), // Both are errors. STRUCT_UNSET_ANY_DEFAULT("TestAllTypes{}.single_any == null", "1 / 0 == 1"), @@ -3188,6 +3260,8 @@ private enum EquivalenceViolationTestCase { CROSS_TYPE_SYMBOLIC_INEQUALITY_INT_UINT("dyn(x) == dyn(u)", "false"), CROSS_TYPE_SYMBOLIC_INEQUALITY_UINT_INT("dyn(u) == dyn(x)", "false"), OPTIONAL_OR_VALUE_VIOLATION("optional.of(x).orValue(y)", "y"), + OPTIONAL_OR_VALUE_FAILING_ALTERNATIVE( + "optional.of(x).orValue(1 / 0)", "optional.of(y).orValue(1 / 0)"), OPTIONAL_VALUE_VIOLATION("optional.of(x).value()", "y"), LIST_OPTIONAL_ELEMENTS_COLLISION("[1, ?opt_var]", "[1, opt_var]"), CROSS_NUMERIC_EQUALITY_INT_DYN_VIOLATION("1 == request", "false"), @@ -3280,7 +3354,7 @@ public void verifyEquivalence_violation_returnsFalse( CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("Equivalence violation detected."); } @@ -3291,7 +3365,7 @@ public void verifyEquivalence_violation_hasCounterexampleMessage() throws Except CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); String message = result.message(); assertThat(message).contains("Equivalence violation detected. Counterexample input:"); assertThat(message).containsMatch(" {2}x = -?\\d+"); @@ -3307,7 +3381,7 @@ public void verifyEquivalence_divergesOnTernaryErrorSemantics() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -3319,7 +3393,7 @@ public void verifyEquivalence_dynamicIndexingWithExplicitMap_hydratesMap() throw CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("string_int_list_map = {"); assertThat(result.message()).contains("\"a\": [1, 2]"); assertThat(result.message()).containsMatch("\"b\": \\[3, [45]]"); @@ -3330,7 +3404,7 @@ public void verifyEquivalence_logicalAndShortCircuitError_isVerified() throws Ex CelAbstractSyntaxTree astA = CEL.compile("false && (1 / 0 == 0)").getAst(); CelAbstractSyntaxTree astB = CEL.compile("false").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3338,7 +3412,7 @@ public void verifyEquivalence_logicalOrShortCircuitUnknown_isVerified() throws E CelAbstractSyntaxTree astA = CEL.compile("true || unknown_var").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3346,7 +3420,7 @@ public void verifyEquivalence_deepListExtensionality_isVerified() throws Excepti CelAbstractSyntaxTree astA = CEL.compile("[[[[1]]]] == [[[[1]]]]").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3355,7 +3429,7 @@ public void verifyEquivalence_stringConcatenationExplosion_isVerified() throws E CEL.compile("(\"a\" + \"b\" + \"c\" + \"d\" + \"e\") == \"abcde\"").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3367,7 +3441,7 @@ public void verifyEquivalence_divergesOnShortCircuitingErrors() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3378,7 +3452,7 @@ public void verifyEquivalence_distinctErrorEquivalence_failsVerification() throw CelAbstractSyntaxTree astB = CEL.compile("role + role == \"abc\"").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } private enum CounterexampleFormatTestCase { @@ -3486,7 +3560,7 @@ public void verifyEquivalence_counterexampleFormat( CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); boolean matched = testCase.expectedFragments.stream() .anyMatch( @@ -3509,7 +3583,7 @@ public void verifyEquivalence_structCounterexampleFormat() throws Exception { CEL.compile("test_all_types == TestAllTypes{single_int32: 2}").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isAnyOf( "Equivalence violation detected. Counterexample input:\n" @@ -3526,7 +3600,7 @@ public void isAlwaysTrue_operationError_counterexampleFormat() throws Exception CelAbstractSyntaxTree ast = CEL.compile("x / x == x / x").getAst(); CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("x = 0"); } @@ -3543,7 +3617,7 @@ public void isAlwaysTrue_dynamicFunction_inconclusive() throws Exception { CelOverloadDecl.newGlobalOverload("request_overload", SimpleType.DYN))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile("request() == request()").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()).contains("doesn't model exactly: calls to `request`."); @@ -3562,7 +3636,7 @@ public void isAlwaysTrue_dynamicFunctionShortCircuited_violated() throws Excepti .build(); CelAbstractSyntaxTree ast = celCompiler.compile("x != 0 && request() == 1").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); assertThat(result.message()).contains("x = 0"); @@ -3579,7 +3653,7 @@ public void isAlwaysTrue_boolFunction_verified() throws Exception { CelOverloadDecl.newGlobalOverload("bool_func_overload", SimpleType.BOOL))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile("bool_func() == bool_func()").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); } @@ -3614,7 +3688,7 @@ public void isAlwaysTrue_distinctFunctionSymbols_inconclusive( "f_list_string", SimpleType.INT, ListType.create(SimpleType.STRING)))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile(expression).getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3630,7 +3704,7 @@ public void isAlwaysTrue_zeroArityFunctionAndSameNamedVariable_inconclusive() th .build(); CelAbstractSyntaxTree ast = celCompiler.compile("g() == g").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3648,9 +3722,9 @@ public void verifyEquivalence_identicalFunctionCalls_equivalent() throws Excepti .build(); CelAbstractSyntaxTree astA = celCompiler.compile("request()").getAst(); CelAbstractSyntaxTree astB = celCompiler.compile("request()").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3669,7 +3743,7 @@ public void isSatisfiable_customOperatorOverload_inconclusive() throws Exception .build(); CelAbstractSyntaxTree ast = celCompiler.compile("!(int_list < int_list_2)").getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3691,7 +3765,7 @@ public void isSatisfiable_divisionByZero_failsInCelWithErrors() throws Exception CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -3704,7 +3778,7 @@ public void isSatisfiable_timeoutReached_throwsCelVerificationException() throws .addVar("d4", SimpleType.DOUBLE) .build(); CelVerifier timeoutVerifier = - withRuntimeOracle(customCel, b -> b.setTimeout(Duration.ofMillis(1))); + checkedVerifier(customCel).setTimeout(Duration.ofMillis(1)).build(); // An overly complex double multiplication to guarantee Z3 FPA theory solver timeouts. CelAbstractSyntaxTree ast = @@ -4001,7 +4075,7 @@ public void isAlwaysTrue_nanAndInfinityConstants_isFalse( CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelAbstractSyntaxTree optimizedAst = optimizer.optimize(ast); - assertThat(VERIFIER.isAlwaysTrue(optimizedAst).status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(VERIFIER.isAlwaysTrue(optimizedAst), VerificationStatus.VIOLATED); } @Test @@ -4024,6 +4098,292 @@ public void isAlwaysTrue_nanAndInfinityConstants_tautology( assertThat(VERIFIER.isAlwaysTrue(optimizedAst).status()).isEqualTo(VerificationStatus.VERIFIED); } + @Test + public void isAlwaysTrue_blockIndexLiteralPastUnrollLimit_verified( + @TestParameter({ + "[1, 2, 3, 4, 5, 6].all(x, x > 0) && [1, 2, 3, 4, 5, 6].exists(x, x == 6)", + "{1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0}.all(k, k > 0)" + + " && {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0}.exists(k, k == 6)" + }) + String expr) + throws Exception { + CelOptimizer optimizer = + CelOptimizerFactory.standardCelOptimizerBuilder(CEL) + .addAstOptimizers(SubexpressionOptimizer.getInstance()) + .build(); + CelAbstractSyntaxTree ast = optimizer.optimize(CEL.compile(expr).getAst()); + + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); + + assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + } + + @FunctionalInterface + @Immutable + private interface VerifierCall { + CelVerificationResult call(CelVerifier verifier) throws Exception; + } + + private static VerifierCall alwaysTrue(String expr) { + return v -> v.isAlwaysTrue(ZERO_CEL.compile(expr).getAst()); + } + + private static VerifierCall satisfiable(String expr) { + return v -> v.isSatisfiable(ZERO_CEL.compile(expr).getAst()); + } + + // Models `zero(x)` as `value`. + private static CelZ3FunctionAxiom zeroAxiom(long value) { + return CelZ3FunctionAxiom.newBuilder(ZERO_DECL) + .addUnaryOverloadTranslator( + "zero_int", + (ctx, typeSystem, constraintSink, arg) -> Optional.of(typeSystem.mkInt(value))) + .build(); + } + + private static VerifierCall equivalent(String exprA, String exprB) { + return v -> + v.verifyEquivalence(ZERO_CEL.compile(exprA).getAst(), ZERO_CEL.compile(exprB).getAst()); + } + + private enum CegarTestCase { + COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("zero(x) == 1"), + VerificationStatus.VIOLATED, + "Condition is not always true. The CEL runtime confirmed a counterexample the verifier" + + " approximated. Counterexample input:"), + COUNTEREXAMPLE_REFUTED( + alwaysTrue("zero(x) == 0"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential counterexample input:"), + ERROR_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("zero(x) / zero(x) == 1"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + // `0 + 'a'` has no overload at runtime. + DISPATCH_ERROR_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("dyn(zero(x)) + 'a' == 'a'"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + BAD_FORMAT_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("int('abc') == zero(x)"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + EXACT_COUNTEREXAMPLE( + alwaysTrue("x != 1"), + VerificationStatus.VIOLATED, + "Condition is not always true. Counterexample input:"), + SATISFYING_INPUT_CONFIRMED( + satisfiable("zero(x) == 0"), + VerificationStatus.VERIFIED, + "Condition is satisfiable. The CEL runtime confirmed a satisfying input the verifier" + + " approximated. Satisfying input:"), + SATISFYING_INPUT_REFUTED( + satisfiable("zero(x) == 1"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential satisfying input:"), + DIVERGENCE_CONFIRMED( + equivalent("zero(x)", "1"), + VerificationStatus.VIOLATED, + "Equivalence violation detected. The CEL runtime confirmed a divergence the verifier" + + " approximated. Counterexample input:"), + DIVERGENCE_REFUTED( + equivalent("zero(x)", "0"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential counterexample input:"), + // Both sides divide by zero at runtime, and all errors are one value. + ERROR_DIVERGENCE_REFUTED( + equivalent("zero(x) / zero(x)", "1 / zero(x)"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found."), + // Maps can't have numerically equal keys, or bytes keys. + NUMERICALLY_EQUAL_KEYS( + satisfiable( + "zero(x) == 0 && 0 in dyn_int_map && 0u in dyn_int_map" + + " && dyn_int_map[0] != dyn_int_map[0u]"), + VerificationStatus.INCONCLUSIVE, + "Potential satisfying input:"), + BYTES_KEY( + satisfiable("zero(x) == 0 && b'a' in dyn_int_map"), + VerificationStatus.INCONCLUSIVE, + "Potential satisfying input:"), + NEGATIVE_KEY( + satisfiable("zero(x) == 0 && -1 in dyn_int_map"), + VerificationStatus.VERIFIED, + "The CEL runtime confirmed a satisfying input the verifier approximated."), + // Proto message inputs aren't exact witness values yet. + INEXACT_WITNESS( + alwaysTrue("zero(test_all_types.single_int64) == 1"), + VerificationStatus.INCONCLUSIVE, + "Potential counterexample input:"); + + private final VerifierCall call; + private final VerificationStatus status; + private final String message; + + CegarTestCase(VerifierCall call, VerificationStatus status, String message) { + this.call = call; + this.status = status; + this.message = message; + } + } + + @Test + public void cegar_checksWitnessWithRuntime(@TestParameter CegarTestCase testCase) + throws Exception { + CelVerifier verifier = checkedVerifier(ZERO_CEL).build(); + + CelVerificationResult result = testCase.call.call(verifier); + + assertThat(result.status()).isEqualTo(testCase.status); + assertThat(result.message()).contains(testCase.message); + } + + private static final CelFunctionDecl MAYBE_DECL = + CelFunctionDecl.newFunctionDeclaration( + "maybe", CelOverloadDecl.newGlobalOverload("maybe_int", SimpleType.BOOL, SimpleType.INT)); + + private enum NotAttemptedCegarTestCase { + OFF(CelVerifierFactory.newVerifier(ZERO_CEL).build()), + UNKNOWN_IDENTIFIERS(checkedVerifier(ZERO_CEL).addUnknownIdentifier("unknown_var").build()); + + private final CelVerifier verifier; + + NotAttemptedCegarTestCase(CelVerifier verifier) { + this.verifier = verifier; + } + } + + @Test + public void cegar_notAttempted_staysInconclusive( + @TestParameter NotAttemptedCegarTestCase testCase) throws Exception { + CelVerificationResult result = + testCase.verifier.isAlwaysTrue(ZERO_CEL.compile("zero(x) == 1").getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()).doesNotContain("CEL runtime"); + } + + private enum CantEvaluateCegarTestCase { + UNBOUND_FUNCTION(UNBOUND_ZERO_CEL, "zero(x) == 1", "No matching overload for function 'zero'"), + LATE_BOUND_FUNCTION( + UNBOUND_ZERO_CEL.toCelBuilder().addLateBoundFunctions("zero").build(), + "zero(x) == 1", + "No matching overload for function 'zero'"), + // The runtime's iteration limit isn't a CEL error. + ITERATION_LIMIT( + ZERO_CEL + .toCelBuilder() + .setOptions(CelOptions.current().comprehensionMaxIterations(0).build()) + .build(), + "[1].all(i, zero(i) == 0)", + "Iteration budget exceeded"), + // Async bindings fail with an internal error when evaluated synchronously. + ASYNC_FUNCTION( + UNBOUND_ZERO_CEL + .toCelBuilder() + .addFunctionBindings(CelFunctionBinding.fromAsync("zero_int", Long.class, x -> null)) + .build(), + "zero(x) == 1", + "evaluated in synchronous mode"), + NULL_RESULT( + CEL.toCelBuilder() + .addFunctionDeclarations(MAYBE_DECL) + .addFunctionBindings(CelFunctionBinding.from("maybe_int", Long.class, x -> null)) + .build(), + "maybe(x)", + "a function returned null"), + UNKNOWN_RESULT( + CEL.toCelBuilder() + .addFunctionDeclarations(MAYBE_DECL) + .addFunctionBindings( + CelFunctionBinding.from("maybe_int", Long.class, x -> CelUnknownSet.create(1L))) + .build(), + "maybe(x)", + "it evaluated to unknowns"); + + private final Cel cel; + private final String expr; + private final String reason; + + CantEvaluateCegarTestCase(Cel cel, String expr, String reason) { + this.cel = cel; + this.expr = expr; + this.reason = reason; + } + } + + @Test + public void cegar_cantEvaluate_saysWhy(@TestParameter CantEvaluateCegarTestCase testCase) + throws Exception { + CelVerifier verifier = checkedVerifier(testCase.cel).build(); + + CelVerificationResult result = + verifier.isAlwaysTrue(testCase.cel.compile(testCase.expr).getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()) + .containsMatch( + "The CEL runtime couldn't check it: .*\\. (Potential counterexample input:|\\(The)"); + assertThat(result.message()).contains(testCase.reason); + } + + @Test + public void cegar_exactMatchCantEvaluate_saysWhy() throws Exception { + // Models `zero(x)` as 0, but the runtime has no binding for it. + CelVerifier verifier = + CelVerifierZ3Impl.newBuilder(UNBOUND_ZERO_CEL) + .addFunctionAxioms(zeroAxiom(0)) + .enableCegar(true) + .build(); + + CelVerificationResult result = + verifier.isAlwaysTrue(UNBOUND_ZERO_CEL.compile("zero(x) == 1").getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertThat(result.message()) + .containsMatch( + "Condition is not always true\\. The CEL runtime couldn't check it: .*No matching" + + " overload for function 'zero'.*\\. Counterexample input:"); + } + + private enum RefutedExactMatchTestCase { + COUNTEREXAMPLE( + alwaysTrue("zero(x) == 0"), + "a counterexample the verifier found. Potential counterexample input:"), + SATISFYING_INPUT( + satisfiable("zero(x) == 1"), + "a satisfying input the verifier found. Potential satisfying input:"), + DIVERGENCE( + equivalent("zero(x)", "0"), + "a divergence the verifier found. Potential counterexample input:"); + + private final VerifierCall call; + private final String message; + + RefutedExactMatchTestCase(VerifierCall call, String message) { + this.call = call; + this.message = message; + } + } + + @Test + public void cegar_runtimeDisagreesWithExactMatch_inconclusive( + @TestParameter RefutedExactMatchTestCase testCase) throws Exception { + // Models `zero(x)` as 1, which the runtime disagrees with. + CelVerifier verifier = + CelVerifierZ3Impl.newBuilder(ZERO_CEL) + .addFunctionAxioms(zeroAxiom(1)) + .enableCegar(true) + .build(); + + CelVerificationResult result = testCase.call.call(verifier); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()) + .contains("Inconclusive: the CEL runtime disagrees with " + testCase.message); + } + @Test public void translateFunctionCall_staticallyResolved_skipsDeadOverloads() throws Exception { CelFunctionDecl spyFuncDecl = @@ -4129,11 +4489,11 @@ public void isAlwaysTrue_largeListCounterexample_truncatesOutput() throws Except listLiteral.append("]"); CelAbstractSyntaxTree ast = cel.compile("!(large_list == " + listLiteral + ")").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b.setTimeout(Duration.ofSeconds(10))); + CelVerifier verifier = checkedVerifier(cel).setTimeout(Duration.ofSeconds(10)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("... (5 more elements)"); } @@ -4153,11 +4513,11 @@ public void isAlwaysTrue_largeMapCounterexample_truncatesOutput() throws Excepti mapLiteral.append("}"); CelAbstractSyntaxTree ast = cel.compile("!(large_map == " + mapLiteral + ")").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b.setTimeout(Duration.ofSeconds(10))); + CelVerifier verifier = checkedVerifier(cel).setTimeout(Duration.ofSeconds(10)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("... (5 more entries)"); } @@ -4173,9 +4533,9 @@ public void counterexample_messageTwoFieldsFormatted() throws Exception { CelAbstractSyntaxTree ast = cel.compile("!(msg == TestAllTypes{single_int32: 1, single_int64: 2})").getAst(); - CelVerificationResult result = withRuntimeOracle(cel, b -> b).isAlwaysTrue(ast); + CelVerificationResult result = checkedVerifier(cel).build().isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("single_int32: 1"); assertThat(result.message()).contains("single_int64: 2"); } @@ -4204,11 +4564,11 @@ public void isAlwaysTrue_customComprehensionWithTrueAccuInit() throws Exception .build(); CelAbstractSyntaxTree ast = cel.compile("dyn_list == [1, 2] ? dyn_list.custom_fold(x) == true : true").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b); + CelVerifier verifier = checkedVerifier(cel).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -4219,7 +4579,7 @@ public void isAlwaysTrue_customComprehensionWithLoopCondition_inconclusive() thr CelAbstractSyntaxTree ast = CEL.compile("int_list == [1, 2] ? int_list.first_elem() == 2 : true").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.reason()) .isEqualTo( @@ -4301,7 +4661,7 @@ public void verifyImplication_exactUnknownAssumption_verified() throws Exception public void isSatisfiable_exactUnknownBesideUnmodeledComprehension_violated() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("(false && int_list.first_elem() == 1) || (b && !b)").getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("b")); + CelVerifier verifier = CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("b").build(); CelVerificationResult result = verifier.isSatisfiable(ast); @@ -4344,10 +4704,10 @@ public void isSatisfiable_maskedByBmcButAlwaysFalse_returnsFailed() throws Excep String expr = "int_list == [1, 2, 3, 4] ? int_list.exists(x, x == 42) && false : false"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(3)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(3).build(); CelVerificationResult result = verifier.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -4359,10 +4719,10 @@ public void verifyEquivalence_maskedByBmcButAlwaysEqual_returnsVerified() throws CelAbstractSyntaxTree astA = CEL.compile(exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(3)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(3).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } private enum EquivalenceZeroUnrollLimitTestCase { @@ -4385,7 +4745,8 @@ public void verifyEquivalence_zeroUnrollLimit_returnsInconclusive( @TestParameter EquivalenceZeroUnrollLimitTestCase testCase) throws Exception { CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4423,7 +4784,7 @@ public void verifyEquivalence_zeroUnrollLimit_twoVarComprehensions_returnsVerifi @TestParameter EquivalenceZeroUnrollLimitVerifiedTestCase testCase) throws Exception { CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4471,7 +4832,8 @@ public void verifyEquivalence_comprehensionScopeShadowing_returnsInconclusive() .build(); CelAbstractSyntaxTree astA = customCel.compile("dyn_list.my_macro_1(true)").getAst(); CelAbstractSyntaxTree astB = customCel.compile("dyn_list.my_macro_2(true)").getAst(); - CelVerifier verifier = withRuntimeOracle(customCel, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(customCel).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4513,7 +4875,8 @@ public void verifyEquivalence_comprehensionResultScopeIsolation_returnsInconclus customCel.compile("cel.bind(x, 10, dyn_list.my_macro(1))").getAst(); CelAbstractSyntaxTree astB = customCel.compile("cel.bind(x, 20, dyn_list.my_macro(1))").getAst(); - CelVerifier verifier = withRuntimeOracle(customCel, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(customCel).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4530,7 +4893,7 @@ public void verifyEquivalence_timeoutReached_throwsCelVerificationException() th .addVar("d4", SimpleType.DOUBLE) .build(); CelVerifier timeoutVerifier = - withRuntimeOracle(customCel, b -> b.setTimeout(Duration.ofMillis(1))); + checkedVerifier(customCel).setTimeout(Duration.ofMillis(1)).build(); CelAbstractSyntaxTree astA = customCel .compile( @@ -4558,7 +4921,7 @@ public void isAlwaysTrue_constantVarDecl_verified() throws Exception { .setConstant(CelConstant.ofValue(42L)) .build()) .build(); - CelVerifier verifier = withRuntimeOracle(celWithConstVar, b -> b); + CelVerifier verifier = checkedVerifier(celWithConstVar).build(); CelAbstractSyntaxTree ast = celWithConstVar .compile("CONST_INT == 42 && [1, 2].all(x, .CONST_INT == 42 && x > 0)") @@ -4591,10 +4954,10 @@ private static CelMacro firstElemMacro() { exprFactory.newIdentifier("accu")))); } - // verifyImplication isn't part of CelVerifier, so it can't go through the runtime oracle. + // verifyImplication isn't part of CelVerifier. private static CelVerificationResult verifyImplication( CelCompiler compiler, String assume, String assertion) throws Exception { - return ((CelVerifierZ3Impl) VERIFIER_WITHOUT_ORACLE) + return ((CelVerifierZ3Impl) UNCHECKED_VERIFIER) .verifyImplication( compiler.compile(assume).getAst(), compiler.compile(assertion).getAst(), diff --git a/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java b/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java deleted file mode 100644 index 67d746a6f..000000000 --- a/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.verifier; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Arrays.stream; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import dev.cel.common.CelAbstractSyntaxTree; -import dev.cel.runtime.CelEvaluationException; -import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntime.Program; -import dev.cel.verifier.CelVerificationResult.VerificationStatus; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; - -/** - * Evaluates every exact witness a verifier returns with the CEL runtime, and fails if the runtime - * disagrees with the verdict. - */ -final class RuntimeOracle implements CelVerifier { - - // All errors compare equal, matching the verifier's single error value. - private enum EvaluationError { - ERROR - } - - @SuppressWarnings("Immutable") // CelRuntime is immutable, just not marked as such - private final CelRuntime runtime; - - private final CelVerifier delegate; - - static CelVerifier wrap(CelRuntime runtime, CelVerifier delegate) { - return new RuntimeOracle(runtime, delegate); - } - - @Override - public CelVerificationResult isSatisfiable(CelAbstractSyntaxTree ast) - throws CelVerificationException { - return check( - delegate.isSatisfiable(ast), VerificationStatus.VERIFIED, r -> r.get(0).equals(true), ast); - } - - @Override - public CelVerificationResult isAlwaysTrue(CelAbstractSyntaxTree ast) - throws CelVerificationException { - return check( - delegate.isAlwaysTrue(ast), VerificationStatus.VIOLATED, r -> !r.get(0).equals(true), ast); - } - - @Override - public CelVerificationResult verifyEquivalence( - CelAbstractSyntaxTree astA, CelAbstractSyntaxTree astB) throws CelVerificationException { - return check( - delegate.verifyEquivalence(astA, astB), - VerificationStatus.VIOLATED, - r -> !r.get(0).equals(r.get(1)), - astA, - astB); - } - - @CanIgnoreReturnValue - private CelVerificationResult check( - CelVerificationResult result, - VerificationStatus checkedStatus, - Predicate> agrees, - CelAbstractSyntaxTree... asts) { - Optional> inputs = exactInputs(result); - if (result.status() != checkedStatus || !inputs.isPresent()) { - return result; - } - ImmutableList results = - stream(asts).map(ast -> evaluate(ast, inputs.get())).collect(toImmutableList()); - if (agrees.test(results)) { - return result; - } - StringBuilder message = - new StringBuilder("The CEL runtime disagrees with ") - .append(result.status()) - .append(" for inputs ") - .append(inputs.get()) - .append(':'); - for (int i = 0; i < asts.length; i++) { - message - .append("\n ") - .append(asts[i].getSource().getContent()) - .append(" evaluates to ") - .append(results.get(i)); - } - throw new AssertionError(message.append('\n').append(result.message()).toString()); - } - - // The witness's inputs, if every value is exact. - private static Optional> exactInputs(CelVerificationResult result) { - return result - .witness() - .map(CelWitness::variables) - .filter(vars -> vars.values().stream().allMatch(v -> v.value().isPresent())) - .map(vars -> ImmutableMap.copyOf(Maps.transformValues(vars, v -> v.value().get()))); - } - - private Object evaluate(CelAbstractSyntaxTree ast, ImmutableMap inputs) { - Program program; - try { - program = runtime.createProgram(ast); - } catch (CelEvaluationException e) { - throw new AssertionError( - "Can't plan " - + ast.getSource().getContent() - + "; verify it with a verifier for its own environment", - e); - } - try { - return program.eval(inputs); - } catch (CelEvaluationException e) { - // A witness missing a variable would otherwise pass as an error result. - if (e.getMessage().contains("No such attribute(s): ")) { - throw new AssertionError( - "Incomplete witness " + inputs + " for " + ast.getSource().getContent(), e); - } - return EvaluationError.ERROR; - } - } - - private RuntimeOracle(CelRuntime runtime, CelVerifier delegate) { - this.runtime = runtime; - this.delegate = delegate; - } -} diff --git a/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java b/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java deleted file mode 100644 index 0bc3a36b9..000000000 --- a/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.verifier; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableMap; -import dev.cel.bundle.Cel; -import dev.cel.bundle.CelFactory; -import dev.cel.common.CelAbstractSyntaxTree; -import dev.cel.common.CelFunctionDecl; -import dev.cel.common.CelOverloadDecl; -import dev.cel.common.types.SimpleType; -import dev.cel.runtime.CelEvaluationException; -import java.util.Optional; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class RuntimeOracleTest { - - private static final Cel CEL = CelFactory.plannerCelBuilder().addVar("x", SimpleType.INT).build(); - - @Test - public void isSatisfiable_runtimeRefutesSatisfyingInput_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.verified("Condition is satisfiable.", witness(2L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isSatisfiable(ast)); - - assertThat(e).hasMessageThat().contains("x == 1 evaluates to false"); - } - - @Test - public void isAlwaysTrue_runtimeRefutesCounterexample_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(1L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isAlwaysTrue(ast)); - - assertThat(e).hasMessageThat().contains("x == 1 evaluates to true"); - } - - @Test - public void verifyEquivalence_runtimeRefutesDivergence_throws() throws Exception { - CelAbstractSyntaxTree astA = CEL.compile("x + 0").getAst(); - CelAbstractSyntaxTree astB = CEL.compile("x").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(1L))); - - AssertionError e = - assertThrows(AssertionError.class, () -> verifier.verifyEquivalence(astA, astB)); - - assertThat(e).hasMessageThat().contains("x + 0 evaluates to 1"); - assertThat(e).hasMessageThat().contains("\n x evaluates to 1"); - } - - @Test - public void isSatisfiable_inexactSatisfyingInput_notChecked() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerificationResult result = - CelVerificationResult.verified( - "Condition is satisfiable.", - CelWitness.create( - ImmutableMap.of("x", CelWitness.Variable.create(Optional.empty(), "Unknown")))); - - assertThat(checking(result).isSatisfiable(ast)).isSameInstanceAs(result); - } - - @Test - public void verifyEquivalence_bothExpressionsError_throws() throws Exception { - CelAbstractSyntaxTree astA = CEL.compile("1 / x").getAst(); - CelAbstractSyntaxTree astB = CEL.compile("2 / x").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(0L))); - - AssertionError e = - assertThrows(AssertionError.class, () -> verifier.verifyEquivalence(astA, astB)); - - assertThat(e).hasMessageThat().contains("1 / x evaluates to ERROR"); - assertThat(e).hasMessageThat().contains("2 / x evaluates to ERROR"); - } - - @Test - public void isSatisfiable_inconclusiveResult_notChecked() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerificationResult result = - CelVerificationResult.inconclusive( - "Inconclusive.", witness(2L), /* isCounterexample= */ false); - - assertThat(checking(result).isSatisfiable(ast)).isSameInstanceAs(result); - } - - @Test - public void isSatisfiable_unplannableExpression_throws() throws Exception { - Cel celWithFunction = - CEL.toCelBuilder() - .addFunctionDeclarations( - CelFunctionDecl.newFunctionDeclaration( - "f", - CelOverloadDecl.newGlobalOverload("f_int", SimpleType.BOOL, SimpleType.INT))) - .build(); - CelAbstractSyntaxTree ast = celWithFunction.compile("f(x)").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.verified("Condition is satisfiable.", witness(1L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isSatisfiable(ast)); - - assertThat(e).hasMessageThat().contains("Can't plan f(x)"); - assertThat(e).hasCauseThat().isInstanceOf(CelEvaluationException.class); - } - - @Test - public void isAlwaysTrue_missingVariableInWitness_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.failed("Violated.", CelWitness.create(ImmutableMap.of()))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isAlwaysTrue(ast)); - - assertThat(e).hasMessageThat().contains("Incomplete witness {} for x == 1"); - assertThat(e).hasCauseThat().isInstanceOf(CelEvaluationException.class); - } - - private static CelWitness witness(long x) { - return CelWitness.create( - ImmutableMap.of("x", CelWitness.Variable.create(Optional.of(x), Long.toString(x)))); - } - - @SuppressWarnings("Immutable") // Test fake; the result is never mutated. - private static CelVerifier checking(CelVerificationResult result) { - return RuntimeOracle.wrap( - CEL, - new CelVerifier() { - @Override - public CelVerificationResult isSatisfiable(CelAbstractSyntaxTree ast) { - return result; - } - - @Override - public CelVerificationResult isAlwaysTrue(CelAbstractSyntaxTree ast) { - return result; - } - - @Override - public CelVerificationResult verifyEquivalence( - CelAbstractSyntaxTree astA, CelAbstractSyntaxTree astB) { - return result; - } - }); - } -}