diff --git a/verifier/README.md b/verifier/README.md index 7d9d7f899..b16461ae6 100644 --- a/verifier/README.md +++ b/verifier/README.md @@ -408,6 +408,18 @@ by the Z3 solver during its search phase. If the solver times out, the verifier will throw a `CelVerificationException` which should be explicitly caught and handled by the caller. +### Validating Counterexamples with the CEL Runtime + +`enableCegar(true)` checks the inputs the solver finds with the CEL +runtime of the verifier's environment: a confirmed `INCONCLUSIVE` counterexample +(or satisfying input) becomes `VIOLATED` (or `VERIFIED`), and any refuted one +becomes `INCONCLUSIVE`. Only exact inputs are checked, when no unknown +identifiers are registered; when the runtime can't evaluate them (e.g. a +function has no binding), the message says why. Policy invariants aren't +checked yet. The runtime must follow the CEL spec (e.g. `CelOptions.DEFAULT`, which has no +iteration limit), and a function with bindings must have one for each overload, +which is synchronous, deterministic and doesn't throw. + ### Comprehensions and Bounded Model Checking Because CEL lists/maps can be dynamically sized, the verifier cannot diff --git a/verifier/src/main/java/dev/cel/verifier/BUILD.bazel b/verifier/src/main/java/dev/cel/verifier/BUILD.bazel index 2d2549456..2ba47739f 100644 --- a/verifier/src/main/java/dev/cel/verifier/BUILD.bazel +++ b/verifier/src/main/java/dev/cel/verifier/BUILD.bazel @@ -170,6 +170,7 @@ java_library( "CelZ3StructTranslator.java", "CelZ3TranslationContext.java", "CelZ3TypeConstraintGenerator.java", + "RuntimeOracle.java", "TranslatedValue.java", ], compatible_with = [], @@ -184,6 +185,7 @@ java_library( "//bundle:cel", "//common:cel_ast", "//common:compiler_common", + "//common:error_codes", "//common:operator", "//common/ast", "//common/ast:cel_block", @@ -198,6 +200,10 @@ java_library( "//optimizer", "//optimizer:optimization_exception", "//optimizer:optimizer_builder", + "//runtime", + "//runtime:function_resolver", + "//runtime:resolved_overload", + "//runtime:unknown_attributes", "//verifier/axioms", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", diff --git a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java index a80d1008d..cf0796cd7 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java @@ -14,6 +14,8 @@ package dev.cel.verifier; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; @@ -111,8 +113,8 @@ Set> getInputVariables() { * Binds an identifier name in the symbol table to an already-translated value. Used to bind * reserved policy symbols such as 'rule.result' to the composed policy graph. */ - void bindSymbol(String varName, TranslatedValue value) { - context.bindSymbol(varName, value); + void bindSymbol(String varName, TranslatedValue value, CelAbstractSyntaxTree ast) { + context.bindSymbol(varName, value, ast); } TranslatedValue translate(CelAbstractSyntaxTree ast) { @@ -439,6 +441,15 @@ private TranslatedValue translateCall(CelExpr expr, CelAbstractSyntaxTree ast) { long exprId = expr.id(); String functionName = call.function(); ImmutableList argsAst = call.args(); + if (functionName.equals(Operator.OPTIONAL_SELECT.getFunction())) { + checkArgument( + !call.target().isPresent() + && argsAst.size() == 2 + && argsAst.get(1).constantOrDefault().getKind() == CelConstant.Kind.STRING_VALUE, + "Malformed AST: operator %s expects an operand and a field name", + Operator.OPTIONAL_SELECT); + return structTranslator.translateSelect(expr, ast); + } List args = new ArrayList<>(); @@ -453,42 +464,41 @@ private TranslatedValue translateCall(CelExpr expr, CelAbstractSyntaxTree ast) { return operatorTranslator .translateFunctionCall(functionName, args, exprId, ast) - .map( - tv -> { - if (mayEmbedArgs(functionName, ast.getTypeOrThrow(exprId))) { - propagateTruncatedTypeConstraint( - tv.z3Expr(), Iterables.transform(args, TranslatedValue::z3Expr)); - } - return TranslatedValue.create(tv.z3Expr(), expr, typeSystem, tv.isApproximate()); - }) - .orElseGet( - () -> { - // Uninterpreted function - Sort[] argSorts = new Sort[args.size()]; - Arrays.fill(argSorts, 0, args.size(), typeSystem.celValueSort()); - FuncDecl funcDecl = - typeSystem.internFuncDecl( - uninterpretedFunctionSymbol(exprId, ast), - argSorts, - typeSystem.celValueSort()); - - Expr[] exprArgs = args.stream().map(TranslatedValue::z3Expr).toArray(Expr[]::new); - Expr callRes = ctx.mkApp(funcDecl, exprArgs); - context.addConstraint( - typeConstraintGenerator.createTypeConstraint(callRes, exprId, ast)); - context.addConstraint(ctx.mkNot(typeSystem.isUnknown(callRes))); - context.addConstraint(ctx.mkNot(typeSystem.isError(callRes))); - - // The solver picks the result freely, so a model depending on it is never exact. - return TranslatedValue.propagateStrict( - ctx, - typeSystem, - functionName, - callRes, - Optional.of(expr), - /* baseTaint= */ context.recordCallApproximation(functionName, ctx.mkTrue()), - args); - }); + .map(tv -> { + if (mayEmbedArgs(functionName, ast.getTypeOrThrow(exprId))) { + propagateTruncatedTypeConstraint( + tv.z3Expr(), Iterables.transform(args, TranslatedValue::z3Expr)); + } + return TranslatedValue.create(tv.z3Expr(), expr, typeSystem, tv.isApproximate()); + }) + .orElseGet(() -> { + // Uninterpreted function + Sort[] argSorts = new Sort[args.size()]; + Arrays.fill(argSorts, 0, args.size(), typeSystem.celValueSort()); + FuncDecl funcDecl = + typeSystem.internFuncDecl( + uninterpretedFunctionSymbol(exprId, ast), argSorts, typeSystem.celValueSort()); + + Expr[] exprArgs = args.stream().map(TranslatedValue::z3Expr).toArray(Expr[]::new); + Expr callRes = ctx.mkApp(funcDecl, exprArgs); + // A narrowed result's static type doesn't constrain it. + if (!context.mayBeNarrowed(expr, ast)) { + context.addConstraint( + typeConstraintGenerator.createTypeConstraint(callRes, exprId, ast)); + } + context.addConstraint(ctx.mkNot(typeSystem.isUnknown(callRes))); + context.addConstraint(ctx.mkNot(typeSystem.isError(callRes))); + + // The solver picks the result freely, so a model depending on it is never exact. + return TranslatedValue.propagateStrict( + ctx, + typeSystem, + functionName, + callRes, + Optional.of(expr), + /* baseTaint= */ context.recordCallApproximation(functionName, ctx.mkTrue()), + args); + }); } /** diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java b/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java index c49bb4343..e78f11a32 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java @@ -82,6 +82,23 @@ public interface CelVerifierBuilder { @CanIgnoreReturnValue CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit); + /** + * Checks the inputs the solver finds by evaluating them with the CEL runtime of the verifier's + * environment (off by default). For now this only validates them, without refining the solver + * (counterexample-guided abstraction refinement, CEGAR). An {@code INCONCLUSIVE} counterexample + * or satisfying input that the runtime confirms becomes {@code VIOLATED} (or {@code VERIFIED} for + * satisfiability), and any counterexample or satisfying input it disagrees with becomes {@code + * INCONCLUSIVE}. + * + *

Inputs are checked only when all their values are exact and no unknown identifiers are + * registered. When the runtime can't evaluate them, e.g. a function has no binding, the message + * says why. Policy invariants aren't checked yet. The runtime must follow the CEL spec (e.g. + * {@code CelOptions.DEFAULT}, which has no iteration limit), and a function with bindings must + * have one for each overload, which is synchronous, deterministic and doesn't throw. + */ + @CanIgnoreReturnValue + CelVerifierBuilder enableCegar(boolean value); + /** Builds the {@link CelVerifier} instance. */ CelVerifier build(); } diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java index 900821e92..c1b52be7d 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; +import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -41,12 +42,15 @@ import dev.cel.optimizer.CelOptimizationException; import dev.cel.optimizer.CelOptimizer; import dev.cel.optimizer.CelOptimizerFactory; +import dev.cel.runtime.CelEvaluationException; import dev.cel.verifier.axioms.CelZ3FunctionAxiom; import dev.cel.verifier.axioms.CelZ3StandardAxioms; import java.time.Duration; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; /** Z3 implementation of the CelVerifier. */ @Immutable @@ -91,6 +95,7 @@ public Optional findType(String typeName) { private final ImmutableSet unknownIdentifiers; private final CelZ3FunctionRegistry functionRegistry; private final CelTypeProvider typeProvider; + private final boolean enableCegar; @SuppressWarnings("Immutable") // Cel environment is immutable, just not marked as such private final Cel cel; @@ -111,6 +116,7 @@ static final class Builder implements CelVerifierBuilder { private final ImmutableList.Builder functionAxioms; private final Cel cel; private CelTypeProvider typeProvider; + private boolean enableCegar; @Override @CanIgnoreReturnValue @@ -144,6 +150,13 @@ public CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit) { return this; } + @Override + @CanIgnoreReturnValue + public CelVerifierBuilder enableCegar(boolean value) { + this.enableCegar = value; + return this; + } + @CanIgnoreReturnValue Builder addFunctionAxioms(CelZ3FunctionAxiom... axioms) { return addFunctionAxioms(Arrays.asList(axioms)); @@ -170,6 +183,7 @@ public CelVerifier build() { unknownIdentifiers.build(), registry, typeProvider, + enableCegar, cel); } @@ -208,17 +222,19 @@ public CelVerificationResult verifyEquivalence( CanonicalizationOptimizer.newInstance( CanonicalizationOptimizer.CanonicalizationOptions.newBuilder().build())) .build(); + CelAbstractSyntaxTree canonicalA = astA; + CelAbstractSyntaxTree canonicalB = astB; try { - astA = optimizer.optimize(astA); - astB = optimizer.optimize(astB); + canonicalA = optimizer.optimize(astA); + canonicalB = optimizer.optimize(astB); } catch (CelOptimizationException e) { // Fall back to original ASTs if canonicalization or re-typechecking fails } try (Context ctx = new Context(ImmutableMap.of("model", "true"))) { CelAstToZ3Translator translator = newTranslator(ctx, /* parameterizeUnknowns= */ true); - TranslatedValue tvA = translator.translate(astA); - TranslatedValue tvB = translator.translate(astB); + TranslatedValue tvA = translator.translate(canonicalA); + TranslatedValue tvB = translator.translate(canonicalB); BoolExpr divergenceCondition = ctx.mkNot(ctx.mkEq(tvA.z3Expr(), tvB.z3Expr())); // Parameterized unknowns are keyed on the operation producing them, whereas CEL identifies @@ -243,7 +259,8 @@ public CelVerificationResult verifyEquivalence( /* unknownCondition= */ ctx.mkFalse(), /* isCounterexample= */ true, /* candidate= */ "a divergence", - /* exactMatchMessage= */ "Equivalence violation detected."); + /* exactMatchMessage= */ "Equivalence violation detected.", + runtimeCheck(results -> !results.get(0).equals(results.get(1)), astA, astB)); } } @@ -267,7 +284,7 @@ CelVerificationResult verifyImplication( for (Map.Entry entry : boundSymbols.entrySet()) { TranslatedValue tv = translator.translate(entry.getValue()); - translator.bindSymbol(entry.getKey(), tv); + translator.bindSymbol(entry.getKey(), tv, entry.getValue()); } TranslatedValue assumeTv = translator.translate(assumeAst); @@ -295,7 +312,8 @@ CelVerificationResult verifyImplication( unknownCondition, /* isCounterexample= */ true, /* candidate= */ "a counterexample", - /* exactMatchMessage= */ String.format("%s violation detected.", subjectName)); + /* exactMatchMessage= */ String.format("%s violation detected.", subjectName), + /* runtimeCheck= */ RuntimeChecker.UNCHECKED); } } @@ -316,7 +334,8 @@ private CelVerificationResult checkSatisfiability( /* unknownCondition= */ ctx.mkFalse(), /* isCounterexample= */ true, /* candidate= */ "a counterexample", - /* exactMatchMessage= */ "Condition is not always true."); + /* exactMatchMessage= */ "Condition is not always true.", + runtimeCheck(results -> !results.get(0).equals(true), ast)); } BoolExpr isUnmodeledUnknown = ctx.mkAnd(tv.isZ3Unknown(), tv.isApproximate()); return search( @@ -327,13 +346,16 @@ private CelVerificationResult checkSatisfiability( isUnmodeledUnknown, /* isCounterexample= */ false, /* candidate= */ "a satisfying input", - /* exactMatchMessage= */ "Condition is satisfiable."); + /* exactMatchMessage= */ "Condition is satisfiable.", + runtimeCheck(results -> results.get(0).equals(true), ast)); } } /** * Searches for a {@code condition} match: first an exact one, then an approximate one, then an - * unmodeled result satisfying {@code unknownCondition} that may stand in for one. + * unmodeled result satisfying {@code unknownCondition} that may stand in for one. {@code + * runtimeCheck} checks a witness with the runtime, noting why if it couldn't. Confirming an + * approximate match doesn't count internal runtime errors. */ private CelVerificationResult search( Context ctx, @@ -343,7 +365,8 @@ private CelVerificationResult search( BoolExpr unknownCondition, boolean isCounterexample, String candidate, - String exactMatchMessage) + String exactMatchMessage, + RuntimeChecker runtimeCheck) throws CelVerificationException { Solver solver = newSolver(ctx); solver.add(translator.getTypeConstraints().toArray(new BoolExpr[0])); @@ -353,9 +376,14 @@ private CelVerificationResult search( Status status = checkSat(solver, typeSystem, condition, ctx.mkNot(taint)); if (status == Status.SATISFIABLE) { CelWitness witness = extractWitness(ctx, translator, solver.getModel()); - return isCounterexample - ? CelVerificationResult.failed(exactMatchMessage, witness) - : CelVerificationResult.verified(exactMatchMessage, witness); + RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ true); + if (!check.isMatch().orElse(true)) { + return CelVerificationResult.inconclusive( + "Inconclusive: the CEL runtime disagrees with " + candidate + " the verifier found.", + witness, + isCounterexample); + } + return match(isCounterexample, exactMatchMessage + check.note(), witness); } else if (status == Status.UNKNOWN) { // Depends on solver internals; not deterministically testable. return solverGaveUp(solver); @@ -365,9 +393,19 @@ private CelVerificationResult search( status = checkSat(solver, typeSystem, condition); if (status == Status.SATISFIABLE) { Model model = solver.getModel(); + CelWitness witness = extractWitness(ctx, translator, model); + RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ false); + if (check.isMatch().orElse(false)) { + return match( + isCounterexample, + String.format( + "%s The CEL runtime confirmed %s the verifier approximated.", + exactMatchMessage, candidate), + witness); + } return CelVerificationResult.inconclusive( - inconclusiveReason(candidate, translator, model, taint), - extractWitness(ctx, translator, model), + inconclusiveReason(candidate, translator, model, taint) + check.note(), + witness, isCounterexample); } else if (status == Status.UNKNOWN) { // Depends on solver internals; not deterministically testable. @@ -389,6 +427,62 @@ private CelVerificationResult search( : CelVerificationResult.failed("Condition is not satisfiable."); } + private static CelVerificationResult match( + boolean isCounterexample, String reason, CelWitness witness) { + return isCounterexample + ? CelVerificationResult.failed(reason, witness) + : CelVerificationResult.verified(reason, witness); + } + + private interface RuntimeChecker { + RuntimeChecker UNCHECKED = (witness, countInternalErrors) -> RuntimeCheck.UNCHECKED; + + RuntimeCheck check(CelWitness witness, boolean countInternalErrors); + } + + // Whether the runtime agrees a witness is a match, if it checked it, and a note for the message. + @AutoValue + abstract static class RuntimeCheck { + static final RuntimeCheck UNCHECKED = + new AutoValue_CelVerifierZ3Impl_RuntimeCheck(Optional.empty(), ""); + + abstract Optional isMatch(); + + abstract String note(); + + static RuntimeCheck of(boolean isMatch) { + return new AutoValue_CelVerifierZ3Impl_RuntimeCheck( + Optional.of(isMatch), isMatch ? "" : " The CEL runtime disagrees with the one found."); + } + + static RuntimeCheck failed(String reason) { + return new AutoValue_CelVerifierZ3Impl_RuntimeCheck( + Optional.empty(), + " The CEL runtime couldn't check it: " + reason + (reason.endsWith(".") ? "" : ".")); + } + } + + // Checks the witness's inputs with the runtime, if refinement is on and they're exact. Unknowns + // aren't inputs the runtime can take. + private RuntimeChecker runtimeCheck( + Predicate> isMatch, CelAbstractSyntaxTree... asts) { + if (!enableCegar || !unknownIdentifiers.isEmpty()) { + return RuntimeChecker.UNCHECKED; + } + return (witness, countInternalErrors) -> { + Optional> inputs = RuntimeOracle.exactInputs(witness); + if (!inputs.isPresent()) { + return RuntimeCheck.UNCHECKED; + } + try { + return RuntimeCheck.of( + isMatch.test(RuntimeOracle.evaluate(cel, inputs.get(), countInternalErrors, asts))); + } catch (CelEvaluationException e) { + return RuntimeCheck.failed(e.getMessage()); + } + }; + } + // Throws on a timeout. Leaves the assertions in place unless they're unsatisfiable, so the model // or the reason the solver gave up can be read. private static Status checkSat(Solver solver, CelZ3TypeSystem typeSystem, BoolExpr... assertions) @@ -468,12 +562,14 @@ private CelVerifierZ3Impl( ImmutableSet unknownIdentifiers, CelZ3FunctionRegistry functionRegistry, CelTypeProvider typeProvider, + boolean enableCegar, Cel cel) { this.timeout = timeout; this.comprehensionUnrollLimit = comprehensionUnrollLimit; this.unknownIdentifiers = unknownIdentifiers; this.functionRegistry = functionRegistry; this.typeProvider = typeProvider; + this.enableCegar = enableCegar; this.cel = cel; } } diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java index 379fba20a..27016ac8d 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java @@ -187,13 +187,32 @@ private TranslatedValue translateDynamicComprehension( ctx.mkAnd(isValidRange, ctx.mkGt(lengthExpr, ctx.mkInt(comprehensionUnrollLimit)))); context.addUnmodeledResult(isTruncated); - if (isAllMacro(comp) || isExistsMacro(comp)) { - return unrollAllAndExists( - celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv); - } else { - return unrollMapAndFilter( - celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv); + TranslatedValue result = + isAllMacro(comp) || isExistsMacro(comp) + ? unrollAllAndExists( + celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv) + : unrollMapAndFilter( + celExpr, ast, seq, lengthExpr, mapPresence, isTruncated, iterRangeTv); + // Only for speed: a range that isn't narrowed always has its static kind. + if (!context.mayBeNarrowed(comp.iterRange(), ast)) { + return result; } + // The type checker may have narrowed the range's type (e.g. `dyn_var.orValue([1])` is typed + // list(int)), so it may be the other kind, or not a list or map at all. + BoolExpr isListOrMap = ctx.mkOr(typeSystem.isList(iterRange), typeSystem.isMap(iterRange)); + BoolExpr isOtherKind = + context.recordApproximation( + "comprehensions over `dyn` values", ctx.mkAnd(ctx.mkNot(isValidRange), isListOrMap)); + context.addUnmodeledResult(isOtherKind); + BoolExpr isNotRange = ctx.mkNot(ctx.mkOr(isListOrMap, typeSystem.isErrorOrUnknown(iterRange))); + return TranslatedValue.create( + CelZ3Exprs.SwitchBuilder.newBuilder(ctx) + .addCase(isOtherKind, mkParameterizedUnknown(celExpr, ast)) + .addCase(isNotRange, typeSystem.mkError()) + .build(result.z3Expr()), + celExpr, + typeSystem, + CelZ3Exprs.mkOrFlattened(ctx, result.isApproximate(), isOtherKind)); } private T withIterationScope( diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java index c7aef6643..e1fc173d2 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3CounterexampleGenerator.java @@ -265,6 +265,7 @@ private Value extractMap(Expr mapRef) { ImmutableMap.Builder values = ImmutableMap.builder(); List entries = new ArrayList<>(); Set> seenKeys = new HashSet<>(); + Set runtimeKeys = new HashSet<>(); ArrayExpr keySet = ctx.mkConstArray(typeSystem.celValueSort(), ctx.mkFalse()); for (long i = 0; i < limit; i++) { Expr key = evaluate(ctx.mkNth(typeSystem.getMapKeys(mapRef), ctx.mkInt(i))); @@ -276,7 +277,9 @@ private Value extractMap(Expr mapRef) { Value keyValue = extract(key); Value value = extract(evaluate(ctx.mkSelect((ArrayExpr) typeSystem.getMapValues(mapRef), key))); - if (keyValue.runtimeValue.isPresent() && value.runtimeValue.isPresent()) { + if (keyValue.runtimeValue.isPresent() + && value.runtimeValue.isPresent() + && addValidKey(keyValue.runtimeValue.get(), runtimeKeys)) { values.put(keyValue.runtimeValue.get(), value.runtimeValue.get()); } else { isExact = false; @@ -294,6 +297,16 @@ private Value extractMap(Expr mapRef) { return isExact ? Value.exact(values.buildOrThrow(), display) : Value.inexact(display); } + // Adds a CEL map key: an int, uint, bool or string, not numerically equal to another. Returns + // whether it's valid. Dyn maps aren't constrained to such keys. + private static boolean addValidKey(Object key, Set keys) { + if (key instanceof Long) { + return keys.add((Long) key >= 0 ? UnsignedLong.valueOf((Long) key) : key); + } + return (key instanceof UnsignedLong || key instanceof Boolean || key instanceof String) + && keys.add(key); + } + private static String join( String prefix, List displayed, long length, String noun, String suffix) { List shown = diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java index e48f1d33b..caa1c8335 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java @@ -177,15 +177,6 @@ Optional translateFunctionCall( CelZ3Exprs.mkOrFlattened( ctx, currentApprox, ctx.mkAnd(isList, CelZ3Exprs.mkOrFlattened(ctx, taints))); } - } else if (op == Operator.OPTIONAL_SELECT) { - // A chained optional select (e.g. `a.?b.?c`) selects from the value of an optional operand. - Expr operand = z3Args.get(0); - Expr target = - ctx.mkITE( - typeSystem.isOptional(operand), - typeSystem.getOptionalValue(typeSystem.getOptionalRef(operand)), - operand); - currentApprox = CelZ3Exprs.mkOrFlattened(ctx, currentApprox, context.isTruncatedMap(target)); } return Optional.of( @@ -335,9 +326,11 @@ private TranslatedValue translateOperatorCall( case LOGICAL_OR: return translateLogicalAndOr(args, false); case LOGICAL_NOT: - return translateLogicalNot(args, ast); + return translateLogicalNot(args); case NEGATE: - return translateNegate(args.get(0), ast); + // NegateAxiom handles the overloads the type checker picked; other kinds error. + return TranslatedValue.propagateStrict( + ctx, typeSystem, op.getFunction(), typeSystem.mkError(), args); case EQUALS: return translateEquality( args.get(0), args.get(1), ast, /* isEquals= */ true, /* shallow= */ false); @@ -359,11 +352,11 @@ private TranslatedValue translateOperatorCall( return TranslatedValue.propagateStrict( ctx, typeSystem, op.getFunction(), typeSystem.mkError(), args); case INDEX: - return translateIndex(args, ast, false); + return translateIndex(args, ast, /* isOptional= */ false); case OPTIONAL_INDEX: - return translateIndex(args, ast, true); + return translateIndex(args, ast, /* isOptional= */ true); case CONDITIONAL: - return translateConditional(args, ast); + return translateConditional(args); case NOT_STRICTLY_FALSE: case OLD_NOT_STRICTLY_FALSE: return translateNotStrictlyFalse(args); @@ -432,48 +425,17 @@ private TranslatedValue translateBinaryLogicalAndOr( return TranslatedValue.create(resultZ3, typeSystem, resultTaint); } - private TranslatedValue translateLogicalNot( - List args, CelAbstractSyntaxTree ast) { + private TranslatedValue translateLogicalNot(List args) { TranslatedValue arg = args.get(0); - CelType type = extractAstTypeOrDefault(arg, ast); - - Expr baseResult = typeSystem.wrapBool(ctx.mkNot((BoolExpr) arg.unwrapZ3Bool())); - if (!type.equals(SimpleType.BOOL)) { - baseResult = typeSystem.withRuntimeError(baseResult, ctx.mkNot(arg.isZ3Bool())); - } + Expr baseResult = + typeSystem.withRuntimeError( + typeSystem.wrapBool(ctx.mkNot((BoolExpr) arg.unwrapZ3Bool())), + ctx.mkNot(arg.isZ3Bool())); return TranslatedValue.propagateStrict( ctx, typeSystem, Operator.LOGICAL_NOT.getFunction(), baseResult, args); } - private TranslatedValue translateNegate(TranslatedValue arg, CelAbstractSyntaxTree ast) { - CelType type = extractAstTypeOrDefault(arg, ast); - Expr z3Expr = arg.z3Expr(); - - Expr result; - if (type.equals(SimpleType.INT)) { - ArithExpr intNeg = ctx.mkUnaryMinus(typeSystem.getInt(z3Expr)); - result = - typeSystem.withRuntimeError( - typeSystem.wrapInt((IntExpr) intNeg), typeSystem.checkIntOverflow(intNeg)); - } else if (type.equals(SimpleType.DOUBLE)) { - result = typeSystem.wrapDouble(ctx.mkFPNeg(typeSystem.getDouble(z3Expr))); - } else { - ArithExpr intNeg = ctx.mkUnaryMinus(typeSystem.getInt(z3Expr)); - Expr intResult = - typeSystem.withRuntimeError( - typeSystem.wrapInt((IntExpr) intNeg), typeSystem.checkIntOverflow(intNeg)); - Expr doubleResult = typeSystem.wrapDouble(ctx.mkFPNeg(typeSystem.getDouble(z3Expr))); - result = - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase(typeSystem.isInt(z3Expr), intResult) - .addCase(typeSystem.isDouble(z3Expr), doubleResult) - .build(typeSystem.mkError()); - } - return TranslatedValue.propagateStrict( - ctx, typeSystem, Operator.NEGATE.getFunction(), result, arg); - } - private BoolExpr isNumeric(Expr arg) { return ctx.mkOr(typeSystem.isInt(arg), typeSystem.isUint(arg), typeSystem.isDouble(arg)); } @@ -564,14 +526,14 @@ private BoolExpr getNumericEquality( boolean approximateIntDouble) { if (arg0.isNumericConstant()) { return getNumericEqualityWithConstant( - arg1.z3Expr(), arg0.celExpr().get().constant(), extractAstTypeOrDefault(arg1, ast)); + arg1.z3Expr(), arg0.celExpr().get().constant(), trustedType(arg1, ast)); } else if (arg1.isNumericConstant()) { return getNumericEqualityWithConstant( - arg0.z3Expr(), arg1.celExpr().get().constant(), extractAstTypeOrDefault(arg0, ast)); + arg0.z3Expr(), arg1.celExpr().get().constant(), trustedType(arg0, ast)); } - CelType type0 = extractAstTypeOrDefault(arg0, ast); - CelType type1 = extractAstTypeOrDefault(arg1, ast); + CelType type0 = trustedType(arg0, ast); + CelType type1 = trustedType(arg1, ast); if (isStaticallyKnown(type0) && isStaticallyKnown(type1) && type0.kind() == type1.kind()) { return getStaticallyKnownNumericEquality(arg0.z3Expr(), type0, arg1.z3Expr()); } @@ -952,8 +914,11 @@ private TranslatedValue translateEquality( Expr z3Arg0 = arg0.z3Expr(); Expr z3Arg1 = arg1.z3Expr(); - CelType type0 = extractAstTypeOrDefault(arg0, ast); - CelType type1 = extractAstTypeOrDefault(arg1, ast); + // A literal's kind is known even if its type may be narrowed. + CelType trusted0 = trustedType(arg0, ast); + CelType trusted1 = trustedType(arg1, ast); + CelType type0 = isContainerLiteral(arg0) ? extractAstTypeOrDefault(arg0, ast) : trusted0; + CelType type1 = isContainerLiteral(arg1) ? extractAstTypeOrDefault(arg1, ast) : trusted1; BoolExpr equality; List taints = new ArrayList<>(); @@ -961,7 +926,7 @@ private TranslatedValue translateEquality( taints.add(mayObserveTruncatedTypeConstraint(ast, arg0, arg1)); } - if (isNumericType(type0) && isNumericType(type1)) { + if (isNumericType(trusted0) && isNumericType(trusted1)) { equality = getNumericEquality(arg0, arg1, ast, taints, /* approximateIntDouble= */ false); } else { BoolExpr bothLists = ctx.mkAnd(typeSystem.isList(z3Arg0), typeSystem.isList(z3Arg1)); @@ -993,12 +958,12 @@ private TranslatedValue translateEquality( ctx.mkITE(bothLists, unrollListEquality(arg0, arg1, ast, taints), structuralEq); } else if (type0.kind() == CelKind.MAP || type1.kind() == CelKind.MAP) { BoolExpr mapEquality = - isExact(type0, type1) + isExact(trusted0, trusted1) && (arg0.isLiteral(ExprKind.Kind.MAP) || arg1.isLiteral(ExprKind.Kind.MAP)) ? mapEqualityAtLiteralKeys(arg0, arg1) : unrollMapEquality(arg0, arg1, ast, taints); structuralEq = (BoolExpr) ctx.mkITE(bothMaps, mapEquality, structuralEq); - } else if (!isStaticallyKnown(type0) && !isStaticallyKnown(type1)) { + } else if (!isStaticallyKnown(trusted0) && !isStaticallyKnown(trusted1)) { // Performance only: a side with a known type isn't a list or map here. // TODO: Unless the type checker narrowed it, e.g. `(int_list + // dyn_var)[0]`. @@ -1034,7 +999,7 @@ private TranslatedValue translateEquality( } private Expr buildListIndex( - Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, boolean isOptional) { + Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, BoolExpr returnOptional) { Expr listRef = typeSystem.getListRef(lhsTrans); SeqExpr seq = typeSystem.getSeq(listRef); Expr index = typeSystem.getInt(rhsTrans); @@ -1043,31 +1008,37 @@ private Expr buildListIndex( ctx.mkGe((ArithExpr) index, ctx.mkInt(0)), ctx.mkLt((ArithExpr) index, ctx.mkLength(seq))); - return finishIndex(inBounds, ctx.mkNth(seq, (ArithExpr) index), typeGuard, isOptional); + return finishIndex(inBounds, ctx.mkNth(seq, (ArithExpr) index), typeGuard, returnOptional); } private Expr buildMapIndex( - Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, boolean isOptional) { + Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, BoolExpr returnOptional) { ImmutableList equalKeys = typeSystem.equalKeys(rhsTrans); return finishIndex( typeSystem.hasEqualKey(lhsTrans, equalKeys), typeSystem.valueAtEqualKey(lhsTrans, equalKeys), typeGuard, - isOptional); + returnOptional); } - // Returns val where found, or else an error (or an empty optional). - private Expr finishIndex(BoolExpr found, Expr val, BoolExpr typeGuard, boolean isOptional) { + // Returns val where found, or else an error, as an optional (empty if not found) where + // returnOptional holds. + private Expr finishIndex( + BoolExpr found, Expr val, BoolExpr typeGuard, BoolExpr returnOptional) { context.constrainValidValue(ctx.mkAnd(typeGuard, found), val); - if (isOptional) { - Expr resultOptRef = ctx.mkApp(typeSystem.optionalOfRefFunc(), val); - context.addConstraint(ctx.mkEq(typeSystem.getOptionalValue(resultOptRef), val)); - context.addConstraint(typeSystem.optHasValue(resultOptRef)); - return ctx.mkITE(found, typeSystem.mkOptionalOf(resultOptRef), typeSystem.mkOptionalNone()); + Expr plain = ctx.mkITE(found, val, typeSystem.mkError()); + // Performance only: skips building an optional that isn't returned. + if (returnOptional.isFalse()) { + return plain; } - return ctx.mkITE(found, val, typeSystem.mkError()); + Expr optional = ctx.mkITE(found, context.mkOptionalOf(val), typeSystem.mkOptionalNone()); + return returnOptional.isTrue() ? optional : ctx.mkITE(returnOptional, optional, plain); } + /** + * Translates {@code lhs[rhs]}, or {@code lhs[?rhs]} if isOptional. An optional lhs is indexed + * optionally (e.g. {@code optional.of([])[0]} is {@code optional.none()}). + */ private TranslatedValue translateIndex( List args, CelAbstractSyntaxTree ast, boolean isOptional) { TranslatedValue lhs = args.get(0); @@ -1081,8 +1052,10 @@ private TranslatedValue translateIndex( BoolExpr isLhsOpt = ctx.mkFalse(); BoolExpr lhsHasValue = ctx.mkFalse(); BoolExpr shouldEvaluate = ctx.mkTrue(); + BoolExpr returnOptional = isOptional ? ctx.mkTrue() : ctx.mkFalse(); - if (isOptional) { + // Performance only: an lhs of a known type other than optional isn't one. + if (lhsType instanceof OptionalType || !isStaticallyKnown(lhsType)) { isLhsOpt = typeSystem.isOptional(lhsTrans); Expr optRef = typeSystem.getOptionalRef(lhsTrans); lhsHasValue = typeSystem.optHasValue(optRef); @@ -1092,21 +1065,24 @@ private TranslatedValue translateIndex( if (lhsType instanceof OptionalType) { lhsType = lhsType.parameters().get(0); + returnOptional = ctx.mkTrue(); + } else if (!isOptional) { + returnOptional = isLhsOpt; } } TranslatedValue element = buildAndConstrainIndex( - lhsTrans, rhsTrans, lhsType, isLhsTypeGuaranteed, shouldEvaluate, isOptional); - Expr actualValue = element.z3Expr(); - - if (isOptional) { - actualValue = - ctx.mkITE( - ctx.mkAnd(isLhsOpt, ctx.mkNot(lhsHasValue)), - typeSystem.mkOptionalNone(), - actualValue); - } + lhsTrans, rhsTrans, lhsType, isLhsTypeGuaranteed, shouldEvaluate, returnOptional); + // An empty optional gives an empty optional, unless no overload takes the index (e.g. a string + // indexing an optional list). + BoolExpr takesIndex = + lhsType.kind() == CelKind.LIST ? typeSystem.isInt(rhsTrans) : ctx.mkTrue(); + Expr actualValue = + ctx.mkITE( + CelZ3Exprs.mkAndFlattened(ctx, isLhsOpt, ctx.mkNot(lhsHasValue), takesIndex), + typeSystem.mkOptionalNone(), + element.z3Expr()); String opName = isOptional ? Operator.OPTIONAL_INDEX.getFunction() : Operator.INDEX.getFunction(); @@ -1137,7 +1113,7 @@ private TranslatedValue buildAndConstrainIndex( CelType lhsType, boolean isLhsTypeGuaranteed, BoolExpr shouldEvaluate, - boolean isOptional) { + BoolExpr returnOptional) { CelType expectedElemType = null; if (lhsType.kind() == CelKind.LIST) { expectedElemType = ((ListType) lhsType).elemType(); @@ -1146,20 +1122,22 @@ private TranslatedValue buildAndConstrainIndex( } if (expectedElemType != null) { - Expr actualValue; - if (lhsType.kind() == CelKind.MAP) { - actualValue = buildMapIndex(lhsTrans, rhsTrans, shouldEvaluate, isOptional); - } else { - BoolExpr isIntIndex = typeSystem.isInt(rhsTrans); - actualValue = - ctx.mkITE( - isIntIndex, - buildListIndex( - lhsTrans, rhsTrans, ctx.mkAnd(shouldEvaluate, isIntIndex), isOptional), - typeSystem.mkError()); - } - - CelType finalType = isOptional ? OptionalType.create(expectedElemType) : expectedElemType; + // The static type picks the overload, which errors on the other kind if it was narrowed. + BoolExpr takesIndex = + lhsType.kind() == CelKind.MAP + ? typeSystem.isMap(lhsTrans) + : ctx.mkAnd(typeSystem.isList(lhsTrans), typeSystem.isInt(rhsTrans)); + BoolExpr guard = ctx.mkAnd(shouldEvaluate, takesIndex); + Expr actualValue = + ctx.mkITE( + takesIndex, + lhsType.kind() == CelKind.MAP + ? buildMapIndex(lhsTrans, rhsTrans, guard, returnOptional) + : buildListIndex(lhsTrans, rhsTrans, guard, returnOptional), + typeSystem.mkError()); + + CelType finalType = + returnOptional.isTrue() ? OptionalType.create(expectedElemType) : expectedElemType; BoolExpr isDefined = ctx.mkAnd(shouldEvaluate, ctx.mkNot(typeSystem.isError(actualValue))); BoolExpr typeConstraint = typeConstraintGenerator.createTypeConstraintForType(actualValue, finalType); @@ -1182,27 +1160,22 @@ private TranslatedValue buildAndConstrainIndex( return TranslatedValue.create( CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase(isListGuard, buildListIndex(lhsTrans, rhsTrans, isListGuard, isOptional)) - .addCase(isMapGuard, buildMapIndex(lhsTrans, rhsTrans, isMapGuard, isOptional)) + .addCase(isListGuard, buildListIndex(lhsTrans, rhsTrans, isListGuard, returnOptional)) + .addCase(isMapGuard, buildMapIndex(lhsTrans, rhsTrans, isMapGuard, returnOptional)) .build(typeSystem.mkError()), typeSystem, ctx.mkFalse()); } - private TranslatedValue translateConditional( - List args, CelAbstractSyntaxTree ast) { + private TranslatedValue translateConditional(List args) { TranslatedValue cond = args.get(0); TranslatedValue trueBranch = args.get(1); TranslatedValue falseBranch = args.get(2); - CelType condType = extractAstTypeOrDefault(cond, ast); BoolExpr condTrue = ctx.mkAnd(cond.isZ3Bool(), (BoolExpr) cond.unwrapZ3Bool()); - BoolExpr hasError = cond.isZ3Error(); + BoolExpr hasError = ctx.mkNot(cond.isZ3Bool()); BoolExpr hasUnknown = cond.isZ3Unknown(); - if (!condType.equals(SimpleType.BOOL)) { - hasError = ctx.mkOr(hasError, ctx.mkNot(cond.isZ3Bool())); - } Expr unknownResult = typeSystem.isParameterizingUnknowns() @@ -1243,6 +1216,12 @@ private static CelType extractAstTypeOrDefault(TranslatedValue val, CelAbstractS return val.celExpr().map(node -> ast.getTypeOrThrow(node.id())).orElse(SimpleType.DYN); } + // The static type of val unless the type checker may have narrowed it (e.g. `dyn_var.orValue(1)` + // is typed int). Overloads are still picked by static types. + private CelType trustedType(TranslatedValue val, CelAbstractSyntaxTree ast) { + return val.celExpr().map(e -> context.trustedType(e, ast)).orElse(SimpleType.DYN); + } + /** * Returns the type that {@code val} is guaranteed to conform to at runtime: a constant's, the * declared type of an input variable, or else dyn. Global constraints may only be derived from @@ -1300,9 +1279,11 @@ private static boolean hasStructuralEquality(CelType type) { } } + // A wrapper's kind is its value's, but it may also be null. private static boolean isNumericType(CelType type) { CelKind kind = type.kind(); - return kind == CelKind.INT || kind == CelKind.UINT || kind == CelKind.DOUBLE; + return !(type instanceof NullableType) + && (kind == CelKind.INT || kind == CelKind.UINT || kind == CelKind.DOUBLE); } /** @@ -1312,7 +1293,7 @@ private static boolean isNumericType(CelType type) { */ private BoolExpr mayObserveTruncatedTypeConstraint( CelAbstractSyntaxTree ast, TranslatedValue... args) { - if (Arrays.stream(args).allMatch(arg -> isConcrete(extractAstTypeOrDefault(arg, ast)))) { + if (Arrays.stream(args).allMatch(arg -> isConcrete(trustedType(arg, ast)))) { return ctx.mkFalse(); } return context.recordUnrollLimitApproximation( diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java index 11cc36191..00ee7e7f2 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java @@ -26,6 +26,7 @@ import com.microsoft.z3.SeqExpr; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.ast.CelExpr; +import dev.cel.common.ast.CelExpr.ExprKind; import dev.cel.common.internal.ProtoTimeUtils; import dev.cel.common.types.CelKind; import dev.cel.common.types.CelType; @@ -536,13 +537,41 @@ private FieldAccess getMsgAccess(Expr operand, String field, BoolExpr typeGua return new FieldAccess(presence, value); } + /** + * Translates {@code e.f}, {@code has(e.f)} or {@code e.?f} (an optional of {@code e.f} if {@code + * has(e.f)}, else empty). + */ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { - CelExpr.CelSelect select = celExpr.select(); + boolean isOptional = celExpr.getKind() == ExprKind.Kind.CALL; + CelExpr operandExpr = isOptional ? celExpr.call().args().get(0) : celExpr.select().operand(); + String field = + isOptional + ? celExpr.call().args().get(1).constant().stringValue() + : celExpr.select().field(); + boolean testOnly = !isOptional && celExpr.select().testOnly(); long exprId = celExpr.id(); - TranslatedValue operandTv = translateExpr(select.operand(), ast); + TranslatedValue operandTv = translateExpr(operandExpr, ast); Expr operand = operandTv.z3Expr(); - String field = select.field(); - CelType operandType = extractAstTypeOrDefault(ast, select.operand().id()); + CelType operandType = context.trustedType(operandExpr, ast); + // Selecting from an optional selects from its value optionally (e.g. `optional.of({}).a` is + // `optional.none()`). Performance only: an operand of a known type other than optional isn't. + boolean mayBeOptional = + operandType instanceof OptionalType + || !CelZ3OperatorTranslator.isStaticallyKnown(operandType); + BoolExpr isOptionalOperand = ctx.mkFalse(); + BoolExpr hasValue = ctx.mkFalse(); + BoolExpr shouldEvaluate = ctx.mkTrue(); + if (mayBeOptional) { + Expr optRef = typeSystem.getOptionalRef(operand); + isOptionalOperand = typeSystem.isOptional(operand); + hasValue = typeSystem.optHasValue(optRef); + shouldEvaluate = (BoolExpr) ctx.mkITE(isOptionalOperand, hasValue, ctx.mkTrue()); + operand = ctx.mkITE(isOptionalOperand, typeSystem.getOptionalValue(optRef), operand); + } + if (operandType instanceof OptionalType) { + // Performance only: reads the value by its static type. + operandType = operandType.parameters().get(0); + } Expr presenceResult; Expr valueResult; @@ -550,11 +579,11 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { BoolExpr presenceError = ctx.mkFalse(); if (operandType instanceof MapType) { - FieldAccess mapAcc = getMapAccess(operand, field, ctx.mkTrue()); + FieldAccess mapAcc = getMapAccess(operand, field, shouldEvaluate); presenceResult = mapAcc.presence; valueResult = ctx.mkITE(mapAcc.presence, mapAcc.value, typeSystem.mkError()); } else if (operandType.kind() == CelKind.STRUCT) { - FieldAccess msgAcc = getMsgAccess(operand, field, ctx.mkTrue()); + FieldAccess msgAcc = getMsgAccess(operand, field, shouldEvaluate); presenceResult = msgAcc.presence; Expr defaultVal = getDefaultValueForType(extractAstTypeOrDefault(ast, exprId)); valueResult = ctx.mkITE(msgAcc.presence, msgAcc.value, defaultVal); @@ -563,8 +592,8 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { BoolExpr isMap = typeSystem.isMap(operand); BoolExpr isMessage = typeSystem.isMessage(operand); - FieldAccess mapAcc = getMapAccess(operand, field, isMap); - FieldAccess msgAcc = getMsgAccess(operand, field, isMessage); + FieldAccess mapAcc = getMapAccess(operand, field, ctx.mkAnd(shouldEvaluate, isMap)); + FieldAccess msgAcc = getMsgAccess(operand, field, ctx.mkAnd(shouldEvaluate, isMessage)); presenceResult = CelZ3Exprs.SwitchBuilder.newBuilder(ctx) @@ -590,13 +619,13 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { msgRead.addCase(isType, typeSystem.mkError()); continue; } - if (select.testOnly()) { + if (testOnly) { continue; } CelType fieldType = declared.get().type(); context.addConstraint( ctx.mkImplies( - ctx.mkAnd(isMessage, isType, msgAcc.presence), + ctx.mkAnd(shouldEvaluate, isMessage, isType, msgAcc.presence), typeConstraintGenerator.createTypeConstraintForType(msgAcc.value, fieldType))); msgRead.addCase( isType, ctx.mkITE(msgAcc.presence, msgAcc.value, getDefaultValueForType(fieldType))); @@ -605,7 +634,10 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { unmodeledMessage = context.recordApproximation( "messages of types the type provider doesn't declare", - ctx.mkAnd(isMessage, ctx.mkNot(CelZ3Exprs.mkOrFlattened(ctx, isModeledType)))); + ctx.mkAnd( + shouldEvaluate, + isMessage, + ctx.mkNot(CelZ3Exprs.mkOrFlattened(ctx, isModeledType)))); Expr mapRead = ctx.mkITE(mapAcc.presence, mapAcc.value, typeSystem.mkError()); // Testing an undeclared field, or a field of something other than a map or message, errors. presenceError = @@ -622,12 +654,33 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { .build(typeSystem.mkError()); } + // has(e.f), else e.f, or for e.?f and an optional e, an optional of e.f if has(e.f). Expr fieldAccess = - select.testOnly() + testOnly ? typeSystem.withRuntimeError(typeSystem.wrapBool(presenceResult), presenceError) : valueResult; + Expr chained = fieldAccess; + if (!testOnly && (isOptional || mayBeOptional)) { + chained = + ctx.mkITE( + presenceError, + typeSystem.mkError(), + ctx.mkITE( + (BoolExpr) presenceResult, + context.mkOptionalOf(valueResult), + typeSystem.mkOptionalNone())); + fieldAccess = isOptional ? chained : valueResult; + } + if (mayBeOptional) { + Expr empty = testOnly ? typeSystem.mkBool(false) : typeSystem.mkOptionalNone(); + fieldAccess = ctx.mkITE(isOptionalOperand, ctx.mkITE(hasValue, chained, empty), fieldAccess); + } - context.addConstraint(typeConstraintGenerator.createTypeConstraint(fieldAccess, exprId, ast)); + context.addConstraint( + ctx.mkOr( + typeSystem.isErrorOrUnknown(fieldAccess), + typeConstraintGenerator.createTypeConstraintForType( + fieldAccess, context.trustedType(celExpr, ast)))); TranslatedValue result = TranslatedValue.propagateStrict( @@ -636,9 +689,11 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { return operandType.kind() == CelKind.STRUCT ? result : result.withApproximation( - CelZ3Exprs.mkOrFlattened(ctx, context.isTruncatedMap(operand), unmodeledMessage)); + CelZ3Exprs.mkOrFlattened( + ctx, ctx.mkAnd(shouldEvaluate, context.isTruncatedMap(operand)), unmodeledMessage)); } + private static CelType extractAstTypeOrDefault(CelAbstractSyntaxTree ast, long id) { return ast.getType(id).orElse(SimpleType.DYN); } diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java index fb6f7270d..686d43100 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java @@ -17,16 +17,24 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.microsoft.z3.BoolExpr; import com.microsoft.z3.Context; import com.microsoft.z3.Expr; import com.microsoft.z3.Model; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.Operator; +import dev.cel.common.ast.CelBlock; +import dev.cel.common.ast.CelExpr; +import dev.cel.common.types.CelKind; import dev.cel.common.types.CelType; import dev.cel.common.types.SimpleType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -49,6 +57,8 @@ final class CelZ3TranslationContext { private final Map> approximationTerms; private final Map symbolTable; private final Map localScope; + private final IdentityHashMap> narrowedExprIds; + private final Set narrowedSymbols; Context ctx() { return ctx; @@ -80,6 +90,14 @@ void constrainValidValue(BoolExpr guard, Expr value) { addConstraint(ctx.mkImplies(guard, typeSystem.isValidValue(value))); } + /** Returns an optional holding {@code value}. */ + Expr mkOptionalOf(Expr value) { + Expr optRef = ctx.mkApp(typeSystem.optionalOfRefFunc(), value); + addConstraint(ctx.mkEq(typeSystem.getOptionalValue(optRef), value)); + addConstraint(typeSystem.optHasValue(optRef)); + return typeSystem.mkOptionalOf(optRef); + } + void addInputVariable(Expr variable, CelType declaredType) { inputVariableTypes.put(variable, declaredType); } @@ -94,6 +112,115 @@ CelType guaranteedType(Expr value) { return inputVariableTypes.getOrDefault(value, SimpleType.DYN); } + /** + * Whether the type checker may have narrowed the static type of {@code expr} by unifying a type + * parameter with dyn or null (e.g. {@code int_list + dyn_var} is typed list(int) but may hold + * doubles). + */ + boolean mayBeNarrowed(CelExpr expr, CelAbstractSyntaxTree ast) { + return narrowedExprIds + .computeIfAbsent( + ast, + a -> { + Set ids = new HashSet<>(); + collectNarrowed(a.getExpr(), a, ImmutableMap.of(), ids); + return ids; + }) + .contains(expr.id()); + } + + /** The static type of {@code expr}, or dyn if the type checker may have narrowed it. */ + CelType trustedType(CelExpr expr, CelAbstractSyntaxTree ast) { + return mayBeNarrowed(expr, ast) + ? SimpleType.DYN + : ast.getType(expr.id()).orElse(SimpleType.DYN); + } + + // Adds expr to ids if it may be narrowed: a subexpression, or a variable it reads, is typed dyn + // or null, or is narrowed. Returns whether expr is either. + private boolean collectNarrowed( + CelExpr expr, CelAbstractSyntaxTree ast, Map boundVars, Set ids) { + boolean narrowed = false; + switch (expr.getKind()) { + case IDENT: + String name = expr.ident().name(); + narrowed = boundVars.getOrDefault(name, narrowedSymbols.contains(name)); + break; + case SELECT: + narrowed = collectNarrowed(expr.select().operand(), ast, boundVars, ids); + break; + case CALL: + CelExpr.CelCall call = expr.call(); + ImmutableList args = call.args(); + if (call.function().equals(CelBlock.FUNCTION_NAME)) { + // `cel.@block([e0, e1, ...], result)` binds @index0 to e0, and so on. + Map scope = new HashMap<>(boundVars); + ImmutableList slots = args.get(0).list().elements(); + for (int i = 0; i < slots.size(); i++) { + scope.put(CelBlock.INDEX_PREFIX + i, collectNarrowed(slots.get(i), ast, scope, ids)); + } + narrowed = collectNarrowed(args.get(1), ast, scope, ids); + break; + } + if (call.target().isPresent()) { + narrowed = collectNarrowed(call.target().get(), ast, boundVars, ids); + } + // The field name of `e.?f` isn't typed. + if (call.function().equals(Operator.OPTIONAL_SELECT.getFunction())) { + args = args.subList(0, 1); + } + for (CelExpr arg : args) { + narrowed |= collectNarrowed(arg, ast, boundVars, ids); + } + break; + case LIST: + for (CelExpr element : expr.list().elements()) { + narrowed |= collectNarrowed(element, ast, boundVars, ids); + } + break; + case MAP: + for (CelExpr.CelMap.Entry entry : expr.map().entries()) { + narrowed |= collectNarrowed(entry.key(), ast, boundVars, ids); + narrowed |= collectNarrowed(entry.value(), ast, boundVars, ids); + } + break; + case STRUCT: + for (CelExpr.CelStruct.Entry entry : expr.struct().entries()) { + narrowed |= collectNarrowed(entry.value(), ast, boundVars, ids); + } + break; + case COMPREHENSION: + // Its value is the result's, which reads the variables. + CelExpr.CelComprehension comp = expr.comprehension(); + boolean range = collectNarrowed(comp.iterRange(), ast, boundVars, ids); + Map scope = new HashMap<>(boundVars); + scope.put(comp.iterVar(), range); + scope.put(comp.iterVar2(), range); + scope.put(comp.accuVar(), collectNarrowed(comp.accuInit(), ast, boundVars, ids)); + // The accumulator also holds what each step computes. + if (collectNarrowed(comp.loopStep(), ast, scope, ids) && !scope.get(comp.accuVar())) { + scope.put(comp.accuVar(), true); + collectNarrowed(comp.loopStep(), ast, scope, ids); + } + collectNarrowed(comp.loopCondition(), ast, scope, ids); + narrowed = collectNarrowed(comp.result(), ast, scope, ids); + break; + default: + break; + } + if (narrowed) { + ids.add(expr.id()); + } + return narrowed || involvesDynOrNull(ast.getType(expr.id()).orElse(SimpleType.DYN)); + } + + private static boolean involvesDynOrNull(CelType type) { + return type.kind().isDyn() + || type.kind().isTypeParam() + || type.kind() == CelKind.NULL_TYPE + || type.parameters().stream().anyMatch(CelZ3TranslationContext::involvesDynOrNull); + } + /** * Whether {@code val} is a map with more keys than the unroll limit, beyond which its presence is * not linked to its keys. @@ -169,8 +296,13 @@ BoolExpr hasUnmodeledResult() { return CelZ3Exprs.mkOrFlattened(ctx, unmodeledResults); } - void bindSymbol(String name, TranslatedValue value) { + /** Binds {@code name} to {@code value}, the result of {@code ast}. */ + void bindSymbol(String name, TranslatedValue value, CelAbstractSyntaxTree ast) { localScope.put(name, value); + // Other expressions read it with its static type, which may be narrowed. + if (mayBeNarrowed(ast.getExpr(), ast) || involvesDynOrNull(ast.getResultType())) { + narrowedSymbols.add(name); + } } Optional lookupSymbol(String name) { @@ -212,5 +344,7 @@ T withScope(String varName, TranslatedValue value, Supplier action) { this.approximationTerms = new LinkedHashMap<>(); this.symbolTable = new HashMap<>(); this.localScope = new HashMap<>(); + this.narrowedExprIds = new IdentityHashMap<>(); + this.narrowedSymbols = new HashSet<>(); } } diff --git a/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java b/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java new file mode 100644 index 000000000..00ac15ca3 --- /dev/null +++ b/verifier/src/main/java/dev/cel/verifier/RuntimeOracle.java @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.verifier; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelErrorCode; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelFunctionResolver; +import dev.cel.runtime.CelResolvedOverload; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntime.Program; +import dev.cel.runtime.CelUnknownSet; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Evaluates expressions with the CEL runtime on a witness's inputs, to check the solver's + * candidates. + */ +final class RuntimeOracle { + + // Stands for every evaluation error, matching the verifier's single error value. + private enum EvaluationError { + ERROR + } + + /** The witness's inputs, if every value is exact. */ + static Optional> exactInputs(CelWitness witness) { + return witness.variables().values().stream().allMatch(v -> v.value().isPresent()) + ? Optional.of( + ImmutableMap.copyOf(Maps.transformValues(witness.variables(), v -> v.value().get()))) + : Optional.empty(); + } + + /** + * Returns the values of {@code asts} for {@code inputs}, with errors as one value. Throws if the + * runtime can't evaluate one, e.g. a function has no binding or its iteration limit is hit, or if + * it fails with an internal error and {@code countInternalErrors} is false. + */ + static ImmutableList evaluate( + CelRuntime runtime, + Map inputs, + boolean countInternalErrors, + CelAbstractSyntaxTree... asts) + throws CelEvaluationException { + ImmutableList.Builder results = ImmutableList.builder(); + for (CelAbstractSyntaxTree ast : asts) { + Program program = runtime.createProgram(ast); + // Notes calls to late-bound functions, which have no binding here. The legacy interpreter + // also consults them after any failed dispatch, so its dispatch errors go unchecked. + AtomicBoolean callsLateBound = new AtomicBoolean(); + CelFunctionResolver lateBoundFunctions = + new CelFunctionResolver() { + @Override + public Optional findOverloadMatchingArgs( + String functionName, Collection overloadIds, Object[] args) { + return findOverloadMatchingArgs(functionName, args); + } + + @Override + public Optional findOverloadMatchingArgs( + String functionName, Object[] args) { + callsLateBound.set(true); + return Optional.empty(); + } + }; + Object result; + try { + result = program.eval(inputs, lateBoundFunctions); + } catch (CelEvaluationException e) { + CelErrorCode code = e.getErrorCode(); + // The runtime reports some CEL errors as internal, but also misused bindings (e.g. async). + if (callsLateBound.get() + || code == CelErrorCode.ITERATION_BUDGET_EXCEEDED + || (!countInternalErrors && code == CelErrorCode.INTERNAL_ERROR)) { + throw e; + } + result = EvaluationError.ERROR; + } + // A binding may return null or unknowns. + if (result == null || result instanceof CelUnknownSet) { + throw new CelEvaluationException( + result == null ? "a function returned null" : "it evaluated to unknowns"); + } + results.add(result); + } + return results.build(); + } + + private RuntimeOracle() {} +} diff --git a/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java b/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java index 7ea52abc8..201739f66 100644 --- a/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java +++ b/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java @@ -199,6 +199,8 @@ private static String extractOpName(CelExpr expr) { switch (expr.exprKind().getKind()) { case SELECT: return "select_" + expr.select().field() + "_" + expr.select().testOnly(); + case CALL: // e.?f + return "optional_select_" + expr.call().args().get(1).constant().stringValue(); case STRUCT: StringBuilder structSb = new StringBuilder("struct_").append(expr.struct().messageName()); for (CelExpr.CelStruct.Entry entry : expr.struct().entries()) { diff --git a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java index dd983d36b..748b6f6ca 100644 --- a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java +++ b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java @@ -20,12 +20,10 @@ import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OF_NON_ZERO_VALUE; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR_VALUE; -import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_SELECT; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_VALUE; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.microsoft.z3.ArrayExpr; import com.microsoft.z3.BoolExpr; import com.microsoft.z3.Context; import com.microsoft.z3.Expr; @@ -33,7 +31,6 @@ import com.microsoft.z3.SeqExpr; import dev.cel.common.CelFunctionDecl; import dev.cel.common.CelOverloadDecl; -import dev.cel.verifier.CelZ3Exprs; import dev.cel.verifier.CelZ3TypeSystem; import java.util.Optional; @@ -88,63 +85,6 @@ final class OptionalAxioms { (ctx, ts, sink, val, other) -> { Expr optRef = ts.getOptionalRef(val); return Optional.of(ctx.mkITE(ts.optHasValue(optRef), val, other)); - }), - createBinaryAxiom( - OPTIONAL_SELECT, - (ctx, ts, sink, operand, field) -> { - Expr optRef = ts.getOptionalRef(operand); - BoolExpr isOpt = ts.isOptional(operand); - BoolExpr hasValue = ts.optHasValue(optRef); - Expr actualOperand = ctx.mkITE(isOpt, ts.getOptionalValue(optRef), operand); - - BoolExpr isMap = ts.isMap(actualOperand); - BoolExpr isMsg = ts.isMessage(actualOperand); - BoolExpr isValidTarget = ctx.mkOr(isMap, isMsg); - - Expr msgFieldZ3Str = ts.getString(field); - Expr mapFieldCelVal = field; - - Expr msgRef = ts.getMessageRef(actualOperand); - Expr mapRef = ts.getMapRef(actualOperand); - - Expr presence = - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase( - isMsg, - ctx.mkSelect((ArrayExpr) ts.getMsgPresence(msgRef), msgFieldZ3Str)) - .addCase( - isMap, - ctx.mkSelect((ArrayExpr) ts.getMapPresence(mapRef), mapFieldCelVal)) - .build(ctx.mkFalse()); - - Expr value = - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase( - isMsg, ctx.mkSelect((ArrayExpr) ts.getMsgValues(msgRef), msgFieldZ3Str)) - .addCase( - isMap, - ctx.mkSelect((ArrayExpr) ts.getMapValues(mapRef), mapFieldCelVal)) - .build(ts.mkError()); - - BoolExpr valNotError = ctx.mkNot(ctx.mkEq(value, ts.mkError())); - BoolExpr shouldEvaluate = (BoolExpr) ctx.mkITE(isOpt, hasValue, ctx.mkTrue()); - sink.accept( - ctx.mkImplies( - CelZ3Exprs.mkAndFlattened( - ctx, shouldEvaluate, isValidTarget, (BoolExpr) presence), - valNotError)); - - Expr resultOptRef = ctx.mkApp(ts.optionalOfRefFunc(), value); - sink.accept(ctx.mkEq(ts.getOptionalValue(resultOptRef), value)); - sink.accept(ts.optHasValue(resultOptRef)); - - Expr optionalResult = - ctx.mkITE( - (BoolExpr) presence, ts.mkOptionalOf(resultOptRef), ts.mkOptionalNone()); - - Expr result = ctx.mkITE(isValidTarget, optionalResult, ts.mkError()); - return Optional.of( - ctx.mkITE(ctx.mkAnd(isOpt, ctx.mkNot(hasValue)), ts.mkOptionalNone(), result)); })); private static BoolExpr isZeroValue(Context ctx, CelZ3TypeSystem ts, Expr val) { diff --git a/verifier/src/test/java/dev/cel/verifier/BUILD.bazel b/verifier/src/test/java/dev/cel/verifier/BUILD.bazel index 91a24c23d..da198b15e 100644 --- a/verifier/src/test/java/dev/cel/verifier/BUILD.bazel +++ b/verifier/src/test/java/dev/cel/verifier/BUILD.bazel @@ -47,8 +47,8 @@ java_library( "//policy:parser_factory", "//policy:validation_exception", "//policy/testing:k8s_test_tag_handler", - "//runtime", - "//runtime:evaluation_exception", + "//runtime:function_binding", + "//runtime:unknown_attributes", "//testing/protos:single_file_java_proto", "@bazel_tools//tools/java/runfiles", "@maven//:com_google_errorprone_error_prone_annotations", diff --git a/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java index 8c18f4da9..1fa2f5947 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelPolicyVerifierImplTest.java @@ -450,6 +450,39 @@ public void verifyInvariants_restrictedDestinationsPolicy_verified() throws Exce "unrestricted_destination_allowed", CelVerificationResult.verified()); } + @Test + public void verifyInvariants_narrowedVariable_violated() throws Exception { + Cel optionalCel = + CEL.toCelBuilder() + .addCompilerLibraries(CelOptionalLibrary.INSTANCE) + .addRuntimeLibraries(CelOptionalLibrary.INSTANCE) + .build(); + CelPolicyVerifier verifier = + CelPolicyVerifierFactory.newVerifier( + CelPolicyCompilerFactory.newPolicyCompiler(optionalCel).build(), + CelVerifierFactory.newVerifier(optionalCel).build()) + .build(); + // variables.v is typed int, but is NaN for request = optional.of(0.0 / 0.0). + String policySource = + "name: narrowed_variable_policy\n" // + + "rule:\n" // + + " match:\n" // + + " - output: 'true'\n" // + + "verification:\n" // + + " variables:\n" // + + " - v: request.orValue(1)\n" // + + " invariants:\n" // + + " - id: variable_equals_itself\n" // + + " assume: request.hasValue()\n" // + + " assert: variables.v == variables.v"; + + ImmutableMap results = + verifier.verifyInvariants(PARSER.parse(policySource)); + + assertThat(results.get("variable_equals_itself").status()) + .isEqualTo(VerificationStatus.VIOLATED); + } + @Test public void verifyInvariants_invariantReferencesPolicyVariable_verified() throws Exception { String policySource = @@ -554,12 +587,12 @@ public void verifyInvariants_workloadAdmissionFlawed_violationsDetected() throws assertThat(results).containsKey("universal_no_unapproved_privileged_prod"); assertThat(results.get("universal_no_unapproved_privileged_prod").status()) .isEqualTo(VerificationStatus.VIOLATED); + // Either the admin or the owner bypass violates it. assertThat(results.get("universal_no_unapproved_privileged_prod").message()) - .isEqualTo( + .containsMatch( "Invariant 'universal_no_unapproved_privileged_prod' violation detected." + " Counterexample input:\n" - + " is_admin = true\n" - + " is_owner = false\n" + + " is_admin = (true\n is_owner = (true|false)|false\n is_owner = true)\n" + " is_privileged = true\n" + " is_prod = true\n" + " has_approval = false"); diff --git a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java index 8951801f7..10239ac09 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.Immutable; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; // import com.google.testing.testsize.MediumTest; @@ -38,6 +39,7 @@ import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelContainer; import dev.cel.common.CelFunctionDecl; +import dev.cel.common.CelOptions; import dev.cel.common.CelOverloadDecl; import dev.cel.common.CelVarDecl; import dev.cel.common.Operator; @@ -58,8 +60,11 @@ import dev.cel.optimizer.CelOptimizer; import dev.cel.optimizer.CelOptimizerFactory; import dev.cel.optimizer.optimizers.ConstantFoldingOptimizer; +import dev.cel.optimizer.optimizers.SubexpressionOptimizer; import dev.cel.parser.CelMacro; import dev.cel.parser.CelStandardMacro; +import dev.cel.runtime.CelFunctionBinding; +import dev.cel.runtime.CelUnknownSet; import dev.cel.verifier.CelVerificationResult.VerificationStatus; import dev.cel.verifier.axioms.CelZ3FunctionAxiom; import dev.cel.verifier.axioms.CelZ3OverloadResult; @@ -69,7 +74,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; -import java.util.function.UnaryOperator; import java.util.stream.IntStream; import org.junit.Before; import org.junit.Test; @@ -141,15 +145,35 @@ public final class CelVerifierZ3ImplTest { TestAllTypes.getDescriptor(), TestAllTypes.NestedMessage.getDescriptor())) .build(); - private static final CelVerifier VERIFIER = withRuntimeOracle(CEL, b -> b); + private static final CelVerifier VERIFIER = checkedVerifier(CEL).build(); - // For ASTs with functions the runtime can't plan, and for verifyImplication. - private static final CelVerifier VERIFIER_WITHOUT_ORACLE = - CelVerifierFactory.newVerifier(CEL).build(); + // Doesn't check witnesses with the runtime: for approximate results, which it may confirm, for + // functions it has no bindings for, and for verifyImplication. + private static final CelVerifier UNCHECKED_VERIFIER = CelVerifierFactory.newVerifier(CEL).build(); - // Builds a verifier whose exact witnesses are checked against the runtime of the same `cel`. - private static CelVerifier withRuntimeOracle(Cel cel, UnaryOperator config) { - return RuntimeOracle.wrap(cel, config.apply(CelVerifierFactory.newVerifier(cel)).build()); + private static final CelFunctionDecl ZERO_DECL = + CelFunctionDecl.newFunctionDeclaration( + "zero", CelOverloadDecl.newGlobalOverload("zero_int", SimpleType.INT, SimpleType.INT)); + + private static final Cel UNBOUND_ZERO_CEL = + CEL.toCelBuilder().addFunctionDeclarations(ZERO_DECL).build(); + + // The verifier doesn't model `zero`, but the runtime has a binding for it. + private static final Cel ZERO_CEL = + UNBOUND_ZERO_CEL + .toCelBuilder() + .addFunctionBindings(CelFunctionBinding.from("zero_int", Long.class, x -> 0L)) + .build(); + + // A verifier builder that checks witnesses with the runtime of the same `cel`. + private static CelVerifierBuilder checkedVerifier(Cel cel) { + return CelVerifierFactory.newVerifier(cel).enableCegar(true); + } + + // An exact verdict the runtime didn't refute or fail to check. + private static void assertExact(CelVerificationResult result, VerificationStatus status) { + assertWithMessage(result.message()).that(result.status()).isEqualTo(status); + assertThat(result.message()).doesNotContain("CEL runtime"); } @Before @@ -161,6 +185,12 @@ private enum IsSatisfiableTestCase { SATISFIABLE("x > 5", "x = (?:[6-9]|[1-9]\\d+)"), DYNAMIC_ARITHMETIC("request == 1 && request + 2 == 3", "request = 1"), DYNAMIC_ARITHMETIC_UNARY("request == 1 && -request == -1", "request = 1"), + // The type checker narrows `dyn_var.orValue(1)` to int, but it may hold any value. + NARROWED_INT_NAN( + "dyn_var.orValue(1) != dyn_var.orValue(1)", + "dyn_var = optional\\.of\\(double\\(\"NaN\"\\)\\)"), + // Only checks that it's satisfiable: the range may be a list of strings. + NARROWED_LIST_RANGE("dyn_var.orValue([1]).exists(x, dyn(x) == 'a')", "dyn_var = "), GREATER_DOUBLE("d > 1.5", "d = "), LESS_EQUALS_UINT64("u <= 5u", "u = [0-5]u"), LESS_EQUALS_DOUBLE("d <= 5.5", "d = "), @@ -253,7 +283,7 @@ public void isSatisfiable_success(@TestParameter IsSatisfiableTestCase testCase) CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); for (String fragment : testCase.expectedFragments) { assertThat(result.message()).containsMatch(fragment); } @@ -265,7 +295,7 @@ public void isSatisfiable_withVariable_returnsSatisfyingModel() throws Exception CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).containsMatch("x = (?:[6-9]|[1-9]\\d+)"); @@ -333,6 +363,7 @@ public void isSatisfiable_witnessValueSatisfiesExpressionAndDisplayRoundTrips( CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); CelWitness.Variable variable = result.witness().get().variables().get(testCase.variableName); assertThat(variable.displayValue()).isEqualTo(testCase.expectedDisplayValue); ImmutableMap input = @@ -370,6 +401,7 @@ public void isSatisfiable_noExactRuntimeValue_witnessValueEmpty( CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.witness().get().variables().get(testCase.variableName).value()).isEmpty(); } @@ -433,6 +465,7 @@ public void isSatisfiable_listAtDisplayLimit_displaysAllElements() throws Except CelVerificationResult result = VERIFIER.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .endsWith("int_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]"); } @@ -440,12 +473,13 @@ public void isSatisfiable_listAtDisplayLimit_displaysAllElements() throws Except @Test public void isSatisfiable_listPastDisplayLimit_elidesRemainingElements() throws Exception { // Within the unroll limit the whole list is extracted, so the display must still elide it. - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(16)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(16).build(); CelAbstractSyntaxTree ast = CEL.compile("int_list == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]").getAst(); CelVerificationResult result = verifier.isSatisfiable(ast); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .endsWith( "int_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ... (1 more" @@ -458,7 +492,7 @@ public void isSatisfiable_mapNoContainerError_returnsSatisfyingModel() throws Ex CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("string_int_map = {"); @@ -471,7 +505,7 @@ public void isSatisfiable_listNoContainerError_returnsSatisfyingModel() throws E CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("dyn_list = ["); @@ -484,7 +518,7 @@ public void isSatisfiable_dynMapNoContainerError_returnsSatisfyingModel() throws CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("Condition is satisfiable."); assertThat(result.message()).contains("Satisfying input:"); assertThat(result.message()).contains("dyn_map = {"); @@ -497,7 +531,7 @@ public void counterexample_nullValueFormattedAsNull() throws Exception { CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).contains("request = null"); } @@ -508,7 +542,7 @@ public void counterexample_mapFormattedCorrectly() throws Exception { CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()).containsMatch("string_int_map = \\{[^}]*,[^}]*\\}"); assertThat(result.message()).contains("\"a\": 1"); } @@ -535,7 +569,7 @@ public void isAlwaysTrue_counterexampleNeverContainsError( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).doesNotContain("Error"); } @@ -545,7 +579,7 @@ public void isSatisfiable_unconditional_returnsUnconditionalMessage() throws Exc CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); assertThat(result.message()) .isEqualTo( "Condition is satisfiable. (The expression is satisfiable unconditionally, regardless" @@ -625,7 +659,7 @@ public void isSatisfiable_approximate_listsApproximationSources( @TestParameter ApproximationSourceTestCase testCase) throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.reason()) @@ -665,7 +699,7 @@ public void isSatisfiable_uninterpretedFunctions_listsApproximationSources( .build(); CelAbstractSyntaxTree ast = celCompiler.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.reason()) @@ -680,7 +714,7 @@ public void isSatisfiable_uninterpretedFunctions_listsApproximationSources( public void isSatisfiable_approximate_returnsPotentialSatisfyingInput() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("int('123') == 123 ? x > 5 : false").getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()).contains("Inconclusive: a satisfying input may exist"); @@ -704,6 +738,9 @@ private enum IsSatisfiableInconclusiveTestCase { + " && dyn_var != ([18446744073709551615u] + [])))"), // Well-known types are never messages, and other message types may have the field. DYN_MESSAGE_FIELD_OF_UNKNOWN_TYPE("type(dyn_var) != map && dyn_var.type_url == ''"), + // The type checker narrows the range to a list, but it may be a map. + NARROWED_LIST_RANGE_OF_MAP( + "type(dyn_var.orValue([1])) == map && dyn_var.orValue([1]).exists(x, true)"), MASKED_BY_BMC("int_list == [1, 2, 3, 4, 5, 6] ? int_list.exists(x, x == 42) : false"), MASKED_BY_BMC_ALL("int_list == [1, 2, 3, 4, 5, 6] ? int_list.all(x, x > 0) : false"), MASKED_BY_BMC_FILTER( @@ -788,7 +825,7 @@ public void isSatisfiable_inconclusive(@TestParameter IsSatisfiableInconclusiveT throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -801,8 +838,10 @@ public void isSatisfiable_maskedByBmcNested_inconclusive() throws Exception { CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelVerifier customVerifier = - withRuntimeOracle( - CEL, b -> b.setComprehensionUnrollLimit(3).setTypeProvider(TYPE_PROVIDER)); + CelVerifierFactory.newVerifier(CEL) + .setComprehensionUnrollLimit(3) + .setTypeProvider(TYPE_PROVIDER) + .build(); CelVerificationResult result = customVerifier.isSatisfiable(ast); @@ -814,7 +853,8 @@ public void isSatisfiable_comprehensionZeroUnrollLimit_inconclusive() throws Exc String expr = "int_list == [1] ? int_list.exists(x, x == 1) : false"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); @@ -822,6 +862,15 @@ public void isSatisfiable_comprehensionZeroUnrollLimit_inconclusive() throws Exc private enum IsUnsatisfiableTestCase { DYN_MESSAGE_PRESENT_UNDECLARED_FIELD("dyn(test_all_types).unknown_field == 42"), + // `-` on `dyn_var.orValue(1)`, typed int, takes only the int overload, so a double errors. + NARROWED_INT_NEGATE_OVERLOAD( + "type(dyn_var.orValue(1)) == double && -dyn_var.orValue(1) == dyn(-1)"), + // The static type picks the overload of `_[_]` or `in`, which errors on the other kind. + NARROWED_LIST_INDEX_OF_MAP("dyn_var == optional.of({0: true}) && dyn_var.orValue([true])[0]"), + NARROWED_MAP_INDEX_OF_LIST("dyn_var == optional.of([true]) && dyn_var.orValue({0: true})[0]"), + NARROWED_MAP_IN_OF_LIST("dyn_var == optional.of([1]) && 1 in dyn_var.orValue({1: 2})"), + // A wrapper is null or an int. + NULLABLE_NULL_AND_ONE("nullable_int == null && nullable_int == 1"), REPEATED_FIELD_FROM_NON_LIST( "dyn_var == 42 && TestAllTypes{repeated_int64: dyn_var} == TestAllTypes{}"), UNSATISFIABLE("x > 5 && x < 3"), @@ -945,7 +994,7 @@ public void isSatisfiable_failure(@TestParameter IsUnsatisfiableTestCase testCas CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).isEqualTo("Condition is not satisfiable."); } @@ -1541,7 +1590,8 @@ public void isAlwaysTrue_withUnknownIdentifier_evaluatesToUnknown( throws Exception { CelAbstractSyntaxTree ast = CEL.compile(expression).getAst(); - CelVerifier verifierWithUnknown = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("x")); + CelVerifier verifierWithUnknown = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("x").build(); // 'x == x' is not a tautology if it can be unknown // i.e: CelUnknown == CelUnknown is unknown. @@ -1575,7 +1625,8 @@ private enum ApproximateBesideUnknownTestCase { public void isAlwaysTrue_approximateBesideUnknown_inconclusive( @TestParameter ApproximateBesideUnknownTestCase testCase) throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("unknown_var")); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("unknown_var").build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); @@ -1596,9 +1647,9 @@ public void isAlwaysTrue_dynamicComprehensionNonBoolYieldsError() throws Excepti + " dyn_list.all(x, x.not_a_bool) == false) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelVerificationResult result = - withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(1)).isAlwaysTrue(ast); + checkedVerifier(CEL).setComprehensionUnrollLimit(1).build().isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .matches( "Condition is not always true\\. Counterexample input:\n" @@ -1610,7 +1661,8 @@ public void isAlwaysTrue_comprehensionExceedsMaxIterations_returnsUnknown() thro String expr = "int_list == [1, 2, 3] ? int_list.all(x, x > 0) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(2)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(2).build(); CelVerificationResult verifiedValue = verifier.isAlwaysTrue(ast); // Truncated loops return Unknown, which negates to Unknown. @@ -1627,7 +1679,7 @@ public void isAlwaysTrue_comprehensionZeroUnrollLimit_emptyList() throws Excepti String expr = "int_list == [] ? int_list.all(x, x > 0) : true"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult verifiedValue = verifier.isAlwaysTrue(ast); assertThat(verifiedValue.status()).isEqualTo(VerificationStatus.VERIFIED); @@ -1657,7 +1709,7 @@ public void isAlwaysTrue_strictCustomFunction_errorPropagates() throws Exception CelAbstractSyntaxTree ast = celWithCustomFunc.compile("custom_func(1 / 0) == 5 || !(custom_func(1 / 0) == 5)").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); // Should fail verification because the error propagates through custom_func, // so the whole expression evaluates to an error, not true. @@ -1722,12 +1774,11 @@ public void verifyEquivalence_withUnknownIdentifiers_inconclusive( CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); CelVerifier verifier = - withRuntimeOracle( - CEL, - b -> - b.addUnknownIdentifier("a") - .addUnknownIdentifier("x") - .addUnknownIdentifier("unknown_var")); + CelVerifierFactory.newVerifier(CEL) + .addUnknownIdentifier("a") + .addUnknownIdentifier("x") + .addUnknownIdentifier("unknown_var") + .build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -1741,7 +1792,7 @@ public void verifyEquivalence_freeVariableIndicesDeduplicated() throws Exception CelAbstractSyntaxTree astB = CEL.compile("x == y && y == port ? dyn_list.all(e, y == port) : false").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -1761,7 +1812,7 @@ public void isSatisfiable_approximateIterRangeInMap_inconclusive() throws Except CelAbstractSyntaxTree ast = celWithCustomFunc.compile("approx_list(1).map(x, x * 2) == []").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -1781,7 +1832,7 @@ public void isSatisfiable_mapWithUnusedApproximateIteration_verified() throws Ex CelAbstractSyntaxTree ast = celWithCustomFunc.compile("y == [] && y.map(x, approx_func(1)) == y").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); } @@ -1795,6 +1846,23 @@ private enum UnconditionalErrorTestCase { "dyn(TestAllTypes.NestedMessage{}).single_int64 == 0" + " || dyn(TestAllTypes.NestedMessage{}).single_int64 != 0"), MAP_MISSING_KEY("{'a': 1}['b'] == 0 || !({'a': 1}['b'] == 0)"), + OPTIONAL_SELECT_UNDECLARED_FIELD_OF_DYN_MESSAGE( + "dyn(TestAllTypes{}).?unknown_field.hasValue()" + + " || !dyn(TestAllTypes{}).?unknown_field.hasValue()"), + OPTIONAL_CHAINING_SELECT_UNDECLARED_FIELD( + "optional.of(dyn(TestAllTypes{})).unknown_field.hasValue()" + + " || !optional.of(dyn(TestAllTypes{})).unknown_field.hasValue()"), + // No overload indexes a list with a string, even an empty optional list. + OPTIONAL_LIST_INDEXED_BY_STRING( + "TestAllTypes{}.?repeated_int64[dyn('a')].hasValue()" + + " || !TestAllTypes{}.?repeated_int64[dyn('a')].hasValue()"), + OPTIONAL_LIST_OPTIONALLY_INDEXED_BY_STRING( + "TestAllTypes{}.?repeated_int64[?dyn('a')].hasValue()" + + " || !TestAllTypes{}.?repeated_int64[?dyn('a')].hasValue()"), + // A present optional's value must be a map or message to select from. + OPTIONAL_CHAINING_SELECT_FROM_NULL( + "{'null_key': dyn(null)}.?null_key.invalid.hasValue()" + + " || !{'null_key': dyn(null)}.?null_key.invalid.hasValue()"), COLLECTION_ERROR("{'a': 1 / 0} == {'a': 1 / 0}"), LIST_ERROR("[1 / 0] == [1 / 0]"), STRICT_LITERAL_ERROR("{'a': 1/0}.exists(k, k == 'a') == {'a': 1/0}.exists(k, k == 'a')"), @@ -1819,7 +1887,7 @@ public void isAlwaysTrue_unconditionalError_failsVerification( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isEqualTo( "Condition is not always true. (The expression fails unconditionally, regardless of" @@ -1834,7 +1902,7 @@ public void verifyEquivalence_unconditionalError_failsVerification( CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isEqualTo( "Equivalence violation detected. (The expression fails unconditionally, regardless of" @@ -1849,7 +1917,7 @@ public void verifyEquivalence_dynamicMapEquality_enforcesExtensionalityOnPresent CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("string_int_map ="); assertThat(result.message()).doesNotContain("string_int_map = {'a': 1}"); } @@ -1871,7 +1939,12 @@ private enum MalformedAstTestCase { Operator.LESS, ImmutableList.of( CelExpr.newBuilder().setId(2).setConstant(CelConstant.ofValue(1L)).build()), - "expects 2 arguments, got 1"); + "expects 2 arguments, got 1"), + OPTIONAL_SELECT( + Operator.OPTIONAL_SELECT, + ImmutableList.of( + CelExpr.newBuilder().setId(2).setConstant(CelConstant.ofValue(1L)).build()), + "expects an operand and a field name"); final Operator operator; final ImmutableList args; @@ -1918,10 +1991,24 @@ public void verifyEquivalence_infinityConstants_notEquivalent() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } private enum IsAlwaysTrueViolationTestCase { + // The type checker narrows `dyn_var.orValue(...)` to the default's type. + NARROWED_MAP_LITERAL_EQUALITY( + "dyn_var == optional.of({'a': 1.0}) ? dyn_var.orValue(string_int_map) != {'a': 1} : true", + "Condition is not always true\\."), + // The type checker narrows `b ? test_all_types : null` to TestAllTypes. + NARROWED_NULL_FIELD( + "b || has((b ? test_all_types : null).single_int64)" + + " || !has((b ? test_all_types : null).single_int64)", + "Condition is not always true\\."), + NARROWED_MESSAGE_FIELD( + "dyn_var == optional.of(dyn_var2) && dyn_var2 == 42" + + " ? dyn_var.orValue(test_all_types).single_int64 == 0" + + " || dyn_var.orValue(test_all_types).single_int64 != 0 : true", + "Condition is not always true\\."), // `[NaN]` differs from itself. DYNAMIC_LIST_VARIABLE_EQUALITY("dyn_list == dyn_list", "dyn_list = \\[double\\(\"NaN\"\\)\\]"), DYNAMIC_MAP_VARIABLE_EQUALITY( @@ -2267,7 +2354,7 @@ public void isAlwaysTrue_violation_returnsFalse( CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); for (String fragment : testCase.expectedFragments) { assertThat(result.message()).containsMatch(fragment); } @@ -2284,11 +2371,11 @@ public void isAlwaysTrue_dynValuesComparedWithIntConstant_verified( CelAbstractSyntaxTree ast = CEL.compile(expression).getAst(); // A tighter timeout than the default, so that removing the fast path for comparing doubles with // exact integer constants fails as a performance regression (~0.3 s with it, 19-45 s without). - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setTimeout(Duration.ofSeconds(5))); + CelVerifier verifier = checkedVerifier(CEL).setTimeout(Duration.ofSeconds(5)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -2301,13 +2388,13 @@ public void isAlwaysTrue_dynamicMapComprehensionNestedEquality_violated() throws // A looser timeout than the default: comparing a dyn map with a literal admits every numeric // representation of its keys and values, a state space that takes 5-25 s to explore remotely // (longer on GitHub runners). - // TODO: Restore 20 s once select on optionals is modeled (`request` may be - // `optional.none()`) and the slowdown since map literal key validation is recovered. - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setTimeout(Duration.ofSeconds(60))); + // TODO: Restore 20 s once the slowdown since map literal key validation is + // recovered. + CelVerifier verifier = checkedVerifier(CEL).setTimeout(Duration.ofSeconds(60)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("request = "); } @@ -2426,7 +2513,7 @@ public void isAlwaysTrue_inconclusive(@TestParameter IsInconclusiveTestCase test throws Exception { CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -2435,7 +2522,7 @@ public void isAlwaysTrue_inconclusive(@TestParameter IsInconclusiveTestCase test public void isAlwaysTrue_inconclusive_containsPotentialCounterexample() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("request.matches('^[a-z]+$') == true").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()) @@ -2536,7 +2623,7 @@ public void verifyEquivalence_dynComprehensionRange_listedAsApproximationSource( CelAbstractSyntaxTree astA = CEL.compile("dyn_var.all(e, e > 0)").getAst(); CelAbstractSyntaxTree astB = CEL.compile("dyn_var.all(e, e < 0)").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.reason()) .isEqualTo( @@ -2548,7 +2635,8 @@ public void verifyEquivalence_dynComprehensionRange_listedAsApproximationSource( public void verifyEquivalence_unknownResults_listedAsApproximationSource() throws Exception { CelAbstractSyntaxTree astA = CEL.compile("unknown_var * 1").getAst(); CelAbstractSyntaxTree astB = CEL.compile("unknown_var + 0").getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("unknown_var")); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("unknown_var").build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -2564,7 +2652,7 @@ public void verifyEquivalence_inconclusive( CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -2811,6 +2899,30 @@ private enum EquivalenceTestCase { OPTIONAL_FIELD_SELECTION_MAP_COMPREHENSION( "{'a': 1, 'b': 2}.transformMap(k, v, v > 1, v).?b", "optional.of(2)"), OPTIONAL_FIELD_SELECTION_BINDER("cel.bind(m, {'a': 1}, m.?a)", "optional.of(1)"), + // Selecting from or indexing an optional does so optionally. + OPTIONAL_CHAINING_SELECT( + "optional.of({'c': {'index': 'goodbye'}}).c.index", "optional.of('goodbye')"), + OPTIONAL_CHAINING_SELECT_MISSING("optional.of({'c': {}}).c.missing", "optional.none()"), + OPTIONAL_CHAINING_SELECT_NONE("optional.none().c", "optional.none()"), + OPTIONAL_CHAINING_SELECT_OPTIONAL_VALUE( + "optional.of({'a': optional.of(1)}).a", "optional.of(optional.of(1))"), + OPTIONAL_CHAINING_HAS_PRESENT("has(optional.of({'c': 1}).c)", "true"), + OPTIONAL_CHAINING_HAS_ABSENT("has(optional.of({'c': 1}).d)", "false"), + OPTIONAL_CHAINING_HAS_NONE("has(optional.none().c)", "false"), + OPTIONAL_CHAINING_SELECT_PROTO3_ZERO( + "optional.of(TestAllTypes{single_int64: 0}).single_int64", "optional.none()"), + OPTIONAL_CHAINING_SELECT_UNSET_WRAPPER( + "optional.of(TestAllTypes{}).single_int64_wrapper", "optional.none()"), + OPTIONAL_CHAINING_INDEX("optional.of(['list-value'])[0]", "optional.of('list-value')"), + OPTIONAL_CHAINING_INDEX_MISSING("optional.of({'c': {}}).c['missing']", "optional.none()"), + OPTIONAL_CHAINING_INDEX_NONE("optional.none()[0]", "optional.none()"), + OPTIONAL_SELECT_THEN_INDEX( + "optional.of(TestAllTypes{repeated_string: ['foo']}).?repeated_string[0]", + "optional.of('foo')"), + OPTIONAL_CHAINING_DYN( + "dyn_var == optional.of({'a': [1]}) ? dyn_var.a[0] : optional.of(1)", "optional.of(1)"), + OPTIONAL_CHAINING_DYN_NONE( + "request == optional.none() ? request.l : optional.none()", "optional.none()"), JSON_VALUE_BOOL("google.protobuf.Value{bool_value: true}", "true"), JSON_VALUE_NUMBER("google.protobuf.Value{number_value: 1.0}", "1.0"), JSON_VALUE_NULL("google.protobuf.Value{null_value: 0}", "null"), @@ -3183,7 +3295,7 @@ public void verifyEquivalence_violation_returnsFalse( CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("Equivalence violation detected."); } @@ -3194,7 +3306,7 @@ public void verifyEquivalence_violation_hasCounterexampleMessage() throws Except CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); String message = result.message(); assertThat(message).contains("Equivalence violation detected. Counterexample input:"); assertThat(message).containsMatch(" {2}x = -?\\d+"); @@ -3210,7 +3322,7 @@ public void verifyEquivalence_divergesOnTernaryErrorSemantics() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -3222,7 +3334,7 @@ public void verifyEquivalence_dynamicIndexingWithExplicitMap_hydratesMap() throw CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("string_int_list_map = {"); assertThat(result.message()).contains("\"a\": [1, 2]"); assertThat(result.message()).containsMatch("\"b\": \\[3, [45]]"); @@ -3233,7 +3345,7 @@ public void verifyEquivalence_logicalAndShortCircuitError_isVerified() throws Ex CelAbstractSyntaxTree astA = CEL.compile("false && (1 / 0 == 0)").getAst(); CelAbstractSyntaxTree astB = CEL.compile("false").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3241,7 +3353,7 @@ public void verifyEquivalence_logicalOrShortCircuitUnknown_isVerified() throws E CelAbstractSyntaxTree astA = CEL.compile("true || unknown_var").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3249,7 +3361,7 @@ public void verifyEquivalence_deepListExtensionality_isVerified() throws Excepti CelAbstractSyntaxTree astA = CEL.compile("[[[[1]]]] == [[[[1]]]]").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3258,7 +3370,7 @@ public void verifyEquivalence_stringConcatenationExplosion_isVerified() throws E CEL.compile("(\"a\" + \"b\" + \"c\" + \"d\" + \"e\") == \"abcde\"").getAst(); CelAbstractSyntaxTree astB = CEL.compile("true").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3270,7 +3382,7 @@ public void verifyEquivalence_divergesOnShortCircuitingErrors() throws Exception CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3281,7 +3393,7 @@ public void verifyEquivalence_distinctErrorEquivalence_failsVerification() throw CelAbstractSyntaxTree astB = CEL.compile("role + role == \"abc\"").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } private enum CounterexampleFormatTestCase { @@ -3389,7 +3501,7 @@ public void verifyEquivalence_counterexampleFormat( CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); boolean matched = testCase.expectedFragments.stream() .anyMatch( @@ -3412,7 +3524,7 @@ public void verifyEquivalence_structCounterexampleFormat() throws Exception { CEL.compile("test_all_types == TestAllTypes{single_int32: 2}").getAst(); CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()) .isAnyOf( "Equivalence violation detected. Counterexample input:\n" @@ -3429,7 +3541,7 @@ public void isAlwaysTrue_operationError_counterexampleFormat() throws Exception CelAbstractSyntaxTree ast = CEL.compile("x / x == x / x").getAst(); CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("x = 0"); } @@ -3446,7 +3558,7 @@ public void isAlwaysTrue_dynamicFunction_inconclusive() throws Exception { CelOverloadDecl.newGlobalOverload("request_overload", SimpleType.DYN))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile("request() == request()").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()).contains("doesn't model exactly: calls to `request`."); @@ -3465,7 +3577,7 @@ public void isAlwaysTrue_dynamicFunctionShortCircuited_violated() throws Excepti .build(); CelAbstractSyntaxTree ast = celCompiler.compile("x != 0 && request() == 1").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); assertThat(result.message()).contains("x = 0"); @@ -3482,7 +3594,7 @@ public void isAlwaysTrue_boolFunction_verified() throws Exception { CelOverloadDecl.newGlobalOverload("bool_func_overload", SimpleType.BOOL))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile("bool_func() == bool_func()").getAst(); - CelVerificationResult result = VERIFIER_WITHOUT_ORACLE.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); } @@ -3517,7 +3629,7 @@ public void isAlwaysTrue_distinctFunctionSymbols_inconclusive( "f_list_string", SimpleType.INT, ListType.create(SimpleType.STRING)))) .build(); CelAbstractSyntaxTree ast = celCompiler.compile(expression).getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3533,7 +3645,7 @@ public void isAlwaysTrue_zeroArityFunctionAndSameNamedVariable_inconclusive() th .build(); CelAbstractSyntaxTree ast = celCompiler.compile("g() == g").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3551,9 +3663,9 @@ public void verifyEquivalence_identicalFunctionCalls_equivalent() throws Excepti .build(); CelAbstractSyntaxTree astA = celCompiler.compile("request()").getAst(); CelAbstractSyntaxTree astB = celCompiler.compile("request()").getAst(); - CelVerificationResult result = VERIFIER.verifyEquivalence(astA, astB); + CelVerificationResult result = UNCHECKED_VERIFIER.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -3572,7 +3684,7 @@ public void isSatisfiable_customOperatorOverload_inconclusive() throws Exception .build(); CelAbstractSyntaxTree ast = celCompiler.compile("!(int_list < int_list_2)").getAst(); - CelVerificationResult result = VERIFIER.isSatisfiable(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } @@ -3594,7 +3706,7 @@ public void isSatisfiable_divisionByZero_failsInCelWithErrors() throws Exception CelVerificationResult result = VERIFIER.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -3607,7 +3719,7 @@ public void isSatisfiable_timeoutReached_throwsCelVerificationException() throws .addVar("d4", SimpleType.DOUBLE) .build(); CelVerifier timeoutVerifier = - withRuntimeOracle(customCel, b -> b.setTimeout(Duration.ofMillis(1))); + checkedVerifier(customCel).setTimeout(Duration.ofMillis(1)).build(); // An overly complex double multiplication to guarantee Z3 FPA theory solver timeouts. CelAbstractSyntaxTree ast = @@ -3904,7 +4016,7 @@ public void isAlwaysTrue_nanAndInfinityConstants_isFalse( CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); CelAbstractSyntaxTree optimizedAst = optimizer.optimize(ast); - assertThat(VERIFIER.isAlwaysTrue(optimizedAst).status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(VERIFIER.isAlwaysTrue(optimizedAst), VerificationStatus.VIOLATED); } @Test @@ -3927,6 +4039,287 @@ public void isAlwaysTrue_nanAndInfinityConstants_tautology( assertThat(VERIFIER.isAlwaysTrue(optimizedAst).status()).isEqualTo(VerificationStatus.VERIFIED); } + @Test + public void isSatisfiable_narrowedBlockIndex_satisfiable() throws Exception { + CelOptimizer optimizer = + CelOptimizerFactory.standardCelOptimizerBuilder(CEL) + .addAstOptimizers(SubexpressionOptimizer.getInstance()) + .build(); + // cel.@block([dyn_var.orValue(1)], @index0 != @index0), where @index0 is typed int. + CelAbstractSyntaxTree ast = + optimizer.optimize(CEL.compile("dyn_var.orValue(1) != dyn_var.orValue(1)").getAst()); + + CelVerificationResult result = UNCHECKED_VERIFIER.isSatisfiable(ast); + + assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + } + + @FunctionalInterface + @Immutable + private interface VerifierCall { + CelVerificationResult call(CelVerifier verifier) throws Exception; + } + + private static VerifierCall alwaysTrue(String expr) { + return v -> v.isAlwaysTrue(ZERO_CEL.compile(expr).getAst()); + } + + private static VerifierCall satisfiable(String expr) { + return v -> v.isSatisfiable(ZERO_CEL.compile(expr).getAst()); + } + + // Models `zero(x)` as `value`. + private static CelZ3FunctionAxiom zeroAxiom(long value) { + return CelZ3FunctionAxiom.newBuilder(ZERO_DECL) + .addUnaryOverloadTranslator( + "zero_int", + (ctx, typeSystem, constraintSink, arg) -> Optional.of(typeSystem.mkInt(value))) + .build(); + } + + private static VerifierCall equivalent(String exprA, String exprB) { + return v -> + v.verifyEquivalence(ZERO_CEL.compile(exprA).getAst(), ZERO_CEL.compile(exprB).getAst()); + } + + private enum CegarTestCase { + COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("zero(x) == 1"), + VerificationStatus.VIOLATED, + "Condition is not always true. The CEL runtime confirmed a counterexample the verifier" + + " approximated. Counterexample input:"), + COUNTEREXAMPLE_REFUTED( + alwaysTrue("zero(x) == 0"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential counterexample input:"), + ERROR_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("zero(x) / zero(x) == 1"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + // `0 + 'a'` has no overload at runtime. + DISPATCH_ERROR_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("dyn(zero(x)) + 'a' == 'a'"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + BAD_FORMAT_COUNTEREXAMPLE_CONFIRMED( + alwaysTrue("int('abc') == zero(x)"), + VerificationStatus.VIOLATED, + "The CEL runtime confirmed a counterexample the verifier approximated."), + EXACT_COUNTEREXAMPLE( + alwaysTrue("x != 1"), + VerificationStatus.VIOLATED, + "Condition is not always true. Counterexample input:"), + SATISFYING_INPUT_CONFIRMED( + satisfiable("zero(x) == 0"), + VerificationStatus.VERIFIED, + "Condition is satisfiable. The CEL runtime confirmed a satisfying input the verifier" + + " approximated. Satisfying input:"), + SATISFYING_INPUT_REFUTED( + satisfiable("zero(x) == 1"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential satisfying input:"), + DIVERGENCE_CONFIRMED( + equivalent("zero(x)", "1"), + VerificationStatus.VIOLATED, + "Equivalence violation detected. The CEL runtime confirmed a divergence the verifier" + + " approximated. Counterexample input:"), + DIVERGENCE_REFUTED( + equivalent("zero(x)", "0"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found. Potential counterexample input:"), + // Both sides divide by zero at runtime, and all errors are one value. + ERROR_DIVERGENCE_REFUTED( + equivalent("zero(x) / zero(x)", "1 / zero(x)"), + VerificationStatus.INCONCLUSIVE, + "The CEL runtime disagrees with the one found."), + // Maps can't have numerically equal keys, or bytes keys. + NUMERICALLY_EQUAL_KEYS( + satisfiable( + "zero(x) == 0 && 0 in dyn_int_map && 0u in dyn_int_map" + + " && dyn_int_map[0] != dyn_int_map[0u]"), + VerificationStatus.INCONCLUSIVE, + "Potential satisfying input:"), + BYTES_KEY( + satisfiable("zero(x) == 0 && b'a' in dyn_int_map"), + VerificationStatus.INCONCLUSIVE, + "Potential satisfying input:"), + NEGATIVE_KEY( + satisfiable("zero(x) == 0 && -1 in dyn_int_map"), + VerificationStatus.VERIFIED, + "The CEL runtime confirmed a satisfying input the verifier approximated."), + // Proto message inputs aren't exact witness values yet. + INEXACT_WITNESS( + alwaysTrue("zero(test_all_types.single_int64) == 1"), + VerificationStatus.INCONCLUSIVE, + "Potential counterexample input:"); + + private final VerifierCall call; + private final VerificationStatus status; + private final String message; + + CegarTestCase(VerifierCall call, VerificationStatus status, String message) { + this.call = call; + this.status = status; + this.message = message; + } + } + + @Test + public void cegar_checksWitnessWithRuntime(@TestParameter CegarTestCase testCase) + throws Exception { + CelVerifier verifier = checkedVerifier(ZERO_CEL).build(); + + CelVerificationResult result = testCase.call.call(verifier); + + assertThat(result.status()).isEqualTo(testCase.status); + assertThat(result.message()).contains(testCase.message); + } + + private static final CelFunctionDecl MAYBE_DECL = + CelFunctionDecl.newFunctionDeclaration( + "maybe", CelOverloadDecl.newGlobalOverload("maybe_int", SimpleType.BOOL, SimpleType.INT)); + + private enum NotAttemptedCegarTestCase { + OFF(CelVerifierFactory.newVerifier(ZERO_CEL).build()), + UNKNOWN_IDENTIFIERS(checkedVerifier(ZERO_CEL).addUnknownIdentifier("unknown_var").build()); + + private final CelVerifier verifier; + + NotAttemptedCegarTestCase(CelVerifier verifier) { + this.verifier = verifier; + } + } + + @Test + public void cegar_notAttempted_staysInconclusive( + @TestParameter NotAttemptedCegarTestCase testCase) throws Exception { + CelVerificationResult result = + testCase.verifier.isAlwaysTrue(ZERO_CEL.compile("zero(x) == 1").getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()).doesNotContain("CEL runtime"); + } + + private enum CantEvaluateCegarTestCase { + UNBOUND_FUNCTION(UNBOUND_ZERO_CEL, "zero(x) == 1", "No matching overload for function 'zero'"), + LATE_BOUND_FUNCTION( + UNBOUND_ZERO_CEL.toCelBuilder().addLateBoundFunctions("zero").build(), + "zero(x) == 1", + "No matching overload for function 'zero'"), + // The runtime's iteration limit isn't a CEL error. + ITERATION_LIMIT( + ZERO_CEL + .toCelBuilder() + .setOptions(CelOptions.current().comprehensionMaxIterations(0).build()) + .build(), + "[1].all(i, zero(i) == 0)", + "Iteration budget exceeded"), + // Async bindings fail with an internal error when evaluated synchronously. + ASYNC_FUNCTION( + UNBOUND_ZERO_CEL + .toCelBuilder() + .addFunctionBindings(CelFunctionBinding.fromAsync("zero_int", Long.class, x -> null)) + .build(), + "zero(x) == 1", + "evaluated in synchronous mode"), + NULL_RESULT( + CEL.toCelBuilder() + .addFunctionDeclarations(MAYBE_DECL) + .addFunctionBindings(CelFunctionBinding.from("maybe_int", Long.class, x -> null)) + .build(), + "maybe(x)", + "a function returned null"), + UNKNOWN_RESULT( + CEL.toCelBuilder() + .addFunctionDeclarations(MAYBE_DECL) + .addFunctionBindings( + CelFunctionBinding.from("maybe_int", Long.class, x -> CelUnknownSet.create(1L))) + .build(), + "maybe(x)", + "it evaluated to unknowns"); + + private final Cel cel; + private final String expr; + private final String reason; + + CantEvaluateCegarTestCase(Cel cel, String expr, String reason) { + this.cel = cel; + this.expr = expr; + this.reason = reason; + } + } + + @Test + public void cegar_cantEvaluate_saysWhy(@TestParameter CantEvaluateCegarTestCase testCase) + throws Exception { + CelVerifier verifier = checkedVerifier(testCase.cel).build(); + + CelVerificationResult result = + verifier.isAlwaysTrue(testCase.cel.compile(testCase.expr).getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()) + .containsMatch( + "The CEL runtime couldn't check it: .*\\. (Potential counterexample input:|\\(The)"); + assertThat(result.message()).contains(testCase.reason); + } + + @Test + public void cegar_exactMatchCantEvaluate_saysWhy() throws Exception { + // Models `zero(x)` as 0, but the runtime has no binding for it. + CelVerifier verifier = + CelVerifierZ3Impl.newBuilder(UNBOUND_ZERO_CEL) + .addFunctionAxioms(zeroAxiom(0)) + .enableCegar(true) + .build(); + + CelVerificationResult result = + verifier.isAlwaysTrue(UNBOUND_ZERO_CEL.compile("zero(x) == 1").getAst()); + + assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertThat(result.message()) + .containsMatch( + "Condition is not always true\\. The CEL runtime couldn't check it: .*No matching" + + " overload for function 'zero'.*\\. Counterexample input:"); + } + + private enum RefutedExactMatchTestCase { + COUNTEREXAMPLE( + alwaysTrue("zero(x) == 0"), + "a counterexample the verifier found. Potential counterexample input:"), + SATISFYING_INPUT( + satisfiable("zero(x) == 1"), + "a satisfying input the verifier found. Potential satisfying input:"), + DIVERGENCE( + equivalent("zero(x)", "0"), + "a divergence the verifier found. Potential counterexample input:"); + + private final VerifierCall call; + private final String message; + + RefutedExactMatchTestCase(VerifierCall call, String message) { + this.call = call; + this.message = message; + } + } + + @Test + public void cegar_runtimeDisagreesWithExactMatch_inconclusive( + @TestParameter RefutedExactMatchTestCase testCase) throws Exception { + // Models `zero(x)` as 1, which the runtime disagrees with. + CelVerifier verifier = + CelVerifierZ3Impl.newBuilder(ZERO_CEL) + .addFunctionAxioms(zeroAxiom(1)) + .enableCegar(true) + .build(); + + CelVerificationResult result = testCase.call.call(verifier); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.message()) + .contains("Inconclusive: the CEL runtime disagrees with " + testCase.message); + } + @Test public void translateFunctionCall_staticallyResolved_skipsDeadOverloads() throws Exception { CelFunctionDecl spyFuncDecl = @@ -4032,11 +4425,11 @@ public void isAlwaysTrue_largeListCounterexample_truncatesOutput() throws Except listLiteral.append("]"); CelAbstractSyntaxTree ast = cel.compile("!(large_list == " + listLiteral + ")").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b.setTimeout(Duration.ofSeconds(10))); + CelVerifier verifier = checkedVerifier(cel).setTimeout(Duration.ofSeconds(10)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("... (5 more elements)"); } @@ -4056,11 +4449,11 @@ public void isAlwaysTrue_largeMapCounterexample_truncatesOutput() throws Excepti mapLiteral.append("}"); CelAbstractSyntaxTree ast = cel.compile("!(large_map == " + mapLiteral + ")").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b.setTimeout(Duration.ofSeconds(10))); + CelVerifier verifier = checkedVerifier(cel).setTimeout(Duration.ofSeconds(10)).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("... (5 more entries)"); } @@ -4076,9 +4469,9 @@ public void counterexample_messageTwoFieldsFormatted() throws Exception { CelAbstractSyntaxTree ast = cel.compile("!(msg == TestAllTypes{single_int32: 1, single_int64: 2})").getAst(); - CelVerificationResult result = withRuntimeOracle(cel, b -> b).isAlwaysTrue(ast); + CelVerificationResult result = checkedVerifier(cel).build().isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); assertThat(result.message()).contains("single_int32: 1"); assertThat(result.message()).contains("single_int64: 2"); } @@ -4107,11 +4500,11 @@ public void isAlwaysTrue_customComprehensionWithTrueAccuInit() throws Exception .build(); CelAbstractSyntaxTree ast = cel.compile("dyn_list == [1, 2] ? dyn_list.custom_fold(x) == true : true").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b); + CelVerifier verifier = checkedVerifier(cel).build(); CelVerificationResult result = verifier.isAlwaysTrue(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } @Test @@ -4122,7 +4515,7 @@ public void isAlwaysTrue_customComprehensionWithLoopCondition_inconclusive() thr CelAbstractSyntaxTree ast = CEL.compile("int_list == [1, 2] ? int_list.first_elem() == 2 : true").getAst(); - CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + CelVerificationResult result = UNCHECKED_VERIFIER.isAlwaysTrue(ast); assertThat(result.reason()) .isEqualTo( @@ -4204,7 +4597,7 @@ public void verifyImplication_exactUnknownAssumption_verified() throws Exception public void isSatisfiable_exactUnknownBesideUnmodeledComprehension_violated() throws Exception { CelAbstractSyntaxTree ast = CEL.compile("(false && int_list.first_elem() == 1) || (b && !b)").getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("b")); + CelVerifier verifier = CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("b").build(); CelVerificationResult result = verifier.isSatisfiable(ast); @@ -4247,10 +4640,10 @@ public void isSatisfiable_maskedByBmcButAlwaysFalse_returnsFailed() throws Excep String expr = "int_list == [1, 2, 3, 4] ? int_list.exists(x, x == 42) && false : false"; CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(3)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(3).build(); CelVerificationResult result = verifier.isSatisfiable(ast); - assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertExact(result, VerificationStatus.VIOLATED); } @Test @@ -4262,10 +4655,10 @@ public void verifyEquivalence_maskedByBmcButAlwaysEqual_returnsVerified() throws CelAbstractSyntaxTree astA = CEL.compile(exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(3)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(3).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); + assertExact(result, VerificationStatus.VERIFIED); } private enum EquivalenceZeroUnrollLimitTestCase { @@ -4288,7 +4681,8 @@ public void verifyEquivalence_zeroUnrollLimit_returnsInconclusive( @TestParameter EquivalenceZeroUnrollLimitTestCase testCase) throws Exception { CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4326,7 +4720,7 @@ public void verifyEquivalence_zeroUnrollLimit_twoVarComprehensions_returnsVerifi @TestParameter EquivalenceZeroUnrollLimitVerifiedTestCase testCase) throws Exception { CelAbstractSyntaxTree astA = CEL.compile(testCase.exprA).getAst(); CelAbstractSyntaxTree astB = CEL.compile(testCase.exprB).getAst(); - CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = checkedVerifier(CEL).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4374,7 +4768,8 @@ public void verifyEquivalence_comprehensionScopeShadowing_returnsInconclusive() .build(); CelAbstractSyntaxTree astA = customCel.compile("dyn_list.my_macro_1(true)").getAst(); CelAbstractSyntaxTree astB = customCel.compile("dyn_list.my_macro_2(true)").getAst(); - CelVerifier verifier = withRuntimeOracle(customCel, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(customCel).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4416,7 +4811,8 @@ public void verifyEquivalence_comprehensionResultScopeIsolation_returnsInconclus customCel.compile("cel.bind(x, 10, dyn_list.my_macro(1))").getAst(); CelAbstractSyntaxTree astB = customCel.compile("cel.bind(x, 20, dyn_list.my_macro(1))").getAst(); - CelVerifier verifier = withRuntimeOracle(customCel, b -> b.setComprehensionUnrollLimit(0)); + CelVerifier verifier = + CelVerifierFactory.newVerifier(customCel).setComprehensionUnrollLimit(0).build(); CelVerificationResult result = verifier.verifyEquivalence(astA, astB); @@ -4433,7 +4829,7 @@ public void verifyEquivalence_timeoutReached_throwsCelVerificationException() th .addVar("d4", SimpleType.DOUBLE) .build(); CelVerifier timeoutVerifier = - withRuntimeOracle(customCel, b -> b.setTimeout(Duration.ofMillis(1))); + checkedVerifier(customCel).setTimeout(Duration.ofMillis(1)).build(); CelAbstractSyntaxTree astA = customCel .compile( @@ -4461,7 +4857,7 @@ public void isAlwaysTrue_constantVarDecl_verified() throws Exception { .setConstant(CelConstant.ofValue(42L)) .build()) .build(); - CelVerifier verifier = withRuntimeOracle(celWithConstVar, b -> b); + CelVerifier verifier = checkedVerifier(celWithConstVar).build(); CelAbstractSyntaxTree ast = celWithConstVar .compile("CONST_INT == 42 && [1, 2].all(x, .CONST_INT == 42 && x > 0)") @@ -4494,10 +4890,10 @@ private static CelMacro firstElemMacro() { exprFactory.newIdentifier("accu")))); } - // verifyImplication isn't part of CelVerifier, so it can't go through the runtime oracle. + // verifyImplication isn't part of CelVerifier. private static CelVerificationResult verifyImplication( CelCompiler compiler, String assume, String assertion) throws Exception { - return ((CelVerifierZ3Impl) VERIFIER_WITHOUT_ORACLE) + return ((CelVerifierZ3Impl) UNCHECKED_VERIFIER) .verifyImplication( compiler.compile(assume).getAst(), compiler.compile(assertion).getAst(), diff --git a/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java b/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java deleted file mode 100644 index 67d746a6f..000000000 --- a/verifier/src/test/java/dev/cel/verifier/RuntimeOracle.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.verifier; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Arrays.stream; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import dev.cel.common.CelAbstractSyntaxTree; -import dev.cel.runtime.CelEvaluationException; -import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntime.Program; -import dev.cel.verifier.CelVerificationResult.VerificationStatus; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; - -/** - * Evaluates every exact witness a verifier returns with the CEL runtime, and fails if the runtime - * disagrees with the verdict. - */ -final class RuntimeOracle implements CelVerifier { - - // All errors compare equal, matching the verifier's single error value. - private enum EvaluationError { - ERROR - } - - @SuppressWarnings("Immutable") // CelRuntime is immutable, just not marked as such - private final CelRuntime runtime; - - private final CelVerifier delegate; - - static CelVerifier wrap(CelRuntime runtime, CelVerifier delegate) { - return new RuntimeOracle(runtime, delegate); - } - - @Override - public CelVerificationResult isSatisfiable(CelAbstractSyntaxTree ast) - throws CelVerificationException { - return check( - delegate.isSatisfiable(ast), VerificationStatus.VERIFIED, r -> r.get(0).equals(true), ast); - } - - @Override - public CelVerificationResult isAlwaysTrue(CelAbstractSyntaxTree ast) - throws CelVerificationException { - return check( - delegate.isAlwaysTrue(ast), VerificationStatus.VIOLATED, r -> !r.get(0).equals(true), ast); - } - - @Override - public CelVerificationResult verifyEquivalence( - CelAbstractSyntaxTree astA, CelAbstractSyntaxTree astB) throws CelVerificationException { - return check( - delegate.verifyEquivalence(astA, astB), - VerificationStatus.VIOLATED, - r -> !r.get(0).equals(r.get(1)), - astA, - astB); - } - - @CanIgnoreReturnValue - private CelVerificationResult check( - CelVerificationResult result, - VerificationStatus checkedStatus, - Predicate> agrees, - CelAbstractSyntaxTree... asts) { - Optional> inputs = exactInputs(result); - if (result.status() != checkedStatus || !inputs.isPresent()) { - return result; - } - ImmutableList results = - stream(asts).map(ast -> evaluate(ast, inputs.get())).collect(toImmutableList()); - if (agrees.test(results)) { - return result; - } - StringBuilder message = - new StringBuilder("The CEL runtime disagrees with ") - .append(result.status()) - .append(" for inputs ") - .append(inputs.get()) - .append(':'); - for (int i = 0; i < asts.length; i++) { - message - .append("\n ") - .append(asts[i].getSource().getContent()) - .append(" evaluates to ") - .append(results.get(i)); - } - throw new AssertionError(message.append('\n').append(result.message()).toString()); - } - - // The witness's inputs, if every value is exact. - private static Optional> exactInputs(CelVerificationResult result) { - return result - .witness() - .map(CelWitness::variables) - .filter(vars -> vars.values().stream().allMatch(v -> v.value().isPresent())) - .map(vars -> ImmutableMap.copyOf(Maps.transformValues(vars, v -> v.value().get()))); - } - - private Object evaluate(CelAbstractSyntaxTree ast, ImmutableMap inputs) { - Program program; - try { - program = runtime.createProgram(ast); - } catch (CelEvaluationException e) { - throw new AssertionError( - "Can't plan " - + ast.getSource().getContent() - + "; verify it with a verifier for its own environment", - e); - } - try { - return program.eval(inputs); - } catch (CelEvaluationException e) { - // A witness missing a variable would otherwise pass as an error result. - if (e.getMessage().contains("No such attribute(s): ")) { - throw new AssertionError( - "Incomplete witness " + inputs + " for " + ast.getSource().getContent(), e); - } - return EvaluationError.ERROR; - } - } - - private RuntimeOracle(CelRuntime runtime, CelVerifier delegate) { - this.runtime = runtime; - this.delegate = delegate; - } -} diff --git a/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java b/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java deleted file mode 100644 index 0bc3a36b9..000000000 --- a/verifier/src/test/java/dev/cel/verifier/RuntimeOracleTest.java +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.verifier; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableMap; -import dev.cel.bundle.Cel; -import dev.cel.bundle.CelFactory; -import dev.cel.common.CelAbstractSyntaxTree; -import dev.cel.common.CelFunctionDecl; -import dev.cel.common.CelOverloadDecl; -import dev.cel.common.types.SimpleType; -import dev.cel.runtime.CelEvaluationException; -import java.util.Optional; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class RuntimeOracleTest { - - private static final Cel CEL = CelFactory.plannerCelBuilder().addVar("x", SimpleType.INT).build(); - - @Test - public void isSatisfiable_runtimeRefutesSatisfyingInput_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.verified("Condition is satisfiable.", witness(2L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isSatisfiable(ast)); - - assertThat(e).hasMessageThat().contains("x == 1 evaluates to false"); - } - - @Test - public void isAlwaysTrue_runtimeRefutesCounterexample_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(1L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isAlwaysTrue(ast)); - - assertThat(e).hasMessageThat().contains("x == 1 evaluates to true"); - } - - @Test - public void verifyEquivalence_runtimeRefutesDivergence_throws() throws Exception { - CelAbstractSyntaxTree astA = CEL.compile("x + 0").getAst(); - CelAbstractSyntaxTree astB = CEL.compile("x").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(1L))); - - AssertionError e = - assertThrows(AssertionError.class, () -> verifier.verifyEquivalence(astA, astB)); - - assertThat(e).hasMessageThat().contains("x + 0 evaluates to 1"); - assertThat(e).hasMessageThat().contains("\n x evaluates to 1"); - } - - @Test - public void isSatisfiable_inexactSatisfyingInput_notChecked() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerificationResult result = - CelVerificationResult.verified( - "Condition is satisfiable.", - CelWitness.create( - ImmutableMap.of("x", CelWitness.Variable.create(Optional.empty(), "Unknown")))); - - assertThat(checking(result).isSatisfiable(ast)).isSameInstanceAs(result); - } - - @Test - public void verifyEquivalence_bothExpressionsError_throws() throws Exception { - CelAbstractSyntaxTree astA = CEL.compile("1 / x").getAst(); - CelAbstractSyntaxTree astB = CEL.compile("2 / x").getAst(); - CelVerifier verifier = checking(CelVerificationResult.failed("Violated.", witness(0L))); - - AssertionError e = - assertThrows(AssertionError.class, () -> verifier.verifyEquivalence(astA, astB)); - - assertThat(e).hasMessageThat().contains("1 / x evaluates to ERROR"); - assertThat(e).hasMessageThat().contains("2 / x evaluates to ERROR"); - } - - @Test - public void isSatisfiable_inconclusiveResult_notChecked() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerificationResult result = - CelVerificationResult.inconclusive( - "Inconclusive.", witness(2L), /* isCounterexample= */ false); - - assertThat(checking(result).isSatisfiable(ast)).isSameInstanceAs(result); - } - - @Test - public void isSatisfiable_unplannableExpression_throws() throws Exception { - Cel celWithFunction = - CEL.toCelBuilder() - .addFunctionDeclarations( - CelFunctionDecl.newFunctionDeclaration( - "f", - CelOverloadDecl.newGlobalOverload("f_int", SimpleType.BOOL, SimpleType.INT))) - .build(); - CelAbstractSyntaxTree ast = celWithFunction.compile("f(x)").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.verified("Condition is satisfiable.", witness(1L))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isSatisfiable(ast)); - - assertThat(e).hasMessageThat().contains("Can't plan f(x)"); - assertThat(e).hasCauseThat().isInstanceOf(CelEvaluationException.class); - } - - @Test - public void isAlwaysTrue_missingVariableInWitness_throws() throws Exception { - CelAbstractSyntaxTree ast = CEL.compile("x == 1").getAst(); - CelVerifier verifier = - checking(CelVerificationResult.failed("Violated.", CelWitness.create(ImmutableMap.of()))); - - AssertionError e = assertThrows(AssertionError.class, () -> verifier.isAlwaysTrue(ast)); - - assertThat(e).hasMessageThat().contains("Incomplete witness {} for x == 1"); - assertThat(e).hasCauseThat().isInstanceOf(CelEvaluationException.class); - } - - private static CelWitness witness(long x) { - return CelWitness.create( - ImmutableMap.of("x", CelWitness.Variable.create(Optional.of(x), Long.toString(x)))); - } - - @SuppressWarnings("Immutable") // Test fake; the result is never mutated. - private static CelVerifier checking(CelVerificationResult result) { - return RuntimeOracle.wrap( - CEL, - new CelVerifier() { - @Override - public CelVerificationResult isSatisfiable(CelAbstractSyntaxTree ast) { - return result; - } - - @Override - public CelVerificationResult isAlwaysTrue(CelAbstractSyntaxTree ast) { - return result; - } - - @Override - public CelVerificationResult verifyEquivalence( - CelAbstractSyntaxTree astA, CelAbstractSyntaxTree astB) { - return result; - } - }); - } -}