diff --git a/verifier/src/main/java/dev/cel/verifier/CelAstAlphaHasher.java b/verifier/src/main/java/dev/cel/verifier/CelAstAlphaHasher.java index f518caf99..6faf6fde0 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelAstAlphaHasher.java +++ b/verifier/src/main/java/dev/cel/verifier/CelAstAlphaHasher.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.jspecify.annotations.Nullable; /** @@ -75,20 +76,25 @@ static AlphaSignature computeSignature(CelExpr expr, CelAbstractSyntaxTree ast) } private static void hashAst(CelExpr expr, @Nullable Scope scope, HasherContext context) { - CelConstant refValue = - context.ast.getReference(expr.id()).flatMap(CelReference::value).orElse(null); + Optional reference = context.ast.getReference(expr.id()); + CelConstant refValue = reference.flatMap(CelReference::value).orElse(null); if (refValue != null) { context.hasher.putString(CelExpr.ExprKind.Kind.CONSTANT.name(), UTF_8); hashConstant(refValue, context); return; } - context.hasher.putString(expr.exprKind().getKind().name(), UTF_8); - switch (expr.exprKind().getKind()) { + CelExpr.ExprKind.Kind kind = expr.exprKind().getKind(); + // Without namespaced declarations, a qualified variable stays a select with a reference. + if (kind == CelExpr.ExprKind.Kind.SELECT && reference.isPresent()) { + kind = CelExpr.ExprKind.Kind.IDENT; + } + context.hasher.putString(kind.name(), UTF_8); + switch (kind) { case CONSTANT: hashConstant(expr.constant(), context); break; case IDENT: - String name = expr.ident().name(); + String name = context.ast.getReferenceOrThrow(expr.id()).name(); int bIdx = scope == null ? -1 : scope.indexOf(name); if (bIdx >= 0) { context.hasher.putByte((byte) 0); // 0 = bound diff --git a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java index 460d48c18..d9c7f5618 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java @@ -86,8 +86,8 @@ Set getTypeConstraints() { return context.constraints(); } - BoolExpr hasTruncation() { - return context.hasTruncation(); + BoolExpr hasUnmodeledResult() { + return context.hasUnmodeledResult(); } ImmutableList approximationSources(Model model, BoolExpr taint) { @@ -183,6 +183,10 @@ private TranslatedValue translateExpr(CelExpr celExpr, CelAbstractSyntaxTree ast case IDENT: return translateIdent(celExpr, ast); case SELECT: + // Without namespaced declarations, a qualified variable stays a select with a reference. + if (ast.getReference(celExpr.id()).isPresent()) { + return translateIdent(celExpr, ast); + } return structTranslator.translateSelect(celExpr, ast); case CALL: return translateCall(celExpr, ast); @@ -231,13 +235,12 @@ private TranslatedValue translateConstant(CelExpr celExpr) { } private TranslatedValue translateIdent(CelExpr celExpr, CelAbstractSyntaxTree ast) { - CelExpr.CelIdent ident = celExpr.ident(); long exprId = celExpr.id(); CelConstant refValue = ast.getReference(exprId).flatMap(CelReference::value).orElse(null); if (refValue != null) { return translateConstant(CelExpr.ofConstant(exprId, refValue)); } - String name = ident.name(); + String name = ast.getReferenceOrThrow(exprId).name(); CelType type = ast.getTypeOrThrow(exprId); if (type instanceof TypeType) { TypeType typeType = (TypeType) type; @@ -266,10 +269,7 @@ private TranslatedValue translateIdent(CelExpr celExpr, CelAbstractSyntaxTree as } return TranslatedValue.create( - v, - CelExpr.newBuilder().setIdent(ident).setId(exprId).build(), - typeSystem, - /* isApproximate= */ ctx.mkFalse()); + v, celExpr, typeSystem, /* isApproximate= */ ctx.mkFalse()); }); return TranslatedValue.create(tv.z3Expr(), celExpr, typeSystem, tv.isApproximate()); diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerificationResult.java b/verifier/src/main/java/dev/cel/verifier/CelVerificationResult.java index f4c4a8123..d4f058515 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerificationResult.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerificationResult.java @@ -28,7 +28,7 @@ public enum VerificationStatus { VERIFIED, /** The property was disproven. A concrete counterexample was found. */ VIOLATED, - /** The property could not be verified due to loop truncation or timeout. */ + /** The property could not be verified because it depends on what the verifier doesn't model. */ INCONCLUSIVE } diff --git a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java index d98e4f0d4..7fade5e17 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java +++ b/verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java @@ -45,10 +45,8 @@ import dev.cel.verifier.axioms.CelZ3StandardAxioms; import java.time.Duration; import java.util.Arrays; -import java.util.Locale; import java.util.Map; import java.util.Optional; -import org.jspecify.annotations.Nullable; /** Z3 implementation of the CelVerifier. */ @Immutable @@ -235,46 +233,17 @@ public CelVerificationResult verifyEquivalence( "unknown results, which are compared only by their origin", ctx.mkAnd(tvA.isZ3Unknown(), tvB.isZ3Unknown()))); - Solver solver = newSolver(ctx); - for (BoolExpr constraint : translator.getTypeConstraints()) { - solver.add(constraint); - } - - // Divergent parameterized Unknowns are naturally caught by Pass 2 (A != B is SAT). - // Identical Unknowns mean the ASTs are alpha-equivalent, so we WANT them to pass. - // We no longer need Pass 3 to conservatively bail out of equivalence checks. - BoolExpr unknownCondition = ctx.mkFalse(); - - SolverRunResult result = - runThreePassVerification( - ctx, - solver, - divergenceCondition, - combinedTaint, - unknownCondition, - translator, - /* checkTruncation= */ false); - - switch (result.outcome) { - case EXACT_MATCH: - return CelVerificationResult.failed( - "Equivalence violation detected.", extractWitness(ctx, translator, result.model)); - case APPROXIMATE_MATCH: - return CelVerificationResult.inconclusive( - inconclusiveReason("a divergence", translator, result.model, combinedTaint), - extractWitness(ctx, translator, result.model), - /* isCounterexample= */ true); - case TRUNCATED: - return CelVerificationResult.inconclusive( - "Inconclusive: expressions are equivalent within the current loop unroll limit, but" - + " may diverge for larger collections."); - case NO_MATCH: - return CelVerificationResult.verified(); - case SOLVER_UNKNOWN: - return CelVerificationResult.inconclusive( - "Inconclusive: the solver returned unknown status (" + result.reason + ")."); - } - throw new AssertionError("Unknown verification outcome: " + result.outcome); + // Divergent parameterized unknowns are caught by Pass 2; identical ones mean the ASTs are + // alpha-equivalent, so there's no unmodeled result to look for. + return search( + ctx, + translator, + divergenceCondition, + combinedTaint, + /* unknownCondition= */ ctx.mkFalse(), + /* isCounterexample= */ true, + /* candidate= */ "a divergence", + /* exactMatchMessage= */ "Equivalence violation detected."); } } @@ -310,49 +279,23 @@ CelVerificationResult verifyImplication( BoolExpr combinedTaint = CelZ3Exprs.mkOrFlattened(ctx, assumeTv.isApproximate(), assertTv.isApproximate()); + // An unknown assertion under a true assumption is already a Pass 2 candidate; an unknown + // assumption may hide a violation unless the assertion holds exactly. BoolExpr unknownCondition = - ctx.mkOr( - translator.getTypeSystem().isUnknown(assumeTv.z3Expr()), - translator.getTypeSystem().isUnknown(assertTv.z3Expr())); - - Solver solver = newSolver(ctx); - for (BoolExpr constraint : translator.getTypeConstraints()) { - solver.add(constraint); - } - - SolverRunResult result = - runThreePassVerification( - ctx, - solver, - violationCondition, - combinedTaint, - unknownCondition, - translator, - /* checkTruncation= */ true); - - switch (result.outcome) { - case EXACT_MATCH: - return CelVerificationResult.failed( - String.format("%s violation detected.", subjectName), - extractWitness(ctx, translator, result.model)); - case APPROXIMATE_MATCH: - return CelVerificationResult.inconclusive( - inconclusiveReason("a counterexample", translator, result.model, combinedTaint), - extractWitness(ctx, translator, result.model), - /* isCounterexample= */ true); - case TRUNCATED: - return CelVerificationResult.inconclusive( - String.format( - "Inconclusive: %s holds within the current loop unroll limit, but" - + " may be violated for larger collections.", - subjectName.toLowerCase(Locale.US))); - case NO_MATCH: - return CelVerificationResult.verified(); - case SOLVER_UNKNOWN: - return CelVerificationResult.inconclusive( - "Inconclusive: the solver returned unknown status (" + result.reason + ")."); - } - throw new AssertionError("Unknown verification outcome: " + result.outcome); + ctx.mkAnd( + assumeTv.isZ3Unknown(), + assumeTv.isApproximate(), + ctx.mkNot(ctx.mkAnd(assertCondition, ctx.mkNot(assertTv.isApproximate())))); + + return search( + ctx, + translator, + violationCondition, + combinedTaint, + unknownCondition, + /* isCounterexample= */ true, + /* candidate= */ "a counterexample", + /* exactMatchMessage= */ String.format("%s violation detected.", subjectName)); } } @@ -362,124 +305,110 @@ private CelVerificationResult checkSatisfiability( CelAstToZ3Translator translator = newTranslator(ctx, /* parameterizeUnknowns= */ false); TranslatedValue tv = translator.translate(ast); - BoolExpr condition = translator.isTrue(tv.z3Expr()); + BoolExpr isTrue = translator.isTrue(tv.z3Expr()); if (searchForCounterexample) { - condition = ctx.mkNot(condition); + return search( + ctx, + translator, + ctx.mkNot(isTrue), + tv.isApproximate(), + // An unknown result is never true, so it's already a Pass 2 counterexample. + /* unknownCondition= */ ctx.mkFalse(), + /* isCounterexample= */ true, + /* candidate= */ "a counterexample", + /* exactMatchMessage= */ "Condition is not always true."); } - - Solver solver = newSolver(ctx); - for (BoolExpr constraint : translator.getTypeConstraints()) { - solver.add(constraint); - } - - SolverRunResult result = - runThreePassVerification( - ctx, - solver, - condition, - tv.isApproximate(), - translator.getTypeSystem().isUnknown(tv.z3Expr()), - translator, - /* checkTruncation= */ true); - - switch (result.outcome) { - case EXACT_MATCH: - CelWitness witness = extractWitness(ctx, translator, result.model); - return searchForCounterexample - ? CelVerificationResult.failed("Condition is not always true.", witness) - : CelVerificationResult.verified("Condition is satisfiable.", witness); - - case APPROXIMATE_MATCH: - return CelVerificationResult.inconclusive( - inconclusiveReason( - searchForCounterexample ? "a counterexample" : "a satisfying input", - translator, - result.model, - tv.isApproximate()), - extractWitness(ctx, translator, result.model), - /* isCounterexample= */ searchForCounterexample); - - case TRUNCATED: - return CelVerificationResult.inconclusive( - searchForCounterexample - ? "Inconclusive: a counterexample may exist beyond the loop unroll limit." - : "Inconclusive: expression is not satisfiable within the current loop unroll" - + " limit, but may be satisfiable for larger collections."); - - case NO_MATCH: - return searchForCounterexample - ? CelVerificationResult.verified() - : CelVerificationResult.failed("Condition is not satisfiable."); - - case SOLVER_UNKNOWN: - return CelVerificationResult.inconclusive( - "Inconclusive: the solver returned unknown status (" + result.reason + ")."); - } - throw new AssertionError("Unknown verification outcome: " + result.outcome); + BoolExpr isUnmodeledUnknown = ctx.mkAnd(tv.isZ3Unknown(), tv.isApproximate()); + return search( + ctx, + translator, + isTrue, + tv.isApproximate(), + isUnmodeledUnknown, + /* isCounterexample= */ false, + /* candidate= */ "a satisfying input", + /* exactMatchMessage= */ "Condition is satisfiable."); } } - private SolverRunResult runThreePassVerification( + /** + * Searches for a {@code condition} match: first an exact one, then an approximate one, then an + * unmodeled result satisfying {@code unknownCondition} that may stand in for one. + */ + private CelVerificationResult search( Context ctx, - Solver solver, + CelAstToZ3Translator translator, BoolExpr condition, BoolExpr taint, BoolExpr unknownCondition, - CelAstToZ3Translator translator, - boolean checkTruncation) + boolean isCounterexample, + String candidate, + String exactMatchMessage) throws CelVerificationException { + Solver solver = newSolver(ctx); + solver.add(translator.getTypeConstraints().toArray(new BoolExpr[0])); - // Pass 1: Search for an exact match/counterexample - solver.push(); - solver.add(condition); - solver.add(ctx.mkNot(taint)); - Status status = solver.check(); - + // Pass 1: Search for an exact match. + Status status = checkSat(solver, condition, ctx.mkNot(taint)); if (status == Status.SATISFIABLE) { - return SolverRunResult.exactMatch(solver.getModel()); + CelWitness witness = extractWitness(ctx, translator, solver.getModel()); + return isCounterexample + ? CelVerificationResult.failed(exactMatchMessage, witness) + : CelVerificationResult.verified(exactMatchMessage, witness); } else if (status == Status.UNKNOWN) { - return SolverRunResult.solverUnknown(checkTimeoutOrGetReason(solver)); + // Depends on solver internals; not deterministically testable. + return solverGaveUp(solver); } - // Pass 2: Search for any tainted match/counterexample - solver.pop(); - solver.push(); - solver.add(condition); - Status approxStatus = solver.check(); - - if (approxStatus == Status.SATISFIABLE) { - return SolverRunResult.approximateMatch(solver.getModel()); - } else if (approxStatus == Status.UNKNOWN) { - return SolverRunResult.solverUnknown(checkTimeoutOrGetReason(solver)); - } - - // If we don't need to check truncation (e.g. for equivalence checks), - // we can pop the solver and return noMatch immediately. - if (!checkTruncation) { - solver.pop(); - return SolverRunResult.noMatch(); + // Pass 2: Search for an approximate match. + status = checkSat(solver, condition); + if (status == Status.SATISFIABLE) { + Model model = solver.getModel(); + return CelVerificationResult.inconclusive( + inconclusiveReason(candidate, translator, model, taint), + extractWitness(ctx, translator, model), + isCounterexample); + } else if (status == Status.UNKNOWN) { + // Depends on solver internals; not deterministically testable. + return solverGaveUp(solver); } - // Pass 3: Check BMC truncation - solver.pop(); - solver.add(unknownCondition); - solver.add(translator.hasTruncation()); - Status truncationStatus = solver.check(); - if (truncationStatus == Status.SATISFIABLE) { - return SolverRunResult.truncated(); - } else if (truncationStatus == Status.UNKNOWN) { - return SolverRunResult.solverUnknown(checkTimeoutOrGetReason(solver)); + // Pass 3: Search for an unmodeled result. + status = checkSat(solver, unknownCondition, translator.hasUnmodeledResult()); + if (status == Status.SATISFIABLE) { + return CelVerificationResult.inconclusive( + inconclusiveReason(candidate, translator, solver.getModel(), taint)); + } else if (status == Status.UNKNOWN) { + // Depends on solver internals; not deterministically testable. + return solverGaveUp(solver); } - return SolverRunResult.noMatch(); + return isCounterexample + ? CelVerificationResult.verified() + : CelVerificationResult.failed("Condition is not satisfiable."); } - private static String checkTimeoutOrGetReason(Solver solver) throws CelVerificationException { - String reason = solver.getReasonUnknown(); - if (reason.equals("timeout") || reason.equals("canceled")) { - throw new CelVerificationException("Verification timed out: " + reason); + // Throws on a timeout. Leaves the assertions in place unless they're unsatisfiable, so the model + // or the reason the solver gave up can be read. + private static Status checkSat(Solver solver, BoolExpr... assertions) + throws CelVerificationException { + solver.push(); + solver.add(assertions); + Status status = solver.check(); + if (status == Status.UNSATISFIABLE) { + solver.pop(); + } else if (status == Status.UNKNOWN) { + String reason = solver.getReasonUnknown(); + if (reason.equals("timeout") || reason.equals("canceled")) { + throw new CelVerificationException("Verification timed out: " + reason); + } } - return reason; + return status; + } + + private static CelVerificationResult solverGaveUp(Solver solver) { + return CelVerificationResult.inconclusive( + "Inconclusive: the solver returned unknown status (" + solver.getReasonUnknown() + ")."); } private Solver newSolver(Context ctx) { @@ -518,46 +447,6 @@ private CelWitness extractWitness(Context ctx, CelAstToZ3Translator translator, translator.getInputVariables()); } - private enum SolverOutcome { - EXACT_MATCH, - APPROXIMATE_MATCH, - TRUNCATED, - NO_MATCH, - SOLVER_UNKNOWN - } - - private static final class SolverRunResult { - private final SolverOutcome outcome; - private final @Nullable Model model; - private final @Nullable String reason; - - static SolverRunResult exactMatch(Model model) { - return new SolverRunResult(SolverOutcome.EXACT_MATCH, model, null); - } - - static SolverRunResult approximateMatch(Model model) { - return new SolverRunResult(SolverOutcome.APPROXIMATE_MATCH, model, null); - } - - static SolverRunResult truncated() { - return new SolverRunResult(SolverOutcome.TRUNCATED, null, null); - } - - static SolverRunResult noMatch() { - return new SolverRunResult(SolverOutcome.NO_MATCH, null, null); - } - - static SolverRunResult solverUnknown(String reason) { - return new SolverRunResult(SolverOutcome.SOLVER_UNKNOWN, null, reason); - } - - private SolverRunResult(SolverOutcome outcome, @Nullable Model model, @Nullable String reason) { - this.outcome = outcome; - this.model = model; - this.reason = reason; - } - } - private CelVerifierZ3Impl( Duration timeout, int comprehensionUnrollLimit, diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java index a047b0a45..c0a2b3ed2 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3ComprehensionTranslator.java @@ -157,9 +157,18 @@ private TranslatedValue translateDynamicComprehension( context.recordApproximation( "comprehensions over `dyn` values", ctx.mkOr(typeSystem.isList(iterRange), typeSystem.isMap(iterRange))); + context.addUnmodeledResult(isRuntimeListOrMap); + // An unknown range (e.g. another unmodeled comprehension) yields an unknown, not an error. Expr result = - ctx.mkITE(isRuntimeListOrMap, mkParameterizedUnknown(celExpr, ast), typeSystem.mkError()); - return TranslatedValue.create(result, celExpr, typeSystem, isRuntimeListOrMap); + ctx.mkITE( + ctx.mkOr(iterRangeTv.isZ3Unknown(), isRuntimeListOrMap), + mkParameterizedUnknown(celExpr, ast), + typeSystem.mkError()); + return TranslatedValue.create( + result, + celExpr, + typeSystem, + CelZ3Exprs.mkOrFlattened(ctx, iterRangeTv.isApproximate(), isRuntimeListOrMap)); } SeqExpr seq = @@ -175,7 +184,7 @@ private TranslatedValue translateDynamicComprehension( BoolExpr isTruncated = context.recordUnrollLimitApproximation( ctx.mkAnd(isValidRange, ctx.mkGt(lengthExpr, ctx.mkInt(comprehensionUnrollLimit)))); - context.addTruncationCondition(isTruncated); + context.addUnmodeledResult(isTruncated); if (isAllMacro(comp) || isExistsMacro(comp)) { return unrollAllAndExists( @@ -341,10 +350,12 @@ private TranslatedValue unrollMapAndFilter( // TODO: Model the loop condition for custom macros; until then, abstract the // result of any non-true condition, since the unrolled value ignores early exits. - BoolExpr isConditionUnmodeled = - context.recordApproximation( - "comprehensions with a loop condition", - ctx.mkBool(!isLiteralTrue(comp.loopCondition()))); + BoolExpr isConditionUnmodeled = ctx.mkFalse(); + if (!isLiteralTrue(comp.loopCondition())) { + isConditionUnmodeled = + context.recordApproximation("comprehensions with a loop condition", ctx.mkTrue()); + context.addUnmodeledResult(isConditionUnmodeled); + } BoolExpr isAbstracted = CelZ3Exprs.mkOrFlattened(ctx, rangeIsUnknown, isTruncated, isConditionUnmodeled); // An error is not final when truncated: an unknown from a later iteration overrides it. @@ -392,8 +403,6 @@ private TranslatedValue reduceAllOrExists( List hasUnknownList = new ArrayList<>(); List hasSafeMatchList = new ArrayList<>(); - List hasSafeErrorList = new ArrayList<>(); - List hasSafeUnknownList = new ArrayList<>(); List activeTaints = new ArrayList<>(); for (BoundedIteration iter : iterations) { @@ -419,12 +428,6 @@ private TranslatedValue reduceAllOrExists( hasSafeMatchList.add( CelZ3Exprs.mkAndFlattened( ctx, isActive, isMatch, CelZ3Exprs.mkNotFlattened(ctx, stepTv.isApproximate()))); - hasSafeErrorList.add( - CelZ3Exprs.mkAndFlattened( - ctx, isActive, isE, CelZ3Exprs.mkNotFlattened(ctx, stepTv.isApproximate()))); - hasSafeUnknownList.add( - CelZ3Exprs.mkAndFlattened( - ctx, isActive, isU, CelZ3Exprs.mkNotFlattened(ctx, stepTv.isApproximate()))); activeTaints.add(CelZ3Exprs.mkAndFlattened(ctx, isActive, stepTv.isApproximate())); } @@ -435,8 +438,6 @@ private TranslatedValue reduceAllOrExists( BoolExpr rangeIsUnknown = typeSystem.isUnknown(iterRangeTv.z3Expr()); BoolExpr hasSafeMatch = CelZ3Exprs.mkOrFlattened(ctx, hasSafeMatchList); - BoolExpr hasSafeError = CelZ3Exprs.mkOrFlattened(ctx, hasSafeErrorList); - BoolExpr hasSafeUnknown = CelZ3Exprs.mkOrFlattened(ctx, hasSafeUnknownList); BoolExpr anyActiveTaint = CelZ3Exprs.mkOrFlattened(ctx, activeTaints); Expr comprehensionUnknown = mkParameterizedUnknown(compExpr, ast); @@ -449,9 +450,9 @@ private TranslatedValue reduceAllOrExists( .addCase(hasError, typeSystem.mkError()) .build(typeSystem.mkBool(isAll)); - BoolExpr baseTaint = CelZ3Exprs.mkOrFlattened(ctx, anyActiveTaint, iterRangeTv.isApproximate()); - - // Taint identically shadows the value flow's short-circuit control structure + // Only an exact match short-circuits; without one, any approximate step may match at runtime. + // The cases mirror the result's to keep the formula easy for Z3 (some versions give up on maps + // with fewer, equivalent cases). BoolExpr resultTaint = (BoolExpr) CelZ3Exprs.SwitchBuilder.newBuilder(ctx) @@ -459,9 +460,9 @@ private TranslatedValue reduceAllOrExists( .addCase(hasMatch, ctx.mkOr(iterRangeTv.isApproximate(), ctx.mkNot(hasSafeMatch))) .addCase( ctx.mkOr(hasUnknown, isTruncated), - ctx.mkOr(iterRangeTv.isApproximate(), isTruncated, ctx.mkNot(hasSafeUnknown))) - .addCase(hasError, ctx.mkOr(iterRangeTv.isApproximate(), ctx.mkNot(hasSafeError))) - .build(baseTaint); + ctx.mkOr(iterRangeTv.isApproximate(), isTruncated, anyActiveTaint)) + .addCase(hasError, ctx.mkOr(iterRangeTv.isApproximate(), anyActiveTaint)) + .build(CelZ3Exprs.mkOrFlattened(ctx, anyActiveTaint, iterRangeTv.isApproximate())); return TranslatedValue.create(result, compExpr, typeSystem, resultTaint); } diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java index e3599d517..203cf5b2c 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java @@ -45,7 +45,7 @@ final class CelZ3TranslationContext { private final int comprehensionUnrollLimit; private final Set constraints; private final Map, CelType> inputVariableTypes; - private final List truncationConditions; + private final List unmodeledResults; private final Map> approximationTerms; private final Map symbolTable; private final Map localScope; @@ -160,12 +160,13 @@ private static boolean holds(Model model, BoolExpr condition) { return model.evaluate(condition, /* completion= */ true).isTrue(); } - void addTruncationCondition(BoolExpr condition) { - truncationConditions.add(condition); + /** Records where a result is an unknown standing in for a value the verifier doesn't compute. */ + void addUnmodeledResult(BoolExpr condition) { + unmodeledResults.add(condition); } - BoolExpr hasTruncation() { - return CelZ3Exprs.mkOrFlattened(ctx, truncationConditions); + BoolExpr hasUnmodeledResult() { + return CelZ3Exprs.mkOrFlattened(ctx, unmodeledResults); } void bindSymbol(String name, TranslatedValue value) { @@ -207,7 +208,7 @@ T withScope(String varName, TranslatedValue value, Supplier action) { this.comprehensionUnrollLimit = comprehensionUnrollLimit; this.constraints = new LinkedHashSet<>(); this.inputVariableTypes = new LinkedHashMap<>(); - this.truncationConditions = new ArrayList<>(); + this.unmodeledResults = new ArrayList<>(); this.approximationTerms = new LinkedHashMap<>(); this.symbolTable = new HashMap<>(); this.localScope = new HashMap<>(); diff --git a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java index d1af0420f..22590e9f9 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java @@ -84,6 +84,7 @@ public final class CelVerifierZ3ImplTest { CelFactory.plannerCelBuilder() .setOptions(CelOptions.current().enableHeterogeneousNumericComparisons(true).build()) .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .addMacros(firstElemMacro()) .setContainer(CelContainer.ofName("cel.expr.conformance.proto3")) .addCompilerLibraries( CelExtensions.bindings(), CelOptionalLibrary.INSTANCE, CelExtensions.comprehensions()) @@ -141,7 +142,7 @@ public final class CelVerifierZ3ImplTest { private static final CelVerifier VERIFIER = withRuntimeOracle(CEL, b -> b); - // For ASTs with functions the runtime can't plan. + // For ASTs with functions the runtime can't plan, and for verifyImplication. private static final CelVerifier VERIFIER_WITHOUT_ORACLE = CelVerifierFactory.newVerifier(CEL).build(); @@ -605,6 +606,18 @@ private enum ApproximationSourceTestCase { UNTAKEN_BRANCH( "size(int_list) == 7 && (b ? int_list[6] == 1 : int(string_var) == 5) && !b", "calls to `int`"), + DYN_RANGE_EXISTS("dyn_var.exists(e, e == 1)", "comprehensions over `dyn` values"), + DYN_RANGE_ALL("dyn_var.all(e, e > 0)", "comprehensions over `dyn` values"), + DYN_RANGE_MAP("dyn_var.map(e, e) == [1]", "comprehensions over `dyn` values"), + DYN_LIST_RANGE("dyn(int_list).exists(e, e == 1)", "comprehensions over `dyn` values"), + DYN_MAP_RANGE("dyn(string_int_map).exists(k, k == 'a')", "comprehensions over `dyn` values"), + SHORT_CIRCUITED_LOOP_CONDITION( + "(false && int_list.first_elem() == 1) || dyn_var.exists(e, e == 1)", + "comprehensions over `dyn` values"), + CHAINED_DYN_RANGE( + "dyn(dyn_var.map(e, e)).exists(x, x == 1)", "comprehensions over `dyn` values"), + CUSTOM_COMPREHENSION_LOOP_CONDITION( + "int_list == [1, 2] && int_list.first_elem() == 1", "comprehensions with a loop condition"), OPTIONAL_SELECT_WITH_AXIOM( "(false && string_int_map.?k.hasValue()) || int(string_var) == 5", "calls to `int`"); @@ -802,6 +815,7 @@ public void isSatisfiable_comprehensionZeroUnrollLimit_inconclusive() throws Exc private enum IsUnsatisfiableTestCase { UNSATISFIABLE("x > 5 && x < 3"), + DYN_RANGE_COMPREHENSION_MASKED_BY_FALSE("dyn_var.exists(e, e == 1) && false"), CONTRADICTORY_TYPES_FOR_DYNAMIC_VARIABLE("unknown_var + 1 == 2 && unknown_var[0] == 1"), TYPE_CONVERSION_UNSATISFIABLE_STRING_TO_INT("type(int('1')) == string"), TYPE_CONVERSION_UNSATISFIABLE_DOUBLE_TO_STRING("type(string(1.5)) == int"), @@ -1402,18 +1416,36 @@ public void isAlwaysTrue_withUnknownIdentifier_evaluatesToUnknown( assertThat(result.witness().get().variables().get("x").value()).isEmpty(); } - @Test - public void isAlwaysTrue_approximateOperandWithUnknown_inconclusive() throws Exception { + private enum ApproximateBesideUnknownTestCase { // The truncated `all` may really be false, which absorbs the unknown, so this is always true. - CelAbstractSyntaxTree ast = - CEL.compile("!(int_list.all(x, x > 0) && unknown_var) || int_list.all(x, x > 0)").getAst(); + APPROXIMATE_OPERAND( + "!(int_list.all(x, x > 0) && unknown_var) || int_list.all(x, x > 0)", + "lists and maps longer than the unroll limit (5)"), + // int('5') == 5 may really be true, which absorbs the unknown in `exists`. + APPROXIMATE_MATCH( + "int_list == [1, 0] ? int_list.exists(x, x == 1 ? int('5') == 5 : unknown_var == 1) : true", + "calls to `int`"); + + private final String expr; + private final String expectedSources; + + ApproximateBesideUnknownTestCase(String expr, String expectedSources) { + this.expr = expr; + this.expectedSources = expectedSources; + } + } + + @Test + public void isAlwaysTrue_approximateBesideUnknown_inconclusive( + @TestParameter ApproximateBesideUnknownTestCase testCase) throws Exception { + CelAbstractSyntaxTree ast = CEL.compile(testCase.expr).getAst(); CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("unknown_var")); CelVerificationResult result = verifier.isAlwaysTrue(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); assertThat(result.message()) - .contains("doesn't model exactly: lists and maps longer than the unroll limit (5)."); + .contains("doesn't model exactly: " + testCase.expectedSources + "."); } @Test @@ -2076,6 +2108,11 @@ public void isAlwaysTrue_violation_returnsFalse( } private enum IsInconclusiveTestCase { + // int('5') == 5 is approximate and may really be true, which absorbs the 1 / 0 error. + APPROXIMATE_MATCH_BEFORE_ERROR( + "int_list == [1, 0] ? int_list.exists(x, x == 1 ? int('5') == 5 : 1 / x == 1) : true"), + // nested_list[5] is past the unroll limit, so it may not be a list. + APPROXIMATE_DYN_RANGE("size(nested_list) == 6 ? dyn(nested_list)[5].all(e, e == e) : true"), // TODO: Implement RFC 3339 spec in conversion TIMESTAMP_STRING_CONVERSION_VALID( "timestamp('2023-01-01T00:00:00Z') == timestamp('2023-01-01T00:00:00Z')"), @@ -3813,36 +3850,168 @@ public void isAlwaysTrue_customComprehensionWithLoopCondition_inconclusive() thr // first_elem() stops folding once the accumulator is set, so it returns 1 for [1, 2]. The loop // condition over a non-literal range is not modeled, so the verifier must not claim that the // result is the last element. + CelAbstractSyntaxTree ast = + CEL.compile("int_list == [1, 2] ? int_list.first_elem() == 2 : true").getAst(); + + CelVerificationResult result = VERIFIER.isAlwaysTrue(ast); + + assertThat(result.reason()) + .isEqualTo( + "Inconclusive: a counterexample may exist, but it depends on what the verifier doesn't" + + " model exactly: comprehensions with a loop condition."); + } + + @Test + public void verifyImplication_dynRangeComprehensionInAssumption_inconclusive() throws Exception { + CelVerificationResult result = verifyImplication(CEL, "dyn_var.exists(e, e == 1)", "x > 0"); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.reason()) + .isEqualTo( + "Inconclusive: a counterexample may exist, but it depends on what the verifier doesn't" + + " model exactly: comprehensions over `dyn` values."); + } + + @Test + public void verifyImplication_approximateAssertionUnderDynRangeComprehension_inconclusive() + throws Exception { + // f(x) == f(x) holds for the uninterpreted f, but the real f may error once the assumption + // holds. + CelCompiler celCompiler = + CEL.toCompilerBuilder() + .addFunctionDeclarations( + CelFunctionDecl.newFunctionDeclaration( + "f", + CelOverloadDecl.newGlobalOverload("f_int", SimpleType.INT, SimpleType.INT))) + .build(); + + CelVerificationResult result = + verifyImplication(celCompiler, "dyn_var.exists(e, e == 1)", "f(x) == f(x)"); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.reason()) + .isEqualTo( + "Inconclusive: a counterexample may exist, but it depends on what the verifier doesn't" + + " model exactly: comprehensions over `dyn` values, calls to `f`."); + } + + @Test + public void verifyImplication_loopExceedsLimit_inconclusive() throws Exception { + CelAbstractSyntaxTree assumeAst = CEL.compile("int_list.all(x, x > 0)").getAst(); + CelAbstractSyntaxTree assertAst = CEL.compile("size(int_list) < 3 || int_list[2] > 0").getAst(); + CelVerifier verifier = + CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(2).build(); + + CelVerificationResult result = + ((CelVerifierZ3Impl) verifier) + .verifyImplication(assumeAst, assertAst, ImmutableMap.of(), "Implication"); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + assertThat(result.reason()) + .isEqualTo( + "Inconclusive: a counterexample may exist, but it depends on what the verifier doesn't" + + " model exactly: lists and maps longer than the unroll limit (2)."); + } + + @Test + public void verifyImplication_exactUnknownAssumption_verified() throws Exception { + // b && !b is never true; with b unknown it is an exact unknown, not an unmodeled result. + CelVerifierZ3Impl verifier = + (CelVerifierZ3Impl) CelVerifierFactory.newVerifier(CEL).addUnknownIdentifier("b").build(); + + CelVerificationResult result = + verifier.verifyImplication( + CEL.compile("b && !b").getAst(), + CEL.compile("dyn_var.exists(e, e == 1)").getAst(), + ImmutableMap.of(), + "Implication"); + + assertWithMessage(result.message()) + .that(result.status()) + .isEqualTo(VerificationStatus.VERIFIED); + } + + @Test + public void isSatisfiable_exactUnknownBesideUnmodeledComprehension_violated() throws Exception { + CelAbstractSyntaxTree ast = + CEL.compile("(false && int_list.first_elem() == 1) || (b && !b)").getAst(); + CelVerifier verifier = withRuntimeOracle(CEL, b -> b.addUnknownIdentifier("b")); + + CelVerificationResult result = verifier.isSatisfiable(ast); + + assertWithMessage(result.message()) + .that(result.status()) + .isEqualTo(VerificationStatus.VIOLATED); + } + + @Test + @SuppressWarnings("deprecation") // enableNamespacedDeclarations(false) + public void isAlwaysTrue_containerQualifiedVariable_witnessUsesQualifiedName( + @TestParameter boolean namespacedDeclarations) throws Exception { Cel cel = CelFactory.plannerCelBuilder() - .addVar("int_list", ListType.create(SimpleType.INT)) - .addMacros( - CelMacro.newReceiverMacro( - "first_elem", - 0, - (exprFactory, target, arguments) -> - Optional.of( - exprFactory.fold( - "x", - target, - "accu", - exprFactory.newIntLiteral(-1), - exprFactory.newGlobalCall( - Operator.EQUALS.getFunction(), - exprFactory.newIdentifier("accu"), - exprFactory.newIntLiteral(-1)), - exprFactory.newIdentifier("x"), - exprFactory.newIdentifier("accu"))))) + .setOptions( + CelOptions.current() + .enableHeterogeneousNumericComparisons(true) + .enableNamespacedDeclarations(namespacedDeclarations) + .build()) + .setContainer(CelContainer.ofName("pkg")) + .addVar("pkg.x", SimpleType.INT) .build(); - CelAbstractSyntaxTree ast = - cel.compile("int_list == [1, 2] ? int_list.first_elem() == 2 : true").getAst(); - CelVerifier verifier = withRuntimeOracle(cel, b -> b); + CelAbstractSyntaxTree ast = cel.compile("x == 1 && pkg.x == 1 && .pkg.x == 1").getAst(); - CelVerificationResult result = verifier.isAlwaysTrue(ast); + CelVerificationResult result = withRuntimeOracle(cel, b -> b).isAlwaysTrue(ast); + + assertThat(result.status()).isEqualTo(VerificationStatus.VIOLATED); + assertThat(result.witness().get().variables().keySet()).containsExactly("pkg.x"); + } + + @Test + @SuppressWarnings("deprecation") // enableNamespacedDeclarations(false) + public void isSatisfiable_containerQualifiedVariableInUnmodeledComprehension_inconclusive( + @TestParameter boolean namespacedDeclarations) throws Exception { + Cel cel = + CelFactory.plannerCelBuilder() + .setOptions( + CelOptions.current() + .enableHeterogeneousNumericComparisons(true) + .enableNamespacedDeclarations(namespacedDeclarations) + .build()) + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .setContainer(CelContainer.ofName("pkg")) + .addVar("pkg.x", SimpleType.INT) + .build(); + CelAbstractSyntaxTree ast = cel.compile("dyn([1]).exists(e, e == pkg.x)").getAst(); + + CelVerificationResult result = withRuntimeOracle(cel, b -> b).isSatisfiable(ast); assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); - assertThat(result.message()) - .contains("doesn't model exactly: comprehensions with a loop condition."); + } + + private enum HoldingImplicationTestCase { + ASSERTION_ALWAYS_TRUE("dyn_var.exists(e, e == 1)", "true"), + ASSUMPTION_NEVER_TRUE("x > 0 && x < 0", "dyn_var.exists(e, e == 1)"), + ASSERTION_IMPLIED_BY_REST_OF_ASSUMPTION("dyn_var.exists(e, e == 1) && x > 0", "x > 0"), + // x < d is false when d is NaN. + SYMBOLIC_NAN_CROSS_NUMERIC_COMPARISON("d != d", "!(x < d)"); + + private final String assume; + private final String assertion; + + HoldingImplicationTestCase(String assume, String assertion) { + this.assume = assume; + this.assertion = assertion; + } + } + + @Test + public void verifyImplication_verified(@TestParameter HoldingImplicationTestCase testCase) + throws Exception { + CelVerificationResult result = verifyImplication(CEL, testCase.assume, testCase.assertion); + + assertWithMessage(result.message()) + .that(result.status()) + .isEqualTo(VerificationStatus.VERIFIED); } @Test @@ -4055,38 +4224,6 @@ public void verifyEquivalence_timeoutReached_throwsCelVerificationException() th assertThat(e).hasMessageThat().containsMatch("timeout|canceled"); } - @Test - public void verifyImplication_loopExceedsLimit_returnsTruncatedInconclusive() throws Exception { - CelAbstractSyntaxTree assumeAst = - CEL.compile("size(int_list) <= 2 && int_list[0] > 0 && int_list[1] > 0").getAst(); - CelAbstractSyntaxTree assertAst = CEL.compile("int_list.all(x, x > 0)").getAst(); - CelVerifier verifier = - CelVerifierFactory.newVerifier(CEL).setComprehensionUnrollLimit(2).build(); - - CelVerificationResult result = - ((CelVerifierZ3Impl) verifier) - .verifyImplication(assumeAst, assertAst, ImmutableMap.of(), "Implication"); - - assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); - assertThat(result.message()) - .contains("implication holds within the current loop unroll limit"); - } - - @Test - public void verifyImplication_symbolicNan_crossNumericComparisonReturnsFalse() throws Exception { - // Assumption: d is NaN (d != d) - CelAbstractSyntaxTree assumeAst = CEL.compile("d != d").getAst(); - // Assertion: x < d is false when d is NaN - CelAbstractSyntaxTree assertAst = CEL.compile("!(x < d)").getAst(); - CelVerifier verifier = CelVerifierFactory.newVerifier(CEL).build(); - - CelVerificationResult result = - ((CelVerifierZ3Impl) verifier) - .verifyImplication(assumeAst, assertAst, ImmutableMap.of(), "Implication"); - - assertThat(result.status()).isEqualTo(VerificationStatus.VERIFIED); - } - @Test public void isAlwaysTrue_constantVarDecl_verified() throws Exception { Cel celWithConstVar = @@ -4111,5 +4248,35 @@ public void isAlwaysTrue_constantVarDecl_verified() throws Exception { .that(result.status()) .isEqualTo(VerificationStatus.VERIFIED); } -} + // `list.first_elem()` folds with a loop condition that stops once the accumulator is set. + private static CelMacro firstElemMacro() { + return CelMacro.newReceiverMacro( + "first_elem", + 0, + (exprFactory, target, arguments) -> + Optional.of( + exprFactory.fold( + "x", + target, + "accu", + exprFactory.newIntLiteral(-1), + exprFactory.newGlobalCall( + Operator.EQUALS.getFunction(), + exprFactory.newIdentifier("accu"), + exprFactory.newIntLiteral(-1)), + exprFactory.newIdentifier("x"), + exprFactory.newIdentifier("accu")))); + } + + // verifyImplication isn't part of CelVerifier, so it can't go through the runtime oracle. + private static CelVerificationResult verifyImplication( + CelCompiler compiler, String assume, String assertion) throws Exception { + return ((CelVerifierZ3Impl) VERIFIER_WITHOUT_ORACLE) + .verifyImplication( + compiler.compile(assume).getAst(), + compiler.compile(assertion).getAst(), + ImmutableMap.of(), + "Implication"); + } +}