diff --git a/checker/checker_options.h b/checker/checker_options.h index da6004d74..e12394f6b 100644 --- a/checker/checker_options.h +++ b/checker/checker_options.h @@ -103,6 +103,24 @@ struct CheckerOptions { // Note: The checked AST will contain the json field names and an extension // tag, but will require runtime support for resolving the json field names. bool use_json_field_names = false; + + // If true, values bound by cel.bind and cel.@block are treated as + // polymorphic (Hindley-Milner let-polymorphism): type parameters of the + // bound expression that remain unconstrained (e.g. the element type of `[]`) + // are generalized, so each reference to the bound variable is inferred + // independently instead of constraining or widening the others. Type + // parameters shared with enclosing variables are not generalized. + // + // Also reports the type of the root expression with its free type + // parameters preserved as named type parameters (e.g. `list(T1)`) via + // ValidationResult::GetGenericResultType(), so bind-like composition outside + // of the checker (e.g. policy variables) can declare variables that are + // instantiated independently at each reference. Only populated if the AST + // was checked with an explicit arena. + // + // Disabled by default; when disabled, type checking is unchanged. The + // resolved type map and the checked AST type map are unaffected either way. + bool use_polymorphic_bind = false; }; } // namespace cel diff --git a/checker/internal/BUILD b/checker/internal/BUILD index a7b2c21ed..7b118b657 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -76,6 +76,7 @@ cc_library( "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", @@ -252,7 +253,9 @@ cc_library( "//common:type_kind", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/container:node_hash_set", "@com_google_absl//absl/functional:function_ref", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", diff --git a/checker/internal/type_check_env.h b/checker/internal/type_check_env.h index 0fda608ab..7abd47529 100644 --- a/checker/internal/type_check_env.h +++ b/checker/internal/type_check_env.h @@ -23,6 +23,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" @@ -64,6 +65,18 @@ class VariableScope { return variables_.insert({decl.name(), std::move(decl)}).second; } + // Updates the type of a variable declared directly in this scope. Returns + // false if absent. The decl is updated in place, so existing pointers to it + // remain valid. + bool UpdateLocalVariableType(absl::string_view name, Type type) { + auto it = variables_.find(name); + if (it == variables_.end()) { + return false; + } + it->second.set_type(std::move(type)); + return true; + } + std::unique_ptr MakeNestedScope() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return absl::WrapUnique(new VariableScope(this)); @@ -72,6 +85,18 @@ class VariableScope { const VariableDecl* absl_nullable LookupLocalVariable( absl::string_view name) const; + // Calls `f` for each variable declared in this scope or its parent scopes. + // Shadowed declarations are included. + void ForEachVisibleVariable( + absl::FunctionRef f) const { + for (const VariableScope* scope = this; scope != nullptr; + scope = scope->parent_) { + for (const auto& entry : scope->variables_) { + f(entry.second); + } + } + } + private: explicit VariableScope( const VariableScope* parent ABSL_ATTRIBUTE_LIFETIME_BOUND) diff --git a/checker/internal/type_checker_builder_impl.h b/checker/internal/type_checker_builder_impl.h index 9895a8aee..7cdf3cef8 100644 --- a/checker/internal/type_checker_builder_impl.h +++ b/checker/internal/type_checker_builder_impl.h @@ -94,6 +94,7 @@ class TypeCheckerBuilderImpl : public TypeCheckerBuilder { ExpressionContainer expression_container) override; const CheckerOptions& options() const override { return options_; } + CheckerOptions& mutable_options() override { return options_; } google::protobuf::Arena* absl_nonnull arena() override { if (type_arena_ == nullptr) { diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index b2aeb72ae..84b7ece1c 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -70,6 +70,29 @@ bool MatchesBlock(const Expr& expr) { call.args()[0].has_list_expr(); } +// Returns true if the comprehension's loop step is just a reference to the +// accumulator (e.g. the cel.bind expansion). The accumulator is then always +// the value of accu_init. +bool IsBindShapedComprehension(const ComprehensionExpr& comprehension) { + return comprehension.loop_step().has_ident_expr() && + comprehension.loop_step().ident_expr().name() == + comprehension.accu_var(); +} + +// Returns the types of the local variables visible from `scope`, i.e. the +// environment for generalizing a binding declared in a child of `scope`. +// +// Declared (global) variables are not included: their types can only contain +// declared type parameters, which are instantiated with fresh type variables +// at each reference, so they never share inference type variables with a +// binding. +std::vector VisibleLocalVariableTypes(const VariableScope& scope) { + std::vector types; + scope.ForEachVisibleVariable( + [&](const VariableDecl& decl) { types.push_back(decl.type()); }); + return types; +} + using AstType = cel::TypeSpec; using Severity = TypeCheckIssue::Severity; @@ -206,7 +229,7 @@ class ResolveVisitor : public AstVisitorBase { const TypeCheckEnv& env, const Ast& ast, TypeInferenceContext& inference_context, std::vector& issues, - google::protobuf::Arena* absl_nonnull arena) + google::protobuf::Arena* absl_nonnull arena, bool use_polymorphic_bind) : namespace_generator_(std::move(namespace_generator)), env_(&env), inference_context_(&inference_context), @@ -214,6 +237,7 @@ class ResolveVisitor : public AstVisitorBase { ast_(&ast), root_scope_(), arena_(arena), + use_polymorphic_bind_(use_polymorphic_bind), current_scope_(&root_scope_) {} void PreVisitExpr(const Expr& expr) override { @@ -454,6 +478,8 @@ class ResolveVisitor : public AstVisitorBase { const Ast* absl_nonnull ast_; VariableScope root_scope_; google::protobuf::Arena* absl_nonnull arena_; + // See CheckerOptions::use_polymorphic_bind. + bool use_polymorphic_bind_; // state tracking for the traversal. const VariableScope* current_scope_; @@ -907,6 +933,21 @@ void ResolveVisitor::PostVisitComprehensionSubexpression( if (!inference_context_->IsAssignable(step_type, accu_type)) { ReportTypeMismatch(comprehension.loop_step().id(), accu_type, step_type); + break; + } + // With polymorphic bind, for bind-shaped comprehensions (the loop step + // is just the accumulator, as in the cel.bind expansion), the + // accumulator is a lazily evaluated alias for accu_init. Generalize its + // type so that each reference in the result is inferred independently, + // like a cel.@block index + // (e.g. `cel.bind(x, [], x + [1] == [1] && x + ['a'] == ['a'])`). + // Type params shared with variables in the enclosing scope are not + // generalized. + if (use_polymorphic_bind_ && IsBindShapedComprehension(comprehension)) { + scope.accu_scope->UpdateLocalVariableType( + comprehension.accu_var(), + inference_context_->Generalize( + accu_type, VisibleLocalVariableTypes(*scope.parent))); } break; } @@ -1328,8 +1369,14 @@ void ResolveVisitor::HandleBlockIndex(const Expr* expr) { // Block is typically manually assembled from logically separate // expressions so fix the type instead of inferring any remaining free type - // params as for normal subexpressions. - auto type = inference_context_->FinalizeType(GetDeducedType(expr)); + // params as for normal subexpressions. With polymorphic bind, generalize any + // remaining free type params instead: each reference to the index is + // inferred independently instead of constraining (or widening) the others. + Type type = use_polymorphic_bind_ + ? inference_context_->Generalize( + GetDeducedType(expr), + VisibleLocalVariableTypes(*current_scope_)) + : inference_context_->FinalizeType(GetDeducedType(expr)); VariableDecl decl = MakeVariableDecl(var_name, std::move(type)); @@ -1451,7 +1498,8 @@ absl::StatusOr TypeCheckerImpl::CheckImpl( TypeInferenceContext type_inference_context( arena, options_.enable_legacy_null_assignment); ResolveVisitor visitor(std::move(generator), env_, *ast, - type_inference_context, issues, arena); + type_inference_context, issues, arena, + options_.use_polymorphic_bind); TraversalOptions opts; opts.use_comprehension_callbacks = true; @@ -1496,6 +1544,14 @@ absl::StatusOr TypeCheckerImpl::CheckImpl( // Apply updates as needed. // Happens in a second pass to simplify validating that pointers haven't // been invalidated by other updates. + std::optional generic_result_type; + if (options_.use_polymorphic_bind && !type_arena.has_value()) { + if (auto it = visitor.types().find(&ast->root_expr()); + it != visitor.types().end()) { + generic_result_type = + type_inference_context.GeneralizeWithCanonicalNames(it->second); + } + } ValidationResult::TypeMap resolved_types; ResolveRewriter rewriter(visitor, type_inference_context, options_, ast->mutable_reference_map(), @@ -1518,6 +1574,9 @@ absl::StatusOr TypeCheckerImpl::CheckImpl( // arena is destructed. Only set the resolved type map if we're using the // caller's arena. result.SetResolvedTypeMap(std::move(resolved_types)); + if (generic_result_type.has_value()) { + result.SetGenericResultType(*std::move(generic_result_type)); + } } return result; diff --git a/checker/internal/type_checker_impl_test.cc b/checker/internal/type_checker_impl_test.cc index 6555066eb..e12500ce8 100644 --- a/checker/internal/type_checker_impl_test.cc +++ b/checker/internal/type_checker_impl_test.cc @@ -887,6 +887,16 @@ class TestExprBuilder { // `__result__`. std::unique_ptr ComprehensionAst(Expr accu_init, Expr loop_condition, Expr loop_step) { + Expr result = Ident("__result__"); + return ComprehensionAst(std::move(accu_init), std::move(loop_condition), + std::move(loop_step), std::move(result)); + } + + // Builds a comprehension over the range `[1, 2]` (iteration variable `i`, + // accumulator `__result__`) with the given accu_init, loop_condition, + // loop_step and result. + std::unique_ptr ComprehensionAst(Expr accu_init, Expr loop_condition, + Expr loop_step, Expr result) { Expr expr = NewExpr(); ComprehensionExpr& comprehension = expr.mutable_comprehension_expr(); comprehension.set_iter_var("i"); @@ -895,7 +905,7 @@ class TestExprBuilder { comprehension.set_accu_init(std::move(accu_init)); comprehension.set_loop_condition(std::move(loop_condition)); comprehension.set_loop_step(std::move(loop_step)); - comprehension.set_result(Ident("__result__")); + comprehension.set_result(std::move(result)); return std::make_unique(std::move(expr), SourceInfo()); } @@ -1055,6 +1065,85 @@ TEST(TypeCheckerImplTest, ComprehensionRecursiveAccumulatorIsError) { "found 'list(list(dyn))'"))); } +// For a bind-shaped comprehension (loop step is the accumulator), references +// to the accumulator in the result are inferred independently. +TEST(TypeCheckerImplTest, BindShapedComprehensionAccumulatorIsGeneralized) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + CheckerOptions options; + options.use_polymorphic_bind = true; + TypeCheckerImpl impl(std::move(env), options); + + // [__result__ + [1], __result__ + ['a']] + TestExprBuilder b; + Expr result_expr = + b.List(b.Call("_+_", b.Ident("__result__"), b.List(b.IntConst(1))), + b.Call("_+_", b.Ident("__result__"), b.List(b.StringConst("a")))); + std::unique_ptr ast = b.ComprehensionAst( + /*accu_init=*/b.List(), /*loop_condition=*/b.BoolConst(false), + /*loop_step=*/b.Ident("__result__"), std::move(result_expr)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + impl.Check(std::move(ast), &arena)); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + const Expr& root = result.GetAst()->root_expr(); + EXPECT_EQ(FormatTypeName(result.GetResolvedTypeMap().at(root.id())), + "list(dyn)"); + // The accumulator init is not constrained by the references. + EXPECT_EQ(FormatTypeName(result.GetResolvedTypeMap().at( + root.comprehension_expr().accu_init().id())), + "list(dyn)"); +} + +// Without CheckerOptions::use_polymorphic_bind, references to the accumulator +// of a bind-shaped comprehension share its type params. +TEST(TypeCheckerImplTest, + BindShapedComprehensionAccumulatorNotGeneralizedByDefault) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + + // [__result__ + [1], __result__ + ['a']] + TestExprBuilder b; + Expr result_expr = + b.List(b.Call("_+_", b.Ident("__result__"), b.List(b.IntConst(1))), + b.Call("_+_", b.Ident("__result__"), b.List(b.StringConst("a")))); + std::unique_ptr ast = b.ComprehensionAst( + /*accu_init=*/b.List(), /*loop_condition=*/b.BoolConst(false), + /*loop_step=*/b.Ident("__result__"), std::move(result_expr)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), + Contains(IsIssueWithSubstring(Severity::kError, + "no matching overload for '_+_'"))); +} + +// Other comprehensions keep a single (monomorphic) accumulator type, even if +// the loop step leaves it unconstrained. +TEST(TypeCheckerImplTest, NonBindComprehensionAccumulatorIsNotGeneralized) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + + // loop step: __result__ + [] + // result: [__result__ + [1], __result__ + ['a']] + TestExprBuilder b; + Expr result_expr = + b.List(b.Call("_+_", b.Ident("__result__"), b.List(b.IntConst(1))), + b.Call("_+_", b.Ident("__result__"), b.List(b.StringConst("a")))); + std::unique_ptr ast = b.ComprehensionAst( + /*accu_init=*/b.List(), /*loop_condition=*/b.BoolConst(true), + /*loop_step=*/b.Call("_+_", b.Ident("__result__"), b.List()), + std::move(result_expr)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), + Contains(IsIssueWithSubstring(Severity::kError, + "no matching overload for '_+_'"))); +} + TEST(TypeCheckerImplTest, MapTypeWithMixedKeys) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 88d43734d..ab730cb19 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -22,6 +22,7 @@ #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" @@ -37,6 +38,7 @@ #include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" +#include "google/protobuf/arena.h" namespace cel::checker_internal { namespace { @@ -759,6 +761,62 @@ Type TypeInferenceContext::FullySubstitute(const Type& type, }); } +Type TypeInferenceContext::SubstituteFreeVars( + const Type& type, + absl::FunctionRef replace_free_var) const { + return TransformTypeRecursively(arena_, type, [&](Type& t) { + if (t.kind() != TypeKind::kTypeParam) { + return; + } + // The substitution may itself contain type params; they are handled as + // the transform recurses into it. + t = Substitute(t, {}); + if (t.kind() == TypeKind::kTypeParam && + IsTypeVar(t.GetTypeParam().name())) { + t = replace_free_var(t.GetTypeParam().name()); + } + }); +} + +Type TypeInferenceContext::Generalize(const Type& type, + absl::Span env_types) { + // Free type vars that occur in the environment stay monomorphic. + absl::flat_hash_set env_free_vars; + for (const Type& env_type : env_types) { + SubstituteFreeVars(env_type, [&](absl::string_view var_name) -> Type { + env_free_vars.insert(var_name); + return DynType(); + }); + } + return SubstituteFreeVars(type, [&](absl::string_view var_name) -> Type { + if (env_free_vars.contains(var_name)) { + // var_name refers to the key in type_parameter_bindings_, so it is + // stable. + return TypeParamType(var_name); + } + // Type vars are named "T%"; the generic parameter is "%G", which + // can't collide with declared type parameter names and is not treated as + // an instantiated type var by InstantiateTypeParams. + auto it = + generic_type_param_names_.insert(absl::StrCat("%G", var_name.substr(2))) + .first; + return TypeParamType(*it); + }); +} + +Type TypeInferenceContext::GeneralizeWithCanonicalNames( + const Type& type) const { + absl::flat_hash_map names; + return SubstituteFreeVars(type, [&](absl::string_view var_name) -> Type { + auto [it, inserted] = names.try_emplace(var_name); + if (inserted) { + it->second = *google::protobuf::Arena::Create( + arena_, absl::StrCat("T", names.size())); + } + return TypeParamType(it->second); + }); +} + bool TypeInferenceContext::AssignabilityContext::IsAssignable(const Type& from, const Type& to) { return inference_context_.IsAssignableInternal(from, to, diff --git a/checker/internal/type_inference_context.h b/checker/internal/type_inference_context.h index c042a7990..9be7a8063 100644 --- a/checker/internal/type_inference_context.h +++ b/checker/internal/type_inference_context.h @@ -22,6 +22,8 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/node_hash_map.h" +#include "absl/container/node_hash_set.h" +#include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" @@ -121,6 +123,29 @@ class TypeInferenceContext { // Recursively apply any substitutions to the given type. Type FullySubstitute(const Type& type, bool free_to_dyn = false) const; + // Returns `type` with bound type variables fully substituted and every + // remaining free type variable that does not occur free in any of + // `env_types` replaced by a generic type parameter (Hindley-Milner + // generalization). + // + // Each subsequent `InstantiateTypeParams` call on the result mints fresh + // type variables for the generic parameters, so references to a variable + // declared with the result are inferred independently (let-polymorphism). + // This is used for lazily evaluated bindings (cel.bind, cel.@block) whose + // initializer has an unconstrained type (e.g. `[]`). + // + // `env_types` are the types of the variables in scope at the binding (e.g. + // an enclosing comprehension's iteration variable). Free type variables + // shared with them are left as is, so constraints from references to the + // binding still flow to the enclosing variables. + Type Generalize(const Type& type, absl::Span env_types = {}); + + // Like `Generalize`, but names the generic type parameters `T1`, `T2`, ... + // in order of first occurrence. Names are allocated on the arena, so the + // result does not depend on the lifetime of this context and is suitable + // for declaring variables in another type checker environment. + Type GeneralizeWithCanonicalNames(const Type& type) const; + // Replace any generic type parameters in the given type with specific type // variables. Internally, type variables are just a unique string parameter // name. @@ -200,6 +225,12 @@ class TypeInferenceContext { Type Substitute(const Type& type, const SubstitutionMap& substitutions) const; + // Fully substitutes `type`, replacing each remaining free type variable + // with the result of `replace_free_var(var_name)`. + Type SubstituteFreeVars( + const Type& type, + absl::FunctionRef replace_free_var) const; + // Types have special unification: type(A) and type(B) unify to type(dyn) if // A and B are not co-assignable. This applies at any depth of matching // parameterized types (e.g. list(type(int)) and list(type(uint)) unify to @@ -232,6 +263,11 @@ class TypeInferenceContext { // // nullopt signifies a free type variable. absl::node_hash_map type_parameter_bindings_; + // Names of generic type parameters introduced by `Generalize`. + // + // node_hash_set is used to preserve pointer stability for use with + // TypeParamType. + absl::node_hash_set generic_type_param_names_; int64_t next_type_parameter_id_ = 0; google::protobuf::Arena* arena_; bool enable_legacy_null_assignment_; diff --git a/checker/internal/type_inference_context_test.cc b/checker/internal/type_inference_context_test.cc index de241e119..10fdd84f9 100644 --- a/checker/internal/type_inference_context_test.cc +++ b/checker/internal/type_inference_context_test.cc @@ -210,6 +210,94 @@ TEST(TypeInferenceContextTest, InstantiateTypeParamsOpaque) { IsTypeParam("T%2"), IsTypeParam("T%1"))); } +TEST(TypeInferenceContextTest, GeneralizeFreeTypeVars) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + Type map_type = context.InstantiateTypeParams( + MapType(&arena, TypeParamType("K"), TypeParamType("K"))); + ASSERT_THAT(map_type, IsMapType(IsTypeParam("T%1"), IsTypeParam("T%1"))); + + Type generic = context.Generalize(map_type); + EXPECT_THAT(generic, IsMapType(IsTypeParam("%G1"), IsTypeParam("%G1"))); + + // Each instantiation mints fresh type vars, preserving links between + // positions. + EXPECT_THAT(context.InstantiateTypeParams(generic), + IsMapType(IsTypeParam("T%2"), IsTypeParam("T%2"))); + EXPECT_THAT(context.InstantiateTypeParams(generic), + IsMapType(IsTypeParam("T%3"), IsTypeParam("T%3"))); +} + +TEST(TypeInferenceContextTest, GeneralizeSubstitutesBoundTypeVars) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + Type map_type = context.InstantiateTypeParams( + MapType(&arena, TypeParamType("K"), TypeParamType("V"))); + ASSERT_THAT(map_type, IsMapType(IsTypeParam("T%1"), IsTypeParam("T%2"))); + ASSERT_TRUE(context.IsAssignable(StringType(), TypeParamType("T%1"))); + + EXPECT_THAT(context.Generalize(map_type), + IsMapType(IsTypeKind(TypeKind::kString), IsTypeParam("%G2"))); +} + +TEST(TypeInferenceContextTest, GeneralizeKeepsEnvironmentTypeVars) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + Type map_type = context.InstantiateTypeParams( + MapType(&arena, TypeParamType("K"), TypeParamType("V"))); + ASSERT_THAT(map_type, IsMapType(IsTypeParam("T%1"), IsTypeParam("T%2"))); + // An enclosing variable's type references T%1 (via a binding to T%3). + Type env_type = + context.InstantiateTypeParams(ListType(&arena, TypeParamType("E"))); + ASSERT_THAT(env_type, IsListType(IsTypeParam("T%3"))); + ASSERT_TRUE(context.IsAssignable(TypeParamType("T%1"), TypeParamType("T%3"))); + + Type env_types[] = {env_type}; + Type generic = context.Generalize(map_type, env_types); + // T%1 is shared with the environment, so it stays monomorphic. Whether it is + // reported as T%1 or T%3 depends on the binding direction. + ASSERT_THAT(generic, IsTypeKind(TypeKind::kMap)); + Type key = generic.AsMap()->GetKey(); + ASSERT_THAT(key, IsTypeKind(TypeKind::kTypeParam)); + EXPECT_TRUE(key.AsTypeParam()->name() == "T%1" || + key.AsTypeParam()->name() == "T%3") + << key.AsTypeParam()->name(); + EXPECT_THAT(generic.AsMap()->GetValue(), IsTypeParam("%G2")); + + // Instantiating keeps the shared var and mints a fresh one for the value. + Type instance = context.InstantiateTypeParams(generic); + EXPECT_EQ(instance.AsMap()->GetKey().AsTypeParam()->name(), + key.AsTypeParam()->name()); + EXPECT_THAT(instance.AsMap()->GetValue(), IsTypeParam("T%4")); +} + +TEST(TypeInferenceContextTest, GeneralizeConcreteTypeIsUnchanged) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + Type list_type = ListType(&arena, IntType()); + + EXPECT_THAT(context.Generalize(list_type), + IsListType(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, GeneralizedTypeIsIndependentOfInstances) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + Type generic = context.Generalize( + context.InstantiateTypeParams(ListType(&arena, TypeParamType("E")))); + + // Binding one instance does not affect the generic type or other instances. + Type instance1 = context.InstantiateTypeParams(generic); + Type instance2 = context.InstantiateTypeParams(generic); + ASSERT_TRUE(context.IsAssignable(ListType(&arena, IntType()), instance1)); + ASSERT_TRUE(context.IsAssignable(ListType(&arena, StringType()), instance2)); + EXPECT_THAT(context.FinalizeType(instance1), + IsListType(IsTypeKind(TypeKind::kInt))); + EXPECT_THAT(context.FinalizeType(instance2), + IsListType(IsTypeKind(TypeKind::kString))); + EXPECT_THAT(context.Generalize(generic), IsListType(IsTypeParam("%G1"))); +} + // TODO(uncreated-issue/72): Does not consider any substitutions based on type // inferences yet. TEST(TypeInferenceContextTest, OpaqueTypeAssignable) { diff --git a/checker/type_checker_builder.h b/checker/type_checker_builder.h index c2d0cbf7b..1011cd87e 100644 --- a/checker/type_checker_builder.h +++ b/checker/type_checker_builder.h @@ -167,6 +167,14 @@ class TypeCheckerBuilder { // The current options for the TypeChecker being built. virtual const CheckerOptions& options() const = 0; + // Mutable access to the options for the TypeChecker being built (e.g. to + // adjust options on a builder obtained from TypeChecker::ToBuilder()). + // + // Options that validate declarations as they are added (e.g. + // enable_type_parameter_name_validation, max_type_decl_nesting) only apply + // to declarations added after the update. + virtual CheckerOptions& mutable_options() = 0; + // Builds a new TypeChecker instance. virtual absl::StatusOr> Build() = 0; diff --git a/checker/validation_result.h b/checker/validation_result.h index 7417e9969..84b9f9645 100644 --- a/checker/validation_result.h +++ b/checker/validation_result.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -89,6 +90,17 @@ class ValidationResult { resolved_type_map_ = std::move(resolved_type_map); } + // Returns the type of the root expression with free type parameters + // preserved as named type parameters (T1, T2, ...). + // + // Only populated if the AST was checked with an explicit arena and + // CheckerOptions::use_polymorphic_bind is enabled. Has the same + // lifetime requirements as the resolved type map. + std::optional GetGenericResultType() const { + return generic_result_type_; + } + void SetGenericResultType(Type type) { generic_result_type_ = type; } + // Returns a string representation of the issues in the result suitable for // display. // @@ -108,6 +120,7 @@ class ValidationResult { private: absl_nullable std::unique_ptr ast_; TypeMap resolved_type_map_; + std::optional generic_result_type_; std::vector issues_; absl_nullable std::unique_ptr source_; }; diff --git a/compiler/BUILD b/compiler/BUILD index 2e3f7a2ba..3444e0f73 100644 --- a/compiler/BUILD +++ b/compiler/BUILD @@ -133,6 +133,34 @@ cc_test( ], ) +cc_test( + name = "free_type_param_test", + srcs = ["free_type_param_test.cc"], + deps = [ + ":compiler", + ":compiler_factory", + ":optional", + ":standard_library", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:expr", + "//common:navigable_ast", + "//common:type", + "//extensions:bindings_ext", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//testutil:test_macros", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "standard_library", srcs = ["standard_library.cc"], diff --git a/compiler/free_type_param_test.cc b/compiler/free_type_param_test.cc new file mode 100644 index 000000000..65736d9cd --- /dev/null +++ b/compiler/free_type_param_test.cc @@ -0,0 +1,437 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for type inference of expressions with free (unconstrained) type +// parameters, in particular when they are bound to a lazily evaluated +// variable (cel.bind, cel.@block). + +#include +#include +#include +#include +#include + +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/navigable_ast.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/optional.h" +#include "compiler/standard_library.h" +#include "extensions/bindings_ext.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "testutil/test_macros.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; + +// Formats a type recursively, e.g. list(list(dyn)). +// +// Leaf types are formatted with their name so that special placeholder types +// are distinguishable from dyn. +std::string DescribeType(const Type& type) { + TypeParameters params = type.GetParameters(); + if (params.empty()) { + return std::string(type.name()); + } + std::vector parts; + for (const Type& param : params) { + parts.push_back(DescribeType(param)); + } + return absl::StrCat(type.name(), "(", absl::StrJoin(parts, ", "), ")"); +} + +// Collects the ids of empty list literals in pre-order. +std::vector CollectEmptyLists(const Expr& expr) { + std::vector ids; + NavigableAst navigable_ast = NavigableAst::Build(expr); + for (const NavigableAstNode& node : + navigable_ast.Root().DescendantsPreorder()) { + if (node.expr()->has_list_expr() && + node.expr()->list_expr().elements().empty()) { + ids.push_back(node.expr()->id()); + } + } + return ids; +} + +absl::StatusOr> NewTestCompiler( + CompilerOptions options = {}) { + CEL_ASSIGN_OR_RETURN( + std::unique_ptr builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool(), + options)); + CEL_RETURN_IF_ERROR(builder->AddLibrary(StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(OptionalCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(extensions::BindingsCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->GetParserBuilder().AddLibrary(test::TestMacrosParserLibrary())); + return builder->Build(); +} + +CompilerOptions PolymorphicBindOptions() { + CompilerOptions options; + options.checker_options.use_polymorphic_bind = true; + return options; +} + +struct FreeTypeParamTestCase { + std::string expr; + // Expected DescribeType of the root resolved type, or empty if the + // expression is expected to fail type checking. + std::string expected_type; + // If non-empty, the expected DescribeType of each empty list literal in the + // checked AST (pre-order), joined with ", ". + std::string expected_empty_list_types = ""; +}; + +void PrintTo(const FreeTypeParamTestCase& test_case, std::ostream* os) { + *os << "{expr: \"" << test_case.expr << "\", expected_type: \"" + << (test_case.expected_type.empty() ? "" : test_case.expected_type) + << "\""; + if (!test_case.expected_empty_list_types.empty()) { + *os << ", expected_empty_list_types: \"" + << test_case.expected_empty_list_types << "\""; + } + *os << "}"; +} + +// Checks `test_case.expr` with the given options and compares the root type +// (and optionally the empty list literal types) with the expectations. +void CheckFreeTypeParamCase(const FreeTypeParamTestCase& test_case, + const CompilerOptions& options) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + NewTestCompiler(options)); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile(test_case.expr, "", &arena)); + + if (test_case.expected_type.empty()) { + EXPECT_FALSE(result.IsValid()); + return; + } + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + const Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + const auto& resolved = result.GetResolvedTypeMap(); + auto it = resolved.find(ast->root_expr().id()); + ASSERT_TRUE(it != resolved.end()); + EXPECT_EQ(DescribeType(it->second), test_case.expected_type); + + if (!test_case.expected_empty_list_types.empty()) { + std::vector types; + for (ExprId id : CollectEmptyLists(ast->root_expr())) { + auto type_it = resolved.find(id); + types.push_back(type_it == resolved.end() + ? "" + : DescribeType(type_it->second)); + } + EXPECT_EQ(absl::StrJoin(types, ", "), test_case.expected_empty_list_types); + } +} + +// With CheckerOptions::use_polymorphic_bind. +class FreeTypeParamTest : public testing::TestWithParam { +}; + +TEST_P(FreeTypeParamTest, Check) { + CheckFreeTypeParamCase(GetParam(), PolymorphicBindOptions()); +} + +INSTANTIATE_TEST_SUITE_P( + FreeTypeParams, FreeTypeParamTest, + testing::ValuesIn(std::vector{ + // References to a bound value with free type params are inferred + // independently. + {"cel.bind(x, [], x + [1] == [1] && x + ['a'] == ['a'])", "bool"}, + {"cel.bind(x, {}, x['foo'] == 1 || x['foo'] == '2')", "bool"}, + {"cel.bind(o, optional.none(), " + "o.orValue(1) == 1 && o.orValue('a') == 'a')", + "bool"}, + {"cel.block([[]], " + "cel.index(0) + [1] == [1] && cel.index(0) + ['a'] == ['a'])", + "bool"}, + {"cel.bind(x, [][0], [x, 1] == [1] && [x, 'a'] == ['a'])", "bool"}, + {"cel.bind(x, {}, x.foo == 1 && x['a'] == 'b')", "bool"}, + // Unconstrained references. + {"cel.bind(x, [], x)", "list(dyn)"}, + {"cel.bind(x, {}, x)", "map(dyn, dyn)"}, + {"cel.bind(o, optional.none(), o)", "optional_type(dyn)"}, + {"cel.bind(x, [][0], x)", "dyn"}, + {"cel.bind(x, {}, x.foo)", "dyn"}, + // A top-level free param does not widen the surrounding expression. + {"[cel.bind(x, [][0], x), 1]", "list(int)"}, + {"cel.block([[][0]], [cel.index(0), 1])", "list(int)"}, + // Constrained references. + {"cel.bind(x, [], x + [1])", "list(int)", + // #unused iter range, accu init. + "list(dyn), list(dyn)"}, + {"cel.bind(x, [1], x + [2])", "list(int)"}, + {"cel.bind(x, [], cel.bind(y, x + [1], y + x))", "list(int)"}, + {"optional.none().optMap(v, v).orValue(1)", "int"}, + // Free type params shared with an enclosing variable are not + // generalized (Hindley-Milner): references to x constrain y. + {"[[]].map(y, cel.bind(x, y, x + [1] == [1] && x + ['a'] == ['a']))", + ""}, + {"[[]].map(y, cel.bind(x, y, x + [1]))", "list(list(int))", + // element of the iter range, map accu init, #unused bind range. + "list(int), list(list(int)), list(dyn)"}, + // An enclosing binding that is itself generalized doesn't prevent + // generalization (each reference to x is a fresh instance). + {"cel.bind(x, [], " + "cel.bind(y, x, y + [1] == [1] && y + ['a'] == ['a']))", + "bool"}, + // Non-bind comprehensions are not generalized. + {"[].map(v, v) + [1]", "list(int)", + // iter range, accu init. + "list(int), list(int)"}, + // Free type params without bindings. + {"([] + ['ab']) + ([1] + [])", ""}, + {"[[], ['ab'], [], [1]]", "list(dyn)"}, + {"[[]] + [[[]]] + [[[[]]]]", "list(list(list(list(dyn))))"}, + {"[]", "list(dyn)"}, + {"{}", "map(dyn, dyn)"}, + {"cel.bind(x, [[]] + [[[]]] + [[[[]]]], x)", + "list(list(list(list(dyn))))"}, + })); + +// Without CheckerOptions::use_polymorphic_bind (the default), bindings are +// not generalized: references share the bound value's type parameters, and +// block indexes are finalized (free type params become dyn). +class PolymorphicBindDisabledTest + : public testing::TestWithParam {}; + +TEST_P(PolymorphicBindDisabledTest, Check) { + CheckFreeTypeParamCase(GetParam(), CompilerOptions()); +} + +INSTANTIATE_TEST_SUITE_P( + PolymorphicBindDisabled, PolymorphicBindDisabledTest, + testing::ValuesIn(std::vector{ + // The first reference binds the shared type param. + {"cel.bind(x, [], x + [1] == [1] && x + ['a'] == ['a'])", ""}, + {"cel.bind(x, {}, x['foo'] == 1 || x['foo'] == '2')", ""}, + // The block index is finalized to list(dyn), which widens. + {"cel.block([[][0]], [cel.index(0), 1])", "list(dyn)"}, + // The init node is narrowed by the (shared) reference. + {"cel.bind(x, [], x + [1])", "list(int)", + // #unused iter range, accu init. + "list(dyn), list(int)"}, + })); + +TEST(GenericResultTypeTest, EnabledViaToBuilder) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, NewTestCompiler()); + std::unique_ptr builder = compiler->ToBuilder(); + builder->GetCheckerBuilder().mutable_options().use_polymorphic_bind = true; + ASSERT_OK_AND_ASSIGN(std::unique_ptr updated, builder->Build()); + EXPECT_TRUE( + updated->GetTypeChecker().ToBuilder()->options().use_polymorphic_bind); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + updated->Compile("[]", "", &arena)); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + ASSERT_TRUE(result.GetGenericResultType().has_value()); + EXPECT_EQ(DescribeType(*result.GetGenericResultType()), "list(T1)"); + + // The type inference changes are enabled too. + ASSERT_OK_AND_ASSIGN( + result, + updated->Compile("cel.bind(x, [], x + [1] == [1] && x + ['a'] == ['a'])", + "", &arena)); + EXPECT_TRUE(result.IsValid()) << result.FormatError(); +} + +struct GenericResultTypeTestCase { + std::string expr; + // Expected DescribeType of the generic result type. + std::string expected_generic_type; + // Expected DescribeType of the root resolved type (unaffected by the + // option). + std::string expected_resolved_type; +}; + +void PrintTo(const GenericResultTypeTestCase& test_case, std::ostream* os) { + *os << "{expr: \"" << test_case.expr << "\", expected_generic_type: \"" + << test_case.expected_generic_type << "\", expected_resolved_type: \"" + << test_case.expected_resolved_type << "\"}"; +} + +class GenericResultTypeTest + : public testing::TestWithParam {}; + +TEST_P(GenericResultTypeTest, Check) { + const GenericResultTypeTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + NewTestCompiler(PolymorphicBindOptions())); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile(test_case.expr, "", &arena)); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + ASSERT_TRUE(result.GetGenericResultType().has_value()); + EXPECT_EQ(DescribeType(*result.GetGenericResultType()), + test_case.expected_generic_type); + EXPECT_EQ(DescribeType(result.GetResolvedTypeMap().at( + result.GetAst()->root_expr().id())), + test_case.expected_resolved_type); +} + +INSTANTIATE_TEST_SUITE_P( + GenericResultTypes, GenericResultTypeTest, + testing::ValuesIn(std::vector{ + {"[]", "list(T1)", "list(dyn)"}, + {"{}", "map(T1, T2)", "map(dyn, dyn)"}, + {"optional.none()", "optional_type(T1)", "optional_type(dyn)"}, + {"[[]] + [[[]]]", "list(list(list(T1)))", "list(list(list(dyn)))"}, + {"cel.bind(x, [], x)", "list(T1)", "list(dyn)"}, + {"[][0]", "T1", "dyn"}, + {"[[], {}]", "list(dyn)", "list(dyn)"}, + {"[] + [1]", "list(int)", "list(int)"}, + {"1", "int", "int"}, + })); + +TEST(GenericResultTypeTest, NotPopulatedByDefault) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, NewTestCompiler()); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("[]", "", &arena)); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_FALSE(result.GetGenericResultType().has_value()); +} + +TEST(GenericResultTypeTest, NotPopulatedWithoutArena) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + NewTestCompiler(PolymorphicBindOptions())); + ASSERT_OK_AND_ASSIGN(ValidationResult result, compiler->Compile("[]")); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_FALSE(result.GetGenericResultType().has_value()); +} + +// A variable declared with a generic result type is instantiated +// independently at each reference (e.g. composing policy variables). +TEST(GenericResultTypeTest, DeclareVariableWithGenericResultType) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + NewTestCompiler(PolymorphicBindOptions())); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult var_result, + compiler->Compile("{}", "", &arena)); + ASSERT_TRUE(var_result.GetGenericResultType().has_value()); + + std::unique_ptr builder = compiler->ToBuilder(); + // Type parameter name validation is enabled by default. + ASSERT_TRUE(builder->GetCheckerBuilder() + .options() + .enable_type_parameter_name_validation); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("v", *var_result.GetGenericResultType())), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr composed, builder->Build()); + + ASSERT_OK_AND_ASSIGN( + ValidationResult result, + composed->Compile("v['foo'] == 1 || v['foo'] == 'a'", "", &arena)); + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + + ASSERT_OK_AND_ASSIGN(result, + composed->Compile("[v, {'a': 1}]", "", &arena)); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_EQ(DescribeType(result.GetResolvedTypeMap().at( + result.GetAst()->root_expr().id())), + "list(map(string, int))"); +} + +// Bindings that reference a declared variable with a parameterized type (e.g. +// a stitched policy variable `v: list(A)`). +// +// Each reference to `v` is instantiated with fresh type variables, so the +// declared type params are never shared with a binding: a binding of `v` is +// generalized like a binding of `[]`. Type variables of an instance that flow +// into an enclosing local variable (here, the iteration variable `e`) are part +// of the environment and stay monomorphic. +struct DeclaredGenericVariableTestCase { + std::string expr; + // Expected DescribeType of the root resolved type, or empty if the + // expression is expected to fail type checking. + std::string expected_type; +}; + +void PrintTo(const DeclaredGenericVariableTestCase& test_case, + std::ostream* os) { + *os << "{expr: \"" << test_case.expr << "\", expected_type: \"" + << (test_case.expected_type.empty() ? "" : test_case.expected_type) + << "\"}"; +} + +class DeclaredGenericVariableTest + : public testing::TestWithParam {}; + +TEST_P(DeclaredGenericVariableTest, Check) { + const DeclaredGenericVariableTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + NewTestCompiler(PolymorphicBindOptions())); + std::unique_ptr builder = compiler->ToBuilder(); + TypeCheckerBuilder& checker_builder = builder->GetCheckerBuilder(); + ASSERT_THAT(checker_builder.AddVariable(MakeVariableDecl( + "v", ListType(checker_builder.arena(), TypeParamType("A")))), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr with_v, builder->Build()); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + with_v->Compile(test_case.expr, "", &arena)); + if (test_case.expected_type.empty()) { + EXPECT_FALSE(result.IsValid()); + return; + } + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_EQ(DescribeType(result.GetResolvedTypeMap().at( + result.GetAst()->root_expr().id())), + test_case.expected_type); +} + +INSTANTIATE_TEST_SUITE_P( + DeclaredGenericVariables, DeclaredGenericVariableTest, + testing::ValuesIn(std::vector{ + // References to v are already independent. + {"v + [1] == [1] && v + ['a'] == ['a']", "bool"}, + // Bindings of v are generalized. + {"cel.bind(x, v, x + [1] == [1] && x + ['a'] == ['a'])", "bool"}, + {"cel.block([v], " + "cel.index(0) + [1] == [1] && cel.index(0) + ['a'] == ['a'])", + "bool"}, + {"cel.bind(x, v, cel.bind(y, x, y + [1] == [1] && y + ['a'] == ['a']))", + "bool"}, + // The element type of this instance of v is shared with the iteration + // variable e, so a binding of e is not generalized. + {"v.map(e, cel.bind(x, e, x + 1))", "list(int)"}, + {"v.map(e, cel.bind(x, e, x == 1 && x == 'a'))", ""}, + })); + +} // namespace +} // namespace cel