Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,20 @@ All notable milestones for the nera project are documented in this file.
mixed maps, arithmetic, nested generators, compound guards, record defaults,
evaluation order, captures and ordinary exceptions.

## Comprehension filters retain bindings across generators

- Names bound in a list or binary comprehension filter remain available to
later generator sources, map keys, binary sizes, filters and templates.
- Generated helpers pass the bound values forward without repeating filters;
later generator shadowing still preserves the values needed to resume outer
iterations.
- Only definitely bound filter values cross helper boundaries; skipped
short-circuit operands do not supply values. Diagnostics recognize earlier
filter bindings in conditional rematches and shadowing.
- `comprehension_filter_bindings` runs unchanged natively and on BEAM, including
mixed generators, closures, deferred bitstring tails, short-circuit filters,
conditional rematches and evaluation traces.

## Unsupported float segment widths compile as on OTP 29

- Literal float segment widths outside the supported encodings no longer reject
Expand Down
66 changes: 52 additions & 14 deletions crates/nera-ir/src/lower/list_comp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,27 +64,25 @@ impl IrBuilder {

// ── 3. Analyse free variables ──
let captured_vars = self.collect_lc_captures(&groups, body_expr);
let num_captured = captured_vars.len() as u32;

// ── 4. Pre-generate helper function names for all generators ──
let mut helpers: Vec<(AtomId, String, u32)> = Vec::new(); // (name_id, mangled, arity)
let mut helpers: Vec<(AtomId, String)> = Vec::new(); // (name_id, mangled)
let prefix = if finalize_as_binary { "bc" } else { "lc" };
for k in 0..n {
for _ in 0..n {
let id = self.lc_counter;
self.lc_counter += 1;
let name_str = format!("__{prefix}_{id}");
let mangled = format!("{}__{prefix}_{id}", self.module_name.0);
let name_id = self.fresh_atom_id();
self.atom_names.insert(name_id, name_str);
// params: iter + acc + k tails + k heads + captured
let arity = 2 + 2 * k as u32 + num_captured;
self.module_functions
.insert((name_id, arity), mangled.clone());
helpers.push((name_id, mangled, arity));
helpers.push((name_id, mangled));
}

// ── 5. Build all helper functions ──
let saved = self.save_state();
// Each completed helper records the names its filters introduced.
// Later helpers carry their values, not expressions to evaluate again.
let mut filter_bindings = Vec::new();

for k in 0..n {
if k > 0 {
Expand All @@ -97,7 +95,7 @@ impl IrBuilder {
self.next_block = 0;
self.scopes = vec![HashMap::new()];
}
self.build_lc_helper(k, n, &groups, &helpers, &captured_vars, body_expr, finalize_as_binary, &bin_flags)?;
self.build_lc_helper(k, n, &groups, &helpers, &captured_vars, body_expr, finalize_as_binary, &bin_flags, &mut filter_bindings)?;
}

self.restore_state(saved);
Expand Down Expand Up @@ -234,7 +232,8 @@ impl IrBuilder {
}

/// Capture expression reads before the generator that shadows their name.
/// Earlier generator values travel through head parameters, not captures.
/// Earlier generator and filter values travel through helper parameters,
/// not captures.
fn collect_lc_captures(
&mut self,
groups: &[GeneratorGroup<'_>],
Expand Down Expand Up @@ -313,7 +312,7 @@ impl IrBuilder {
/// Build one LC helper function (helper_k of n total).
///
/// Parameters for helper_k:
/// iter_k, acc, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…
/// iter_k, acc, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…, filter_values…
///
/// The accumulator collects elements in reverse order. When the outermost
/// helper (k==0) reaches the nil case, it reverses the accumulator via
Expand All @@ -324,13 +323,14 @@ impl IrBuilder {
k: usize,
n: usize,
groups: &[GeneratorGroup<'_>],
helpers: &[(AtomId, String, u32)], // (name_id, mangled, arity)
helpers: &[(AtomId, String)], // (name_id, mangled)
captured_vars: &[(AtomId, VarId)],
body_expr: &ast::Expr,
finalize_as_binary: bool,
bin_gen_flags: &[bool],
filter_bindings: &mut Vec<Vec<AtomId>>,
) -> Result<(), LowerError> {
let (helper_name_id, ref helper_mangled, helper_arity) = helpers[k];
let (helper_name_id, ref helper_mangled) = helpers[k];
// Each helper is its own function. Deferred tails recorded while
// building the previous helper name that helper's variables.
self.match_contexts.clear();
Expand All @@ -355,6 +355,18 @@ impl IrBuilder {
self.bind_var(*name, param);
}

// Preserve one slot per binding layer, even if a later generator
// shadows the same name. Resuming an outer helper needs its old value.
let filter_params: Vec<Vec<(AtomId, VarId)>> = filter_bindings.iter()
.map(|names| names.iter().map(|name| (*name, self.fresh_var())).collect())
.collect();
let carried_filters: Vec<VarId> = filter_params.iter().flatten()
.map(|(_, var)| *var).collect();
let helper_arity = 2 + 2 * k as u32 + env_params.len() as u32
+ carried_filters.len() as u32;
self.module_functions.insert((helper_name_id, helper_arity), helper_mangled.clone());
let mut new_filter_names = Vec::new();

// ── Nil block ──
let nil_block = self.fresh_block_with_id();
self.start_block(nil_block);
Expand Down Expand Up @@ -390,6 +402,7 @@ impl IrBuilder {
chain_args.push(*item);
}
chain_args.extend_from_slice(&env_params);
chain_args.extend(filter_params[..k - 1].iter().flatten().map(|(_, var)| *var));
self.set_terminator(Terminator::TailCall(
CallTarget::Local(helpers[k - 1].0),
chain_args,
Expand All @@ -414,6 +427,9 @@ impl IrBuilder {
// segment size), so they are extracted before it runs.
for j in 0..k {
self.bind_generator_pattern(&groups[j].0, head_params[j], false);
for (name, var) in &filter_params[j] {
self.bind_var(*name, *var);
}
}

let generator_inputs = self.collect_all_bindings();
Expand Down Expand Up @@ -442,11 +458,12 @@ impl IrBuilder {
(generator_scopes, generator_match_contexts)
};

let before_filters = self.collect_all_bindings();
let filters = &groups[k].2;
let is_innermost = k == n - 1;

// Helper: build args for a self tail-call.
// self(tail_var, acc_var, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…)
// self(tail_var, acc_var, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…, filter_values…)
let build_self_args = |tail_params: &[VarId],
head_params: &[VarId],
env_params: &[VarId],
Expand All @@ -457,6 +474,7 @@ impl IrBuilder {
a.extend_from_slice(tail_params);
a.extend_from_slice(head_params);
a.extend_from_slice(env_params);
a.extend_from_slice(&carried_filters);
a
};

Expand Down Expand Up @@ -487,6 +505,7 @@ impl IrBuilder {
inner_args.extend_from_slice(&head_params);
inner_args.push(head_var);
inner_args.extend_from_slice(&env_params);
inner_args.extend_from_slice(&carried_filters);
self.set_terminator(Terminator::TailCall(
CallTarget::Local(helpers[k + 1].0),
inner_args,
Expand Down Expand Up @@ -531,6 +550,21 @@ impl IrBuilder {
self_args,
));
} else {
// Lowering may retain names from skipped short-circuit
// operands. Only definite exports can cross helper boundaries.
let mut definite = HashSet::new();
for filter in filters {
crate::warnings::collect_exported_bindings(filter, &mut definite);
}
new_filter_names = definite.into_iter()
.filter(|name| !before_filters.contains_key(name)
&& self.lookup_var(*name).is_some()).collect();
new_filter_names.sort_by_key(|name| name.0);
// Materialize deferred tails before passing them to a helper.
let new_filter_values: Vec<VarId> = new_filter_names.iter().map(|name| {
let var = self.lookup_var(*name).expect("filter binding exists");
self.materialize_bound_variable(*name, var)
}).collect();
let next_iter = self.lower_expr(groups[k + 1].1, false)?;

let mut inner_args = vec![next_iter, acc_param];
Expand All @@ -539,6 +573,8 @@ impl IrBuilder {
inner_args.extend_from_slice(&head_params);
inner_args.push(head_var);
inner_args.extend_from_slice(&env_params);
inner_args.extend_from_slice(&carried_filters);
inner_args.extend_from_slice(&new_filter_values);
self.set_terminator(Terminator::TailCall(
CallTarget::Local(helpers[k + 1].0),
inner_args,
Expand Down Expand Up @@ -627,6 +663,7 @@ impl IrBuilder {
all_params.extend_from_slice(&tail_params);
all_params.extend_from_slice(&head_params);
all_params.extend_from_slice(&env_params);
all_params.extend_from_slice(&carried_filters);
let mut helper_func = IrFunction {
name: helper_name_id,
arity: helper_arity,
Expand All @@ -643,6 +680,7 @@ impl IrBuilder {
#[cfg(test)]
crate::passes::assert_valid(&helper_func, "after helper yield insertion");
self.functions.push(helper_func);
filter_bindings.push(new_filter_names);

Ok(())
}
Expand Down
6 changes: 6 additions & 0 deletions crates/nera-ir/src/lower/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1925,6 +1925,12 @@ fn test_binary_bound_segment_lowers() {
.expect("bound binary segment values must lower to valid IR");
}

#[test]
fn test_comprehension_filter_bindings_lowers() {
lower_source(include_str!("../../../../tests/comprehension_filter_bindings.erl"))
.expect("filter bindings must reach later helpers as initialized values");
}

#[test]
fn test_generator_size_rebind_lowers() {
// NERA-52: the final binary-consumption branch must preserve initialized
Expand Down
1 change: 1 addition & 0 deletions crates/nera-ir/src/warnings/exported_vars.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@ fn check_exported_in_subexprs(
}
ast::LcQualifier::Filter(filter_expr, _) => {
check_exported_in_subexprs(filter_expr, &lc_scope, atom_names, diagnostics);
collect_exported_bindings(filter_expr, &mut lc_scope);
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions crates/nera-ir/src/warnings/match_underscore_var.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use nera_syntax::error::Diagnostic;
use nera_syntax::span::Span;

use super::ast_walkers::{collect_pattern_bindings, collect_pattern_bindings_with_spans};
use super::unbound_vars::collect_exported_bindings;

/// Check a function for `_`-prefixed variables reused in pattern position.
pub(super) fn check_match_underscore_vars_fn(
Expand Down Expand Up @@ -231,6 +232,7 @@ fn check_expr(
}
ast::LcQualifier::Filter(filter_expr, _) => {
check_expr(filter_expr, &lc_scope, atom_names, warnings);
collect_exported_bindings(filter_expr, &mut lc_scope);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/nera-ir/src/warnings/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ use nera_syntax::error::Diagnostic;
pub(crate) use ast_walkers::{collect_pattern_bindings, collect_pattern_refs, collect_expr_refs, collect_guard_refs};
pub(crate) use binary_segments::first_refused_segment_error;
pub(crate) use illegal_guards::is_guard_expr;
pub(crate) use unbound_vars::collect_exported_bindings;

// Re-export for cross-module unused record detection.
pub use unused_records::collect_used_record_names;
Expand Down
11 changes: 8 additions & 3 deletions crates/nera-ir/src/warnings/shadowed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use nera_syntax::ast::{self, AtomId};
use nera_syntax::error::Diagnostic;

use super::ast_walkers::{collect_pattern_bindings, collect_pattern_bindings_with_spans};
use super::unbound_vars::collect_exported_bindings;

/// Check a function for shadowed variables in nested scopes.
pub(super) fn check_shadowed_variables_fn(
Expand Down Expand Up @@ -107,11 +108,12 @@ fn check_shadowed_in_expr(
| ast::LcQualifier::BinGenerator(pat, source_expr, _) => {
// The source expression is evaluated in the current scope.
check_shadowed_in_expr(source_expr, &lc_scope, atom_names, warnings);
// Generator pattern vars can shadow outer scope.
// Generator patterns can shadow outer bindings,
// including earlier generators and filters.
let mut bound = Vec::new();
collect_pattern_bindings_with_spans(pat, &mut bound);
for (atom_id, span) in &bound {
if outer_scope.contains(atom_id) {
if lc_scope.contains(atom_id) {
let name = atom_names
.get(atom_id)
.map(|s| s.as_str())
Expand All @@ -126,11 +128,14 @@ fn check_shadowed_in_expr(
).with_code("shadowed_variable"));
}
}
lc_scope.insert(*atom_id);
}
// Repeated names within this pattern are comparisons,
// not shadowing of a name this same pattern introduces.
lc_scope.extend(bound.iter().map(|(name, _)| *name));
}
ast::LcQualifier::Filter(filter_expr, _) => {
check_shadowed_in_expr(filter_expr, &lc_scope, atom_names, warnings);
collect_exported_bindings(filter_expr, &mut lc_scope);
}
}
}
Expand Down
63 changes: 63 additions & 0 deletions crates/nera-ir/src/warnings/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1456,6 +1456,69 @@ fn pattern_inputs_keep_exported_bindings() {
}
}

#[test]
fn comprehension_filter_bindings_are_available_to_later_qualifiers() {
let (module, names) = parse_module(include_str!(
"../../../../tests/comprehension_filter_bindings.erl"
));
let diagnostics = analyze_module(&module, &names, None);
assert!(
!diagnostics.iter().any(|d| d.severity == nera_syntax::error::Severity::Error),
"filter bindings must be valid later inputs: {diagnostics:?}"
);
}

#[test]
fn comprehension_filter_bindings_reach_scope_sensitive_diagnostics() {
for (open, close) in [("[", "]"), ("<<", ">>")] {
for (template, qualifiers, expected) in [
("case V of N -> <<1>>; _ -> <<0>> end",
"X <- [1], begin N = X, true end", None),
("begin case V of true -> N = 1; false -> ok end, <<N>> end",
"X <- [1], begin N = X, true end", None),
("begin case V of true -> N = 1; false -> N = 1 end, <<N>> end",
"X <- [1], begin N = X, true end", None),
("begin F = fun(N) -> <<N>> end, F(V) end",
"X <- [1], begin N = X, true end", Some("shadowed_variable")),
("<<N>>", "X <- [1], begin N = X, true end, N <- [V]",
Some("shadowed_variable")),
("<<_N>>", "X <- [V], begin _N = X, true end, begin _N = 1, true end",
Some("match_underscore_var")),
("<<_N>>", "X <- [true], X orelse begin _N = 1, true end, _N <- [V]", None),
("<<N>>", "_ <- [V], {N, N} <- [{1, 1}]", None),
] {
let source = format!("-module(test). -export([f/1]). f(V) ->
{open} ({template}) || {qualifiers} {close}.");
let (module, names) = parse_module(&source);
let diagnostics = analyze_module(&module, &names, None);
assert!(
!diagnostics.iter().any(|d| d.severity == nera_syntax::error::Severity::Error),
"{source}\n{diagnostics:?}"
);
let relevant: Vec<_> = diagnostics.iter().filter_map(|d| d.code.as_deref())
.filter(|code| ["unsafe_variable", "exported_var", "unreachable_clause",
"unused_variable", "shadowed_variable", "match_underscore_var"].contains(code))
.collect();
assert_eq!(relevant, expected.into_iter().collect::<Vec<_>>(),
"{source}\n{diagnostics:?}");
}
}
}

#[test]
fn comprehension_filters_do_not_export_local_or_partial_bindings() {
for expr in [
"[N || X <- [1], begin N = X, true end, _ <- [ok]], N",
"[N || X <- [1], case X of 1 -> N = 1, true; _ -> true end, _ <- [ok]]",
"[N || X <- [1], X > 0 andalso begin N = 1, true end, _ <- [ok]]",
"[N || _ <- begin N = 1, [ok] end]",
] {
assert_unbound_pattern_input(
&format!("-module(test). -export([f/0]). f() -> {expr}."), "N",
);
}
}

#[test]
fn unbound_in_guard() {
let src = r#"
Expand Down
Loading
Loading