From 18c08e5efaec9d0f467489beeb55f592555acc6b Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 15:05:53 +0000 Subject: [PATCH 01/12] yeast: Parse query choices A choice is a collection of separate queries. These will be attempted in turn, and the first one that succeeds is picked. (This differs from the choice operator in tree-sitter, which can match multiple times.) If a capture is present in all alternatives of a choice, then it is directly available in the rule body. If it's absent in some branches, it becomes an Option type. Likewise if repetition is present, the capture can end up with a Vec type (even if it is guaranteed to be present in some alternatives). It is an error to mix raw and translated captures across alternatives. In this case, then only sensible thing to do is to use a raw capture and manually translate things later. Finally, the check that root captures are raw (because the alternative would lead to infinite recursion) is extended to support choice operators as well. --- shared/yeast-macros/src/ast.rs | 5 ++ shared/yeast-macros/src/rule_parse.rs | 104 ++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/shared/yeast-macros/src/ast.rs b/shared/yeast-macros/src/ast.rs index f534fe4d83a7..36e138e1970c 100644 --- a/shared/yeast-macros/src/ast.rs +++ b/shared/yeast-macros/src/ast.rs @@ -78,6 +78,8 @@ pub(crate) enum Pattern { pattern: Box, cardinality: Cardinality, }, + /// A choice that tries its alternatives from left to right. + Choice { alternatives: Vec }, /// An ordered sequence of patterns within a field. Sequence(Vec), /// A repeated or optional pattern. @@ -92,6 +94,9 @@ impl Pattern { match self { Pattern::Any { .. } | Pattern::Node { .. } | Pattern::Unnamed(_) => true, Pattern::Capture { cardinality, .. } => *cardinality == Cardinality::SINGLE, + Pattern::Choice { alternatives } => { + alternatives.iter().all(Pattern::matches_single_node) + } Pattern::Sequence(_) | Pattern::Repeated { .. } => false, } } diff --git a/shared/yeast-macros/src/rule_parse.rs b/shared/yeast-macros/src/rule_parse.rs index 7339a431134b..babde73b1ce2 100644 --- a/shared/yeast-macros/src/rule_parse.rs +++ b/shared/yeast-macros/src/rule_parse.rs @@ -1,11 +1,13 @@ use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree}; -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::iter::Peekable; use crate::ast::{Cardinality, Pattern, Replacement, ReturnAnnotation, Rule}; type Tokens = Peekable; type Result = std::result::Result; +/// Maps each capture name to whether it uses `@@` and its identifier for diagnostics. +type CaptureNames = BTreeMap; pub(crate) fn parse_pattern(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); @@ -16,13 +18,13 @@ pub(crate) fn parse_pattern(input: TokenStream) -> Result { "unexpected token after query", )); } - validate_unique_capture_names(&pattern, &mut BTreeSet::new())?; + validate_unique_capture_names(&pattern, &mut CaptureNames::new())?; Ok(pattern) } fn validate_unique_capture_names( pattern: &Pattern, - capture_names: &mut BTreeSet, + capture_names: &mut CaptureNames, ) -> Result<()> { match pattern { Pattern::Any { .. } | Pattern::Unnamed(_) => {} @@ -31,10 +33,12 @@ fn validate_unique_capture_names( validate_unique_capture_names(pattern, capture_names)?; } } - Pattern::Capture { name, pattern, .. } => { + Pattern::Capture { + name, raw, pattern, .. + } => { validate_unique_capture_names(pattern, capture_names)?; let name_string = name.to_string(); - if !capture_names.insert(name_string.clone()) { + if capture_names.contains_key(&name_string) { return Err(syn::Error::new_spanned( name, format!( @@ -43,6 +47,30 @@ fn validate_unique_capture_names( ), )); } + capture_names.insert(name_string, (*raw, name.clone())); + } + Pattern::Choice { alternatives } => { + let mut unified_names = capture_names.clone(); + for alternative in alternatives { + let mut alternative_names = capture_names.clone(); + validate_unique_capture_names(alternative, &mut alternative_names)?; + for (name, (raw, identifier)) in alternative_names { + if let Some((unified_raw, _)) = unified_names.get(&name) { + if *unified_raw != raw { + return Err(syn::Error::new_spanned( + identifier, + format!( + "capture `{name}` must consistently use `@` or `@@` \ + across query choice alternatives" + ), + )); + } + } else { + unified_names.insert(name, (raw, identifier)); + } + } + } + *capture_names = unified_names; } Pattern::Sequence(patterns) => { for pattern in patterns { @@ -60,15 +88,7 @@ pub(crate) fn parse_rule(input: TokenStream) -> Result { let (pattern_and_guard, replacement) = split_arrow(input)?; let (pattern, guard) = split_guard(pattern_and_guard)?; let pattern = parse_pattern(pattern)?; - if let Pattern::Capture { - name, raw: false, .. - } = &pattern - { - return Err(syn::Error::new_spanned( - name, - "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated", - )); - } + validate_rule_root_captures(&pattern)?; Ok(Rule { pattern, guard, @@ -76,6 +96,25 @@ pub(crate) fn parse_rule(input: TokenStream) -> Result { }) } +fn validate_rule_root_captures(pattern: &Pattern) -> Result<()> { + match pattern { + Pattern::Capture { + name, raw: false, .. + } => Err(syn::Error::new_spanned( + name, + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated", + )), + Pattern::Capture { pattern, .. } => validate_rule_root_captures(pattern), + Pattern::Choice { alternatives } => { + for alternative in alternatives { + validate_rule_root_captures(alternative)?; + } + Ok(()) + } + _ => Ok(()), + } +} + fn parse_pattern_with_capture(tokens: &mut Tokens) -> Result { let pattern = parse_pattern_atom(tokens)?; maybe_capture(tokens, pattern, Cardinality::SINGLE) @@ -92,6 +131,10 @@ fn parse_pattern_atom(tokens: &mut Tokens) -> Result { let mut inner = group.stream().into_iter().peekable(); parse_parenthesized_pattern(&mut inner, group.span()) } + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Bracket => { + let group = expect_group(tokens, Delimiter::Bracket)?; + parse_choice(group) + } Some(TokenTree::Ident(identifier)) if *identifier == "_" => { tokens.next(); Ok(Pattern::Any { @@ -101,11 +144,29 @@ fn parse_pattern_atom(tokens: &mut Tokens) -> Result { Some(TokenTree::Literal(_)) => Ok(Pattern::Unnamed(expect_string_literal(tokens)?)), Some(token) => Err(syn::Error::new_spanned( token.clone(), - "expected `(`, `_`, or string literal in query", + "expected `(`, `[`, `_`, or string literal in query", )), } } +fn parse_choice(group: proc_macro2::Group) -> Result { + let span = group.span(); + let mut tokens = group.stream().into_iter().peekable(); + let mut alternatives = Vec::new(); + while tokens.peek().is_some() { + let pattern = parse_pattern_atom(&mut tokens)?; + let pattern = maybe_repeat(&mut tokens, pattern)?; + alternatives.push(pattern); + } + if alternatives.is_empty() { + return Err(syn::Error::new( + span, + "choice must contain at least one query", + )); + } + Ok(Pattern::Choice { alternatives }) +} + fn parse_parenthesized_pattern(tokens: &mut Tokens, span: Span) -> Result { match tokens.peek() { None => Err(syn::Error::new(span, "empty query groups are not allowed")), @@ -148,7 +209,12 @@ fn parse_parenthesized_pattern(tokens: &mut Tokens, span: Span) -> Result { + Some(TokenTree::Group(group)) + if matches!( + group.delimiter(), + Delimiter::Parenthesis | Delimiter::Bracket + ) => + { let patterns = parse_pattern_list(tokens)?; finish_sequence(tokens, patterns, span) } @@ -222,6 +288,12 @@ fn push_field_pattern(fields: &mut Vec<(String, Pattern)>, name: String, pattern fn parse_pattern_list(tokens: &mut Tokens) -> Result> { let mut patterns = Vec::new(); while tokens.peek().is_some() { + if peek_is_group(tokens, Delimiter::Bracket) { + let pattern = parse_pattern_atom(tokens)?; + patterns.push(maybe_repeat(tokens, pattern)?); + continue; + } + if peek_is_group(tokens, Delimiter::Parenthesis) { let group = expect_group(tokens, Delimiter::Parenthesis)?; let mut inner = group.stream().into_iter().peekable(); From 0e7bfb3bcdfd5e07fd41e7d20283b7cdc386cf5c Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 15:15:47 +0000 Subject: [PATCH 02/12] yeast: Test query choice parsing Adds a few tests for various aspects of choice parsing. --- shared/yeast-macros/src/rule_parse.rs | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/shared/yeast-macros/src/rule_parse.rs b/shared/yeast-macros/src/rule_parse.rs index babde73b1ce2..d213ffb23d3e 100644 --- a/shared/yeast-macros/src/rule_parse.rs +++ b/shared/yeast-macros/src/rule_parse.rs @@ -640,6 +640,60 @@ mod tests { assert!(matches!(rule.replacement, Replacement::Templates(_))); } + #[test] + fn parses_quantified_query_choice_alternatives() { + let pattern = parse_pattern(quote! { + [ + (identifier)+ @items + (integer)? + ","* + ] + }) + .unwrap(); + + let Pattern::Choice { alternatives } = pattern else { + panic!("expected query choice"); + }; + assert!(matches!( + &alternatives[0], + Pattern::Capture { + cardinality: Cardinality::ONE_OR_MORE, + .. + } + )); + assert!(matches!( + &alternatives[1], + Pattern::Repeated { + cardinality: Cardinality::OPTIONAL, + .. + } + )); + assert!(matches!( + &alternatives[2], + Pattern::Repeated { + cardinality: Cardinality::ZERO_OR_MORE, + .. + } + )); + } + + #[test] + fn rejects_mixed_capture_modes_in_query_choice() { + for pattern in [ + quote!([(identifier) @value (integer) @@value]), + quote!([(identifier) @@value (integer) @value]), + ] { + let result = parse_pattern(pattern); + let Err(error) = result else { + panic!("expected mixed capture modes to be rejected"); + }; + assert_eq!( + error.to_string(), + "capture `value` must consistently use `@` or `@@` across query choice alternatives" + ); + } + } + #[test] fn rejects_eager_capture_of_rule_root() { let result = parse_rule(quote! { @@ -656,6 +710,25 @@ mod tests { ); } + #[test] + fn rejects_eager_capture_in_rule_root_choice() { + let result = parse_rule(quote! { + [ + (identifier) @name + (integer) + ] + => + (identifier "replacement") + }); + let Err(error) = result else { + panic!("expected eager capture in rule-root choice to be rejected"); + }; + assert_eq!( + error.to_string(), + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated" + ); + } + #[test] fn repeated_capture_owns_its_cardinality() { let pattern = parse_pattern(quote!((array (identifier)* @items))).unwrap(); @@ -798,6 +871,31 @@ mod tests { )); } + #[test] + fn parses_choice_led_repeated_sequence() { + let pattern = parse_pattern(quote!((root ([(identifier) (integer)] ",")*))).unwrap(); + let Pattern::Node { fields, .. } = pattern else { + panic!("expected root node"); + }; + let Pattern::Sequence(children) = &fields[0].1 else { + panic!("expected child sequence"); + }; + let Pattern::Repeated { + pattern: repeated, .. + } = &children[0] + else { + panic!("expected repetition"); + }; + assert!(matches!( + repeated.as_ref(), + Pattern::Sequence(patterns) + if matches!( + patterns.as_slice(), + [Pattern::Choice { .. }, Pattern::Unnamed(comma)] if comma == "," + ) + )); + } + #[test] fn rejects_unconsumed_node_pattern_tokens() { for query in [quote!((root (foo . (bar) @x))), quote!((root(bar @ x)))] { From ac9a7506434fd81d40a9964dec71a5f61d3f9439 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 16:03:04 +0000 Subject: [PATCH 03/12] yeast: Match query choices Adds the logic that handles matching query choices, extending the QueryNode type with a matching Choice branch. --- shared/yeast/src/lib.rs | 8 +++- shared/yeast/src/query.rs | 78 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 917a39e81f5d..8c8d1b81a5fb 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -1287,8 +1287,12 @@ impl<'a, C> RuleIndex<'a, C> { let mut by_kind: BTreeMap<&'static str, Vec<&'a Rule>> = BTreeMap::new(); let mut wildcard = Vec::new(); for rule in rules { - match rule.query.root_kind() { - Some(kind) => by_kind.entry(kind).or_default().push(rule), + match rule.query.root_kinds() { + Some(kinds) => { + for kind in kinds { + by_kind.entry(kind).or_default().push(rule); + } + } None => wildcard.push(rule), } } diff --git a/shared/yeast/src/query.rs b/shared/yeast/src/query.rs index c8785ab00b27..c724d137a39e 100644 --- a/shared/yeast/src/query.rs +++ b/shared/yeast/src/query.rs @@ -16,6 +16,9 @@ pub enum QueryNode { UnnamedNode { kind: &'static str, }, + Choice { + alternatives: Vec, + }, Capture { capture: &'static str, node: Box, @@ -23,13 +26,32 @@ pub enum QueryNode { } impl QueryNode { - /// Returns the root node kind this query matches, if it's specific. - /// Returns None for wildcards (Any) and captures wrapping wildcards. + /// Returns the root node kind this query matches when it has exactly one. + /// Choices whose alternatives share a root kind return that kind; wildcards + /// and choices spanning multiple root kinds return `None`. pub fn root_kind(&self) -> Option<&'static str> { + let kinds = self.root_kinds()?; + (kinds.len() == 1).then_some(kinds[0]) + } + + /// Returns all root node kinds this query can match when they are known. + /// Returns `None` for wildcards and choices containing a wildcard. + pub(crate) fn root_kinds(&self) -> Option> { match self { - QueryNode::Node { kind, .. } => Some(kind), - QueryNode::UnnamedNode { kind } => Some(kind), - QueryNode::Capture { node, .. } => node.root_kind(), + QueryNode::Node { kind, .. } => Some(vec![kind]), + QueryNode::UnnamedNode { kind } => Some(vec![kind]), + QueryNode::Choice { alternatives } => { + let mut kinds = Vec::new(); + for alternative in alternatives { + for kind in alternative.root_kinds()? { + if !kinds.contains(&kind) { + kinds.push(kind); + } + } + } + Some(kinds) + } + QueryNode::Capture { node, .. } => node.root_kinds(), QueryNode::Any { .. } => None, } } @@ -41,6 +63,9 @@ pub enum QueryListElem { children: Vec, rep: Rep, }, + Choice { + alternatives: Vec>, + }, SingleNode(QueryNode), } @@ -60,6 +85,9 @@ impl QueryNode { QueryNode::Any { match_unnamed } => !match_unnamed, QueryNode::Node { .. } => true, QueryNode::UnnamedNode { .. } => false, + QueryNode::Choice { alternatives } => { + alternatives.iter().all(QueryNode::matches_named_only) + } QueryNode::Capture { node, .. } => node.matches_named_only(), } } @@ -105,6 +133,14 @@ impl QueryNode { .ok_or_else(|| format!("unnamed Node kind {kind} not found in language"))?; Ok(node.kind == target_kind) } + QueryNode::Choice { alternatives } => { + if let Some(captures) = match_choice(alternatives, ast, node)? { + matches.merge(&captures); + Ok(true) + } else { + Ok(false) + } + } QueryNode::Capture { capture, node: sub_query, @@ -119,6 +155,20 @@ impl QueryNode { } } +fn match_choice( + alternatives: &[QueryNode], + ast: &Ast, + node: Id, +) -> Result, String> { + for alternative in alternatives { + let mut captures = Captures::new(); + if alternative.do_match(ast, node, &mut captures)? { + return Ok(Some(captures)); + } + } + Ok(None) +} + fn match_children<'a>( child_matchers: impl Iterator, ast: &Ast, @@ -177,6 +227,24 @@ impl QueryListElem { Ok(true) } } + QueryListElem::Choice { alternatives } => { + let start = remaining_children.clone(); + for alternative in alternatives { + let mut candidate_children = start.clone(); + let mut candidate_captures = Captures::new(); + if match_children( + alternative.iter(), + ast, + &mut candidate_children, + &mut candidate_captures, + )? { + *remaining_children = candidate_children; + matches.merge(&candidate_captures); + return Ok(true); + } + } + Ok(false) + } QueryListElem::SingleNode(sub_query) => { // Forward-scan semantics: advance through the iterator until // we find a child that matches `sub_query`. Skip ahead past From c9c0bfdd62227e29e3c3ba53466ef867a3cf1b59 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 16:17:13 +0000 Subject: [PATCH 04/12] yeast: Lower query choices Lowering is fairly straightforward, since we have essentially the same structure in the AST and in the QueryNode type. --- shared/yeast-macros/src/lower.rs | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/shared/yeast-macros/src/lower.rs b/shared/yeast-macros/src/lower.rs index dcb232a38052..ed564304673d 100644 --- a/shared/yeast-macros/src/lower.rs +++ b/shared/yeast-macros/src/lower.rs @@ -159,6 +159,14 @@ impl Pattern { } } } + Pattern::Choice { alternatives } => { + let alternatives = alternatives.iter().map(Pattern::lower); + quote! { + yeast::query::QueryNode::Choice { + alternatives: vec![#(#alternatives),*], + } + } + } Pattern::Sequence(_) | Pattern::Repeated { .. } => { unreachable!("sequence and repeated patterns cannot match a single node") } @@ -168,6 +176,9 @@ impl Pattern { fn lower_list(&self) -> Vec { match self { Pattern::Sequence(patterns) => patterns.iter().flat_map(Pattern::lower_list).collect(), + Pattern::Choice { .. } if !self.matches_single_node() => { + vec![self.lower_list_choice()] + } Pattern::Capture { name, pattern, @@ -226,6 +237,21 @@ impl Pattern { } } + fn lower_list_choice(&self) -> TokenStream { + let Pattern::Choice { alternatives } = self else { + unreachable!("expected query choice pattern"); + }; + let alternatives = alternatives.iter().map(|alternative| { + let children = alternative.lower_list(); + quote! { vec![#(#children),*] } + }); + quote! { + yeast::query::QueryListElem::Choice { + alternatives: vec![#(#alternatives),*], + } + } + } + fn captures(&self) -> Vec { let mut captures = Vec::new(); self.collect_captures(Cardinality::SINGLE, &mut captures); @@ -254,6 +280,9 @@ impl Pattern { cardinality, }); } + Pattern::Choice { alternatives } => { + captures.extend(unify_choice_captures(alternatives, cardinality)); + } Pattern::Sequence(patterns) => { for pattern in patterns { pattern.collect_captures(cardinality, captures); @@ -269,6 +298,41 @@ impl Pattern { } } +fn unify_choice_captures( + alternatives: &[Pattern], + parent: Cardinality, +) -> Vec { + let alternative_count = alternatives.len(); + let mut order = Vec::new(); + let mut by_name = std::collections::BTreeMap::>::new(); + + for alternative in alternatives { + for capture in alternative.captures() { + let name = capture.name.to_string(); + if !by_name.contains_key(&name) { + order.push(name.clone()); + } + by_name.entry(name).or_default().push(capture); + } + } + + order + .into_iter() + .map(|name| { + let captures = by_name.remove(&name).unwrap(); + let present_in_every_alternative = captures.len() == alternative_count; + let mut capture = captures[0].clone(); + for other in captures.iter().skip(1) { + capture.cardinality.multiple |= other.cardinality.multiple; + capture.cardinality.required &= other.cardinality.required; + } + capture.cardinality.required &= present_in_every_alternative; + capture.cardinality = combine_cardinality(parent, capture.cardinality); + capture + }) + .collect() +} + fn combine_cardinality(parent: Cardinality, child: Cardinality) -> Cardinality { Cardinality { multiple: parent.multiple || child.multiple, From ddd1bd8e033957909b8bdfd0ebfc256975adc47c Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 16:24:39 +0000 Subject: [PATCH 05/12] yeast: Test query choice behavior --- shared/yeast/tests/test.rs | 221 +++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 11c1b1996ee6..f0e32fe699b9 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -415,6 +415,96 @@ fn test_query_no_match() { assert!(!matched); } +#[test] +fn test_query_choice_checks_presence() { + for input in ["x = 1", "x = y"] { + let ast = parse_input(input); + let mut cursor = AstCursor::new(&ast); + cursor.goto_first_child(); + let assignment_id = cursor.node_id(); + + let query = yeast::query!( + (assignment + right: [ + (integer) + (identifier) + ] + ) + ); + let mut captures = yeast::captures::Captures::new(); + assert!( + query.do_match(&ast, assignment_id, &mut captures).unwrap(), + "query choice should match {input}" + ); + } +} + +#[test] +fn test_repeated_query_choice_accumulates_present_captures() { + let ast = parse_input("[x, 1, y]"); + + let mut cursor = AstCursor::new(&ast); + cursor.goto_first_child(); + let array_id = cursor.node_id(); + + let query = yeast::query!( + (array + [ + (identifier) @names + (integer) + ]* + ) + ); + let mut captures = yeast::captures::Captures::new(); + assert!(query.do_match(&ast, array_id, &mut captures).unwrap()); + assert_eq!(captures.get_all("names").len(), 2); +} + +#[test] +fn test_query_choice_accepts_quantified_alternative() { + let ast = parse_input("[x, y]"); + + let mut cursor = AstCursor::new(&ast); + cursor.goto_first_child(); + let array_id = cursor.node_id(); + + let query = yeast::query!( + (array + [ + (identifier)+ @names + (integer) + ] + ) + ); + let mut captures = yeast::captures::Captures::new(); + assert!(query.do_match(&ast, array_id, &mut captures).unwrap()); + assert_eq!(captures.get_all("names").len(), 2); +} + +#[test] +fn test_query_sequence_can_start_with_choice() { + let ast = parse_input("[x, 1]"); + + let mut cursor = AstCursor::new(&ast); + cursor.goto_first_child(); + let array_id = cursor.node_id(); + + let query = yeast::query!( + (array + ( + [ + (identifier) @value + (integer) @value + ] + "," + ) + ) + ); + let mut captures = yeast::captures::Captures::new(); + assert!(query.do_match(&ast, array_id, &mut captures).unwrap()); + assert_eq!(captures.get_all("value").len(), 1); +} + #[test] fn test_query_skips_extras_in_positional_match() { // Regression test: positional wildcards `(_)` must not bind to @@ -1673,6 +1763,137 @@ fn test_raw_capture_marker_explicit_translate() { ); } +#[test] +fn test_query_choice_makes_branch_specific_captures_optional() { + fn rules() -> Vec { + vec![ + yeast::rule!( + (program (_)* @stmts) + => + (program stmt: {stmts}) + ), + yeast::rule!( + (assignment + left: (_) @left + right: [ + (integer) @number + (identifier) @@name + ]) + => + assignment { + let right = match (number, name) { + (Some(number), None) => number, + (None, Some(name)) => { + let spelling = ctx.source_text(name); + tree!((identifier #{spelling})) + } + _ => return Err("exactly one alternative should match".to_string()), + }; + tree!((assignment left: {left} right: {right})) + } + ), + yeast::rule!((identifier) => (identifier "ID")), + yeast::rule!((integer) => (integer "INT")), + ] + } + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let phases = vec![Phase::new("translate", rules())]; + let runner: Runner = Runner::with_schema(lang, &schema, &phases); + + for (input, expected_right) in [("x = 1", "integer \"INT\""), ("x = y", "identifier \"y\"")] { + let ast = runner.run(input).unwrap(); + let dump = dump_ast(&ast, ast.get_root(), input); + assert!( + dump.contains(expected_right), + "unexpected dump for {input}: {dump}" + ); + } +} + +#[test] +fn test_query_choice_keeps_shared_capture_required() { + fn rules() -> Vec { + vec![ + yeast::rule!( + (program (_)* @stmts) + => + (program stmt: {stmts}) + ), + yeast::rule!( + (assignment + left: (_) @left + right: [ + (integer) @value + (identifier) @value + ]) + => + assignment { + let value: yeast::OutputId = value; + tree!((assignment left: {left} right: {value})) + } + ), + yeast::rule!((identifier) => (identifier "ID")), + yeast::rule!((integer) => (integer "INT")), + ] + } + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let phases = vec![Phase::new("translate", rules())]; + let runner: Runner = Runner::with_schema(lang, &schema, &phases); + + for input in ["x = 1", "x = y"] { + runner.run(input).unwrap(); + } +} + +#[test] +fn test_query_choice_promotes_mixed_multiplicity_to_repeated() { + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let rules: Vec = vec![ + yeast::rule!( + (program (_)* @stmts) + => + (program stmt: {stmts}) + ), + yeast::rule!( + (assignment + left: [ + (identifier) @values + (left_assignment_list (identifier)* @values) + ] + right: (_) @right) + => + assignment { + let values: Vec = values; + tree!((assignment + left: (left_assignment_list item: {values}) + right: {right})) + } + ), + yeast::rule!((identifier) => (identifier "ID")), + yeast::rule!((integer) => (integer "INT")), + ]; + let phases = vec![Phase::new("translate", rules)]; + let runner: Runner = Runner::with_schema(lang, &schema, &phases); + + for (input, expected_count) in [("x = 1", 1), ("x, y = 1", 2)] { + let ast = runner.run(input).unwrap(); + let dump = dump_ast(&ast, ast.get_root(), input); + assert_eq!( + dump.matches("identifier \"ID\"").count(), + expected_count, + "unexpected dump for {input}: {dump}" + ); + } +} + // ---- Cursor tests ---- #[test] From 54ed04b06b2aa3d47aaadabb8276c3c2e9fc9079 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 16:29:26 +0000 Subject: [PATCH 06/12] yeast: Add documentation for query choices --- shared/yeast-macros/src/lib.rs | 4 ++++ shared/yeast/doc/yeast.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 868ebe15cdff..cbe27198cfc3 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -26,6 +26,7 @@ mod template_parse; /// (kind)* @capture - capture each repeated named-node match /// "literal"* @capture - capture each repeated unnamed-token match /// (kind)? - zero or one named node +/// [(pattern) (pattern)] - query choice: try alternatives in order /// ``` /// /// Named fields and bare child patterns may be intermixed in any order. @@ -161,6 +162,9 @@ pub fn trees(input: TokenStream) -> TokenStream { /// - `@name` (no quantifier) → `name: OutputId` /// - `@name` (after `*`/`+`) → `name: Vec` /// - `@name` (after `?`) → `name: Option` +/// - captures in a query choice are unified by name: a capture in every branch +/// remains required, a missing/optional capture becomes optional, and any +/// repeated occurrence makes the binding a vector /// /// Raw `@@name` captures use the corresponding input `Id` types instead. /// diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 99d3424c1a52..10230e644541 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -178,6 +178,36 @@ Single-pattern and empty query groups are rejected as redundant or meaningless. This does not affect the empty output template `()`, which intentionally means “emit no replacement nodes.” +### Query choices + +A query choice tries its alternatives from left to right and merges captures +from the selected alternative: + +```rust +[ + (integer) @number + (identifier) @@name +] +``` + +An alternative may contain any query pattern. Parenthesize a sequence of +sibling patterns to use it as one alternative: + +```rust +[ + ( + (item)* @items + (separator) @separator + ) + (single_item) @item +] +``` + +Capture bindings are unified by name. A capture present in every alternative +remains required; one missing or optional in any alternative becomes optional; +and any repeated occurrence makes the combined binding a vector. The same name +must consistently use `@` or `@@`. + ## Template language Templates construct new AST nodes using the `tree!` and `trees!` macros. From 0b2d2cef6ac8e27d2fe6927213654d4365203986 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 4 Nov 2026 13:17:42 +0000 Subject: [PATCH 07/12] Unified: Remove redundant `declReferenceExpr` rule This one is essentially identical to the rule that directly follows it, so we can just get rid of it. --- unified/extractor/src/languages/swift/swift.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index af05d93ad172..f8873734fe69 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -206,9 +206,6 @@ fn translation_rules() -> Vec> { => (unsupported_node) ), - rule!((declReferenceExpr baseName: (identifier) @@name) => expr { - tree!((identifier #{name})) - }), // A bare name reference (`x`), and an operator used as a value (`+` in // `reduce(0, +)`), are both `declReferenceExpr`; its `baseName` is the // referenced identifier / operator symbol. From f0f22ab3b0b16f2541993885b14624fe711cb15e Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 4 Nov 2026 13:18:05 +0000 Subject: [PATCH 08/12] Unified: Consolidate unresolved operator rules --- unified/extractor/src/languages/swift/swift.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index f8873734fe69..2df3294ce953 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -260,9 +260,15 @@ fn translation_rules() -> Vec> { // to an opaque `unsupported_node`. These bare nodes only occur inside an // unresolved sequence — folded forms are handled by the dedicated rules // above (assignment) and below (`ternaryExpr`). - rule!((assignmentExpr) @@op => (infix_operator #{op})), - rule!((unresolvedAsExpr) @@op => (infix_operator #{op})), - rule!((unresolvedIsExpr) @@op => (infix_operator #{op})), + rule!( + [ + (assignmentExpr) + (unresolvedAsExpr) + (unresolvedIsExpr) + ] @@op + => + (infix_operator #{op}) + ), // The ternary is a three-part operator (`? thenExpr :`) that *wraps* a // nested expression. Splice it into `?`, the then-expression, `:` so the // then-expression survives as a real (traversable) operand rather than From 42f0fc931c6301e0806492eb6a05a18da4bbf17f Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 4 Nov 2026 13:19:15 +0000 Subject: [PATCH 09/12] Unified: Simplify enum case raw values An optional capture takes care of the potentially missing value (meaning we don't need a separate rule for it). --- unified/extractor/src/languages/swift/swift.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 2df3294ce953..3525102b3a22 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -539,7 +539,9 @@ fn translation_rules() -> Vec> { } ), rule!( - (enumCaseElement name: @@name rawValue: (initializerClause value: @val)) + (enumCaseElement + name: @@name + rawValue: (initializerClause value: @val)?) => (variable_declaration modifier: {ctx.outer_modifiers.clone()} @@ -548,15 +550,6 @@ fn translation_rules() -> Vec> { pattern: (identifier #{name}) value: {val}) ), - rule!( - (enumCaseElement name: @@name) - => - (variable_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: [(modifier "chained_declaration") if { ctx.is_chained }] - modifier: (modifier "enum_case") - pattern: (identifier #{name})) - ), // Enum cases. A single `case` declaration may carry modifiers // (e.g. `indirect`) and list several comma-separated elements; each // becomes its own declaration carrying those shared modifiers, and From 756169140c9c343199f40b7202830840f9326676 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 4 Nov 2026 13:19:39 +0000 Subject: [PATCH 10/12] Unified: Consolidate conditional pattern rules I'm pleasantly surprised that the new syntax for template choices supports having a Rust block as an alternative. We may need to annotate these with the "return type" at some point, in order to make type checking work out, but that can wait. --- .../extractor/src/languages/swift/swift.rs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 3525102b3a22..f66e79b190e6 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -922,13 +922,23 @@ fn translation_rules() -> Vec> { ), // A single case item unwraps to its pattern, possibly boxed in conditional_pattern rule!( - (switchCaseItem - pattern: @p - whereClause: (whereClause condition: @cond)) + [ + (switchCaseItem + pattern: @pattern + whereClause: (whereClause condition: @condition)?) + (catchItem + pattern: @pattern + whereClause: (whereClause condition: @condition)?) + ] => - (conditional_pattern pattern: {p} condition: {cond}) + [ + (conditional_pattern + pattern: {pattern} + condition: {condition}) + if { condition.is_some() } + {pattern} + ] ), - rule!((switchCaseItem pattern: @p) => pattern { p }), // A pattern-matching condition (`if case let x = e`, `if case .foo(let x) // = e`) becomes a `pattern_guard_expr`: the matched pattern and the // scrutinee value are translated recursively. @@ -1093,18 +1103,6 @@ fn translation_rules() -> Vec> { body: {body} catch_clause: {catches}) ), - rule!( - (catchItem - pattern: @pattern - whereClause: (whereClause condition: @guard)) - => - (conditional_pattern pattern: {pattern} condition: {guard}) - ), - rule!( - (catchItem pattern: @pattern) - => - pattern {pattern} - ), // Catch block with one or more patterns (which have been translated by the catchItem rules) rule!( (catchClause From 066ab02b898a329cc03c09fca6e4d190e9cbe06f Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 22:18:48 +0000 Subject: [PATCH 11/12] Unified: Consolidate modifier rules --- unified/extractor/src/languages/swift/swift.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index f66e79b190e6..756fadbfd0d1 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1172,8 +1172,14 @@ fn translation_rules() -> Vec> { // A leading attribute (`@objc`) or access/function/member/mutation/ // ownership modifier (swift-syntax models each as a single `declModifier`) // becomes a `modifier`; its source text is the modifier spelling. - rule!((attribute) @@m => (modifier #{m})), - rule!((declModifier) @@m => (modifier #{m})), + rule!( + [ + (attribute) + (declModifier) + ] @@m + => + (modifier #{m}) + ), // Type specifiers (`inout`, `sending`, ...) are mapped to modifiers. rule!((simpleTypeSpecifier) @@m => (modifier #{m})), rule!((lifetimeTypeSpecifier) @@m => (modifier #{m})), From 4e071b229b820cacc1bccb981788698f3eea158d Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 2 Nov 2026 22:22:58 +0000 Subject: [PATCH 12/12] Unified: Consolidate unsupported root rules --- .../extractor/src/languages/swift/swift.rs | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 756fadbfd0d1..fcd927f8b8a3 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1502,35 +1502,41 @@ fn translation_rules() -> Vec> { // these roots is discarded with its parent and needs no separate rule. // Explicit empty content preserves their existing representation // instead of inheriting the matched source text. - rule!((actorDecl) => (unsupported_node "")), - rule!((borrowExpr) => (unsupported_node "")), - rule!((classRestrictionType) => (unsupported_node "")), - rule!((compositionType) => (unsupported_node "")), - rule!((consumeExpr) => (unsupported_node "")), - rule!((copyExpr) => (unsupported_node "")), - rule!((discardStmt) => (unsupported_node "")), - rule!((fallThroughStmt) => (unsupported_node "")), - rule!((ifConfigDecl) => (unsupported_node "")), - rule!((implicitlyUnwrappedOptionalType) => (unsupported_node "")), - rule!((inlineArrayType) => (unsupported_node "")), - rule!((keyPathExpr) => (unsupported_node "")), - rule!((macroDecl) => (unsupported_node "")), - rule!((metatypeType) => (unsupported_node "")), - rule!((namedOpaqueReturnType) => (unsupported_node "")), - rule!((operatorDecl) => (unsupported_node "")), - rule!((packElementExpr) => (unsupported_node "")), - rule!((packElementType) => (unsupported_node "")), - rule!((packExpansionExpr) => (unsupported_node "")), - rule!((packExpansionType) => (unsupported_node "")), - rule!((postfixIfConfigExpr) => (unsupported_node "")), - rule!((poundSourceLocation) => (unsupported_node "")), - rule!((precedenceGroupDecl) => (unsupported_node "")), - rule!((someOrAnyType) => (unsupported_node "")), - rule!((subscriptDecl) => (unsupported_node "")), - rule!((suppressedType) => (unsupported_node "")), - rule!((typeExpr) => (unsupported_node "")), - rule!((unsafeExpr) => (unsupported_node "")), - rule!((yieldStmt) => (unsupported_node "")), + rule!( + [ + (actorDecl) + (borrowExpr) + (classRestrictionType) + (compositionType) + (consumeExpr) + (copyExpr) + (discardStmt) + (fallThroughStmt) + (ifConfigDecl) + (implicitlyUnwrappedOptionalType) + (inlineArrayType) + (keyPathExpr) + (macroDecl) + (metatypeType) + (namedOpaqueReturnType) + (operatorDecl) + (packElementExpr) + (packElementType) + (packExpansionExpr) + (packExpansionType) + (postfixIfConfigExpr) + (poundSourceLocation) + (precedenceGroupDecl) + (someOrAnyType) + (subscriptDecl) + (suppressedType) + (typeExpr) + (unsafeExpr) + (yieldStmt) + ] + => + (unsupported_node "") + ), // Anything reaching this final generic handler has neither a // dedicated rule nor an explicit unsupported entry. rule!(