From bc1c106285b6489a01f5f8a7e1aa5326d4030ade Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Mon, 26 Oct 2026 09:58:34 -0700 Subject: [PATCH 1/3] bench: add mixed SQL benchmarks and identifier length sweep Add a mixed_sql workload with 22 TPC-H files and eight short statements. Measure tokenize_identifiers at lengths 1, 2, 4, 8, 16, 32, 64 and 128. Keep the existing query shape, dialect, spans and output-drop behavior. --- sqlparser_bench/benches/sqlparser_bench.rs | 93 +++++++++++++++++++++- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/sqlparser_bench/benches/sqlparser_bench.rs b/sqlparser_bench/benches/sqlparser_bench.rs index 13274f4c0b..32e223c446 100644 --- a/sqlparser_bench/benches/sqlparser_bench.rs +++ b/sqlparser_bench/benches/sqlparser_bench.rs @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -use criterion::{criterion_group, criterion_main, Criterion}; -use sqlparser::dialect::{GenericDialect, PostgreSqlDialect, SQLiteDialect}; +use criterion::{criterion_group, criterion_main, Criterion, Throughput}; +use sqlparser::dialect::{GenericDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect}; use sqlparser::keywords::Keyword; use sqlparser::parser::Parser; -use sqlparser::tokenizer::{Span, Word}; +use sqlparser::tokenizer::{Span, Tokenizer, Word}; fn basic_queries(c: &mut Criterion) { let mut group = c.benchmark_group("sqlparser-rs parsing benchmark"); @@ -297,9 +297,96 @@ fn parse_function_arg_call_chain(c: &mut Criterion) { group.finish(); } +fn tokenize_identifiers(c: &mut Criterion) { + let mut group = c.benchmark_group("tokenize_identifiers"); + let dialect = MySqlDialect {}; + for len in [1, 2, 4, 8, 16, 32, 64, 128] { + let word = "a".repeat(len); + let sql = format!("SELECT {word}, {word} FROM {word} WHERE {word} = 42"); + group.bench_function(format!("length_{len}"), |b| { + b.iter(|| { + std::hint::black_box( + Tokenizer::new(&dialect, std::hint::black_box(&sql)) + .tokenize_with_location() + .unwrap(), + ); + }); + }); + } + group.finish(); +} + +fn mixed_queries() -> Vec<&'static str> { + vec![ + include_str!("../../tests/queries/tpch/1.sql"), + include_str!("../../tests/queries/tpch/2.sql"), + include_str!("../../tests/queries/tpch/3.sql"), + include_str!("../../tests/queries/tpch/4.sql"), + include_str!("../../tests/queries/tpch/5.sql"), + include_str!("../../tests/queries/tpch/6.sql"), + include_str!("../../tests/queries/tpch/7.sql"), + include_str!("../../tests/queries/tpch/8.sql"), + include_str!("../../tests/queries/tpch/9.sql"), + include_str!("../../tests/queries/tpch/10.sql"), + include_str!("../../tests/queries/tpch/11.sql"), + include_str!("../../tests/queries/tpch/12.sql"), + include_str!("../../tests/queries/tpch/13.sql"), + include_str!("../../tests/queries/tpch/14.sql"), + include_str!("../../tests/queries/tpch/15.sql"), + include_str!("../../tests/queries/tpch/16.sql"), + include_str!("../../tests/queries/tpch/17.sql"), + include_str!("../../tests/queries/tpch/18.sql"), + include_str!("../../tests/queries/tpch/19.sql"), + include_str!("../../tests/queries/tpch/20.sql"), + include_str!("../../tests/queries/tpch/21.sql"), + include_str!("../../tests/queries/tpch/22.sql"), + "SELECT name, email FROM customers WHERE id = ?", + "INSERT INTO events (id, message) VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')", + "UPDATE accounts SET balance = balance + 10.25 WHERE id = 42", + "DELETE FROM sessions WHERE expires_at < '2026-01-01'", + "CREATE TABLE items (id BIGINT PRIMARY KEY, name VARCHAR(100), price DECIMAL(10, 2))", + "-- lookup\nSELECT \"display name\", 'literal' FROM customers WHERE active = TRUE /* end */", + "VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')", + "SELECT value FROM settings WHERE name IN ('first', 'second') ORDER BY value LIMIT 10", + ] +} + +fn mixed_sql(c: &mut Criterion) { + let mut group = c.benchmark_group("mixed_sql"); + let dialect = GenericDialect {}; + let queries = mixed_queries(); + for sql in &queries { + Parser::parse_sql(&dialect, sql).expect(sql); + } + group.throughput(Throughput::Elements(queries.len() as u64)); + group.bench_function("tokenize", |b| { + b.iter(|| { + for sql in &queries { + std::hint::black_box( + Tokenizer::new(&dialect, std::hint::black_box(sql)) + .tokenize_with_location() + .unwrap(), + ); + } + }); + }); + group.bench_function("parse", |b| { + b.iter(|| { + for sql in &queries { + std::hint::black_box( + Parser::parse_sql(&dialect, std::hint::black_box(sql)).unwrap(), + ); + } + }); + }); + group.finish(); +} + criterion_group!( benches, basic_queries, + tokenize_identifiers, + mixed_sql, word_to_ident, parse_many_identifiers, parse_compound_chain, From 9b6e281fcc9e2f55b3209bcf0e4dda753b6977da Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Fri, 23 Oct 2026 09:55:49 -0700 Subject: [PATCH 2/3] tokenizer: move peeking_take_while onto State Keep the existing character-reading helper with the cursor it operates on and update its callers. Leave its scanning behavior unchanged. --- src/tokenizer.rs | 53 +++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9276f91ded..91c5c21b2e 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -834,6 +834,21 @@ impl State<'_> { self.peekable.peek() } + /// Collect matching characters into a new string, leaving the first + /// non-matching character unread. + fn peeking_take_while(&mut self, mut predicate: impl FnMut(char) -> bool) -> String { + let mut s = String::new(); + while let Some(&ch) = self.peek() { + if predicate(ch) { + self.next(); // consume + s.push(ch); + } else { + break; + } + } + s + } + /// Return the current `Location` (line and column) pub fn location(&self) -> Location { Location { @@ -1267,8 +1282,8 @@ impl<'a> Tokenizer<'a> { line: 0, col: 0, }; - let mut s = peeking_take_while(&mut inner_state, |ch| matches!(ch, '0'..='9' | '.')); - let s2 = peeking_take_while(chars, |ch| matches!(ch, '0'..='9' | '.')); + let mut s = inner_state.peeking_take_while(|ch| matches!(ch, '0'..='9' | '.')); + let s2 = chars.peeking_take_while(|ch| matches!(ch, '0'..='9' | '.')); s += s2.as_str(); return Ok(Some(Token::Number(s, false))); } @@ -1539,7 +1554,7 @@ impl<'a> Tokenizer<'a> { let error_loc = chars.location(); chars.next(); // skip the first delimiter - peeking_take_while(chars, |ch| ch.is_whitespace()); + chars.peeking_take_while(|ch| ch.is_whitespace()); if chars.peek() != Some(&nested_quote_start) { return self.tokenizer_error( error_loc, @@ -1549,7 +1564,7 @@ impl<'a> Tokenizer<'a> { word.push(nested_quote_start.into()); word.push(self.tokenize_quoted_identifier(nested_quote_end, chars)?); word.push(nested_quote_end.into()); - peeking_take_while(chars, |ch| ch.is_whitespace()); + chars.peeking_take_while(|ch| ch.is_whitespace()); if chars.peek() != Some("e_end) { return self.tokenizer_error( error_loc, @@ -1680,7 +1695,7 @@ impl<'a> Tokenizer<'a> { // If it is not a number with an exponent, it may be // an identifier starting with digits. let word = - peeking_take_while(chars, |ch| self.dialect.is_identifier_part(ch)); + chars.peeking_take_while(|ch| self.dialect.is_identifier_part(ch)); if !word.is_empty() { s += word.as_str(); @@ -2091,7 +2106,7 @@ impl<'a> Tokenizer<'a> { } '?' => { chars.next(); - let s = peeking_take_while(chars, |ch| ch.is_numeric()); + let s = chars.peeking_take_while(|ch| ch.is_numeric()); Ok(Some(Token::Placeholder(format!("?{s}")))) } @@ -2199,7 +2214,7 @@ impl<'a> Tokenizer<'a> { })) }; } else { - value.push_str(&peeking_take_while(chars, |ch| { + value.push_str(&chars.peeking_take_while(|ch| { ch.is_alphanumeric() || ch == '_' // Allow $ as a placeholder character if the dialect supports it @@ -2215,7 +2230,7 @@ impl<'a> Tokenizer<'a> { { value.push('.'); chars.next(); - value.push_str(&peeking_take_while(chars, |ch| ch.is_ascii_digit())); + value.push_str(&chars.peeking_take_while(|ch| ch.is_ascii_digit())); return Ok(Token::Placeholder(format!("${value}"))); } @@ -2304,7 +2319,7 @@ impl<'a> Tokenizer<'a> { // Consume characters until newline fn tokenize_single_line_comment(&self, chars: &mut State) -> String { - peeking_take_while(chars, |ch| match ch { + chars.peeking_take_while(|ch| match ch { '\n' => false, // Always stop at \n '\r' if dialect_of!(self is PostgreSqlDialect) => false, // Stop at \r for Postgres _ => true, // Keep consuming for other characters @@ -2314,9 +2329,7 @@ impl<'a> Tokenizer<'a> { /// Tokenize an identifier or keyword, after the first char is already consumed. fn tokenize_word(&self, first_chars: impl Into, chars: &mut State) -> String { let mut s = first_chars.into(); - s.push_str(&peeking_take_while(chars, |ch| { - self.dialect.is_identifier_part(ch) - })); + s.push_str(&chars.peeking_take_while(|ch| self.dialect.is_identifier_part(ch))); s } @@ -2668,22 +2681,6 @@ impl<'a> Tokenizer<'a> { } } -/// Read from `chars` until `predicate` returns `false` or EOF is hit. -/// Return the characters read as String, and keep the first non-matching -/// char available as `chars.next()`. -fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool) -> String { - let mut s = String::new(); - while let Some(&ch) = chars.peek() { - if predicate(ch) { - chars.next(); // consume - s.push(ch); - } else { - break; - } - } - s -} - fn unescape_single_quoted_string(chars: &mut State<'_>) -> Option { Unescape::new(chars).unescape() } From dafd04e7b565b87bedad57471c8166baf6e0c325 Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Wed, 21 Oct 2026 11:33:33 -0700 Subject: [PATCH 3/3] tokenizer: avoid temporary strings when building tokens Some token scans collect text in a temporary string, copy it into the string they are building, then discard the temporary. Append directly instead, using the same scanning loop. --- src/tokenizer.rs | 179 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 163 insertions(+), 16 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 91c5c21b2e..b33f5ee17a 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -834,18 +834,24 @@ impl State<'_> { self.peekable.peek() } - /// Collect matching characters into a new string, leaving the first - /// non-matching character unread. - fn peeking_take_while(&mut self, mut predicate: impl FnMut(char) -> bool) -> String { - let mut s = String::new(); + /// Append matching characters to `output`, leaving the first non-matching + /// character unread. + fn peeking_read_while(&mut self, output: &mut String, mut predicate: impl FnMut(char) -> bool) { while let Some(&ch) = self.peek() { if predicate(ch) { - self.next(); // consume - s.push(ch); + self.next(); + output.push(ch); } else { break; } } + } + + /// Collect matching characters into a new string, leaving the first + /// non-matching character unread. + fn peeking_take_while(&mut self, predicate: impl FnMut(char) -> bool) -> String { + let mut s = String::new(); + self.peeking_read_while(&mut s, predicate); s } @@ -1283,8 +1289,7 @@ impl<'a> Tokenizer<'a> { col: 0, }; let mut s = inner_state.peeking_take_while(|ch| matches!(ch, '0'..='9' | '.')); - let s2 = chars.peeking_take_while(|ch| matches!(ch, '0'..='9' | '.')); - s += s2.as_str(); + chars.peeking_read_while(&mut s, |ch| matches!(ch, '0'..='9' | '.')); return Ok(Some(Token::Number(s, false))); } @@ -1694,11 +1699,12 @@ impl<'a> Tokenizer<'a> { if exponent_part.is_empty() { // If it is not a number with an exponent, it may be // an identifier starting with digits. - let word = - chars.peeking_take_while(|ch| self.dialect.is_identifier_part(ch)); + let prefix_len = s.len(); + chars.peeking_read_while(&mut s, |ch| { + self.dialect.is_identifier_part(ch) + }); - if !word.is_empty() { - s += word.as_str(); + if s.len() != prefix_len { return Ok(Some(Token::make_word_owned(s, None))); } } else if prev_token == Some(&Token::Period) { @@ -2214,12 +2220,12 @@ impl<'a> Tokenizer<'a> { })) }; } else { - value.push_str(&chars.peeking_take_while(|ch| { + chars.peeking_read_while(&mut value, |ch| { ch.is_alphanumeric() || ch == '_' // Allow $ as a placeholder character if the dialect supports it || matches!(ch, '$' if self.dialect.supports_dollar_placeholder()) - })); + }); // If the dialect supports a dollar sign as a money prefix (e.g. SQL Server), // and the value so far is all digits, check for a decimal part, e.g. `$123.45` @@ -2230,7 +2236,7 @@ impl<'a> Tokenizer<'a> { { value.push('.'); chars.next(); - value.push_str(&chars.peeking_take_while(|ch| ch.is_ascii_digit())); + chars.peeking_read_while(&mut value, |ch| ch.is_ascii_digit()); return Ok(Token::Placeholder(format!("${value}"))); } @@ -2329,7 +2335,7 @@ impl<'a> Tokenizer<'a> { /// Tokenize an identifier or keyword, after the first char is already consumed. fn tokenize_word(&self, first_chars: impl Into, chars: &mut State) -> String { let mut s = first_chars.into(); - s.push_str(&chars.peeking_take_while(|ch| self.dialect.is_identifier_part(ch))); + chars.peeking_read_while(&mut s, |ch| self.dialect.is_identifier_part(ch)); s } @@ -5115,4 +5121,145 @@ mod tests { ); } } + #[test] + fn peeking_read_while_appends_and_preserves_boundaries() { + for (sql, suffix, location, inspected, remainder) in [ + ("", "", Location::new(7, 3), "", ""), + ("!tail", "", Location::new(7, 3), "!", "!tail"), + ("abc", "abc", Location::new(7, 6), "abc", ""), + ("abc,def", "abc", Location::new(7, 6), "abc,", ",def"), + ("é\nz!tail", "é\nz", Location::new(8, 2), "é\nz!", "!tail"), + ("é\nz", "é\nz", Location::new(8, 2), "é\nz", ""), + ] { + let mut chars = State { + peekable: sql.chars().peekable(), + line: 7, + col: 3, + }; + let mut output = String::from("prefix:"); + let mut seen = String::new(); + chars.peeking_read_while(&mut output, |ch| { + seen.push(ch); + ch != '!' && ch != ',' + }); + assert_eq!(output, format!("prefix:{suffix}"), "{sql:?}"); + assert_eq!(seen, inspected, "{sql:?}"); + assert_eq!(chars.location(), location, "{sql:?}"); + assert_eq!( + core::iter::from_fn(|| chars.next()).collect::(), + remainder, + "{sql:?}" + ); + } + } + + #[test] + fn tokenize_identifier_prefixes_and_unicode_spans() { + let suffixes = [ + String::new(), + "_name".to_owned(), + "alpha".to_owned(), + "é".to_owned(), + "漢字".to_owned(), + "λ".to_owned(), + "\u{0301}".to_owned(), + "a".repeat(128), + ]; + for dialect in all_dialects().dialects { + for prefix in [ + "a", "b", "B", "n", "N", "nq", "NQ", "nQ", "Nq", "q", "Q", "r", "R", "x", "X", "e", + "E", "u", "U", "é", "漢", "λ", + ] { + for suffix in &suffixes { + let word = format!("{prefix}{suffix}"); + let mut chars = word.chars(); + if !dialect.is_identifier_start(chars.next().unwrap()) + || !chars.all(|ch| dialect.is_identifier_part(ch)) + { + continue; + } + let sql = format!("\n{word}\t/*tail*/"); + let tokens = Tokenizer::new(&*dialect, &sql) + .tokenize_with_location() + .unwrap(); + assert_eq!( + tokens.iter().map(|t| t.token.clone()).collect::>(), + vec![ + Token::Whitespace(Whitespace::Newline), + Token::make_word(&word, None), + Token::Whitespace(Whitespace::Tab), + Token::Whitespace(Whitespace::MultiLineComment("tail".to_owned())), + ], + "{dialect:?}: {sql:?}" + ); + assert_eq!( + tokens[1].span, + Span::new( + Location::new(2, 1), + Location::new(2, word.chars().count() as u64 + 1) + ), + "{dialect:?}: {sql:?}" + ); + } + } + } + } + + #[test] + fn tokenize_identifier_at_eof_and_before_error() { + for dialect in all_dialects().dialects { + let word = "nQalpha"; + let tokens = Tokenizer::new(&*dialect, word) + .tokenize_with_location() + .unwrap(); + assert_eq!(tokens.len(), 1, "{dialect:?}"); + assert_eq!(tokens[0].token, Token::make_word(word, None)); + assert_eq!( + tokens[0].span, + Span::new(Location::new(1, 1), Location::new(1, 8)) + ); + + let mut buf = tokens.clone(); + let error = Tokenizer::new(&*dialect, "nQalpha '") + .tokenize_with_location_into_buf(&mut buf) + .unwrap_err(); + assert_eq!(error.message, "Unterminated string literal", "{dialect:?}"); + // Quote error positions differ by dialect. + let quote_error = Tokenizer::new(&*dialect, "'").tokenize().unwrap_err(); + assert_eq!( + error.location, + Location::new(1, quote_error.location.column + word.len() as u64 + 1), + "{dialect:?}" + ); + assert_eq!(buf.len(), 3, "{dialect:?}"); + assert_eq!(buf[0], tokens[0]); + assert_eq!(buf[1], tokens[0]); + assert_eq!(buf[2].token, Token::Whitespace(Whitespace::Space)); + assert_eq!( + buf[2].span, + Span::new(Location::new(1, 8), Location::new(1, 9)) + ); + } + } + + #[test] + fn tokenize_numeric_word_preserves_prefix_and_stops_before_suffix() { + let dialect = GenericDialect {}; + for (sql, number, remaining, column) in + [("12", "12", None, 3), ("12.34tail", "12.34", Some('t'), 6)] + { + let tokenizer = Tokenizer::new(&dialect, sql); + let mut state = State { + peekable: sql.chars().peekable(), + line: 1, + col: 1, + }; + let token = tokenizer + .tokenize_identifier_or_keyword(['1'], &mut state) + .unwrap(); + assert_eq!(token, Some(Token::Number(number.to_owned(), false))); + assert_eq!(state.peek().copied(), remaining); + assert_eq!((state.line, state.col), (1, column)); + } + } }