diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index 8d9ce90172..eb9349cfce 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use crate::ast::Expr; +use crate::dialect::spark::parse_div_infix; use crate::dialect::{Dialect, Precedence}; use crate::keywords::{self, Keyword}; use crate::parser::{Parser, ParserError}; @@ -66,6 +68,13 @@ impl Dialect for DatabricksDialect { true } + /// `1L` is a `BIGINT` literal. + /// + /// See + fn supports_long_suffix_before_numeric_prefix(&self) -> bool { + true + } + /// `'%\_%'` keeps its backslash, as Hive's `unescapeSQLString` does, so a /// LIKE pattern survives the literal. fn ignores_wildcard_escapes(&self) -> bool { @@ -105,6 +114,11 @@ impl Dialect for DatabricksDialect { true } + /// See + fn supports_exists_function(&self) -> bool { + true + } + // https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select.html#syntax fn supports_select_wildcard_except(&self) -> bool { true @@ -187,4 +201,26 @@ impl Dialect for DatabricksDialect { fn supports_pipe_operator(&self) -> bool { true } + + /// See + fn parse_infix( + &self, + parser: &mut Parser, + expr: &Expr, + precedence: u8, + ) -> Option> { + parse_div_infix(parser, expr, precedence) + } + + /// See + fn supports_create_table_using(&self) -> bool { + true + } + + /// `LONG` is an alias for `BIGINT`. + /// + /// See + fn supports_long_type_as_bigint(&self) -> bool { + true + } } diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index 3a4207bec5..1300149765 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -75,6 +75,11 @@ impl Dialect for HiveDialect { true } + /// `1L` is a `BIGINT` literal. + fn supports_long_suffix_before_numeric_prefix(&self) -> bool { + true + } + fn require_interval_qualifier(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 8e4c664783..ec98dc5efe 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -497,6 +497,13 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if an `L` ending a number, as in `1L`, makes it a long + /// literal even when [`Dialect::supports_numeric_prefix`] would otherwise + /// read `1L` as an identifier. `1La` is still an identifier. + fn supports_long_suffix_before_numeric_prefix(&self) -> bool { + false + } + /// Returns true if the dialect supports a type suffix on a numeric literal, /// as Spark SQL does: `1Y` (tinyint), `1S` (smallint), `1L` (bigint), /// `1.0F` (float), `1.0D` (double), `1.0BD` (decimal). The suffix is kept @@ -563,6 +570,13 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if `exists(...)` is a call to a function named `exists`, + /// as in `exists(array(1, 2), x -> x > 1)`, unless a query follows the + /// parenthesis, as in `EXISTS (SELECT 1)`. + fn supports_exists_function(&self) -> bool { + false + } + /// Returns true if the dialect supports multiple variable assignment /// using parentheses in a `SET` variable declaration. /// diff --git a/src/dialect/spark.rs b/src/dialect/spark.rs index c574993b1d..ae5fd0b10e 100644 --- a/src/dialect/spark.rs +++ b/src/dialect/spark.rs @@ -106,6 +106,11 @@ impl Dialect for SparkSqlDialect { true } + /// See + fn supports_exists_function(&self) -> bool { + true + } + /// See fn supports_select_wildcard_except(&self) -> bool { true @@ -191,19 +196,33 @@ impl Dialect for SparkSqlDialect { expr: &Expr, precedence: u8, ) -> Option> { - if parser.parse_keyword(Keyword::DIV) { - let left = Box::new(expr.clone()); - let right = Box::new(match parser.parse_subexpr(precedence) { - Ok(expr) => expr, - Err(e) => return Some(Err(e)), - }); - Some(Ok(Expr::BinaryOp { - left, - op: BinaryOperator::MyIntegerDivide, - right, - })) - } else { - None - } + parse_div_infix(parser, expr, precedence) + } + + /// `'it\'s'` and `'a\tb'`: Spark reads backslash escapes in string literals. + /// + /// See + fn supports_string_literal_backslash_escape(&self) -> bool { + true } } + +/// `a DIV b`, Spark's integer division, shared with [`DatabricksDialect`](crate::dialect::DatabricksDialect). +pub(crate) fn parse_div_infix( + parser: &mut Parser, + expr: &Expr, + precedence: u8, +) -> Option> { + if !parser.parse_keyword(Keyword::DIV) { + return None; + } + let right = match parser.parse_subexpr(precedence) { + Ok(expr) => expr, + Err(e) => return Some(Err(e)), + }; + Some(Ok(Expr::BinaryOp { + left: Box::new(expr.clone()), + op: BinaryOperator::MyIntegerDivide, + right: Box::new(right), + })) +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 2af8b620c6..cfc8bb9cd2 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1592,18 +1592,8 @@ impl<'a> Parser<'a> { Keyword::TRY_CAST => Ok(Some(self.parse_cast_expr(CastKind::TryCast)?)), Keyword::SAFE_CAST => Ok(Some(self.parse_cast_expr(CastKind::SafeCast)?)), Keyword::EXISTS - // Support parsing Databricks has a function named `exists`. - if !dialect_of!(self is DatabricksDialect) - || matches!( - self.peek_nth_token_ref(1).token, - Token::Word(Word { - keyword: Keyword::SELECT | Keyword::WITH, - .. - }) - ) => - { - Ok(Some(self.parse_exists_expr(false)?)) - } + // `None` lets a dialect's `exists` function parse as a function call. + => self.maybe_parse_exists_expr(false), Keyword::EXTRACT => Ok(Some(self.parse_extract_expr()?)), Keyword::CEIL => Ok(Some(self.parse_ceil_floor_expr(true)?)), Keyword::FLOOR => Ok(Some(self.parse_ceil_floor_expr(false)?)), @@ -2992,6 +2982,16 @@ impl<'a> Parser<'a> { Ok(exists_node) } + /// Parse `EXISTS ()` after the `EXISTS` keyword, or return `None` when + /// the dialect reads the parentheses as the arguments of an `exists` function. + fn maybe_parse_exists_expr(&mut self, negated: bool) -> Result, ParserError> { + if self.dialect.supports_exists_function() { + self.maybe_parse(|parser| parser.parse_exists_expr(negated)) + } else { + self.parse_exists_expr(negated).map(Some) + } + } + /// Parse a SQL `EXTRACT` expression e.g. `EXTRACT(YEAR FROM date)`. pub fn parse_extract_expr(&mut self) -> Result { self.expect_token(&Token::LParen)?; @@ -3339,25 +3339,16 @@ impl<'a> Parser<'a> { /// /// Represented in the AST as `Expr::UnaryOp` with `UnaryOperator::Not`. pub fn parse_not(&mut self) -> Result { - match &self.peek_token_ref().token { - Token::Word(w) => match w.keyword { - Keyword::EXISTS => { - let negated = true; - let _ = self.parse_keyword(Keyword::EXISTS); - self.parse_exists_expr(negated) - } - _ => Ok(Expr::UnaryOp { - op: UnaryOperator::Not, - expr: Box::new( - self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?, - ), - }), - }, - _ => Ok(Expr::UnaryOp { - op: UnaryOperator::Not, - expr: Box::new(self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?), - }), + if self.parse_keyword(Keyword::EXISTS) { + if let Some(expr) = self.maybe_parse_exists_expr(true)? { + return Ok(expr); + } + self.prev_token(); } + Ok(Expr::UnaryOp { + op: UnaryOperator::Not, + expr: Box::new(self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?), + }) } /// Parse expression types that start with a left brace '{'. diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 78b38acdd6..a77fe4d145 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1438,6 +1438,20 @@ impl<'a> Tokenizer<'a> { } } + if self.dialect.supports_long_suffix_before_numeric_prefix() + && chars.peek() == Some(&'L') + { + let mut ahead = chars.peekable.clone(); + ahead.next(); + if !ahead + .peek() + .is_some_and(|ch| self.dialect.is_identifier_part(*ch)) + { + chars.next(); + return Ok(Some(Token::Number(s, true))); + } + } + // If the dialect supports identifiers that start with a numeric prefix, // we need to check if the value is in fact an identifier and must thus // be tokenized as a word. diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 684180f9c6..293fa788e3 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -8671,7 +8671,7 @@ fn parse_exists_subquery() { verified_stmt("SELECT * FROM t WHERE EXISTS (WITH u AS (SELECT 1) SELECT * FROM u)"); verified_stmt("SELECT EXISTS (SELECT 1)"); - let res = all_dialects_except(|d| d.is::()) + let res = all_dialects_except(|d| d.supports_exists_function()) .parse_sql_statements("SELECT EXISTS ("); assert_eq!( ParserError::ParserError( @@ -8680,7 +8680,7 @@ fn parse_exists_subquery() { res.unwrap_err(), ); - let res = all_dialects_except(|d| d.is::()) + let res = all_dialects_except(|d| d.supports_exists_function()) .parse_sql_statements("SELECT EXISTS (NULL)"); assert_eq!( ParserError::ParserError( @@ -8690,6 +8690,52 @@ fn parse_exists_subquery() { ); } +#[test] +fn parse_exists_function() { + let dialects = all_dialects_where(|d| d.supports_exists_function()); + match dialects.verified_expr("exists(array(1, 2), x -> x > 1)") { + Expr::Function(f) => assert_eq!(f.name.to_string(), "exists"), + other => panic!("expected a function, got {other:?}"), + } + for sql in [ + "EXISTS (SELECT 1)", + "EXISTS (WITH t AS (SELECT 1) SELECT * FROM t)", + "EXISTS (VALUES (1))", + "EXISTS ((SELECT 1))", + "EXISTS ((SELECT 1) UNION (SELECT 2))", + ] { + assert!( + matches!(dialects.verified_expr(sql), Expr::Exists { .. }), + "{sql}" + ); + } + assert!(matches!( + dialects.verified_expr("NOT EXISTS (SELECT 1)"), + Expr::Exists { negated: true, .. } + )); + match dialects.verified_expr("NOT exists(array(1, 2), x -> x > 1)") { + Expr::UnaryOp { + op: UnaryOperator::Not, + expr, + } => assert!(matches!(*expr, Expr::Function(_))), + other => panic!("expected NOT of a function, got {other:?}"), + } + let dialects = + all_dialects_where(|d| d.supports_exists_function() && d.supports_from_first_select()); + assert!(matches!( + dialects.verified_expr("EXISTS (FROM t SELECT a)"), + Expr::Exists { .. } + )); + + assert_eq!( + all_dialects_except(|d| d.supports_exists_function()) + .parse_sql_statements("SELECT exists(array(1, 2), x -> x > 1)"), + Err(ParserError::ParserError( + "Expected: SELECT, VALUES, or a subquery in the query body, found: array".to_string() + )) + ); +} + #[test] fn parse_create_database() { let sql = "CREATE DATABASE mydb"; diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 6bbcfc9d81..ff87cf3983 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -814,3 +814,46 @@ fn test_interval_preserves_column_query() { )))] ); } + +#[test] +fn test_databricks_div() { + databricks().one_statement_parses_to("SELECT 7 div 2", "SELECT 7 DIV 2"); + assert_eq!( + databricks().verified_expr("7 DIV 2 + 1"), + Expr::BinaryOp { + left: Box::new(Expr::BinaryOp { + left: Box::new(Expr::value(number("7"))), + op: BinaryOperator::MyIntegerDivide, + right: Box::new(Expr::value(number("2"))), + }), + op: BinaryOperator::Plus, + right: Box::new(Expr::value(number("1"))), + } + ); +} + +#[test] +fn test_databricks_create_table_using_and_long() { + databricks().verified_stmt("CREATE TABLE t (i INT) USING DELTA"); + databricks().one_statement_parses_to( + "CREATE TABLE t (id LONG) USING parquet", + "CREATE TABLE t (id BIGINT) USING parquet", + ); +} + +#[test] +fn test_databricks_long_literal() { + // `1L` is a BIGINT literal, not an identifier that starts with a digit. + assert_eq!( + databricks().verified_only_select("SELECT 1L").projection, + vec![SelectItem::UnnamedExpr(Expr::value(Value::Number( + "1".parse().unwrap(), + true + )))] + ); + // A longer word is still an identifier. + assert_eq!( + databricks().verified_only_select("SELECT 1La").projection, + vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1La")))] + ); +} diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 21e76e23b6..5cfa9b76a3 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -304,6 +304,14 @@ fn columns_after_partition() { fn long_numerics() { let query = r#"SELECT MIN(MIN(10, 5), 1L) AS a"#; hive().verified_stmt(query); + // `1L` is a BIGINT literal, not an identifier that starts with a digit. + assert_eq!( + hive().verified_only_select("SELECT 1L").projection, + vec![SelectItem::UnnamedExpr(Expr::value(Value::Number( + "1".parse().unwrap(), + true + )))] + ); } #[test] diff --git a/tests/sqlparser_spark.rs b/tests/sqlparser_spark.rs index 4f425e6426..d4c05da273 100644 --- a/tests/sqlparser_spark.rs +++ b/tests/sqlparser_spark.rs @@ -589,3 +589,50 @@ fn test_interval_preserves_column_query() { )))] ); } + +#[test] +fn test_exists_function() { + // `exists(arr, lambda)` is a function; `EXISTS ()` stays a subquery test. + match spark().verified_expr("exists(array(1, 2), x -> x > 1)") { + Expr::Function(f) => assert_eq!(f.name.to_string(), "exists"), + other => panic!("expected a function, got {other:?}"), + } + // `TABLE t` prints as `SELECT * FROM t`. + for (subquery, canonical) in [ + ("EXISTS (SELECT 1)", "EXISTS (SELECT 1)"), + ( + "EXISTS (WITH t AS (SELECT 1) SELECT * FROM t)", + "EXISTS (WITH t AS (SELECT 1) SELECT * FROM t)", + ), + ("EXISTS (TABLE t)", "EXISTS (SELECT * FROM t)"), + ("EXISTS (VALUES (1))", "EXISTS (VALUES (1))"), + ] { + assert!( + matches!( + spark().expr_parses_to(subquery, canonical), + Expr::Exists { negated: false, .. } + ), + "{subquery}" + ); + } + // A pipe query after the parenthesis. + spark().one_statement_parses_to( + "SELECT * FROM t WHERE EXISTS (TABLE u |> WHERE u.a = t.a)", + "SELECT * FROM t WHERE EXISTS (SELECT * FROM u |> WHERE u.a = t.a)", + ); + spark().verified_stmt("SELECT exists(arr, x -> x > 2), NOT exists(arr, x -> x < 0) FROM t"); + spark().verified_stmt("SELECT * FROM t WHERE EXISTS (SELECT 1) AND NOT EXISTS ((SELECT 2))"); +} + +#[test] +fn test_string_literal_backslash_escapes() { + let select = spark() + .verified_only_select_with_canonical(r"SELECT 'it\'s', 'a\tb'", "SELECT 'it''s', 'a\tb'"); + assert_eq!( + select.projection, + vec![ + SelectItem::UnnamedExpr(Expr::value(Value::SingleQuotedString("it's".to_string()))), + SelectItem::UnnamedExpr(Expr::value(Value::SingleQuotedString("a\tb".to_string()))), + ] + ); +}