Spark, Databricks: parse DIV, USING, LONG, 1L, exists() and backslash escapes in both

src/dialect/databricks.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use crate::ast::Expr;
+use crate::dialect::spark::parse_div_infix;
 use crate::dialect::{Dialect, Precedence};
 use crate::keywords::{self, Keyword};
 use crate::parser::{Parser, ParserError};
@@ -66,6 +68,13 @@ impl Dialect for DatabricksDialect {
         true
     }
 
+    /// `1L` is a `BIGINT` literal.
+    ///
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/data-types/bigint-type>
+    fn supports_long_suffix_before_numeric_prefix(&self) -> bool {
+        true
+    }
+
     /// `'%\_%'` keeps its backslash, as Hive's `unescapeSQLString` does, so a
     /// LIKE pattern survives the literal.
     fn ignores_wildcard_escapes(&self) -> bool {
@@ -105,6 +114,11 @@ impl Dialect for DatabricksDialect {
         true
     }
 
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/functions/exists>
+    fn supports_exists_function(&self) -> bool {
+        true
+    }
+
     // https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select.html#syntax
     fn supports_select_wildcard_except(&self) -> bool {
         true
@@ -187,4 +201,26 @@ impl Dialect for DatabricksDialect {
     fn supports_pipe_operator(&self) -> bool {
         true
     }
+
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/functions/div>
+    fn parse_infix(
+        &self,
+        parser: &mut Parser,
+        expr: &Expr,
+        precedence: u8,
+    ) -> Option<Result<Expr, ParserError>> {
+        parse_div_infix(parser, expr, precedence)
+    }
+
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using>
+    fn supports_create_table_using(&self) -> bool {
+        true
+    }
+
+    /// `LONG` is an alias for `BIGINT`.
+    ///
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/data-types/bigint-type>
+    fn supports_long_type_as_bigint(&self) -> bool {
+        true
+    }
 }

src/dialect/hive.rs

@@ -75,6 +75,11 @@ impl Dialect for HiveDialect {
         true
     }
 
+    /// `1L` is a `BIGINT` literal.
+    fn supports_long_suffix_before_numeric_prefix(&self) -> bool {
+        true
+    }
+
     fn require_interval_qualifier(&self) -> bool {
         true
     }

src/dialect/mod.rs

@@ -497,6 +497,13 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if an `L` ending a number, as in `1L`, makes it a long
+    /// literal even when [`Dialect::supports_numeric_prefix`] would otherwise
+    /// read `1L` as an identifier. `1La` is still an identifier.
+    fn supports_long_suffix_before_numeric_prefix(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialect supports a type suffix on a numeric literal,
     /// as Spark SQL does: `1Y` (tinyint), `1S` (smallint), `1L` (bigint),
     /// `1.0F` (float), `1.0D` (double), `1.0BD` (decimal). The suffix is kept
@@ -563,6 +570,13 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if `exists(...)` is a call to a function named `exists`,
+    /// as in `exists(array(1, 2), x -> x > 1)`, unless a query follows the
+    /// parenthesis, as in `EXISTS (SELECT 1)`.
+    fn supports_exists_function(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialect supports multiple variable assignment
     /// using parentheses in a `SET` variable declaration.
     ///

src/dialect/spark.rs

@@ -106,6 +106,11 @@ impl Dialect for SparkSqlDialect {
         true
     }
 
+    /// See <https://spark.apache.org/docs/latest/sql-ref-functions-builtin.html#collection-functions>
+    fn supports_exists_function(&self) -> bool {
+        true
+    }
+
     /// See <https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select.html>
     fn supports_select_wildcard_except(&self) -> bool {
         true
@@ -191,19 +196,33 @@ impl Dialect for SparkSqlDialect {
         expr: &Expr,
         precedence: u8,
     ) -> Option<Result<Expr, ParserError>> {
-        if parser.parse_keyword(Keyword::DIV) {
-            let left = Box::new(expr.clone());
-            let right = Box::new(match parser.parse_subexpr(precedence) {
-                Ok(expr) => expr,
-                Err(e) => return Some(Err(e)),
-            });
-            Some(Ok(Expr::BinaryOp {
-                left,
-                op: BinaryOperator::MyIntegerDivide,
-                right,
-            }))
-        } else {
-            None
-        }
+        parse_div_infix(parser, expr, precedence)
+    }
+
+    /// `'it\'s'` and `'a\tb'`: Spark reads backslash escapes in string literals.
+    ///
+    /// See <https://spark.apache.org/docs/latest/sql-ref-literals.html#string-literal>
+    fn supports_string_literal_backslash_escape(&self) -> bool {
+        true
     }
 }
+
+/// `a DIV b`, Spark's integer division, shared with [`DatabricksDialect`](crate::dialect::DatabricksDialect).
+pub(crate) fn parse_div_infix(
+    parser: &mut Parser,
+    expr: &Expr,
+    precedence: u8,
+) -> Option<Result<Expr, ParserError>> {
+    if !parser.parse_keyword(Keyword::DIV) {
+        return None;
+    }
+    let right = match parser.parse_subexpr(precedence) {
+        Ok(expr) => expr,
+        Err(e) => return Some(Err(e)),
+    };
+    Some(Ok(Expr::BinaryOp {
+        left: Box::new(expr.clone()),
+        op: BinaryOperator::MyIntegerDivide,
+        right: Box::new(right),
+    }))
+}

src/parser/mod.rs

@@ -1592,18 +1592,8 @@ impl<'a> Parser<'a> {
             Keyword::TRY_CAST => Ok(Some(self.parse_cast_expr(CastKind::TryCast)?)),
             Keyword::SAFE_CAST => Ok(Some(self.parse_cast_expr(CastKind::SafeCast)?)),
             Keyword::EXISTS
-            // Support parsing Databricks has a function named `exists`.
-            if !dialect_of!(self is DatabricksDialect)
-                || matches!(
-                        self.peek_nth_token_ref(1).token,
-                        Token::Word(Word {
-                            keyword: Keyword::SELECT | Keyword::WITH,
-                            ..
-                        })
-                    ) =>
-                {
-                    Ok(Some(self.parse_exists_expr(false)?))
-                }
+            // `None` lets a dialect's `exists` function parse as a function call.
+            => self.maybe_parse_exists_expr(false),
             Keyword::EXTRACT => Ok(Some(self.parse_extract_expr()?)),
             Keyword::CEIL => Ok(Some(self.parse_ceil_floor_expr(true)?)),
             Keyword::FLOOR => Ok(Some(self.parse_ceil_floor_expr(false)?)),
@@ -2992,6 +2982,16 @@ impl<'a> Parser<'a> {
         Ok(exists_node)
     }
 
+    /// Parse `EXISTS (<query>)` after the `EXISTS` keyword, or return `None` when
+    /// the dialect reads the parentheses as the arguments of an `exists` function.
+    fn maybe_parse_exists_expr(&mut self, negated: bool) -> Result<Option<Expr>, ParserError> {
+        if self.dialect.supports_exists_function() {
+            self.maybe_parse(|parser| parser.parse_exists_expr(negated))
+        } else {
+            self.parse_exists_expr(negated).map(Some)
+        }
+    }
+
     /// Parse a SQL `EXTRACT` expression e.g. `EXTRACT(YEAR FROM date)`.
     pub fn parse_extract_expr(&mut self) -> Result<Expr, ParserError> {
         self.expect_token(&Token::LParen)?;
@@ -3339,25 +3339,16 @@ impl<'a> Parser<'a> {
     ///
     /// Represented in the AST as `Expr::UnaryOp` with `UnaryOperator::Not`.
     pub fn parse_not(&mut self) -> Result<Expr, ParserError> {
-        match &self.peek_token_ref().token {
-            Token::Word(w) => match w.keyword {
-                Keyword::EXISTS => {
-                    let negated = true;
-                    let _ = self.parse_keyword(Keyword::EXISTS);
-                    self.parse_exists_expr(negated)
-                }
-                _ => Ok(Expr::UnaryOp {
-                    op: UnaryOperator::Not,
-                    expr: Box::new(
-                        self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?,
-                    ),
-                }),
-            },
-            _ => Ok(Expr::UnaryOp {
-                op: UnaryOperator::Not,
-                expr: Box::new(self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?),
-            }),
+        if self.parse_keyword(Keyword::EXISTS) {
+            if let Some(expr) = self.maybe_parse_exists_expr(true)? {
+                return Ok(expr);
+            }
+            self.prev_token();
         }
+        Ok(Expr::UnaryOp {
+            op: UnaryOperator::Not,
+            expr: Box::new(self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?),
+        })
     }
 
     /// Parse expression types that start with a left brace '{'.

src/tokenizer.rs

@@ -1438,6 +1438,20 @@ impl<'a> Tokenizer<'a> {
                         }
                     }
 
+                    if self.dialect.supports_long_suffix_before_numeric_prefix()
+                        && chars.peek() == Some(&'L')
+                    {
+                        let mut ahead = chars.peekable.clone();
+                        ahead.next();
+                        if !ahead
+                            .peek()
+                            .is_some_and(|ch| self.dialect.is_identifier_part(*ch))
+                        {
+                            chars.next();
+                            return Ok(Some(Token::Number(s, true)));
+                        }
+                    }
+
                     // If the dialect supports identifiers that start with a numeric prefix,
                     // we need to check if the value is in fact an identifier and must thus
                     // be tokenized as a word.

tests/sqlparser_common.rs

@@ -8671,7 +8671,7 @@ fn parse_exists_subquery() {
     verified_stmt("SELECT * FROM t WHERE EXISTS (WITH u AS (SELECT 1) SELECT * FROM u)");
     verified_stmt("SELECT EXISTS (SELECT 1)");
 
-    let res = all_dialects_except(|d| d.is::<DatabricksDialect>())
+    let res = all_dialects_except(|d| d.supports_exists_function())
         .parse_sql_statements("SELECT EXISTS (");
     assert_eq!(
         ParserError::ParserError(
@@ -8680,7 +8680,7 @@ fn parse_exists_subquery() {
         res.unwrap_err(),
     );
 
-    let res = all_dialects_except(|d| d.is::<DatabricksDialect>())
+    let res = all_dialects_except(|d| d.supports_exists_function())
         .parse_sql_statements("SELECT EXISTS (NULL)");
     assert_eq!(
         ParserError::ParserError(
@@ -8690,6 +8690,52 @@ fn parse_exists_subquery() {
     );
 }
 
+#[test]
+fn parse_exists_function() {
+    let dialects = all_dialects_where(|d| d.supports_exists_function());
+    match dialects.verified_expr("exists(array(1, 2), x -> x > 1)") {
+        Expr::Function(f) => assert_eq!(f.name.to_string(), "exists"),
+        other => panic!("expected a function, got {other:?}"),
+    }
+    for sql in [
+        "EXISTS (SELECT 1)",
+        "EXISTS (WITH t AS (SELECT 1) SELECT * FROM t)",
+        "EXISTS (VALUES (1))",
+        "EXISTS ((SELECT 1))",
+        "EXISTS ((SELECT 1) UNION (SELECT 2))",
+    ] {
+        assert!(
+            matches!(dialects.verified_expr(sql), Expr::Exists { .. }),
+            "{sql}"
+        );
+    }
+    assert!(matches!(
+        dialects.verified_expr("NOT EXISTS (SELECT 1)"),
+        Expr::Exists { negated: true, .. }
+    ));
+    match dialects.verified_expr("NOT exists(array(1, 2), x -> x > 1)") {
+        Expr::UnaryOp {
+            op: UnaryOperator::Not,
+            expr,
+        } => assert!(matches!(*expr, Expr::Function(_))),
+        other => panic!("expected NOT of a function, got {other:?}"),
+    }
+    let dialects =
+        all_dialects_where(|d| d.supports_exists_function() && d.supports_from_first_select());
+    assert!(matches!(
+        dialects.verified_expr("EXISTS (FROM t SELECT a)"),
+        Expr::Exists { .. }
+    ));
+
+    assert_eq!(
+        all_dialects_except(|d| d.supports_exists_function())
+            .parse_sql_statements("SELECT exists(array(1, 2), x -> x > 1)"),
+        Err(ParserError::ParserError(
+            "Expected: SELECT, VALUES, or a subquery in the query body, found: array".to_string()
+        ))
+    );
+}
+
 #[test]
 fn parse_create_database() {
     let sql = "CREATE DATABASE mydb";

tests/sqlparser_databricks.rs

@@ -814,3 +814,46 @@ fn test_interval_preserves_column_query() {
         )))]
     );
 }
+
+#[test]
+fn test_databricks_div() {
+    databricks().one_statement_parses_to("SELECT 7 div 2", "SELECT 7 DIV 2");
+    assert_eq!(
+        databricks().verified_expr("7 DIV 2 + 1"),
+        Expr::BinaryOp {
+            left: Box::new(Expr::BinaryOp {
+                left: Box::new(Expr::value(number("7"))),
+                op: BinaryOperator::MyIntegerDivide,
+                right: Box::new(Expr::value(number("2"))),
+            }),
+            op: BinaryOperator::Plus,
+            right: Box::new(Expr::value(number("1"))),
+        }
+    );
+}
+
+#[test]
+fn test_databricks_create_table_using_and_long() {
+    databricks().verified_stmt("CREATE TABLE t (i INT) USING DELTA");
+    databricks().one_statement_parses_to(
+        "CREATE TABLE t (id LONG) USING parquet",
+        "CREATE TABLE t (id BIGINT) USING parquet",
+    );
+}
+
+#[test]
+fn test_databricks_long_literal() {
+    // `1L` is a BIGINT literal, not an identifier that starts with a digit.
+    assert_eq!(
+        databricks().verified_only_select("SELECT 1L").projection,
+        vec![SelectItem::UnnamedExpr(Expr::value(Value::Number(
+            "1".parse().unwrap(),
+            true
+        )))]
+    );
+    // A longer word is still an identifier.
+    assert_eq!(
+        databricks().verified_only_select("SELECT 1La").projection,
+        vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1La")))]
+    );
+}

tests/sqlparser_hive.rs

@@ -304,6 +304,14 @@ fn columns_after_partition() {
 fn long_numerics() {
     let query = r#"SELECT MIN(MIN(10, 5), 1L) AS a"#;
     hive().verified_stmt(query);
+    // `1L` is a BIGINT literal, not an identifier that starts with a digit.
+    assert_eq!(
+        hive().verified_only_select("SELECT 1L").projection,
+        vec![SelectItem::UnnamedExpr(Expr::value(Value::Number(
+            "1".parse().unwrap(),
+            true
+        )))]
+    );
 }
 
 #[test]