Extend implementation & testing for parsing without semicolons

- introduce `statements_without_semicolons_parse_to` test helper
- for the implementation, `RETURN` parsing needs to be tightened up to avoid ambiguity & tests that formerly asserted "end of statement" now maybe need to assert "an SQL statement"
- a new `assert_err_parse_statements` splits the dialects based on semicolon requirements & asserts the expected error message accordingly

Author: aharpervc

src/dialect/mod.rs

@@ -1122,6 +1122,11 @@ pub trait Dialect: Debug + Any {
     ) -> bool {
         false
     }
+
+    /// Returns true if the dialect supports parsing statements without a semicolon delimiter.
+    fn supports_statements_without_semicolon_delimiter(&self) -> bool {
+        false
+    }
 }
 
 /// This represents the operators for which precedence must be defined

src/dialect/mssql.rs

@@ -67,7 +67,7 @@ impl Dialect for MsSqlDialect {
     }
 
     fn supports_connect_by(&self) -> bool {
-        true
+        false
     }
 
     fn supports_eq_alias_assignment(&self) -> bool {
@@ -280,6 +280,9 @@ impl MsSqlDialect {
     ) -> Result<Vec<Statement>, ParserError> {
         let mut stmts = Vec::new();
         loop {
+            while let Token::SemiColon = parser.peek_token_ref().token {
+                parser.advance_token();
+            }
             if let Token::EOF = parser.peek_token_ref().token {
                 break;
             }

src/keywords.rs

@@ -1071,6 +1071,7 @@ pub const RESERVED_FOR_TABLE_ALIAS: &[Keyword] = &[
     Keyword::ANTI,
     Keyword::SEMI,
     Keyword::RETURNING,
+    Keyword::RETURN,
     Keyword::ASOF,
     Keyword::MATCH_CONDITION,
     // for MSSQL-specific OUTER APPLY (seems reserved in most dialects)
@@ -1125,6 +1126,7 @@ pub const RESERVED_FOR_COLUMN_ALIAS: &[Keyword] = &[
     Keyword::CLUSTER,
     Keyword::DISTRIBUTE,
     Keyword::RETURNING,
+    Keyword::RETURN,
     // Reserved only as a column alias in the `SELECT` clause
     Keyword::FROM,
     Keyword::INTO,
@@ -1139,6 +1141,7 @@ pub const RESERVED_FOR_TABLE_FACTOR: &[Keyword] = &[
     Keyword::LIMIT,
     Keyword::HAVING,
     Keyword::WHERE,
+    Keyword::RETURN,
 ];
 
 /// Global list of reserved keywords that cannot be parsed as identifiers
@@ -1149,4 +1152,5 @@ pub const RESERVED_FOR_IDENTIFIER: &[Keyword] = &[
     Keyword::INTERVAL,
     Keyword::STRUCT,
     Keyword::TRIM,
+    Keyword::RETURN,
 ];

src/parser/mod.rs

@@ -265,6 +265,22 @@ impl ParserOptions {
         self.unescape = unescape;
         self
     }
+
+    /// Set if semicolon statement delimiters are required.
+    ///
+    /// If this option is `true`, the following SQL will not parse. If the option is `false`, the SQL will parse.
+    ///
+    /// ```sql
+    /// SELECT 1
+    /// SELECT 2
+    /// ```
+    pub fn with_require_semicolon_stmt_delimiter(
+        mut self,
+        require_semicolon_stmt_delimiter: bool,
+    ) -> Self {
+        self.require_semicolon_stmt_delimiter = require_semicolon_stmt_delimiter;
+        self
+    }
 }
 
 #[derive(Copy, Clone)]
@@ -355,7 +371,11 @@ impl<'a> Parser<'a> {
             state: ParserState::Normal,
             dialect,
             recursion_counter: RecursionCounter::new(DEFAULT_REMAINING_DEPTH),
-            options: ParserOptions::new().with_trailing_commas(dialect.supports_trailing_commas()),
+            options: ParserOptions::new()
+                .with_trailing_commas(dialect.supports_trailing_commas())
+                .with_require_semicolon_stmt_delimiter(
+                    !dialect.supports_statements_without_semicolon_delimiter(),
+                ),
         }
     }
 
@@ -478,10 +498,10 @@ impl<'a> Parser<'a> {
             match self.peek_token().token {
                 Token::EOF => break,
 
-                // end of statement
-                Token::Word(word) => {
-                    if expecting_statement_delimiter && word.keyword == Keyword::END {
-                        break;
+                // don't expect a semicolon statement delimiter after a newline when not otherwise required
+                Token::Whitespace(Whitespace::Newline) => {
+                    if !self.options.require_semicolon_stmt_delimiter {
+                        expecting_statement_delimiter = false;
                     }
                 }
                 _ => {}
@@ -493,7 +513,7 @@ impl<'a> Parser<'a> {
 
             let statement = self.parse_statement()?;
             stmts.push(statement);
-            expecting_statement_delimiter = true;
+            expecting_statement_delimiter = self.options.require_semicolon_stmt_delimiter;
         }
         Ok(stmts)
     }
@@ -4533,6 +4553,9 @@ impl<'a> Parser<'a> {
     ) -> Result<Vec<Statement>, ParserError> {
         let mut values = vec![];
         loop {
+            // ignore empty statements (between successive statement delimiters)
+            while self.consume_token(&Token::SemiColon) {}
+
             match &self.peek_nth_token_ref(0).token {
                 Token::EOF => break,
                 Token::Word(w) => {
@@ -4544,7 +4567,13 @@ impl<'a> Parser<'a> {
             }
 
             values.push(self.parse_statement()?);
-            self.expect_token(&Token::SemiColon)?;
+
+            if self.options.require_semicolon_stmt_delimiter {
+                self.expect_token(&Token::SemiColon)?;
+            }
+
+            // ignore empty statements (between successive statement delimiters)
+            while self.consume_token(&Token::SemiColon) {}
         }
         Ok(values)
     }
@@ -16370,7 +16399,28 @@ impl<'a> Parser<'a> {
 
     /// Parse [Statement::Return]
     fn parse_return(&mut self) -> Result<Statement, ParserError> {
-        match self.maybe_parse(|p| p.parse_expr())? {
+        let rs = self.maybe_parse(|p| {
+            let expr = p.parse_expr()?;
+
+            match &expr {
+                Expr::Value(_)
+                | Expr::Function(_)
+                | Expr::UnaryOp { .. }
+                | Expr::BinaryOp { .. }
+                | Expr::Case { .. }
+                | Expr::Cast { .. }
+                | Expr::Convert { .. }
+                | Expr::Subquery(_) => Ok(expr),
+                // todo: how to retstrict to variables?
+                Expr::Identifier(id) if id.value.starts_with('@') => Ok(expr),
+                _ => parser_err!(
+                    "Non-returnable expression found following RETURN",
+                    p.peek_token().span.start
+                ),
+            }
+        })?;
+
+        match rs {
             Some(expr) => Ok(Statement::Return(ReturnStatement {
                 value: Some(ReturnStatementValue::Expr(expr)),
             })),

src/test_utils.rs

@@ -186,6 +186,37 @@ impl TestedDialects {
         statements
     }
 
+    /// The same as [`statements_parse_to`] but it will strip semicolons from the SQL text.
+    pub fn statements_without_semicolons_parse_to(
+        &self,
+        sql: &str,
+        canonical: &str,
+    ) -> Vec<Statement> {
+        let sql_without_semicolons = sql
+            .replace("; ", " ")
+            .replace(" ;", " ")
+            .replace(";\n", "\n")
+            .replace("\n;", "\n")
+            .replace(";", " ");
+        let statements = self
+            .parse_sql_statements(&sql_without_semicolons)
+            .expect(&sql_without_semicolons);
+        if !canonical.is_empty() && sql != canonical {
+            assert_eq!(self.parse_sql_statements(canonical).unwrap(), statements);
+        } else {
+            assert_eq!(
+                sql,
+                statements
+                    .iter()
+                    // note: account for format_statement_list manually inserted semicolons
+                    .map(|s| s.to_string().trim_end_matches(";").to_string())
+                    .collect::<Vec<_>>()
+                    .join("; ")
+            );
+        }
+        statements
+    }
+
     /// Ensures that `sql` parses as an [`Expr`], and that
     /// re-serializing the parse result produces canonical
     pub fn expr_parses_to(&self, sql: &str, canonical: &str) -> Expr {
@@ -318,6 +349,43 @@ where
     all_dialects_where(|d| !except(d))
 }
 
+/// Returns all dialects that don't support statements without semicolon delimiters.
+/// (i.e. dialects that require semicolon delimiters.)
+pub fn all_dialects_requiring_semicolon_statement_delimiter() -> TestedDialects {
+    let tested_dialects =
+        all_dialects_except(|d| d.supports_statements_without_semicolon_delimiter());
+    assert_ne!(tested_dialects.dialects.len(), 0);
+    tested_dialects
+}
+
+/// Returns all dialects that do support statements without semicolon delimiters.
+/// (i.e. dialects not requiring semicolon delimiters.)
+pub fn all_dialects_not_requiring_semicolon_statement_delimiter() -> TestedDialects {
+    let tested_dialects =
+        all_dialects_where(|d| d.supports_statements_without_semicolon_delimiter());
+    assert_ne!(tested_dialects.dialects.len(), 0);
+    tested_dialects
+}
+
+/// Asserts an error for `parse_sql_statements`:
+/// - "end of statement" for dialects that require semicolon delimiters
+/// - "an SQL statement" for dialects that don't require semicolon delimiters.
+pub fn assert_err_parse_statements(sql: &str, found: &str) {
+    assert_eq!(
+        ParserError::ParserError(format!("Expected: end of statement, found: {}", found)),
+        all_dialects_requiring_semicolon_statement_delimiter()
+            .parse_sql_statements(sql)
+            .unwrap_err()
+    );
+
+    assert_eq!(
+        ParserError::ParserError(format!("Expected: an SQL statement, found: {}", found)),
+        all_dialects_not_requiring_semicolon_statement_delimiter()
+            .parse_sql_statements(sql)
+            .unwrap_err()
+    );
+}
+
 pub fn assert_eq_vec<T: ToString>(expected: &[&str], actual: &[T]) {
     assert_eq!(
         expected,