Merge remote-tracking branch 'apache/main' into Nyrox/main

Author: alamb

examples/cli.rs

@@ -17,9 +17,11 @@
 
 #![warn(clippy::all)]
 
-/// A small command-line app to run the parser.
-/// Run with `cargo run --example cli`
+//! A small command-line app to run the parser.
+//! Run with `cargo run --example cli`
+
 use std::fs;
+use std::io::{stdin, Read};
 
 use simple_logger::SimpleLogger;
 use sqlparser::dialect::*;
@@ -38,6 +40,9 @@ $ cargo run --example cli FILENAME.sql [--dialectname]
 To print the parse results as JSON:
 $ cargo run --feature json_example --example cli FILENAME.sql [--dialectname]
 
+To read from stdin instead of a file:
+$ cargo run --example cli - [--dialectname]
+
 "#,
     );
 
@@ -57,9 +62,18 @@ $ cargo run --feature json_example --example cli FILENAME.sql [--dialectname]
         s => panic!("Unexpected parameter: {s}"),
     };
 
-    println!("Parsing from file '{}' using {:?}", &filename, dialect);
-    let contents = fs::read_to_string(&filename)
-        .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename));
+    let contents = if filename == "-" {
+        println!("Parsing from stdin using {:?}", dialect);
+        let mut buf = Vec::new();
+        stdin()
+            .read_to_end(&mut buf)
+            .expect("failed to read from stdin");
+        String::from_utf8(buf).expect("stdin content wasn't valid utf8")
+    } else {
+        println!("Parsing from file '{}' using {:?}", &filename, dialect);
+        fs::read_to_string(&filename)
+            .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename))
+    };
     let without_bom = if contents.chars().next().unwrap() as u64 != 0xfeff {
         contents.as_str()
     } else {

src/ast/mod.rs

@@ -3406,6 +3406,13 @@ pub enum Statement {
     /// See Postgres <https://www.postgresql.org/docs/current/sql-listen.html>
     LISTEN { channel: Ident },
     /// ```sql
+    /// UNLISTEN
+    /// ```
+    /// stop listening for a notification
+    ///
+    /// See Postgres <https://www.postgresql.org/docs/current/sql-unlisten.html>
+    UNLISTEN { channel: Ident },
+    /// ```sql
     /// NOTIFY channel [ , payload ]
     /// ```
     /// send a notification event together with an optional “payload” string to channel
@@ -5014,6 +5021,10 @@ impl fmt::Display for Statement {
                 write!(f, "LISTEN {channel}")?;
                 Ok(())
             }
+            Statement::UNLISTEN { channel } => {
+                write!(f, "UNLISTEN {channel}")?;
+                Ok(())
+            }
             Statement::NOTIFY { channel, payload } => {
                 write!(f, "NOTIFY {channel}")?;
                 if let Some(payload) = payload {

src/ast/query.rs

@@ -999,6 +999,8 @@ pub enum TableFactor {
         with_ordinality: bool,
         /// [Partition selection](https://dev.mysql.com/doc/refman/8.0/en/partitioning-selection.html), supported by MySQL.
         partitions: Vec<Ident>,
+        /// Optional PartiQL JsonPath: <https://partiql.org/dql/from.html>
+        json_path: Option<JsonPath>,
     },
     Derived {
         lateral: bool,
@@ -1400,8 +1402,12 @@ impl fmt::Display for TableFactor {
                 version,
                 partitions,
                 with_ordinality,
+                json_path,
             } => {
                 write!(f, "{name}")?;
+                if let Some(json_path) = json_path {
+                    write!(f, "{json_path}")?;
+                }
                 if !partitions.is_empty() {
                     write!(f, "PARTITION ({})", display_comma_separated(partitions))?;
                 }

src/ast/spans.rs

@@ -371,6 +371,7 @@ impl Spanned for Statement {
                     .chain(core::iter::once(query.span()))
                     .chain(with_options.iter().map(|i| i.span())),
             ),
+            // These statements need to be implemented
             Statement::AlterRole { .. } => Span::empty(),
             Statement::AttachDatabase { .. } => Span::empty(),
             Statement::AttachDuckDBDatabase { .. } => Span::empty(),
@@ -441,6 +442,7 @@ impl Spanned for Statement {
             Statement::LISTEN { .. } => Span::empty(),
             Statement::NOTIFY { .. } => Span::empty(),
             Statement::LoadData { .. } => Span::empty(),
+            Statement::UNLISTEN { .. } => Span::empty(),
         }
     }
 }
@@ -1636,6 +1638,7 @@ impl Spanned for TableFactor {
                 version: _,
                 with_ordinality: _,
                 partitions: _,
+                json_path: _,
             } => union_spans(
                 name.0
                     .iter()

src/dialect/mod.rs

@@ -633,13 +633,8 @@ pub trait Dialect: Debug + Any {
         false
     }
 
-    /// Returns true if the dialect supports the `LISTEN` statement
-    fn supports_listen(&self) -> bool {
-        false
-    }
-
-    /// Returns true if the dialect supports the `NOTIFY` statement
-    fn supports_notify(&self) -> bool {
+    /// Returns true if the dialect supports the `LISTEN`, `UNLISTEN` and `NOTIFY` statements
+    fn supports_listen_notify(&self) -> bool {
         false
     }
 
@@ -680,6 +675,12 @@ pub trait Dialect: Debug + Any {
     fn supports_create_table_select(&self) -> bool {
         false
     }
+
+    /// Returns true if the dialect supports PartiQL for querying semi-structured data
+    /// <https://partiql.org/index.html>
+    fn supports_partiql(&self) -> bool {
+        false
+    }
 }
 
 /// This represents the operators for which precedence must be defined

src/dialect/postgresql.rs

@@ -191,12 +191,9 @@ impl Dialect for PostgreSqlDialect {
     }
 
     /// see <https://www.postgresql.org/docs/current/sql-listen.html>
-    fn supports_listen(&self) -> bool {
-        true
-    }
-
+    /// see <https://www.postgresql.org/docs/current/sql-unlisten.html>
     /// see <https://www.postgresql.org/docs/current/sql-notify.html>
-    fn supports_notify(&self) -> bool {
+    fn supports_listen_notify(&self) -> bool {
         true
     }
 
@@ -209,6 +206,11 @@ impl Dialect for PostgreSqlDialect {
     fn supports_comment_on(&self) -> bool {
         true
     }
+
+    /// See <https://www.postgresql.org/docs/current/sql-load.html>
+    fn supports_load_extension(&self) -> bool {
+        true
+    }
 }
 
 pub fn parse_create(parser: &mut Parser) -> Option<Result<Statement, ParserError>> {

src/dialect/redshift.rs

@@ -74,4 +74,9 @@ impl Dialect for RedshiftSqlDialect {
     fn supports_top_before_distinct(&self) -> bool {
         true
     }
+
+    /// Redshift supports PartiQL: <https://docs.aws.amazon.com/redshift/latest/dg/super-overview.html>
+    fn supports_partiql(&self) -> bool {
+        true
+    }
 }

src/keywords.rs

@@ -799,6 +799,7 @@ define_keywords!(
     UNION,
     UNIQUE,
     UNKNOWN,
+    UNLISTEN,
     UNLOAD,
     UNLOCK,
     UNLOGGED,

src/parser/mod.rs

@@ -533,10 +533,11 @@ impl<'a> Parser<'a> {
                 Keyword::EXECUTE | Keyword::EXEC => self.parse_execute(),
                 Keyword::PREPARE => self.parse_prepare(),
                 Keyword::MERGE => self.parse_merge(),
-                // `LISTEN` and `NOTIFY` are Postgres-specific
+                // `LISTEN`, `UNLISTEN` and `NOTIFY` are Postgres-specific
                 // syntaxes. They are used for Postgres statement.
-                Keyword::LISTEN if self.dialect.supports_listen() => self.parse_listen(),
-                Keyword::NOTIFY if self.dialect.supports_notify() => self.parse_notify(),
+                Keyword::LISTEN if self.dialect.supports_listen_notify() => self.parse_listen(),
+                Keyword::UNLISTEN if self.dialect.supports_listen_notify() => self.parse_unlisten(),
+                Keyword::NOTIFY if self.dialect.supports_listen_notify() => self.parse_notify(),
                 // `PRAGMA` is sqlite specific https://www.sqlite.org/pragma.html
                 Keyword::PRAGMA => self.parse_pragma(),
                 Keyword::UNLOAD => self.parse_unload(),
@@ -1003,6 +1004,21 @@ impl<'a> Parser<'a> {
         Ok(Statement::LISTEN { channel })
     }
 
+    pub fn parse_unlisten(&mut self) -> Result<Statement, ParserError> {
+        let channel = if self.consume_token(&Token::Mul) {
+            Ident::new(Expr::Wildcard(AttachedToken::empty()).to_string())
+        } else {
+            match self.parse_identifier(false) {
+                Ok(expr) => expr,
+                _ => {
+                    self.prev_token();
+                    return self.expected("wildcard or identifier", self.peek_token());
+                }
+            }
+        };
+        Ok(Statement::UNLISTEN { channel })
+    }
+
     pub fn parse_notify(&mut self) -> Result<Statement, ParserError> {
         let channel = self.parse_identifier(false)?;
         let payload = if self.consume_token(&Token::Comma) {
@@ -2927,7 +2943,7 @@ impl<'a> Parser<'a> {
         } else if Token::LBracket == tok {
             if dialect_of!(self is PostgreSqlDialect | DuckDbDialect | GenericDialect) {
                 self.parse_subscript(expr)
-            } else if dialect_of!(self is SnowflakeDialect) {
+            } else if dialect_of!(self is SnowflakeDialect) || self.dialect.supports_partiql() {
                 self.prev_token();
                 self.parse_json_access(expr)
             } else {
@@ -3063,6 +3079,14 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_json_access(&mut self, expr: Expr) -> Result<Expr, ParserError> {
+        let path = self.parse_json_path()?;
+        Ok(Expr::JsonAccess {
+            value: Box::new(expr),
+            path,
+        })
+    }
+
+    fn parse_json_path(&mut self) -> Result<JsonPath, ParserError> {
         let mut path = Vec::new();
         loop {
             match self.next_token().token {
@@ -3086,10 +3110,7 @@ impl<'a> Parser<'a> {
         }
 
         debug_assert!(!path.is_empty());
-        Ok(Expr::JsonAccess {
-            value: Box::new(expr),
-            path: JsonPath { path },
-        })
+        Ok(JsonPath { path })
     }
 
     pub fn parse_map_access(&mut self, expr: Expr) -> Result<Expr, ParserError> {
@@ -3522,16 +3543,11 @@ impl<'a> Parser<'a> {
         // e.g. `SELECT 1, 2, FROM t`
         // https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#trailing_commas
         // https://docs.snowflake.com/en/release-notes/2024/8_11#select-supports-trailing-commas
-        //
-        // This pattern could be captured better with RAII type semantics, but it's quite a bit of
-        // code to add for just one case, so we'll just do it manually here.
-        let old_value = self.options.trailing_commas;
-        self.options.trailing_commas |= self.dialect.supports_projection_trailing_commas();
 
-        let ret = self.parse_comma_separated(|p| p.parse_select_item());
-        self.options.trailing_commas = old_value;
+        let trailing_commas =
+            self.options.trailing_commas | self.dialect.supports_projection_trailing_commas();
 
-        ret
+        self.parse_comma_separated_with_trailing_commas(|p| p.parse_select_item(), trailing_commas)
     }
 
     pub fn parse_actions_list(&mut self) -> Result<Vec<ParsedAction>, ParserError> {
@@ -3558,11 +3574,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse the comma of a comma-separated syntax element.
+    /// Allows for control over trailing commas
     /// Returns true if there is a next element
-    fn is_parse_comma_separated_end(&mut self) -> bool {
+    fn is_parse_comma_separated_end_with_trailing_commas(&mut self, trailing_commas: bool) -> bool {
         if !self.consume_token(&Token::Comma) {
             true
-        } else if self.options.trailing_commas {
+        } else if trailing_commas {
             let token = self.peek_token().token;
             match token {
                 Token::Word(ref kw)
@@ -3580,15 +3597,34 @@ impl<'a> Parser<'a> {
         }
     }
 
+    /// Parse the comma of a comma-separated syntax element.
+    /// Returns true if there is a next element
+    fn is_parse_comma_separated_end(&mut self) -> bool {
+        self.is_parse_comma_separated_end_with_trailing_commas(self.options.trailing_commas)
+    }
+
     /// Parse a comma-separated list of 1+ items accepted by `F`
-    pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> Result<Vec<T>, ParserError>
+    pub fn parse_comma_separated<T, F>(&mut self, f: F) -> Result<Vec<T>, ParserError>
+    where
+        F: FnMut(&mut Parser<'a>) -> Result<T, ParserError>,
+    {
+        self.parse_comma_separated_with_trailing_commas(f, self.options.trailing_commas)
+    }
+
+    /// Parse a comma-separated list of 1+ items accepted by `F`
+    /// Allows for control over trailing commas
+    fn parse_comma_separated_with_trailing_commas<T, F>(
+        &mut self,
+        mut f: F,
+        trailing_commas: bool,
+    ) -> Result<Vec<T>, ParserError>
     where
         F: FnMut(&mut Parser<'a>) -> Result<T, ParserError>,
     {
         let mut values = vec![];
         loop {
             values.push(f(self)?);
-            if self.is_parse_comma_separated_end() {
+            if self.is_parse_comma_separated_end_with_trailing_commas(trailing_commas) {
                 break;
             }
         }
@@ -10326,6 +10362,11 @@ impl<'a> Parser<'a> {
         } else {
             let name = self.parse_object_name(true)?;
 
+            let json_path = match self.peek_token().token {
+                Token::LBracket if self.dialect.supports_partiql() => Some(self.parse_json_path()?),
+                _ => None,
+            };
+
             let partitions: Vec<Ident> = if dialect_of!(self is MySqlDialect | GenericDialect)
                 && self.parse_keyword(Keyword::PARTITION)
             {
@@ -10368,6 +10409,7 @@ impl<'a> Parser<'a> {
                 version,
                 partitions,
                 with_ordinality,
+                json_path,
             };
 
             while let Some(kw) = self.parse_one_of_keywords(&[Keyword::PIVOT, Keyword::UNPIVOT]) {

src/test_utils.rs

@@ -345,6 +345,7 @@ pub fn table(name: impl Into<String>) -> TableFactor {
         version: None,
         partitions: vec![],
         with_ordinality: false,
+        json_path: None,
     }
 }
 
@@ -360,6 +361,7 @@ pub fn table_with_alias(name: impl Into<String>, alias: impl Into<String>) -> Ta
         version: None,
         partitions: vec![],
         with_ordinality: false,
+        json_path: None,
     }
 }
 

src/tokenizer.rs

@@ -786,8 +786,9 @@ impl<'a> Tokenizer<'a> {
                     }
                     Ok(Some(Token::Whitespace(Whitespace::Newline)))
                 }
-                // BigQuery uses b or B for byte string literal
-                b @ 'B' | b @ 'b' if dialect_of!(self is BigQueryDialect | GenericDialect) => {
+                // BigQuery and MySQL use b or B for byte string literal, Postgres for bit strings
+                b @ 'B' | b @ 'b' if dialect_of!(self is BigQueryDialect | PostgreSqlDialect | MySqlDialect | GenericDialect) =>
+                {
                     chars.next(); // consume
                     match chars.peek() {
                         Some('\'') => {