From 5b9fa35942abe8846794df092f77053e92ab0fb3 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 25 Feb 2025 16:31:52 +0200 Subject: [PATCH 1/3] Align SQL formatting and add all missing table options to be handled in any order --- src/ast/dml.rs | 183 +++++- src/ast/helpers/stmt_create_table.rs | 237 +++++++- src/ast/spans.rs | 24 + src/keywords.rs | 17 + src/parser/mod.rs | 799 +++++++++++++++++++++++++-- tests/sqlparser_duckdb.rs | 24 + tests/sqlparser_mssql.rs | 48 ++ tests/sqlparser_mysql.rs | 168 +++++- tests/sqlparser_postgres.rs | 24 + 9 files changed, 1464 insertions(+), 60 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 9cdb1ca86..810c02934 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -29,6 +29,11 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +use crate::parser::{ + Compression, DelayKeyWrite, DirectoryOption, Encryption, InsertMethod, OptionState, RowFormat, + StorageType, TablespaceOption, +}; + pub use super::ddl::{ColumnDef, TableConstraint}; use super::{ @@ -157,6 +162,30 @@ pub struct CreateTable { pub engine: Option, pub comment: Option, pub auto_increment_offset: Option, + pub key_block_size: Option, + pub max_rows: Option, + pub min_rows: Option, + pub autoextend_size: Option, + pub avg_row_length: Option, + pub checksum: Option, + pub connection: Option, + pub engine_attribute: Option, + pub password: Option, + pub secondary_engine_attribute: Option, + pub tablespace_option: Option, + pub row_format: Option, + pub insert_method: Option, + pub compression: Option, + pub delay_key_write: Option, + pub encryption: Option, + pub pack_keys: Option, + pub stats_auto_recalc: Option, + pub stats_persistent: Option, + pub stats_sample_pages: Option, + pub start_transaction: Option, + pub union_tables: Option>, + pub data_directory: Option, + pub index_directory: Option, pub default_charset: Option, pub collation: Option, pub on_commit: Option, @@ -402,7 +431,7 @@ impl Display for CreateTable { } if let Some(auto_increment_offset) = self.auto_increment_offset { - write!(f, " AUTO_INCREMENT {auto_increment_offset}")?; + write!(f, " AUTO_INCREMENT={auto_increment_offset}")?; } if let Some(primary_key) = &self.primary_key { write!(f, " PRIMARY KEY {}", primary_key)?; @@ -510,6 +539,158 @@ impl Display for CreateTable { write!(f, " COLLATE={collation}")?; } + if let Some(insert_method) = &self.insert_method { + match insert_method { + InsertMethod::No => write!(f, " INSERT_METHOD=NO")?, + InsertMethod::First => write!(f, " INSERT_METHOD=FIRST")?, + InsertMethod::Last => write!(f, " INSERT_METHOD=LAST")?, + } + } + + if let Some(key_block_size) = self.key_block_size { + write!(f, " KEY_BLOCK_SIZE={key_block_size}")?; + } + + if let Some(row_format) = &self.row_format { + match row_format { + RowFormat::Default => write!(f, " ROW_FORMAT=DEFAULT")?, + RowFormat::Dynamic => write!(f, " ROW_FORMAT=DYNAMIC")?, + RowFormat::Fixed => write!(f, " ROW_FORMAT=FIXED")?, + RowFormat::Compressed => write!(f, " ROW_FORMAT=COMPRESSED")?, + RowFormat::Redundant => write!(f, " ROW_FORMAT=REDUNDANT")?, + RowFormat::Compact => write!(f, " ROW_FORMAT=COMPACT")?, + } + } + + if let Some(data_dir) = &self.data_directory { + write!(f, " DATA DIRECTORY='{}'", data_dir.path)?; + } + + if let Some(index_dir) = &self.index_directory { + write!(f, " INDEX DIRECTORY='{}'", index_dir.path)?; + } + + if let Some(pack_keys) = &self.pack_keys { + match pack_keys { + OptionState::Default => write!(f, " PACK_KEYS=DEFAULT")?, + OptionState::One => write!(f, " PACK_KEYS=1")?, + OptionState::Zero => write!(f, " PACK_KEYS=0")?, + } + } + + if let Some(stats_auto_recalc) = &self.stats_auto_recalc { + match stats_auto_recalc { + OptionState::Default => write!(f, " STATS_AUTO_RECALC=DEFAULT")?, + OptionState::One => write!(f, " STATS_AUTO_RECALC=1")?, + OptionState::Zero => write!(f, " STATS_AUTO_RECALC=0")?, + } + } + + if let Some(stats_persistent) = &self.stats_persistent { + match stats_persistent { + OptionState::Default => write!(f, " STATS_PERSISTENT=DEFAULT")?, + OptionState::One => write!(f, " STATS_PERSISTENT=1")?, + OptionState::Zero => write!(f, " STATS_PERSISTENT=0")?, + } + } + + if let Some(stats_sample_pages) = self.stats_sample_pages { + write!(f, " STATS_SAMPLE_PAGES={stats_sample_pages}")?; + } + + if let Some(delay_key_write) = &self.delay_key_write { + match delay_key_write { + DelayKeyWrite::Enabled => write!(f, " DELAY_KEY_WRITE=1")?, + DelayKeyWrite::Disabled => write!(f, " DELAY_KEY_WRITE=0")?, + } + } + + if let Some(compression) = &self.compression { + match compression { + Compression::Lz4 => write!(f, " COMPRESSION=LZ4")?, + Compression::Zlib => write!(f, " COMPRESSION=ZLIB")?, + Compression::None => write!(f, " COMPRESSION=NONE")?, + } + } + + if let Some(encryption) = &self.encryption { + match encryption { + Encryption::Yes => write!(f, " ENCRYPTION='Y'")?, + Encryption::No => write!(f, " ENCRYPTION='N'")?, + } + } + + if let Some(max_rows) = &self.max_rows { + write!(f, " MAX_ROWS={}", max_rows)?; + } + + if let Some(min_rows) = &self.min_rows { + write!(f, " MIN_ROWS={}", min_rows)?; + } + + if let Some(autoextend_size) = &self.autoextend_size { + write!(f, " AUTOEXTEND_SIZE={}", autoextend_size)?; + } + + if let Some(avg_row_length) = &self.avg_row_length { + write!(f, " AVG_ROW_LENGTH={}", avg_row_length)?; + } + + if let Some(checksum) = &self.checksum { + match checksum { + true => write!(f, " CHECKSUM=1")?, + false => write!(f, " CHECKSUM=0")?, + } + } + + if let Some(connection) = &self.connection { + write!(f, " CONNECTION='{}'", connection)?; + } + + if let Some(engine_attribute) = &self.engine_attribute { + write!(f, " ENGINE_ATTRIBUTE='{}'", engine_attribute)?; + } + + if let Some(password) = &self.password { + write!(f, " PASSWORD='{}'", password)?; + } + + if let Some(secondary_engine_attribute) = &self.secondary_engine_attribute { + write!( + f, + " SECONDARY_ENGINE_ATTRIBUTE='{}'", + secondary_engine_attribute + )?; + } + + if self.start_transaction.unwrap_or(false) { + write!(f, " START TRANSACTION")?; + } + + if let Some(tablespace_option) = &self.tablespace_option { + write!(f, " TABLESPACE {}", tablespace_option.name)?; + if let Some(storage) = &tablespace_option.storage { + match storage { + StorageType::Disk => write!(f, " STORAGE DISK")?, + StorageType::Memory => write!(f, " STORAGE MEMORY")?, + } + } + } + + if let Some(union_tables) = &self.union_tables { + if !union_tables.is_empty() { + write!( + f, + " UNION=({})", + union_tables + .iter() + .map(|table| table.to_string()) + .collect::>() + .join(", ") + )?; + } + } + if self.on_commit.is_some() { let on_commit = match self.on_commit { Some(OnCommit::DeleteRows) => "ON COMMIT DELETE ROWS", diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index 1c50cb843..b9de2db6d 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -30,7 +30,10 @@ use crate::ast::{ ObjectName, OnCommit, OneOrManyWithParens, Query, RowAccessPolicy, SqlOption, Statement, StorageSerializationPolicy, TableConstraint, TableEngine, Tag, WrappedCollection, }; -use crate::parser::ParserError; +use crate::parser::{ + Compression, DelayKeyWrite, DirectoryOption, Encryption, InsertMethod, OptionState, + ParserError, RowFormat, TablespaceOption, +}; /// Builder for create table statement variant ([1]). /// @@ -89,6 +92,30 @@ pub struct CreateTableBuilder { pub auto_increment_offset: Option, pub default_charset: Option, pub collation: Option, + pub key_block_size: Option, + pub max_rows: Option, + pub min_rows: Option, + pub autoextend_size: Option, + pub avg_row_length: Option, + pub checksum: Option, + pub row_format: Option, + pub compression: Option, + pub encryption: Option, + pub insert_method: Option, + pub pack_keys: Option, + pub stats_auto_recalc: Option, + pub stats_persistent: Option, + pub stats_sample_pages: Option, + pub delay_key_write: Option, + pub connection: Option, + pub engine_attribute: Option, + pub password: Option, + pub secondary_engine_attribute: Option, + pub start_transaction: Option, + pub tablespace_option: Option, + pub union_tables: Option>, + pub data_directory: Option, + pub index_directory: Option, pub on_commit: Option, pub on_cluster: Option, pub primary_key: Option>, @@ -142,6 +169,30 @@ impl CreateTableBuilder { engine: None, comment: None, auto_increment_offset: None, + compression: None, + encryption: None, + insert_method: None, + key_block_size: None, + row_format: None, + data_directory: None, + index_directory: None, + pack_keys: None, + stats_auto_recalc: None, + stats_persistent: None, + stats_sample_pages: None, + delay_key_write: None, + max_rows: None, + min_rows: None, + autoextend_size: None, + avg_row_length: None, + checksum: None, + connection: None, + engine_attribute: None, + password: None, + secondary_engine_attribute: None, + start_transaction: None, + tablespace_option: None, + union_tables: None, default_charset: None, collation: None, on_commit: None, @@ -292,6 +343,118 @@ impl CreateTableBuilder { self.collation = collation; self } + pub fn compression(mut self, compression: Option) -> Self { + self.compression = compression; + self + } + pub fn encryption(mut self, encryption: Option) -> Self { + self.encryption = encryption; + self + } + pub fn delay_key_write(mut self, delay_key_write: Option) -> Self { + self.delay_key_write = delay_key_write; + self + } + pub fn insert_method(mut self, insert_method: Option) -> Self { + self.insert_method = insert_method; + self + } + + pub fn key_block_size(mut self, key_block_size: Option) -> Self { + self.key_block_size = key_block_size; + self + } + pub fn max_rows(mut self, max_rows: Option) -> Self { + self.max_rows = max_rows; + self + } + + pub fn min_rows(mut self, min_rows: Option) -> Self { + self.min_rows = min_rows; + self + } + + pub fn autoextend_size(mut self, autoextend_size: Option) -> Self { + self.autoextend_size = autoextend_size; + self + } + pub fn avg_row_length(mut self, avg_row_length: Option) -> Self { + self.avg_row_length = avg_row_length; + self + } + + pub fn checksum(mut self, checksum: Option) -> Self { + self.checksum = checksum; + self + } + pub fn connection(mut self, connection: Option) -> Self { + self.connection = connection; + self + } + pub fn engine_attribute(mut self, engine_attribute: Option) -> Self { + self.engine_attribute = engine_attribute; + self + } + pub fn password(mut self, password: Option) -> Self { + self.password = password; + self + } + pub fn secondary_engine_attribute( + mut self, + secondary_engine_attribute: Option, + ) -> Self { + self.secondary_engine_attribute = secondary_engine_attribute; + self + } + + pub fn start_transaction(mut self, start_transaction: Option) -> Self { + self.start_transaction = start_transaction; + self + } + + pub fn tablespace_option(mut self, tablespace_option: Option) -> Self { + self.tablespace_option = tablespace_option; + self + } + pub fn union_tables(mut self, union_tables: Option>) -> Self { + self.union_tables = union_tables; + self + } + + pub fn row_format(mut self, row_format: Option) -> Self { + self.row_format = row_format; + self + } + + pub fn data_directory(mut self, directory_option: Option) -> Self { + self.data_directory = directory_option; + self + } + + pub fn index_directory(mut self, directory_option: Option) -> Self { + self.index_directory = directory_option; + self + } + + pub fn pack_keys(mut self, pack_keys: Option) -> Self { + self.pack_keys = pack_keys; + self + } + + pub fn stats_auto_recalc(mut self, stats_auto_recalc: Option) -> Self { + self.stats_auto_recalc = stats_auto_recalc; + self + } + + pub fn stats_persistent(mut self, stats_persistent: Option) -> Self { + self.stats_persistent = stats_persistent; + self + } + + pub fn stats_sample_pages(mut self, stats_sample_pages: Option) -> Self { + self.stats_sample_pages = stats_sample_pages; + self + } pub fn on_commit(mut self, on_commit: Option) -> Self { self.on_commit = on_commit; @@ -450,6 +613,30 @@ impl CreateTableBuilder { auto_increment_offset: self.auto_increment_offset, default_charset: self.default_charset, collation: self.collation, + compression: self.compression, + encryption: self.encryption, + insert_method: self.insert_method, + key_block_size: self.key_block_size, + row_format: self.row_format, + data_directory: self.data_directory, + index_directory: self.index_directory, + pack_keys: self.pack_keys, + stats_auto_recalc: self.stats_auto_recalc, + stats_persistent: self.stats_persistent, + stats_sample_pages: self.stats_sample_pages, + delay_key_write: self.delay_key_write, + max_rows: self.max_rows, + min_rows: self.min_rows, + autoextend_size: self.autoextend_size, + avg_row_length: self.avg_row_length, + checksum: self.checksum, + connection: self.connection, + engine_attribute: self.engine_attribute, + password: self.password, + secondary_engine_attribute: self.secondary_engine_attribute, + start_transaction: self.start_transaction, + tablespace_option: self.tablespace_option, + union_tables: self.union_tables, on_commit: self.on_commit, on_cluster: self.on_cluster, primary_key: self.primary_key, @@ -510,6 +697,30 @@ impl TryFrom for CreateTableBuilder { engine, comment, auto_increment_offset, + compression, + encryption, + insert_method, + key_block_size, + row_format, + data_directory, + index_directory, + pack_keys, + stats_auto_recalc, + stats_persistent, + stats_sample_pages, + delay_key_write, + max_rows, + min_rows, + autoextend_size, + avg_row_length, + checksum, + connection, + engine_attribute, + password, + secondary_engine_attribute, + start_transaction, + tablespace_option, + union_tables, default_charset, collation, on_commit, @@ -559,6 +770,30 @@ impl TryFrom for CreateTableBuilder { engine, comment, auto_increment_offset, + compression, + encryption, + insert_method, + key_block_size, + row_format, + data_directory, + index_directory, + pack_keys, + stats_auto_recalc, + stats_persistent, + stats_sample_pages, + delay_key_write, + max_rows, + min_rows, + autoextend_size, + avg_row_length, + checksum, + connection, + engine_attribute, + password, + secondary_engine_attribute, + start_transaction, + tablespace_option, + union_tables, default_charset, collation, on_commit, diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 836f229a2..eeec3edeb 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -580,6 +580,30 @@ impl Spanned for CreateTable { auto_increment_offset: _, // u32, no span default_charset: _, // string, no span collation: _, // string, no span + compression: _, // enum + encryption: _, // enum + key_block_size: _, // u32, no span + insert_method: _, // enum + row_format: _, // enum + data_directory: _, // string, no span + index_directory: _, // string, no span + stats_auto_recalc: _, // enum + stats_persistent: _, // enum + pack_keys: _, // enum + stats_sample_pages: _, // u32, no span + delay_key_write: _, // enum + max_rows: _, // u32, no span + min_rows: _, // u32, no span + avg_row_length: _, // u32, no span + autoextend_size: _, // u32, no span + checksum: _, // bool, no span + connection: _, // string, no span + password: _, // string, no span + engine_attribute: _, // string, no span + secondary_engine_attribute: _, // string, no span + start_transaction: _, // bool + tablespace_option: _, // enum + union_tables: _, // list of strings, no span on_commit: _, // enum on_cluster: _, // todo, clickhouse specific primary_key: _, // todo, clickhouse specific diff --git a/src/keywords.rs b/src/keywords.rs index bf8a1915d..ddb786650 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -116,9 +116,11 @@ define_keywords!( AUTHENTICATION, AUTHORIZATION, AUTO, + AUTOEXTEND_SIZE, AUTOINCREMENT, AUTO_INCREMENT, AVG, + AVG_ROW_LENGTH, AVRO, BACKWARD, BASE64, @@ -180,6 +182,7 @@ define_keywords!( CHARSET, CHAR_LENGTH, CHECK, + CHECKSUM, CIRCLE, CLEAR, CLOB, @@ -269,6 +272,7 @@ define_keywords!( DEFINED, DEFINER, DELAYED, + DELAY_KEY_WRITE, DELETE, DELIMITED, DELIMITER, @@ -313,6 +317,7 @@ define_keywords!( END_PARTITION, ENFORCED, ENGINE, + ENGINE_ATTRIBUTE, ENUM, ENUM16, ENUM8, @@ -444,6 +449,7 @@ define_keywords!( INPUTFORMAT, INSENSITIVE, INSERT, + INSERT_METHOD, INSTALL, INSTANT, INSTEAD, @@ -480,6 +486,7 @@ define_keywords!( JULIAN, KEY, KEYS, + KEY_BLOCK_SIZE, KILL, LAG, LANGUAGE, @@ -533,6 +540,7 @@ define_keywords!( MAX, MAXVALUE, MAX_DATA_EXTENSION_TIME_IN_DAYS, + MAX_ROWS, MEASURES, MEDIUMBLOB, MEDIUMINT, @@ -554,6 +562,7 @@ define_keywords!( MINUTE, MINUTES, MINVALUE, + MIN_ROWS, MOD, MODE, MODIFIES, @@ -651,6 +660,7 @@ define_keywords!( OWNERSHIP, PACKAGE, PACKAGES, + PACK_KEYS, PARALLEL, PARAMETER, PARQUET, @@ -773,6 +783,7 @@ define_keywords!( ROW, ROWID, ROWS, + ROW_FORMAT, ROW_NUMBER, RULE, RUN, @@ -787,6 +798,7 @@ define_keywords!( SEARCH, SECOND, SECONDARY, + SECONDARY_ENGINE_ATTRIBUTE, SECONDS, SECRET, SECURITY, @@ -838,12 +850,16 @@ define_keywords!( STATEMENT, STATIC, STATISTICS, + STATS_AUTO_RECALC, + STATS_PERSISTENT, + STATS_SAMPLE_PAGES, STATUS, STDDEV_POP, STDDEV_SAMP, STDIN, STDOUT, STEP, + STORAGE, STORAGE_INTEGRATION, STORAGE_SERIALIZATION_POLICY, STORED, @@ -870,6 +886,7 @@ define_keywords!( TABLE, TABLES, TABLESAMPLE, + TABLESPACE, TAG, TARGET, TASK, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index cbd464c34..c05e932d6 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -48,6 +48,130 @@ pub enum ParserError { RecursionLimitExceeded, } +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Compression { + Zlib, + Lz4, + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum DelayKeyWrite { + Disabled, + Enabled, +} +impl fmt::Display for DelayKeyWrite { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DelayKeyWrite::Disabled => write!(f, "DISABLED"), + DelayKeyWrite::Enabled => write!(f, "ENABLED"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Encryption { + Yes, + No, +} + +impl fmt::Display for Encryption { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Encryption::Yes => write!(f, "Y"), + Encryption::No => write!(f, "N"), + } + } +} + +impl fmt::Display for Compression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Compression::Zlib => write!(f, "ZLIB"), + Compression::Lz4 => write!(f, "LZ4"), + Compression::None => write!(f, "NONE"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum InsertMethod { + No, + First, + Last, +} +impl fmt::Display for InsertMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + InsertMethod::No => write!(f, "NO"), + InsertMethod::First => write!(f, "FIRST"), + InsertMethod::Last => write!(f, "LAST"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum RowFormat { + Default, + Dynamic, + Fixed, + Compressed, + Redundant, + Compact, +} +impl fmt::Display for RowFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RowFormat::Default => write!(f, "DEFAULT"), + RowFormat::Dynamic => write!(f, "DYNAMIC"), + RowFormat::Fixed => write!(f, "FIXED"), + RowFormat::Compressed => write!(f, "COMPRESSED"), + RowFormat::Redundant => write!(f, "REDUNDANT"), + RowFormat::Compact => write!(f, "COMPACT"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum DirectoryType { + Data, + Index, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DirectoryOption { + pub directory_type: DirectoryType, + pub path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum OptionState { + Zero, + One, + Default, +} +impl fmt::Display for OptionState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OptionState::Zero => write!(f, "0"), + OptionState::One => write!(f, "1"), + OptionState::Default => write!(f, "DEFAULT"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum StorageType { + Disk, + Memory, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct TablespaceOption { + pub name: String, + pub storage: Option, +} + // Use `Parser::expected` instead, if possible macro_rules! parser_err { ($MSG:expr, $loc:expr) => { @@ -7063,35 +7187,616 @@ impl<'a> Parser<'a> { let with_options = self.parse_options(Keyword::WITH)?; let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; - let engine = if self.parse_keyword(Keyword::ENGINE) { - self.expect_token(&Token::Eq)?; - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => { - let name = w.value; - let parameters = if self.peek_token() == Token::LParen { - Some(self.parse_parenthesized_identifiers()?) + let mut engine = None; + let mut auto_increment_offset = None; + let mut default_charset = None; + let mut collation = None; + let mut insert_method = None; + let mut key_block_size = None; + let mut row_format = None; + let mut data_directory = None; + let mut index_directory = None; + let mut pack_keys = None; + let mut stats_auto_recalc = None; + let mut stats_persistent = None; + let mut stats_sample_pages = None; + let mut delay_key_write = None; + let mut compression = None; + let mut encryption = None; + let mut max_rows = None; + let mut min_rows = None; + let mut autoextend_size = None; + let mut avg_row_length = None; + let mut checksum = None; + let mut connection = None; + let mut engine_attribute = None; + let mut password = None; + let mut secondary_engine_attribute = None; + let mut start_transaction = None; + let mut tablespace_option = None; + let mut union_tables = None; + + // Parse table options in any order + while let Some(keyword) = self.parse_one_of_keywords(&[ + Keyword::ENGINE, + Keyword::AUTO_INCREMENT, + Keyword::DEFAULT, + Keyword::CHARSET, + Keyword::COLLATE, + Keyword::INSERT_METHOD, + Keyword::KEY_BLOCK_SIZE, + Keyword::ROW_FORMAT, + Keyword::DATA, + Keyword::INDEX, + Keyword::PACK_KEYS, + Keyword::STATS_AUTO_RECALC, + Keyword::STATS_PERSISTENT, + Keyword::STATS_SAMPLE_PAGES, + Keyword::DELAY_KEY_WRITE, + Keyword::COMPRESSION, + Keyword::ENCRYPTION, + Keyword::MAX_ROWS, + Keyword::MIN_ROWS, + Keyword::AUTOEXTEND_SIZE, + Keyword::AVG_ROW_LENGTH, + Keyword::CHECKSUM, + Keyword::CONNECTION, + Keyword::ENGINE_ATTRIBUTE, + Keyword::PASSWORD, + Keyword::SECONDARY_ENGINE_ATTRIBUTE, + Keyword::START, + Keyword::TABLESPACE, + Keyword::UNION, + ]) { + match keyword { + Keyword::ENGINE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => { + let name = w.value; + let parameters = if self.peek_token() == Token::LParen { + Some(self.parse_parenthesized_identifiers()?) + } else { + None + }; + engine = Some(TableEngine { name, parameters }); + } + _ => self.expected("identifier", next_token)?, + } + } + + Keyword::AUTO_INCREMENT => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + auto_increment_offset = + Some(Self::parse::(s, next_token.span.start)?) + } + _ => self.expected("literal int", next_token)?, + } + } + + Keyword::DEFAULT => { + if self.parse_keyword(Keyword::CHARSET) { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => default_charset = Some(w.value), + _ => self.expected("identifier", next_token)?, + } + } else if self.parse_keyword(Keyword::COLLATE) { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => collation = Some(w.value), + _ => self.expected("identifier", next_token)?, + } + } else { + return Err(ParserError::ParserError( + "Expected CHARACTER SET or COLLATE after DEFAULT".to_string(), + )); + } + } + + Keyword::CHARSET => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => default_charset = Some(w.value), + _ => self.expected("identifier", next_token)?, + } + } + + Keyword::COLLATE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => collation = Some(w.value), + _ => self.expected("identifier", next_token)?, + } + } + + Keyword::INSERT_METHOD => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::Word(w) => match w.value.to_uppercase().as_str() { + "NO" => insert_method = Some(InsertMethod::No), + "FIRST" => insert_method = Some(InsertMethod::First), + "LAST" => insert_method = Some(InsertMethod::Last), + _ => { + return Err(ParserError::ParserError( + "Invalid INSERT_METHOD value".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected INSERT_METHOD value".to_string(), + )) + } + } + } + + Keyword::KEY_BLOCK_SIZE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + key_block_size = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected KEY_BLOCK_SIZE value".to_string(), + )) + } + } + } + + Keyword::ROW_FORMAT => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::Word(w) => match w.value.to_uppercase().as_str() { + "DEFAULT" => row_format = Some(RowFormat::Default), + "DYNAMIC" => row_format = Some(RowFormat::Dynamic), + "FIXED" => row_format = Some(RowFormat::Fixed), + "COMPRESSED" => row_format = Some(RowFormat::Compressed), + "REDUNDANT" => row_format = Some(RowFormat::Redundant), + "COMPACT" => row_format = Some(RowFormat::Compact), + _ => { + return Err(ParserError::ParserError( + "Invalid ROW_FORMAT value".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected ROW_FORMAT value".to_string(), + )) + } + } + } + + Keyword::DATA => { + if self.parse_keyword(Keyword::DIRECTORY) { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::SingleQuotedString(path) => { + data_directory = Some(DirectoryOption { + directory_type: DirectoryType::Data, + path, + }) + } + _ => { + return Err(ParserError::ParserError( + "Expected path for DATA DIRECTORY".to_string(), + )); + } + } + } else { + return Err(ParserError::ParserError( + "Expected DIRECTORY after DATA".to_string(), + )); + } + } + + Keyword::INDEX => { + if self.parse_keyword(Keyword::DIRECTORY) { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::SingleQuotedString(path) => { + index_directory = Some(DirectoryOption { + directory_type: DirectoryType::Index, + path, + }) + } + _ => { + return Err(ParserError::ParserError( + "Expected path for INDEX DIRECTORY".to_string(), + )); + } + } + } else { + return Err(ParserError::ParserError( + "Expected DIRECTORY after INDEX".to_string(), + )); + } + } + + Keyword::PACK_KEYS => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => match s.as_str() { + "0" => pack_keys = Some(OptionState::Zero), + "1" => pack_keys = Some(OptionState::One), + _ => { + return Err(ParserError::ParserError(format!( + "PACK_KEYS can only be 0, 1, or DEFAULT, found: {}", + s + ))) + } + }, + Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { + pack_keys = Some(OptionState::Default) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for PACK_KEYS".to_string(), + )) + } + } + } + + Keyword::STATS_AUTO_RECALC => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => match s.as_str() { + "0" => stats_auto_recalc = Some(OptionState::Zero), + "1" => stats_auto_recalc = Some(OptionState::One), + _ => { + return Err(ParserError::ParserError(format!( + "STATS_AUTO_RECALC can only be 0, 1, or DEFAULT, found: {}", + s + ))) + } + }, + Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { + stats_auto_recalc = Some(OptionState::Default) + } + _ => { + return Err(ParserError::ParserError( + "Expected STATS_AUTO_RECALC value (0, 1, or DEFAULT)".to_string(), + )) + } + } + } + + Keyword::STATS_PERSISTENT => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => match s.as_str() { + "0" => stats_persistent = Some(OptionState::Zero), + "1" => stats_persistent = Some(OptionState::One), + _ => { + return Err(ParserError::ParserError(format!( + "STATS_PERSISTENT can only be 0, 1, or DEFAULT, found: {}", + s + ))) + } + }, + Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { + stats_persistent = Some(OptionState::Default) + } + _ => { + return Err(ParserError::ParserError( + "Expected STATS_PERSISTENT value (0, 1, or DEFAULT)".to_string(), + )) + } + } + } + + Keyword::STATS_SAMPLE_PAGES => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + stats_sample_pages = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for STATS_SAMPLE_PAGES".to_string(), + )) + } + } + } + + Keyword::DELAY_KEY_WRITE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => match s.to_uppercase().as_str() { + "0" => delay_key_write = Some(DelayKeyWrite::Disabled), + "1" => delay_key_write = Some(DelayKeyWrite::Enabled), + _ => { + return Err(ParserError::ParserError( + "Invalid DELAY_KEY_WRITE value (expected 0 or 1)".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected DELAY_KEY_WRITE value".to_string(), + )) + } + } + } + + Keyword::COMPRESSION => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::Word(w) => match w.value.to_uppercase().as_str() { + "ZLIB" => compression = Some(Compression::Zlib), + "LZ4" => compression = Some(Compression::Lz4), + "NONE" => compression = Some(Compression::None), + _ => { + return Err(ParserError::ParserError( + "Invalid COMPRESSION value".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected COMPRESSION value".to_string(), + )) + } + } + } + + Keyword::ENCRYPTION => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::SingleQuotedString(s) => match s.to_uppercase().as_str() { + "Y" => encryption = Some(Encryption::Yes), + "N" => encryption = Some(Encryption::No), + _ => { + return Err(ParserError::ParserError( + "Invalid ENCRYPTION value (expected 'Y' or 'N')".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected ENCRYPTION value".to_string(), + )) + } + } + } + + Keyword::MAX_ROWS => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + max_rows = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for MAX_ROWS".to_string(), + )) + } + } + } + + Keyword::MIN_ROWS => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + min_rows = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for MIN_ROWS".to_string(), + )) + } + } + } + + Keyword::AUTOEXTEND_SIZE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + autoextend_size = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for AUTOEXTEND_SIZE".to_string(), + )) + } + } + } + + Keyword::AVG_ROW_LENGTH => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => { + avg_row_length = Some(Self::parse::(s, next_token.span.start)?) + } + _ => { + return Err(ParserError::ParserError( + "Expected numeric value for AVG_ROW_LENGTH".to_string(), + )) + } + } + } + + Keyword::CHECKSUM => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::Number(s, _) => match s.as_str() { + "0" => checksum = Some(false), + "1" => checksum = Some(true), + _ => { + return Err(ParserError::ParserError( + "Invalid CHECKSUM value (expected 0 or 1)".to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected a numeric value for CHECKSUM".to_string(), + )) + } + } + } + + Keyword::CONNECTION => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::SingleQuotedString(s) => connection = Some(s), + _ => { + return Err(ParserError::ParserError( + "Expected CONNECTION value as a single-quoted string".to_string(), + )) + } + } + } + + Keyword::ENGINE_ATTRIBUTE => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::SingleQuotedString(s) => engine_attribute = Some(s), + _ => { + return Err(ParserError::ParserError( + "Expected ENGINE_ATTRIBUTE value as a single-quoted string" + .to_string(), + )) + } + } + } + + Keyword::PASSWORD => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::SingleQuotedString(s) => password = Some(s), + _ => { + return Err(ParserError::ParserError( + "Expected PASSWORD value as a single-quoted string".to_string(), + )) + } + } + } + + Keyword::SECONDARY_ENGINE_ATTRIBUTE => { + let _ = self.consume_token(&Token::Eq); + let token = self.next_token(); + match token.token { + Token::SingleQuotedString(s) => secondary_engine_attribute = Some(s), + _ => return Err(ParserError::ParserError( + "Expected SECONDARY_ENGINE_ATTRIBUTE value as a single-quoted string" + .to_string(), + )), + } + } + + Keyword::START => { + if self.parse_keyword(Keyword::TRANSACTION) { + start_transaction = Some(true); + } else { + return Err(ParserError::ParserError( + "Expected TRANSACTION after START".to_string(), + )); + } + } + + Keyword::TABLESPACE => { + let _ = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + + let name = match next_token.token { + Token::Word(w) => w.value.clone(), + Token::SingleQuotedString(s) => s.clone(), + _ => { + return Err(ParserError::ParserError( + "Expected TABLESPACE name".to_string(), + )); + } + }; + + let storage = if self.parse_keyword(Keyword::STORAGE) { + let _ = self.consume_token(&Token::Eq); + let storage_token = self.next_token(); + match storage_token.token { + Token::Word(w) => match w.value.to_uppercase().as_str() { + "DISK" => Some(StorageType::Disk), + "MEMORY" => Some(StorageType::Memory), + _ => { + return Err(ParserError::ParserError( + "Invalid STORAGE type (expected DISK or MEMORY)" + .to_string(), + )) + } + }, + _ => { + return Err(ParserError::ParserError( + "Expected STORAGE type".to_string(), + )) + } + } } else { None }; - Some(TableEngine { name, parameters }) + + tablespace_option = Some(TablespaceOption { name, storage }); } - _ => self.expected("identifier", next_token)?, - } - } else { - None - }; - let auto_increment_offset = if self.parse_keyword(Keyword::AUTO_INCREMENT) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => Some(Self::parse::(s, next_token.span.start)?), - _ => self.expected("literal int", next_token)?, + Keyword::UNION => { + let _ = self.consume_token(&Token::Eq); + self.expect_token(&Token::LParen)?; + let mut tables = Vec::new(); + + loop { + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => tables.push(w.value), + Token::Comma => continue, + Token::RParen => break, + _ => { + return Err(ParserError::ParserError( + "Expected table name or ')' in UNION".to_string(), + )); + } + } + } + union_tables = Some(tables); + } + + _ => { + return Err(ParserError::ParserError(format!( + "Unexpected keyword: {:?}", + keyword + ))); + } } - } else { - None - }; + } + + let create_table_config = self.parse_optional_create_table_config()?; // ClickHouse supports `PRIMARY KEY`, before `ORDER BY` // https://clickhouse.com/docs/en/sql-reference/statements/create/table#primary-key @@ -7119,30 +7824,6 @@ impl<'a> Parser<'a> { None }; - let create_table_config = self.parse_optional_create_table_config()?; - - let default_charset = if self.parse_keywords(&[Keyword::DEFAULT, Keyword::CHARSET]) { - self.expect_token(&Token::Eq)?; - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } else { - None - }; - - let collation = if self.parse_keywords(&[Keyword::COLLATE]) { - self.expect_token(&Token::Eq)?; - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } else { - None - }; - let on_commit = if self.parse_keywords(&[Keyword::ON, Keyword::COMMIT]) { Some(self.parse_create_table_on_commit()?) } else { @@ -7192,6 +7873,30 @@ impl<'a> Parser<'a> { .order_by(order_by) .default_charset(default_charset) .collation(collation) + .compression(compression) + .insert_method(insert_method) + .encryption(encryption) + .key_block_size(key_block_size) + .row_format(row_format) + .data_directory(data_directory) + .index_directory(index_directory) + .pack_keys(pack_keys) + .stats_auto_recalc(stats_auto_recalc) + .stats_persistent(stats_persistent) + .stats_sample_pages(stats_sample_pages) + .delay_key_write(delay_key_write) + .max_rows(max_rows) + .min_rows(min_rows) + .autoextend_size(autoextend_size) + .avg_row_length(avg_row_length) + .checksum(checksum) + .connection(connection) + .engine_attribute(engine_attribute) + .password(password) + .secondary_engine_attribute(secondary_engine_attribute) + .start_transaction(start_transaction) + .tablespace_option(tablespace_option) + .union_tables(union_tables) .on_commit(on_commit) .on_cluster(on_cluster) .clustered_by(clustered_by) diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index 320583248..d1a87e0cf 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -747,6 +747,30 @@ fn test_duckdb_union_datatype() { comment: Default::default(), auto_increment_offset: Default::default(), default_charset: Default::default(), + insert_method: Default::default(), + data_directory: Default::default(), + key_block_size: Default::default(), + pack_keys: Default::default(), + row_format: Default::default(), + stats_auto_recalc: Default::default(), + stats_persistent: Default::default(), + stats_sample_pages: Default::default(), + max_rows: Default::default(), + min_rows: Default::default(), + autoextend_size: Default::default(), + avg_row_length: Default::default(), + checksum: Default::default(), + connection: Default::default(), + engine_attribute: Default::default(), + password: Default::default(), + secondary_engine_attribute: Default::default(), + tablespace_option: Default::default(), + compression: Default::default(), + delay_key_write: Default::default(), + encryption: Default::default(), + start_transaction: Default::default(), + union_tables: Default::default(), + index_directory: Default::default(), collation: Default::default(), on_commit: Default::default(), on_cluster: Default::default(), diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 88e7a1f17..65aae0812 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1759,6 +1759,30 @@ fn parse_create_table_with_valid_options() { comment: None, auto_increment_offset: None, default_charset: None, + insert_method: None, + data_directory: None, + key_block_size: None, + pack_keys: None, + row_format: None, + stats_auto_recalc: None, + stats_persistent: None, + stats_sample_pages: None, + max_rows: None, + min_rows: None, + autoextend_size: None, + avg_row_length: None, + checksum: None, + connection: None, + engine_attribute: None, + password: None, + secondary_engine_attribute: None, + tablespace_option: None, + compression: None, + delay_key_write: None, + encryption: None, + start_transaction: None, + union_tables: None, + index_directory: None, collation: None, on_commit: None, on_cluster: None, @@ -1930,6 +1954,30 @@ fn parse_create_table_with_identity_column() { comment: None, auto_increment_offset: None, default_charset: None, + insert_method: None, + data_directory: None, + key_block_size: None, + pack_keys: None, + row_format: None, + stats_auto_recalc: None, + stats_persistent: None, + stats_sample_pages: None, + max_rows: None, + min_rows: None, + autoextend_size: None, + avg_row_length: None, + checksum: None, + connection: None, + engine_attribute: None, + password: None, + secondary_engine_attribute: None, + tablespace_option: None, + compression: None, + delay_key_write: None, + encryption: None, + start_transaction: None, + union_tables: None, + index_directory: None, collation: None, on_commit: None, on_cluster: None, diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 990107b25..d5ab25e6b 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -25,7 +25,7 @@ use matches::assert_matches; use sqlparser::ast::MysqlInsertPriority::{Delayed, HighPriority, LowPriority}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MySqlDialect}; -use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::parser::{ParserError, ParserOptions, RowFormat, StorageType, TablespaceOption}; use sqlparser::tokenizer::Span; use sqlparser::tokenizer::Token; use test_utils::*; @@ -859,29 +859,175 @@ fn parse_create_table_comment() { #[test] fn parse_create_table_auto_increment_offset() { - let canonical = - "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT 123"; - let with_equal = - "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123"; + let sql = "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123"; - for sql in [canonical, with_equal] { - match mysql().one_statement_parses_to(sql, canonical) { + match mysql().verified_stmt(sql) { + Statement::CreateTable(CreateTable { + name, + auto_increment_offset, + .. + }) => { + assert_eq!(name.to_string(), "foo"); + assert_eq!( + auto_increment_offset.expect("Should exist").to_string(), + "123" + ); + } + _ => unreachable!(), + } +} + +#[test] +fn parse_create_table_multiple_options_order_independent() { + let sql1 = "CREATE TABLE mytable (id INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8"; + let sql2 = "CREATE TABLE mytable (id INT) KEY_BLOCK_SIZE=8 ENGINE=InnoDB ROW_FORMAT=DYNAMIC"; + let sql3 = "CREATE TABLE mytable (id INT) ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 ENGINE=InnoDB"; + + for sql in [sql1, sql2, sql3] { + match mysql().parse_sql_statements(sql).unwrap().pop().unwrap() { Statement::CreateTable(CreateTable { name, - auto_increment_offset, + engine, + row_format, + key_block_size, .. }) => { - assert_eq!(name.to_string(), "foo"); + assert_eq!(name.to_string(), "mytable"); assert_eq!( - auto_increment_offset.expect("Should exist").to_string(), - "123" + engine, + Some(TableEngine { + name: "InnoDB".to_string(), + parameters: None + }) ); + assert_eq!(row_format, Some(RowFormat::Dynamic)); + assert_eq!(key_block_size.expect("Should exist").to_string(), "8"); } _ => unreachable!(), } } } +#[test] +fn parse_create_table_with_all_table_options() { + let sql = + "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci INSERT_METHOD=FIRST KEY_BLOCK_SIZE=8 ROW_FORMAT=DYNAMIC DATA DIRECTORY='/var/lib/mysql/data' INDEX DIRECTORY='/var/lib/mysql/index' PACK_KEYS=1 STATS_AUTO_RECALC=1 STATS_PERSISTENT=0 STATS_SAMPLE_PAGES=128 DELAY_KEY_WRITE=1 COMPRESSION=ZLIB ENCRYPTION='Y' MAX_ROWS=10000 MIN_ROWS=10 AUTOEXTEND_SIZE=64 AVG_ROW_LENGTH=128 CHECKSUM=1 CONNECTION='mysql://localhost' ENGINE_ATTRIBUTE='primary' PASSWORD='secure_password' SECONDARY_ENGINE_ATTRIBUTE='secondary_attr' START TRANSACTION TABLESPACE my_tablespace STORAGE DISK UNION=(table1, table2, table3)"; + + match mysql().verified_stmt(sql) { + Statement::CreateTable(CreateTable { + name, + engine, + default_charset, + auto_increment_offset, + key_block_size, + row_format, + data_directory, + index_directory, + pack_keys, + stats_auto_recalc, + stats_persistent, + stats_sample_pages, + compression, + insert_method, + encryption, + max_rows, + min_rows, + collation, + autoextend_size, + avg_row_length, + checksum, + connection, + engine_attribute, + password, + secondary_engine_attribute, + start_transaction, + tablespace_option, + union_tables, + .. + }) => { + assert_eq!(name.to_string(), "foo"); + assert_eq!( + engine, + Some(TableEngine { + name: "InnoDB".to_string(), + parameters: None + }) + ); + assert_eq!(default_charset, Some("utf8mb4".to_string())); + assert_eq!(collation, Some("utf8mb4_0900_ai_ci".to_string())); + assert_eq!( + auto_increment_offset.expect("Should exist").to_string(), + "123" + ); + assert_eq!(key_block_size.expect("Should exist").to_string(), "8"); + assert_eq!(row_format.expect("Should exist").to_string(), "DYNAMIC"); + assert_eq!(pack_keys.expect("Should exist").to_string(), "1"); + assert_eq!(stats_auto_recalc.expect("Should exist").to_string(), "1"); + assert_eq!(stats_persistent.expect("Should exist").to_string(), "0"); + assert_eq!(stats_sample_pages.expect("Should exist").to_string(), "128"); + assert_eq!(insert_method.expect("Should exist").to_string(), "FIRST"); + assert_eq!(compression.expect("Should exist").to_string(), "ZLIB"); + assert_eq!(encryption.expect("Should exist").to_string(), "Y"); + assert_eq!(max_rows.expect("Should exist").to_string(), "10000"); + assert_eq!(min_rows.expect("Should exist").to_string(), "10"); + assert_eq!(autoextend_size.expect("Should exist").to_string(), "64"); + assert_eq!(avg_row_length.expect("Should exist").to_string(), "128"); + assert_eq!( + if checksum.expect("Should exist") { + "1" + } else { + "0" + }, + "1" + ); + assert_eq!( + connection.expect("Should exist").to_string(), + "mysql://localhost" + ); + assert_eq!( + engine_attribute.expect("Should exist").to_string(), + "primary" + ); + assert_eq!( + password.expect("Should exist").to_string(), + "secure_password" + ); + assert_eq!( + secondary_engine_attribute + .expect("Should exist") + .to_string(), + "secondary_attr" + ); + + assert_eq!(start_transaction.expect("Should exist"), true); + assert_eq!( + tablespace_option.expect("Should exist"), + TablespaceOption { + name: "my_tablespace".to_string(), + storage: Some(StorageType::Disk), + } + ); + assert_eq!( + union_tables.expect("Should exist"), + vec![ + "table1".to_string(), + "table2".to_string(), + "table3".to_string() + ] + ); + assert_eq!( + data_directory.expect("Should exist").path, + "/var/lib/mysql/data" + ); + assert_eq!( + index_directory.expect("Should exist").path, + "/var/lib/mysql/index" + ); + } + _ => unreachable!(), + } +} + #[test] fn parse_create_table_set_enum() { let sql = "CREATE TABLE foo (bar SET('a', 'b'), baz ENUM('a', 'b'))"; diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 6c008c84d..d7d874f5d 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -5518,6 +5518,30 @@ fn parse_trigger_related_functions() { comment: None, auto_increment_offset: None, default_charset: None, + insert_method: None, + data_directory: None, + key_block_size: None, + pack_keys: None, + row_format: None, + stats_auto_recalc: None, + stats_persistent: None, + stats_sample_pages: None, + max_rows: None, + min_rows: None, + autoextend_size: None, + avg_row_length: None, + checksum: None, + connection: None, + engine_attribute: None, + password: None, + secondary_engine_attribute: None, + tablespace_option: None, + compression: None, + delay_key_write: None, + encryption: None, + start_transaction: None, + union_tables: None, + index_directory: None, collation: None, on_commit: None, on_cluster: None, From eff0f629409374d771afcf235f1abdeb408f5786 Mon Sep 17 00:00:00 2001 From: Tomer Shani Date: Tue, 11 Mar 2025 18:39:12 +0200 Subject: [PATCH 2/3] Generalized support for table options create table parsing - comment parsing to generic parameter parsing create table parsing - unify the different parameter options into the CreateTableOptions enum create table parsing - consolidate SqlOption::Union & SqlOption::TableEngine into SqlOption::NamedParenthesizedList --- src/ast/dml.rs | 258 +------ src/ast/helpers/stmt_create_table.rs | 340 +-------- src/ast/helpers/stmt_data_loading.rs | 2 - src/ast/mod.rs | 105 ++- src/ast/spans.rs | 63 +- src/dialect/snowflake.rs | 17 +- src/parser/mod.rs | 1060 +++++++------------------- tests/sqlparser_bigquery.rs | 6 +- tests/sqlparser_clickhouse.rs | 37 +- tests/sqlparser_common.rs | 47 +- tests/sqlparser_duckdb.rs | 34 +- tests/sqlparser_hive.rs | 9 +- tests/sqlparser_mssql.rs | 70 +- tests/sqlparser_mysql.rs | 386 ++++++---- tests/sqlparser_postgres.rs | 48 +- tests/sqlparser_snowflake.rs | 17 +- 16 files changed, 750 insertions(+), 1749 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 810c02934..94589e085 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -29,20 +29,15 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::parser::{ - Compression, DelayKeyWrite, DirectoryOption, Encryption, InsertMethod, OptionState, RowFormat, - StorageType, TablespaceOption, -}; - pub use super::ddl::{ColumnDef, TableConstraint}; use super::{ display_comma_separated, display_separated, query::InputFormatClause, Assignment, ClusteredBy, - CommentDef, Expr, FileFormat, FromTable, HiveDistributionStyle, HiveFormat, HiveIOFormat, - HiveRowFormat, Ident, IndexType, InsertAliases, MysqlInsertPriority, ObjectName, OnCommit, - OnInsert, OneOrManyWithParens, OrderByExpr, Query, RowAccessPolicy, SelectItem, Setting, - SqlOption, SqliteOnConflict, StorageSerializationPolicy, TableEngine, TableObject, - TableWithJoins, Tag, WrappedCollection, + CommentDef, CreateTableOptions, Expr, FileFormat, FromTable, HiveDistributionStyle, HiveFormat, + HiveIOFormat, HiveRowFormat, Ident, IndexType, InsertAliases, MysqlInsertPriority, ObjectName, + OnCommit, OnInsert, OneOrManyWithParens, OrderByExpr, Query, RowAccessPolicy, SelectItem, + Setting, SqliteOnConflict, StorageSerializationPolicy, TableObject, TableWithJoins, Tag, + WrappedCollection, }; /// Index column type. @@ -151,43 +146,17 @@ pub struct CreateTable { pub constraints: Vec, pub hive_distribution: HiveDistributionStyle, pub hive_formats: Option, - pub table_properties: Vec, - pub with_options: Vec, + pub table_options: CreateTableOptions, pub file_format: Option, pub location: Option, pub query: Option>, pub without_rowid: bool, pub like: Option, pub clone: Option, - pub engine: Option, - pub comment: Option, - pub auto_increment_offset: Option, - pub key_block_size: Option, - pub max_rows: Option, - pub min_rows: Option, - pub autoextend_size: Option, - pub avg_row_length: Option, - pub checksum: Option, - pub connection: Option, - pub engine_attribute: Option, - pub password: Option, - pub secondary_engine_attribute: Option, - pub tablespace_option: Option, - pub row_format: Option, - pub insert_method: Option, - pub compression: Option, - pub delay_key_write: Option, - pub encryption: Option, - pub pack_keys: Option, - pub stats_auto_recalc: Option, - pub stats_persistent: Option, - pub stats_sample_pages: Option, - pub start_transaction: Option, - pub union_tables: Option>, - pub data_directory: Option, - pub index_directory: Option, - pub default_charset: Option, - pub collation: Option, + // For Hive dialect, the table comment is after the column definitions without `=`, + // so we need to add an extra variant to allow to identify this case when displaying. + // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) + pub comment_after_column_def: Option, pub on_commit: Option, /// ClickHouse "ON CLUSTER" clause: /// @@ -208,9 +177,6 @@ pub struct CreateTable { /// Hive: Table clustering column list. /// pub clustered_by: Option, - /// BigQuery: Table options list. - /// - pub options: Option>, /// Postgres `INHERITs` clause, which contains the list of tables from which /// the new table inherits. /// @@ -311,7 +277,7 @@ impl Display for CreateTable { // Hive table comment should be after column definitions, please refer to: // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) - if let Some(CommentDef::AfterColumnDefsWithoutEq(comment)) = &self.comment { + if let Some(comment) = &self.comment_after_column_def { write!(f, " COMMENT '{comment}'")?; } @@ -404,35 +370,14 @@ impl Display for CreateTable { } write!(f, " LOCATION '{}'", self.location.as_ref().unwrap())?; } - if !self.table_properties.is_empty() { - write!( - f, - " TBLPROPERTIES ({})", - display_comma_separated(&self.table_properties) - )?; - } - if !self.with_options.is_empty() { - write!(f, " WITH ({})", display_comma_separated(&self.with_options))?; - } - if let Some(engine) = &self.engine { - write!(f, " ENGINE={engine}")?; - } - if let Some(comment_def) = &self.comment { - match comment_def { - CommentDef::WithEq(comment) => { - write!(f, " COMMENT = '{comment}'")?; - } - CommentDef::WithoutEq(comment) => { - write!(f, " COMMENT '{comment}'")?; - } - // For CommentDef::AfterColumnDefsWithoutEq will be displayed after column definition - CommentDef::AfterColumnDefsWithoutEq(_) => (), - } - } - if let Some(auto_increment_offset) = self.auto_increment_offset { - write!(f, " AUTO_INCREMENT={auto_increment_offset}")?; + match &self.table_options { + options @ CreateTableOptions::With(_) + | options @ CreateTableOptions::Plain(_) + | options @ CreateTableOptions::TableProperties(_) => write!(f, " {}", options)?, + _ => (), } + if let Some(primary_key) = &self.primary_key { write!(f, " PRIMARY KEY {}", primary_key)?; } @@ -448,15 +393,9 @@ impl Display for CreateTable { if let Some(cluster_by) = self.cluster_by.as_ref() { write!(f, " CLUSTER BY {cluster_by}")?; } - - if let Some(options) = self.options.as_ref() { - write!( - f, - " OPTIONS({})", - display_comma_separated(options.as_slice()) - )?; + if let options @ CreateTableOptions::Options(_) = &self.table_options { + write!(f, " {}", options)?; } - if let Some(external_volume) = self.external_volume.as_ref() { write!(f, " EXTERNAL_VOLUME = '{external_volume}'")?; } @@ -532,165 +471,6 @@ impl Display for CreateTable { write!(f, " WITH TAG ({})", display_comma_separated(tag.as_slice()))?; } - if let Some(default_charset) = &self.default_charset { - write!(f, " DEFAULT CHARSET={default_charset}")?; - } - if let Some(collation) = &self.collation { - write!(f, " COLLATE={collation}")?; - } - - if let Some(insert_method) = &self.insert_method { - match insert_method { - InsertMethod::No => write!(f, " INSERT_METHOD=NO")?, - InsertMethod::First => write!(f, " INSERT_METHOD=FIRST")?, - InsertMethod::Last => write!(f, " INSERT_METHOD=LAST")?, - } - } - - if let Some(key_block_size) = self.key_block_size { - write!(f, " KEY_BLOCK_SIZE={key_block_size}")?; - } - - if let Some(row_format) = &self.row_format { - match row_format { - RowFormat::Default => write!(f, " ROW_FORMAT=DEFAULT")?, - RowFormat::Dynamic => write!(f, " ROW_FORMAT=DYNAMIC")?, - RowFormat::Fixed => write!(f, " ROW_FORMAT=FIXED")?, - RowFormat::Compressed => write!(f, " ROW_FORMAT=COMPRESSED")?, - RowFormat::Redundant => write!(f, " ROW_FORMAT=REDUNDANT")?, - RowFormat::Compact => write!(f, " ROW_FORMAT=COMPACT")?, - } - } - - if let Some(data_dir) = &self.data_directory { - write!(f, " DATA DIRECTORY='{}'", data_dir.path)?; - } - - if let Some(index_dir) = &self.index_directory { - write!(f, " INDEX DIRECTORY='{}'", index_dir.path)?; - } - - if let Some(pack_keys) = &self.pack_keys { - match pack_keys { - OptionState::Default => write!(f, " PACK_KEYS=DEFAULT")?, - OptionState::One => write!(f, " PACK_KEYS=1")?, - OptionState::Zero => write!(f, " PACK_KEYS=0")?, - } - } - - if let Some(stats_auto_recalc) = &self.stats_auto_recalc { - match stats_auto_recalc { - OptionState::Default => write!(f, " STATS_AUTO_RECALC=DEFAULT")?, - OptionState::One => write!(f, " STATS_AUTO_RECALC=1")?, - OptionState::Zero => write!(f, " STATS_AUTO_RECALC=0")?, - } - } - - if let Some(stats_persistent) = &self.stats_persistent { - match stats_persistent { - OptionState::Default => write!(f, " STATS_PERSISTENT=DEFAULT")?, - OptionState::One => write!(f, " STATS_PERSISTENT=1")?, - OptionState::Zero => write!(f, " STATS_PERSISTENT=0")?, - } - } - - if let Some(stats_sample_pages) = self.stats_sample_pages { - write!(f, " STATS_SAMPLE_PAGES={stats_sample_pages}")?; - } - - if let Some(delay_key_write) = &self.delay_key_write { - match delay_key_write { - DelayKeyWrite::Enabled => write!(f, " DELAY_KEY_WRITE=1")?, - DelayKeyWrite::Disabled => write!(f, " DELAY_KEY_WRITE=0")?, - } - } - - if let Some(compression) = &self.compression { - match compression { - Compression::Lz4 => write!(f, " COMPRESSION=LZ4")?, - Compression::Zlib => write!(f, " COMPRESSION=ZLIB")?, - Compression::None => write!(f, " COMPRESSION=NONE")?, - } - } - - if let Some(encryption) = &self.encryption { - match encryption { - Encryption::Yes => write!(f, " ENCRYPTION='Y'")?, - Encryption::No => write!(f, " ENCRYPTION='N'")?, - } - } - - if let Some(max_rows) = &self.max_rows { - write!(f, " MAX_ROWS={}", max_rows)?; - } - - if let Some(min_rows) = &self.min_rows { - write!(f, " MIN_ROWS={}", min_rows)?; - } - - if let Some(autoextend_size) = &self.autoextend_size { - write!(f, " AUTOEXTEND_SIZE={}", autoextend_size)?; - } - - if let Some(avg_row_length) = &self.avg_row_length { - write!(f, " AVG_ROW_LENGTH={}", avg_row_length)?; - } - - if let Some(checksum) = &self.checksum { - match checksum { - true => write!(f, " CHECKSUM=1")?, - false => write!(f, " CHECKSUM=0")?, - } - } - - if let Some(connection) = &self.connection { - write!(f, " CONNECTION='{}'", connection)?; - } - - if let Some(engine_attribute) = &self.engine_attribute { - write!(f, " ENGINE_ATTRIBUTE='{}'", engine_attribute)?; - } - - if let Some(password) = &self.password { - write!(f, " PASSWORD='{}'", password)?; - } - - if let Some(secondary_engine_attribute) = &self.secondary_engine_attribute { - write!( - f, - " SECONDARY_ENGINE_ATTRIBUTE='{}'", - secondary_engine_attribute - )?; - } - - if self.start_transaction.unwrap_or(false) { - write!(f, " START TRANSACTION")?; - } - - if let Some(tablespace_option) = &self.tablespace_option { - write!(f, " TABLESPACE {}", tablespace_option.name)?; - if let Some(storage) = &tablespace_option.storage { - match storage { - StorageType::Disk => write!(f, " STORAGE DISK")?, - StorageType::Memory => write!(f, " STORAGE MEMORY")?, - } - } - } - - if let Some(union_tables) = &self.union_tables { - if !union_tables.is_empty() { - write!( - f, - " UNION=({})", - union_tables - .iter() - .map(|table| table.to_string()) - .collect::>() - .join(", ") - )?; - } - } - if self.on_commit.is_some() { let on_commit = match self.on_commit { Some(OnCommit::DeleteRows) => "ON COMMIT DELETE ROWS", diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index b9de2db6d..fe86007e5 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -26,15 +26,14 @@ use sqlparser_derive::{Visit, VisitMut}; use super::super::dml::CreateTable; use crate::ast::{ - ClusteredBy, ColumnDef, CommentDef, Expr, FileFormat, HiveDistributionStyle, HiveFormat, Ident, - ObjectName, OnCommit, OneOrManyWithParens, Query, RowAccessPolicy, SqlOption, Statement, - StorageSerializationPolicy, TableConstraint, TableEngine, Tag, WrappedCollection, -}; -use crate::parser::{ - Compression, DelayKeyWrite, DirectoryOption, Encryption, InsertMethod, OptionState, - ParserError, RowFormat, TablespaceOption, + ClusteredBy, ColumnDef, CommentDef, CreateTableOptions, Expr, FileFormat, + HiveDistributionStyle, HiveFormat, Ident, ObjectName, OnCommit, OneOrManyWithParens, Query, + RowAccessPolicy, Statement, StorageSerializationPolicy, TableConstraint, Tag, + WrappedCollection, }; +use crate::parser::ParserError; + /// Builder for create table statement variant ([1]). /// /// This structure helps building and accessing a create table with more ease, without needing to: @@ -79,43 +78,13 @@ pub struct CreateTableBuilder { pub constraints: Vec, pub hive_distribution: HiveDistributionStyle, pub hive_formats: Option, - pub table_properties: Vec, - pub with_options: Vec, pub file_format: Option, pub location: Option, pub query: Option>, pub without_rowid: bool, pub like: Option, pub clone: Option, - pub engine: Option, - pub comment: Option, - pub auto_increment_offset: Option, - pub default_charset: Option, - pub collation: Option, - pub key_block_size: Option, - pub max_rows: Option, - pub min_rows: Option, - pub autoextend_size: Option, - pub avg_row_length: Option, - pub checksum: Option, - pub row_format: Option, - pub compression: Option, - pub encryption: Option, - pub insert_method: Option, - pub pack_keys: Option, - pub stats_auto_recalc: Option, - pub stats_persistent: Option, - pub stats_sample_pages: Option, - pub delay_key_write: Option, - pub connection: Option, - pub engine_attribute: Option, - pub password: Option, - pub secondary_engine_attribute: Option, - pub start_transaction: Option, - pub tablespace_option: Option, - pub union_tables: Option>, - pub data_directory: Option, - pub index_directory: Option, + pub comment_after_column_def: Option, pub on_commit: Option, pub on_cluster: Option, pub primary_key: Option>, @@ -123,7 +92,6 @@ pub struct CreateTableBuilder { pub partition_by: Option>, pub cluster_by: Option>>, pub clustered_by: Option, - pub options: Option>, pub inherits: Option>, pub strict: bool, pub copy_grants: bool, @@ -140,6 +108,7 @@ pub struct CreateTableBuilder { pub catalog: Option, pub catalog_sync: Option, pub storage_serialization_policy: Option, + pub table_options: CreateTableOptions, } impl CreateTableBuilder { @@ -158,43 +127,13 @@ impl CreateTableBuilder { constraints: vec![], hive_distribution: HiveDistributionStyle::NONE, hive_formats: None, - table_properties: vec![], - with_options: vec![], file_format: None, location: None, query: None, without_rowid: false, like: None, clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - compression: None, - encryption: None, - insert_method: None, - key_block_size: None, - row_format: None, - data_directory: None, - index_directory: None, - pack_keys: None, - stats_auto_recalc: None, - stats_persistent: None, - stats_sample_pages: None, - delay_key_write: None, - max_rows: None, - min_rows: None, - autoextend_size: None, - avg_row_length: None, - checksum: None, - connection: None, - engine_attribute: None, - password: None, - secondary_engine_attribute: None, - start_transaction: None, - tablespace_option: None, - union_tables: None, - default_charset: None, - collation: None, + comment_after_column_def: None, on_commit: None, on_cluster: None, primary_key: None, @@ -202,7 +141,6 @@ impl CreateTableBuilder { partition_by: None, cluster_by: None, clustered_by: None, - options: None, inherits: None, strict: false, copy_grants: false, @@ -219,6 +157,7 @@ impl CreateTableBuilder { catalog: None, catalog_sync: None, storage_serialization_policy: None, + table_options: CreateTableOptions::None, } } pub fn or_replace(mut self, or_replace: bool) -> Self { @@ -281,15 +220,6 @@ impl CreateTableBuilder { self } - pub fn table_properties(mut self, table_properties: Vec) -> Self { - self.table_properties = table_properties; - self - } - - pub fn with_options(mut self, with_options: Vec) -> Self { - self.with_options = with_options; - self - } pub fn file_format(mut self, file_format: Option) -> Self { self.file_format = file_format; self @@ -319,140 +249,8 @@ impl CreateTableBuilder { self } - pub fn engine(mut self, engine: Option) -> Self { - self.engine = engine; - self - } - - pub fn comment(mut self, comment: Option) -> Self { - self.comment = comment; - self - } - - pub fn auto_increment_offset(mut self, offset: Option) -> Self { - self.auto_increment_offset = offset; - self - } - - pub fn default_charset(mut self, default_charset: Option) -> Self { - self.default_charset = default_charset; - self - } - - pub fn collation(mut self, collation: Option) -> Self { - self.collation = collation; - self - } - pub fn compression(mut self, compression: Option) -> Self { - self.compression = compression; - self - } - pub fn encryption(mut self, encryption: Option) -> Self { - self.encryption = encryption; - self - } - pub fn delay_key_write(mut self, delay_key_write: Option) -> Self { - self.delay_key_write = delay_key_write; - self - } - pub fn insert_method(mut self, insert_method: Option) -> Self { - self.insert_method = insert_method; - self - } - - pub fn key_block_size(mut self, key_block_size: Option) -> Self { - self.key_block_size = key_block_size; - self - } - pub fn max_rows(mut self, max_rows: Option) -> Self { - self.max_rows = max_rows; - self - } - - pub fn min_rows(mut self, min_rows: Option) -> Self { - self.min_rows = min_rows; - self - } - - pub fn autoextend_size(mut self, autoextend_size: Option) -> Self { - self.autoextend_size = autoextend_size; - self - } - pub fn avg_row_length(mut self, avg_row_length: Option) -> Self { - self.avg_row_length = avg_row_length; - self - } - - pub fn checksum(mut self, checksum: Option) -> Self { - self.checksum = checksum; - self - } - pub fn connection(mut self, connection: Option) -> Self { - self.connection = connection; - self - } - pub fn engine_attribute(mut self, engine_attribute: Option) -> Self { - self.engine_attribute = engine_attribute; - self - } - pub fn password(mut self, password: Option) -> Self { - self.password = password; - self - } - pub fn secondary_engine_attribute( - mut self, - secondary_engine_attribute: Option, - ) -> Self { - self.secondary_engine_attribute = secondary_engine_attribute; - self - } - - pub fn start_transaction(mut self, start_transaction: Option) -> Self { - self.start_transaction = start_transaction; - self - } - - pub fn tablespace_option(mut self, tablespace_option: Option) -> Self { - self.tablespace_option = tablespace_option; - self - } - pub fn union_tables(mut self, union_tables: Option>) -> Self { - self.union_tables = union_tables; - self - } - - pub fn row_format(mut self, row_format: Option) -> Self { - self.row_format = row_format; - self - } - - pub fn data_directory(mut self, directory_option: Option) -> Self { - self.data_directory = directory_option; - self - } - - pub fn index_directory(mut self, directory_option: Option) -> Self { - self.index_directory = directory_option; - self - } - - pub fn pack_keys(mut self, pack_keys: Option) -> Self { - self.pack_keys = pack_keys; - self - } - - pub fn stats_auto_recalc(mut self, stats_auto_recalc: Option) -> Self { - self.stats_auto_recalc = stats_auto_recalc; - self - } - - pub fn stats_persistent(mut self, stats_persistent: Option) -> Self { - self.stats_persistent = stats_persistent; - self - } - - pub fn stats_sample_pages(mut self, stats_sample_pages: Option) -> Self { - self.stats_sample_pages = stats_sample_pages; + pub fn comment_after_column_def(mut self, comment: Option) -> Self { + self.comment_after_column_def = comment; self } @@ -491,11 +289,6 @@ impl CreateTableBuilder { self } - pub fn options(mut self, options: Option>) -> Self { - self.options = options; - self - } - pub fn inherits(mut self, inherits: Option>) -> Self { self.inherits = inherits; self @@ -585,6 +378,11 @@ impl CreateTableBuilder { self } + pub fn table_options(mut self, table_options: CreateTableOptions) -> Self { + self.table_options = table_options; + self + } + pub fn build(self) -> Statement { Statement::CreateTable(CreateTable { or_replace: self.or_replace, @@ -600,43 +398,13 @@ impl CreateTableBuilder { constraints: self.constraints, hive_distribution: self.hive_distribution, hive_formats: self.hive_formats, - table_properties: self.table_properties, - with_options: self.with_options, file_format: self.file_format, location: self.location, query: self.query, without_rowid: self.without_rowid, like: self.like, clone: self.clone, - engine: self.engine, - comment: self.comment, - auto_increment_offset: self.auto_increment_offset, - default_charset: self.default_charset, - collation: self.collation, - compression: self.compression, - encryption: self.encryption, - insert_method: self.insert_method, - key_block_size: self.key_block_size, - row_format: self.row_format, - data_directory: self.data_directory, - index_directory: self.index_directory, - pack_keys: self.pack_keys, - stats_auto_recalc: self.stats_auto_recalc, - stats_persistent: self.stats_persistent, - stats_sample_pages: self.stats_sample_pages, - delay_key_write: self.delay_key_write, - max_rows: self.max_rows, - min_rows: self.min_rows, - autoextend_size: self.autoextend_size, - avg_row_length: self.avg_row_length, - checksum: self.checksum, - connection: self.connection, - engine_attribute: self.engine_attribute, - password: self.password, - secondary_engine_attribute: self.secondary_engine_attribute, - start_transaction: self.start_transaction, - tablespace_option: self.tablespace_option, - union_tables: self.union_tables, + comment_after_column_def: self.comment_after_column_def, on_commit: self.on_commit, on_cluster: self.on_cluster, primary_key: self.primary_key, @@ -644,7 +412,6 @@ impl CreateTableBuilder { partition_by: self.partition_by, cluster_by: self.cluster_by, clustered_by: self.clustered_by, - options: self.options, inherits: self.inherits, strict: self.strict, copy_grants: self.copy_grants, @@ -661,6 +428,7 @@ impl CreateTableBuilder { catalog: self.catalog, catalog_sync: self.catalog_sync, storage_serialization_policy: self.storage_serialization_policy, + table_options: self.table_options, }) } } @@ -686,43 +454,13 @@ impl TryFrom for CreateTableBuilder { constraints, hive_distribution, hive_formats, - table_properties, - with_options, file_format, location, query, without_rowid, like, clone, - engine, - comment, - auto_increment_offset, - compression, - encryption, - insert_method, - key_block_size, - row_format, - data_directory, - index_directory, - pack_keys, - stats_auto_recalc, - stats_persistent, - stats_sample_pages, - delay_key_write, - max_rows, - min_rows, - autoextend_size, - avg_row_length, - checksum, - connection, - engine_attribute, - password, - secondary_engine_attribute, - start_transaction, - tablespace_option, - union_tables, - default_charset, - collation, + comment_after_column_def: comment, on_commit, on_cluster, primary_key, @@ -730,7 +468,6 @@ impl TryFrom for CreateTableBuilder { partition_by, cluster_by, clustered_by, - options, inherits, strict, copy_grants, @@ -747,6 +484,7 @@ impl TryFrom for CreateTableBuilder { catalog, catalog_sync, storage_serialization_policy, + table_options, }) => Ok(Self { or_replace, temporary, @@ -759,43 +497,13 @@ impl TryFrom for CreateTableBuilder { constraints, hive_distribution, hive_formats, - table_properties, - with_options, file_format, location, query, without_rowid, like, clone, - engine, - comment, - auto_increment_offset, - compression, - encryption, - insert_method, - key_block_size, - row_format, - data_directory, - index_directory, - pack_keys, - stats_auto_recalc, - stats_persistent, - stats_sample_pages, - delay_key_write, - max_rows, - min_rows, - autoextend_size, - avg_row_length, - checksum, - connection, - engine_attribute, - password, - secondary_engine_attribute, - start_transaction, - tablespace_option, - union_tables, - default_charset, - collation, + comment_after_column_def: comment, on_commit, on_cluster, primary_key, @@ -803,7 +511,6 @@ impl TryFrom for CreateTableBuilder { partition_by, cluster_by, clustered_by, - options, inherits, strict, iceberg, @@ -822,6 +529,7 @@ impl TryFrom for CreateTableBuilder { catalog, catalog_sync, storage_serialization_policy, + table_options, }), _ => Err(ParserError::ParserError(format!( "Expected create table statement, but received: {stmt}" @@ -835,8 +543,8 @@ impl TryFrom for CreateTableBuilder { pub(crate) struct CreateTableConfiguration { pub partition_by: Option>, pub cluster_by: Option>>, - pub options: Option>, pub inherits: Option>, + pub table_options: CreateTableOptions, } #[cfg(test)] diff --git a/src/ast/helpers/stmt_data_loading.rs b/src/ast/helpers/stmt_data_loading.rs index e960bb05b..92a727279 100644 --- a/src/ast/helpers/stmt_data_loading.rs +++ b/src/ast/helpers/stmt_data_loading.rs @@ -21,8 +21,6 @@ #[cfg(not(feature = "std"))] use alloc::string::String; -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; use core::fmt; #[cfg(feature = "serde")] diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 582922a3e..b5e1a5ba4 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2681,6 +2681,18 @@ pub enum CreateTableOptions { /// /// Options(Vec), + + /// Plain options, options which are not part on any declerative statement e.g. WITH/OPTIONS/... + /// + Plain(Vec), + + TableProperties(Vec), +} + +impl Default for CreateTableOptions { + fn default() -> Self { + Self::None + } } impl fmt::Display for CreateTableOptions { @@ -2692,6 +2704,12 @@ impl fmt::Display for CreateTableOptions { CreateTableOptions::Options(options) => { write!(f, "OPTIONS({})", display_comma_separated(options)) } + CreateTableOptions::TableProperties(options) => { + write!(f, "TBLPROPERTIES ({})", display_comma_separated(options)) + } + CreateTableOptions::Plain(options) => { + write!(f, "{}", display_separated(options, " ")) + } CreateTableOptions::None => Ok(()), } } @@ -7548,7 +7566,10 @@ pub enum SqlOption { /// Any option that consists of a key value pair where the value is an expression. e.g. /// /// WITH(DISTRIBUTION = ROUND_ROBIN) - KeyValue { key: Ident, value: Expr }, + KeyValue { + key: Ident, + value: Expr, + }, /// One or more table partitions and represents which partition the boundary values belong to, /// e.g. /// @@ -7560,6 +7581,16 @@ pub enum SqlOption { range_direction: Option, for_values: Vec, }, + + Comment(CommentDef), + + TableSpace(TablespaceOption), + + // Advanced parameter formations + // UNION = (tbl_name[,tbl_name]...) + // ENGINE = ReplicatedMergeTree('/table_name','{replica}', ver) + // ENGINE = SummingMergeTree([columns]) + NamedParenthesizedList(NamedParenthesizedList), } impl fmt::Display for SqlOption { @@ -7591,10 +7622,52 @@ impl fmt::Display for SqlOption { display_comma_separated(for_values) ) } + SqlOption::TableSpace(tablespace_option) => { + write!(f, "TABLESPACE {}", tablespace_option.name)?; + match tablespace_option.storage { + Some(StorageType::Disk) => write!(f, " STORAGE DISK"), + Some(StorageType::Memory) => write!(f, " STORAGE MEMORY"), + _ => Ok(()), + } + } + SqlOption::Comment(comment) => match comment { + CommentDef::WithEq(comment) => { + write!(f, "COMMENT = '{comment}'") + } + CommentDef::WithoutEq(comment) => { + write!(f, "COMMENT '{comment}'") + } + }, + SqlOption::NamedParenthesizedList(value) => { + write!(f, "{} = ", value.key)?; + if let Some(key) = &value.value { + write!(f, "{}", key)?; + } + if !value.parameters.is_empty() { + write!(f, "({})", display_comma_separated(&value.parameters))? + } + Ok(()) + } } } } +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum StorageType { + Disk, + Memory, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct TablespaceOption { + pub name: String, + pub storage: Option, +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] @@ -8860,27 +8933,13 @@ impl Display for CreateViewParams { } } -/// Engine of DB. Some warehouse has parameters of engine, e.g. [ClickHouse] -/// -/// [ClickHouse]: https://clickhouse.com/docs/en/engines/table-engines #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub struct TableEngine { - pub name: String, - pub parameters: Option>, -} - -impl Display for TableEngine { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.name)?; - - if let Some(parameters) = self.parameters.as_ref() { - write!(f, "({})", display_comma_separated(parameters))?; - } - - Ok(()) - } +pub struct NamedParenthesizedList { + pub key: Ident, + pub value: Option, + pub parameters: Vec, } /// Snowflake `WITH ROW ACCESS POLICY policy_name ON (identifier, ...)` @@ -8944,18 +9003,12 @@ pub enum CommentDef { /// Does not include `=` when printing the comment, as `COMMENT 'comment'` WithEq(String), WithoutEq(String), - // For Hive dialect, the table comment is after the column definitions without `=`, - // so we need to add an extra variant to allow to identify this case when displaying. - // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) - AfterColumnDefsWithoutEq(String), } impl Display for CommentDef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - CommentDef::WithEq(comment) - | CommentDef::WithoutEq(comment) - | CommentDef::AfterColumnDefsWithoutEq(comment) => write!(f, "{comment}"), + CommentDef::WithEq(comment) | CommentDef::WithoutEq(comment) => write!(f, "{comment}"), } } } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index eeec3edeb..5eeed7e42 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -30,10 +30,10 @@ use super::{ Function, FunctionArg, FunctionArgExpr, FunctionArgumentClause, FunctionArgumentList, FunctionArguments, GroupByExpr, HavingBound, IfStatement, IlikeSelectItem, Insert, Interpolate, InterpolateExpr, Join, JoinConstraint, JoinOperator, JsonPath, JsonPathElem, LateralView, - LimitClause, MatchRecognizePattern, Measure, NamedWindowDefinition, ObjectName, ObjectNamePart, - Offset, OnConflict, OnConflictAction, OnInsert, OpenStatement, OrderBy, OrderByExpr, - OrderByKind, Partition, PivotValueSource, ProjectionSelect, Query, RaiseStatement, - RaiseStatementValue, ReferentialAction, RenameSelectItem, ReplaceSelectElement, + LimitClause, MatchRecognizePattern, Measure, NamedParenthesizedList, NamedWindowDefinition, + ObjectName, ObjectNamePart, Offset, OnConflict, OnConflictAction, OnInsert, OpenStatement, + OrderBy, OrderByExpr, OrderByKind, Partition, PivotValueSource, ProjectionSelect, Query, + RaiseStatement, RaiseStatementValue, ReferentialAction, RenameSelectItem, ReplaceSelectElement, ReplaceSelectItem, Select, SelectInto, SelectItem, SetExpr, SqlOption, Statement, Subscript, SymbolDefinition, TableAlias, TableAliasColumnDef, TableConstraint, TableFactor, TableObject, TableOptionsClustered, TableWithJoins, UpdateTableFromKind, Use, Value, Values, ViewColumnDef, @@ -567,51 +567,20 @@ impl Spanned for CreateTable { constraints, hive_distribution: _, // hive specific hive_formats: _, // hive specific - table_properties, - with_options, - file_format: _, // enum - location: _, // string, no span + file_format: _, // enum + location: _, // string, no span query, without_rowid: _, // bool like, clone, - engine: _, // todo - comment: _, // todo, no span - auto_increment_offset: _, // u32, no span - default_charset: _, // string, no span - collation: _, // string, no span - compression: _, // enum - encryption: _, // enum - key_block_size: _, // u32, no span - insert_method: _, // enum - row_format: _, // enum - data_directory: _, // string, no span - index_directory: _, // string, no span - stats_auto_recalc: _, // enum - stats_persistent: _, // enum - pack_keys: _, // enum - stats_sample_pages: _, // u32, no span - delay_key_write: _, // enum - max_rows: _, // u32, no span - min_rows: _, // u32, no span - avg_row_length: _, // u32, no span - autoextend_size: _, // u32, no span - checksum: _, // bool, no span - connection: _, // string, no span - password: _, // string, no span - engine_attribute: _, // string, no span - secondary_engine_attribute: _, // string, no span - start_transaction: _, // bool - tablespace_option: _, // enum - union_tables: _, // list of strings, no span - on_commit: _, // enum + comment_after_column_def: _, // todo, no span + on_commit: _, on_cluster: _, // todo, clickhouse specific primary_key: _, // todo, clickhouse specific order_by: _, // todo, clickhouse specific partition_by: _, // todo, BigQuery specific cluster_by: _, // todo, BigQuery specific clustered_by: _, // todo, Hive specific - options: _, // todo, BigQuery specific inherits: _, // todo, PostgreSQL specific strict: _, // bool copy_grants: _, // bool @@ -627,15 +596,15 @@ impl Spanned for CreateTable { base_location: _, // todo, Snowflake specific catalog: _, // todo, Snowflake specific catalog_sync: _, // todo, Snowflake specific - storage_serialization_policy: _, // todo, Snowflake specific + storage_serialization_policy: _, + table_options, } = self; union_spans( core::iter::once(name.span()) + .chain(core::iter::once(table_options.span())) .chain(columns.iter().map(|i| i.span())) .chain(constraints.iter().map(|i| i.span())) - .chain(table_properties.iter().map(|i| i.span())) - .chain(with_options.iter().map(|i| i.span())) .chain(query.iter().map(|i| i.span())) .chain(like.iter().map(|i| i.span())) .chain(clone.iter().map(|i| i.span())), @@ -1028,6 +997,14 @@ impl Spanned for SqlOption { } => union_spans( core::iter::once(column_name.span).chain(for_values.iter().map(|i| i.span())), ), + SqlOption::TableSpace(_) => Span::empty(), + SqlOption::Comment(_) => Span::empty(), + SqlOption::NamedParenthesizedList(NamedParenthesizedList { + key: name, + value, + parameters: values, + }) => union_spans(core::iter::once(name.span).chain(values.iter().map(|i| i.span))) + .union_opt(&value.as_ref().map(|i| i.span)), } } } @@ -1065,6 +1042,8 @@ impl Spanned for CreateTableOptions { CreateTableOptions::None => Span::empty(), CreateTableOptions::With(vec) => union_spans(vec.iter().map(|i| i.span())), CreateTableOptions::Options(vec) => union_spans(vec.iter().map(|i| i.span())), + CreateTableOptions::Plain(vec) => union_spans(vec.iter().map(|i| i.span())), + CreateTableOptions::TableProperties(vec) => union_spans(vec.iter().map(|i| i.span())), } } } diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index c4d6a5ad5..ccce16198 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -25,8 +25,8 @@ use crate::ast::helpers::stmt_data_loading::{ use crate::ast::{ ColumnOption, ColumnPolicy, ColumnPolicyProperty, CopyIntoSnowflakeKind, Ident, IdentityParameters, IdentityProperty, IdentityPropertyFormatKind, IdentityPropertyKind, - IdentityPropertyOrder, ObjectName, RowAccessPolicy, ShowObjects, Statement, TagsColumnOption, - WrappedCollection, + IdentityPropertyOrder, ObjectName, RowAccessPolicy, ShowObjects, SqlOption, Statement, + TagsColumnOption, WrappedCollection, }; use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; @@ -417,6 +417,8 @@ pub fn parse_create_table( // "CREATE TABLE x COPY GRANTS (c INT)" and "CREATE TABLE x (c INT) COPY GRANTS" are both // accepted by Snowflake + let mut plain_options = vec![]; + loop { let next_token = parser.next_token(); match &next_token.token { @@ -428,7 +430,9 @@ pub fn parse_create_table( Keyword::COMMENT => { // Rewind the COMMENT keyword parser.prev_token(); - builder = builder.comment(parser.parse_optional_inline_comment()?); + if let Some(comment_def) = parser.parse_optional_inline_comment()? { + plain_options.push(SqlOption::Comment(comment_def)) + } } Keyword::AS => { let query = parser.parse_query()?; @@ -589,6 +593,13 @@ pub fn parse_create_table( } } } + let table_options = if !plain_options.is_empty() { + crate::ast::CreateTableOptions::Plain(plain_options) + } else { + crate::ast::CreateTableOptions::None + }; + + builder = builder.table_options(table_options); if iceberg && builder.base_location.is_none() { return Err(ParserError::ParserError( diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c05e932d6..d6bca70c8 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -48,130 +48,6 @@ pub enum ParserError { RecursionLimitExceeded, } -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum Compression { - Zlib, - Lz4, - None, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum DelayKeyWrite { - Disabled, - Enabled, -} -impl fmt::Display for DelayKeyWrite { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - DelayKeyWrite::Disabled => write!(f, "DISABLED"), - DelayKeyWrite::Enabled => write!(f, "ENABLED"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum Encryption { - Yes, - No, -} - -impl fmt::Display for Encryption { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Encryption::Yes => write!(f, "Y"), - Encryption::No => write!(f, "N"), - } - } -} - -impl fmt::Display for Compression { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Compression::Zlib => write!(f, "ZLIB"), - Compression::Lz4 => write!(f, "LZ4"), - Compression::None => write!(f, "NONE"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum InsertMethod { - No, - First, - Last, -} -impl fmt::Display for InsertMethod { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - InsertMethod::No => write!(f, "NO"), - InsertMethod::First => write!(f, "FIRST"), - InsertMethod::Last => write!(f, "LAST"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum RowFormat { - Default, - Dynamic, - Fixed, - Compressed, - Redundant, - Compact, -} -impl fmt::Display for RowFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - RowFormat::Default => write!(f, "DEFAULT"), - RowFormat::Dynamic => write!(f, "DYNAMIC"), - RowFormat::Fixed => write!(f, "FIXED"), - RowFormat::Compressed => write!(f, "COMPRESSED"), - RowFormat::Redundant => write!(f, "REDUNDANT"), - RowFormat::Compact => write!(f, "COMPACT"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum DirectoryType { - Data, - Index, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct DirectoryOption { - pub directory_type: DirectoryType, - pub path: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum OptionState { - Zero, - One, - Default, -} -impl fmt::Display for OptionState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - OptionState::Zero => write!(f, "0"), - OptionState::One => write!(f, "1"), - OptionState::Default => write!(f, "DEFAULT"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum StorageType { - Disk, - Memory, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct TablespaceOption { - pub name: String, - pub storage: Option, -} - // Use `Parser::expected` instead, if possible macro_rules! parser_err { ($MSG:expr, $loc:expr) => { @@ -5648,12 +5524,17 @@ impl<'a> Parser<'a> { }; let location = hive_formats.location.clone(); let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; + let table_options = if !table_properties.is_empty() { + CreateTableOptions::TableProperties(table_properties) + } else { + CreateTableOptions::None + }; Ok(CreateTableBuilder::new(table_name) .columns(columns) .constraints(constraints) .hive_distribution(hive_distribution) .hive_formats(Some(hive_formats)) - .table_properties(table_properties) + .table_options(table_options) .or_replace(or_replace) .if_not_exists(if_not_exists) .external(true) @@ -7165,17 +7046,16 @@ impl<'a> Parser<'a> { // parse optional column list (schema) let (columns, constraints) = self.parse_columns()?; - let mut comment = if dialect_of!(self is HiveDialect) - && self.parse_keyword(Keyword::COMMENT) - { - let next_token = self.next_token(); - match next_token.token { - Token::SingleQuotedString(str) => Some(CommentDef::AfterColumnDefsWithoutEq(str)), - _ => self.expected("comment", next_token)?, - } - } else { - None - }; + let comment_after_column_def = + if dialect_of!(self is HiveDialect) && self.parse_keyword(Keyword::COMMENT) { + let next_token = self.next_token(); + match next_token.token { + Token::SingleQuotedString(str) => Some(CommentDef::WithoutEq(str)), + _ => self.expected("comment", next_token)?, + } + } else { + None + }; // SQLite supports `WITHOUT ROWID` at the end of `CREATE TABLE` let without_rowid = self.parse_keywords(&[Keyword::WITHOUT, Keyword::ROWID]); @@ -7183,618 +7063,6 @@ impl<'a> Parser<'a> { let hive_distribution = self.parse_hive_distribution()?; let clustered_by = self.parse_optional_clustered_by()?; let hive_formats = self.parse_hive_formats()?; - // PostgreSQL supports `WITH ( options )`, before `AS` - let with_options = self.parse_options(Keyword::WITH)?; - let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; - - let mut engine = None; - let mut auto_increment_offset = None; - let mut default_charset = None; - let mut collation = None; - let mut insert_method = None; - let mut key_block_size = None; - let mut row_format = None; - let mut data_directory = None; - let mut index_directory = None; - let mut pack_keys = None; - let mut stats_auto_recalc = None; - let mut stats_persistent = None; - let mut stats_sample_pages = None; - let mut delay_key_write = None; - let mut compression = None; - let mut encryption = None; - let mut max_rows = None; - let mut min_rows = None; - let mut autoextend_size = None; - let mut avg_row_length = None; - let mut checksum = None; - let mut connection = None; - let mut engine_attribute = None; - let mut password = None; - let mut secondary_engine_attribute = None; - let mut start_transaction = None; - let mut tablespace_option = None; - let mut union_tables = None; - - // Parse table options in any order - while let Some(keyword) = self.parse_one_of_keywords(&[ - Keyword::ENGINE, - Keyword::AUTO_INCREMENT, - Keyword::DEFAULT, - Keyword::CHARSET, - Keyword::COLLATE, - Keyword::INSERT_METHOD, - Keyword::KEY_BLOCK_SIZE, - Keyword::ROW_FORMAT, - Keyword::DATA, - Keyword::INDEX, - Keyword::PACK_KEYS, - Keyword::STATS_AUTO_RECALC, - Keyword::STATS_PERSISTENT, - Keyword::STATS_SAMPLE_PAGES, - Keyword::DELAY_KEY_WRITE, - Keyword::COMPRESSION, - Keyword::ENCRYPTION, - Keyword::MAX_ROWS, - Keyword::MIN_ROWS, - Keyword::AUTOEXTEND_SIZE, - Keyword::AVG_ROW_LENGTH, - Keyword::CHECKSUM, - Keyword::CONNECTION, - Keyword::ENGINE_ATTRIBUTE, - Keyword::PASSWORD, - Keyword::SECONDARY_ENGINE_ATTRIBUTE, - Keyword::START, - Keyword::TABLESPACE, - Keyword::UNION, - ]) { - match keyword { - Keyword::ENGINE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => { - let name = w.value; - let parameters = if self.peek_token() == Token::LParen { - Some(self.parse_parenthesized_identifiers()?) - } else { - None - }; - engine = Some(TableEngine { name, parameters }); - } - _ => self.expected("identifier", next_token)?, - } - } - - Keyword::AUTO_INCREMENT => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - auto_increment_offset = - Some(Self::parse::(s, next_token.span.start)?) - } - _ => self.expected("literal int", next_token)?, - } - } - - Keyword::DEFAULT => { - if self.parse_keyword(Keyword::CHARSET) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => default_charset = Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } else if self.parse_keyword(Keyword::COLLATE) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => collation = Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } else { - return Err(ParserError::ParserError( - "Expected CHARACTER SET or COLLATE after DEFAULT".to_string(), - )); - } - } - - Keyword::CHARSET => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => default_charset = Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } - - Keyword::COLLATE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => collation = Some(w.value), - _ => self.expected("identifier", next_token)?, - } - } - - Keyword::INSERT_METHOD => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::Word(w) => match w.value.to_uppercase().as_str() { - "NO" => insert_method = Some(InsertMethod::No), - "FIRST" => insert_method = Some(InsertMethod::First), - "LAST" => insert_method = Some(InsertMethod::Last), - _ => { - return Err(ParserError::ParserError( - "Invalid INSERT_METHOD value".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected INSERT_METHOD value".to_string(), - )) - } - } - } - - Keyword::KEY_BLOCK_SIZE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - key_block_size = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected KEY_BLOCK_SIZE value".to_string(), - )) - } - } - } - - Keyword::ROW_FORMAT => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::Word(w) => match w.value.to_uppercase().as_str() { - "DEFAULT" => row_format = Some(RowFormat::Default), - "DYNAMIC" => row_format = Some(RowFormat::Dynamic), - "FIXED" => row_format = Some(RowFormat::Fixed), - "COMPRESSED" => row_format = Some(RowFormat::Compressed), - "REDUNDANT" => row_format = Some(RowFormat::Redundant), - "COMPACT" => row_format = Some(RowFormat::Compact), - _ => { - return Err(ParserError::ParserError( - "Invalid ROW_FORMAT value".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected ROW_FORMAT value".to_string(), - )) - } - } - } - - Keyword::DATA => { - if self.parse_keyword(Keyword::DIRECTORY) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::SingleQuotedString(path) => { - data_directory = Some(DirectoryOption { - directory_type: DirectoryType::Data, - path, - }) - } - _ => { - return Err(ParserError::ParserError( - "Expected path for DATA DIRECTORY".to_string(), - )); - } - } - } else { - return Err(ParserError::ParserError( - "Expected DIRECTORY after DATA".to_string(), - )); - } - } - - Keyword::INDEX => { - if self.parse_keyword(Keyword::DIRECTORY) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::SingleQuotedString(path) => { - index_directory = Some(DirectoryOption { - directory_type: DirectoryType::Index, - path, - }) - } - _ => { - return Err(ParserError::ParserError( - "Expected path for INDEX DIRECTORY".to_string(), - )); - } - } - } else { - return Err(ParserError::ParserError( - "Expected DIRECTORY after INDEX".to_string(), - )); - } - } - - Keyword::PACK_KEYS => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => match s.as_str() { - "0" => pack_keys = Some(OptionState::Zero), - "1" => pack_keys = Some(OptionState::One), - _ => { - return Err(ParserError::ParserError(format!( - "PACK_KEYS can only be 0, 1, or DEFAULT, found: {}", - s - ))) - } - }, - Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { - pack_keys = Some(OptionState::Default) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for PACK_KEYS".to_string(), - )) - } - } - } - - Keyword::STATS_AUTO_RECALC => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => match s.as_str() { - "0" => stats_auto_recalc = Some(OptionState::Zero), - "1" => stats_auto_recalc = Some(OptionState::One), - _ => { - return Err(ParserError::ParserError(format!( - "STATS_AUTO_RECALC can only be 0, 1, or DEFAULT, found: {}", - s - ))) - } - }, - Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { - stats_auto_recalc = Some(OptionState::Default) - } - _ => { - return Err(ParserError::ParserError( - "Expected STATS_AUTO_RECALC value (0, 1, or DEFAULT)".to_string(), - )) - } - } - } - - Keyword::STATS_PERSISTENT => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => match s.as_str() { - "0" => stats_persistent = Some(OptionState::Zero), - "1" => stats_persistent = Some(OptionState::One), - _ => { - return Err(ParserError::ParserError(format!( - "STATS_PERSISTENT can only be 0, 1, or DEFAULT, found: {}", - s - ))) - } - }, - Token::Word(w) if w.value.to_uppercase() == "DEFAULT" => { - stats_persistent = Some(OptionState::Default) - } - _ => { - return Err(ParserError::ParserError( - "Expected STATS_PERSISTENT value (0, 1, or DEFAULT)".to_string(), - )) - } - } - } - - Keyword::STATS_SAMPLE_PAGES => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - stats_sample_pages = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for STATS_SAMPLE_PAGES".to_string(), - )) - } - } - } - - Keyword::DELAY_KEY_WRITE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => match s.to_uppercase().as_str() { - "0" => delay_key_write = Some(DelayKeyWrite::Disabled), - "1" => delay_key_write = Some(DelayKeyWrite::Enabled), - _ => { - return Err(ParserError::ParserError( - "Invalid DELAY_KEY_WRITE value (expected 0 or 1)".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected DELAY_KEY_WRITE value".to_string(), - )) - } - } - } - - Keyword::COMPRESSION => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::Word(w) => match w.value.to_uppercase().as_str() { - "ZLIB" => compression = Some(Compression::Zlib), - "LZ4" => compression = Some(Compression::Lz4), - "NONE" => compression = Some(Compression::None), - _ => { - return Err(ParserError::ParserError( - "Invalid COMPRESSION value".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected COMPRESSION value".to_string(), - )) - } - } - } - - Keyword::ENCRYPTION => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::SingleQuotedString(s) => match s.to_uppercase().as_str() { - "Y" => encryption = Some(Encryption::Yes), - "N" => encryption = Some(Encryption::No), - _ => { - return Err(ParserError::ParserError( - "Invalid ENCRYPTION value (expected 'Y' or 'N')".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected ENCRYPTION value".to_string(), - )) - } - } - } - - Keyword::MAX_ROWS => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - max_rows = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for MAX_ROWS".to_string(), - )) - } - } - } - - Keyword::MIN_ROWS => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - min_rows = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for MIN_ROWS".to_string(), - )) - } - } - } - - Keyword::AUTOEXTEND_SIZE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - autoextend_size = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for AUTOEXTEND_SIZE".to_string(), - )) - } - } - } - - Keyword::AVG_ROW_LENGTH => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => { - avg_row_length = Some(Self::parse::(s, next_token.span.start)?) - } - _ => { - return Err(ParserError::ParserError( - "Expected numeric value for AVG_ROW_LENGTH".to_string(), - )) - } - } - } - - Keyword::CHECKSUM => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - match next_token.token { - Token::Number(s, _) => match s.as_str() { - "0" => checksum = Some(false), - "1" => checksum = Some(true), - _ => { - return Err(ParserError::ParserError( - "Invalid CHECKSUM value (expected 0 or 1)".to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected a numeric value for CHECKSUM".to_string(), - )) - } - } - } - - Keyword::CONNECTION => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::SingleQuotedString(s) => connection = Some(s), - _ => { - return Err(ParserError::ParserError( - "Expected CONNECTION value as a single-quoted string".to_string(), - )) - } - } - } - - Keyword::ENGINE_ATTRIBUTE => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::SingleQuotedString(s) => engine_attribute = Some(s), - _ => { - return Err(ParserError::ParserError( - "Expected ENGINE_ATTRIBUTE value as a single-quoted string" - .to_string(), - )) - } - } - } - - Keyword::PASSWORD => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::SingleQuotedString(s) => password = Some(s), - _ => { - return Err(ParserError::ParserError( - "Expected PASSWORD value as a single-quoted string".to_string(), - )) - } - } - } - - Keyword::SECONDARY_ENGINE_ATTRIBUTE => { - let _ = self.consume_token(&Token::Eq); - let token = self.next_token(); - match token.token { - Token::SingleQuotedString(s) => secondary_engine_attribute = Some(s), - _ => return Err(ParserError::ParserError( - "Expected SECONDARY_ENGINE_ATTRIBUTE value as a single-quoted string" - .to_string(), - )), - } - } - - Keyword::START => { - if self.parse_keyword(Keyword::TRANSACTION) { - start_transaction = Some(true); - } else { - return Err(ParserError::ParserError( - "Expected TRANSACTION after START".to_string(), - )); - } - } - - Keyword::TABLESPACE => { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - - let name = match next_token.token { - Token::Word(w) => w.value.clone(), - Token::SingleQuotedString(s) => s.clone(), - _ => { - return Err(ParserError::ParserError( - "Expected TABLESPACE name".to_string(), - )); - } - }; - - let storage = if self.parse_keyword(Keyword::STORAGE) { - let _ = self.consume_token(&Token::Eq); - let storage_token = self.next_token(); - match storage_token.token { - Token::Word(w) => match w.value.to_uppercase().as_str() { - "DISK" => Some(StorageType::Disk), - "MEMORY" => Some(StorageType::Memory), - _ => { - return Err(ParserError::ParserError( - "Invalid STORAGE type (expected DISK or MEMORY)" - .to_string(), - )) - } - }, - _ => { - return Err(ParserError::ParserError( - "Expected STORAGE type".to_string(), - )) - } - } - } else { - None - }; - - tablespace_option = Some(TablespaceOption { name, storage }); - } - - Keyword::UNION => { - let _ = self.consume_token(&Token::Eq); - self.expect_token(&Token::LParen)?; - let mut tables = Vec::new(); - - loop { - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => tables.push(w.value), - Token::Comma => continue, - Token::RParen => break, - _ => { - return Err(ParserError::ParserError( - "Expected table name or ')' in UNION".to_string(), - )); - } - } - } - union_tables = Some(tables); - } - - _ => { - return Err(ParserError::ParserError(format!( - "Unexpected keyword: {:?}", - keyword - ))); - } - } - } let create_table_config = self.parse_optional_create_table_config()?; @@ -7832,13 +7100,6 @@ impl<'a> Parser<'a> { let strict = self.parse_keyword(Keyword::STRICT); - // Excludes Hive dialect here since it has been handled after table column definitions. - if !dialect_of!(self is HiveDialect) && self.parse_keyword(Keyword::COMMENT) { - // rewind the COMMENT keyword - self.prev_token(); - comment = self.parse_optional_inline_comment()? - }; - // Parse optional `AS ( query )` let query = if self.parse_keyword(Keyword::AS) { Some(self.parse_query()?) @@ -7855,8 +7116,6 @@ impl<'a> Parser<'a> { .temporary(temporary) .columns(columns) .constraints(constraints) - .with_options(with_options) - .table_properties(table_properties) .or_replace(or_replace) .if_not_exists(if_not_exists) .transient(transient) @@ -7867,43 +7126,15 @@ impl<'a> Parser<'a> { .without_rowid(without_rowid) .like(like) .clone_clause(clone) - .engine(engine) - .comment(comment) - .auto_increment_offset(auto_increment_offset) + .comment_after_column_def(comment_after_column_def) .order_by(order_by) - .default_charset(default_charset) - .collation(collation) - .compression(compression) - .insert_method(insert_method) - .encryption(encryption) - .key_block_size(key_block_size) - .row_format(row_format) - .data_directory(data_directory) - .index_directory(index_directory) - .pack_keys(pack_keys) - .stats_auto_recalc(stats_auto_recalc) - .stats_persistent(stats_persistent) - .stats_sample_pages(stats_sample_pages) - .delay_key_write(delay_key_write) - .max_rows(max_rows) - .min_rows(min_rows) - .autoextend_size(autoextend_size) - .avg_row_length(avg_row_length) - .checksum(checksum) - .connection(connection) - .engine_attribute(engine_attribute) - .password(password) - .secondary_engine_attribute(secondary_engine_attribute) - .start_transaction(start_transaction) - .tablespace_option(tablespace_option) - .union_tables(union_tables) .on_commit(on_commit) .on_cluster(on_cluster) .clustered_by(clustered_by) .partition_by(create_table_config.partition_by) .cluster_by(create_table_config.cluster_by) - .options(create_table_config.options) .inherits(create_table_config.inherits) + .table_options(create_table_config.table_options) .primary_key(primary_key) .strict(strict) .build()) @@ -7927,17 +7158,29 @@ impl<'a> Parser<'a> { /// Parse configuration like inheritance, partitioning, clustering information during the table creation. /// /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#syntax_2) - /// [PostgreSQL Partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) - /// [PostgreSQL Inheritance](https://www.postgresql.org/docs/current/ddl-inherit.html) + /// [PostgreSQL](https://www.postgresql.org/docs/current/ddl-partitioning.html) + /// [MySql](https://dev.mysql.com/doc/refman/8.4/en/create-table.html) fn parse_optional_create_table_config( &mut self, ) -> Result { + let mut table_options = CreateTableOptions::None; + let inherits = if self.parse_keyword(Keyword::INHERITS) { Some(self.parse_parenthesized_qualified_column_list(IsOptional::Mandatory, false)?) } else { None }; + // PostgreSQL supports `WITH ( options )`, before `AS` + let with_options = self.parse_options(Keyword::WITH)?; + if !with_options.is_empty() { + table_options = CreateTableOptions::With(with_options) + } + + let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; + if !table_properties.is_empty() { + table_options = CreateTableOptions::TableProperties(table_properties); + } let partition_by = if dialect_of!(self is BigQueryDialect | PostgreSqlDialect | GenericDialect) && self.parse_keywords(&[Keyword::PARTITION, Keyword::BY]) { @@ -7947,7 +7190,6 @@ impl<'a> Parser<'a> { }; let mut cluster_by = None; - let mut options = None; if dialect_of!(self is BigQueryDialect | GenericDialect) { if self.parse_keywords(&[Keyword::CLUSTER, Keyword::BY]) { cluster_by = Some(WrappedCollection::NoWrapping( @@ -7957,19 +7199,253 @@ impl<'a> Parser<'a> { if let Token::Word(word) = self.peek_token().token { if word.keyword == Keyword::OPTIONS { - options = Some(self.parse_options(Keyword::OPTIONS)?); + table_options = + CreateTableOptions::Options(self.parse_options(Keyword::OPTIONS)?) } }; } + if !dialect_of!(self is HiveDialect) && table_options == CreateTableOptions::None { + let plain_options = self.parse_plain_options()?; + if !plain_options.is_empty() { + table_options = CreateTableOptions::Plain(plain_options) + } + }; + Ok(CreateTableConfiguration { partition_by, cluster_by, - options, inherits, + table_options, }) } + fn parse_plain_option(&mut self) -> Result, ParserError> { + // Single parameter option + if self.parse_keywords(&[Keyword::START, Keyword::TRANSACTION]) { + return Ok(Some(SqlOption::Ident(Ident::new("START TRANSACTION")))); + } + + // Custom option + if self.parse_keywords(&[Keyword::COMMENT]) { + let has_eq = self.consume_token(&Token::Eq); + let value = self.next_token(); + + let comment = match (has_eq, value.token) { + (true, Token::SingleQuotedString(s)) => { + Ok(Some(SqlOption::Comment(CommentDef::WithEq(s)))) + } + (false, Token::SingleQuotedString(s)) => { + Ok(Some(SqlOption::Comment(CommentDef::WithoutEq(s)))) + } + (_, token) => { + self.expected("Token::SingleQuotedString", TokenWithSpan::wrap(token)) + } + }; + return comment; + } + + if self.parse_keywords(&[Keyword::ENGINE]) { + let _ = self.consume_token(&Token::Eq); + let value = self.next_token(); + + let engine = match value.token { + Token::Word(w) => { + let parameters = if self.peek_token() == Token::LParen { + self.parse_parenthesized_identifiers()? + } else { + vec![] + }; + + Ok(Some(SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("ENGINE"), + value: Some(Ident::new(w.value)), + parameters, + }, + ))) + } + _ => { + return self.expected("Token::Word", value)?; + } + }; + + return engine; + } + + if self.parse_keywords(&[Keyword::TABLESPACE]) { + let _ = self.consume_token(&Token::Eq); + let value = self.next_token(); + + let tablespace = match value.token { + // TABLESPACE tablespace_name [STORAGE DISK] | [TABLESPACE tablespace_name] STORAGE MEMORY + Token::Word(Word { value: name, .. }) | Token::SingleQuotedString(name) => { + let storage = match self.parse_keyword(Keyword::STORAGE) { + true => { + let _ = self.consume_token(&Token::Eq); + let storage_token = self.next_token(); + match &storage_token.token { + Token::Word(w) => match w.value.to_uppercase().as_str() { + "DISK" => Some(StorageType::Disk), + "MEMORY" => Some(StorageType::Memory), + _ => self + .expected("Storage type (DISK or MEMORY)", storage_token)?, + }, + _ => self.expected("Token::Word", storage_token)?, + } + } + false => None, + }; + + Ok(Some(SqlOption::TableSpace(TablespaceOption { + name, + storage, + }))) + } + _ => { + return self.expected("Token::Word", value)?; + } + }; + + return tablespace; + } + + if self.parse_keyword(Keyword::UNION) { + let _ = self.consume_token(&Token::Eq); + let value = self.next_token(); + + match value.token { + // UNION [=] (tbl_name[,tbl_name]...) + Token::LParen => { + let tables: Vec = + self.parse_comma_separated0(Parser::parse_identifier, Token::RParen)?; + self.expect_token(&Token::RParen)?; + + return Ok(Some(SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("UNION"), + value: None, + parameters: tables, + }, + ))); + } + _ => { + return self.expected("Token::LParen", value)?; + } + } + } + + // Key/Value parameter option + let key = if self.parse_keywords(&[Keyword::DEFAULT, Keyword::CHARSET]) { + // [DEFAULT] CHARACTER SET [=] charset_name + Ident::new("DEFAULT CHARSET") + } else if self.parse_keyword(Keyword::CHARSET) { + // [DEFAULT] CHARACTER SET [=] charset_name + Ident::new("CHARSET") + } else if self.parse_keywords(&[Keyword::DEFAULT, Keyword::CHARACTER, Keyword::SET]) { + // [DEFAULT] CHARACTER SET [=] charset_name + Ident::new("DEFAULT CHARACTER SET") + } else if self.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) { + // [DEFAULT] CHARACTER SET [=] charset_name + Ident::new("CHARACTER SET") + } else if self.parse_keywords(&[Keyword::DEFAULT, Keyword::COLLATE]) { + // [DEFAULT] COLLATE [=] collation_name + Ident::new("DEFAULT COLLATE") + } else if self.parse_keyword(Keyword::COLLATE) { + // [DEFAULT] COLLATE [=] collation_name + Ident::new("COLLATE") + } else if self.parse_keywords(&[Keyword::DATA, Keyword::DIRECTORY]) { + // {DATA | INDEX} DIRECTORY [=] 'absolute path to directory' + Ident::new("DATA DIRECTORY") + } else if self.parse_keywords(&[Keyword::INDEX, Keyword::DIRECTORY]) { + // {DATA | INDEX} DIRECTORY [=] 'absolute path to directory' + Ident::new("INDEX DIRECTORY") + } else if self.parse_keyword(Keyword::KEY_BLOCK_SIZE) { + // KEY_BLOCK_SIZE [=] value + Ident::new("KEY_BLOCK_SIZE") + } else if self.parse_keyword(Keyword::ROW_FORMAT) { + // ROW_FORMAT [=] {DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT} + Ident::new("ROW_FORMAT") + } else if self.parse_keyword(Keyword::PACK_KEYS) { + // PACK_KEYS [=] {0 | 1 | DEFAULT} + Ident::new("PACK_KEYS") + } else if self.parse_keyword(Keyword::STATS_AUTO_RECALC) { + // STATS_AUTO_RECALC [=] {DEFAULT | 0 | 1} + Ident::new("STATS_AUTO_RECALC") + } else if self.parse_keyword(Keyword::STATS_PERSISTENT) { + //STATS_PERSISTENT [=] {DEFAULT | 0 | 1} + Ident::new("STATS_PERSISTENT") + } else if self.parse_keyword(Keyword::STATS_SAMPLE_PAGES) { + // STATS_SAMPLE_PAGES [=] value + Ident::new("STATS_SAMPLE_PAGES") + } else if self.parse_keyword(Keyword::DELAY_KEY_WRITE) { + // DELAY_KEY_WRITE [=] {0 | 1} + Ident::new("DELAY_KEY_WRITE") + } else if self.parse_keyword(Keyword::COMPRESSION) { + // COMPRESSION [=] {'ZLIB' | 'LZ4' | 'NONE'} + Ident::new("COMPRESSION") + } else if self.parse_keyword(Keyword::ENCRYPTION) { + // ENCRYPTION [=] {'Y' | 'N'} + Ident::new("ENCRYPTION") + } else if self.parse_keyword(Keyword::MAX_ROWS) { + // MAX_ROWS [=] value + Ident::new("MAX_ROWS") + } else if self.parse_keyword(Keyword::MIN_ROWS) { + // MIN_ROWS [=] value + Ident::new("MIN_ROWS") + } else if self.parse_keyword(Keyword::AUTOEXTEND_SIZE) { + // AUTOEXTEND_SIZE [=] value + Ident::new("AUTOEXTEND_SIZE") + } else if self.parse_keyword(Keyword::AVG_ROW_LENGTH) { + // AVG_ROW_LENGTH [=] value + Ident::new("AVG_ROW_LENGTH") + } else if self.parse_keyword(Keyword::CHECKSUM) { + // CHECKSUM [=] {0 | 1} + Ident::new("CHECKSUM") + } else if self.parse_keyword(Keyword::CONNECTION) { + // CONNECTION [=] 'connect_string' + Ident::new("CONNECTION") + } else if self.parse_keyword(Keyword::ENGINE_ATTRIBUTE) { + // ENGINE_ATTRIBUTE [=] 'string' + Ident::new("ENGINE_ATTRIBUTE") + } else if self.parse_keyword(Keyword::PASSWORD) { + // PASSWORD [=] 'string' + Ident::new("PASSWORD") + } else if self.parse_keyword(Keyword::SECONDARY_ENGINE_ATTRIBUTE) { + // SECONDARY_ENGINE_ATTRIBUTE [=] 'string' + Ident::new("SECONDARY_ENGINE_ATTRIBUTE") + } else if self.parse_keyword(Keyword::INSERT_METHOD) { + // INSERT_METHOD [=] { NO | FIRST | LAST } + Ident::new("INSERT_METHOD") + } else if self.parse_keyword(Keyword::AUTO_INCREMENT) { + Ident::new("AUTO_INCREMENT") + } else { + return Ok(None); + }; + + let _ = self.consume_token(&Token::Eq); + + let value = match self + .maybe_parse(|parser| parser.parse_value())? + .map(Expr::Value) + { + Some(expr) => expr, + None => Expr::Identifier(self.parse_identifier()?), + }; + + Ok(Some(SqlOption::KeyValue { key, value })) + } + + pub fn parse_plain_options(&mut self) -> Result, ParserError> { + let mut options = Vec::new(); + + while let Some(option) = self.parse_plain_option()? { + options.push(option); + } + + Ok(options) + } + pub fn parse_optional_inline_comment(&mut self) -> Result, ParserError> { let comment = if self.parse_keyword(Keyword::COMMENT) { let has_eq = self.consume_token(&Token::Eq); diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 416d2e435..8f54f3c97 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -484,7 +484,7 @@ fn parse_create_table_with_options() { columns, partition_by, cluster_by, - options, + table_options, .. }) => { assert_eq!( @@ -539,7 +539,7 @@ fn parse_create_table_with_options() { Ident::new("userid"), Ident::new("age"), ])), - Some(vec![ + CreateTableOptions::Options(vec![ SqlOption::KeyValue { key: Ident::new("partition_expiration_days"), value: Expr::Value( @@ -561,7 +561,7 @@ fn parse_create_table_with_options() { }, ]) ), - (partition_by, cluster_by, options) + (partition_by, cluster_by, table_options) ) } _ => unreachable!(), diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index c56f98860..38e10f1ea 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -219,10 +219,10 @@ fn parse_delimited_identifiers() { #[test] fn parse_create_table() { - clickhouse().verified_stmt(r#"CREATE TABLE "x" ("a" "int") ENGINE=MergeTree ORDER BY ("x")"#); - clickhouse().verified_stmt(r#"CREATE TABLE "x" ("a" "int") ENGINE=MergeTree ORDER BY "x""#); + clickhouse().verified_stmt(r#"CREATE TABLE "x" ("a" "int") ENGINE = MergeTree ORDER BY ("x")"#); + clickhouse().verified_stmt(r#"CREATE TABLE "x" ("a" "int") ENGINE = MergeTree ORDER BY "x""#); clickhouse().verified_stmt( - r#"CREATE TABLE "x" ("a" "int") ENGINE=MergeTree ORDER BY "x" AS SELECT * FROM "t" WHERE true"#, + r#"CREATE TABLE "x" ("a" "int") ENGINE = MergeTree ORDER BY "x" AS SELECT * FROM "t" WHERE true"#, ); } @@ -589,7 +589,7 @@ fn parse_clickhouse_data_types() { #[test] fn parse_create_table_with_nullable() { - let sql = r#"CREATE TABLE table (k UInt8, `a` Nullable(String), `b` Nullable(DateTime64(9, 'UTC')), c Nullable(DateTime64(9)), d Date32 NULL) ENGINE=MergeTree ORDER BY (`k`)"#; + let sql = r#"CREATE TABLE table (k UInt8, `a` Nullable(String), `b` Nullable(DateTime64(9, 'UTC')), c Nullable(DateTime64(9)), d Date32 NULL) ENGINE = MergeTree ORDER BY (`k`)"#; // ClickHouse has a case-sensitive definition of data type, but canonical representation is not let canonical_sql = sql.replace("String", "STRING"); @@ -714,14 +714,14 @@ fn parse_create_table_with_nested_data_types() { fn parse_create_table_with_primary_key() { match clickhouse_and_generic().verified_stmt(concat!( r#"CREATE TABLE db.table (`i` INT, `k` INT)"#, - " ENGINE=SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')", + " ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')", " PRIMARY KEY tuple(i)", " ORDER BY tuple(i)", )) { Statement::CreateTable(CreateTable { name, columns, - engine, + table_options, primary_key, order_by, .. @@ -742,16 +742,23 @@ fn parse_create_table_with_primary_key() { ], columns ); - assert_eq!( - engine, - Some(TableEngine { - name: "SharedMergeTree".to_string(), - parameters: Some(vec![ + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("ENGINE"), + value: Some(Ident::new("SharedMergeTree")), + parameters: vec![ Ident::with_quote('\'', "/clickhouse/tables/{uuid}/{shard}"), Ident::with_quote('\'', "{replica}"), - ]), - }) - ); + ] + } + ))); + fn assert_function(actual: &Function, name: &str, arg: &str) -> bool { assert_eq!(actual.name, ObjectName::from(vec![Ident::new(name)])); assert_eq!( @@ -798,7 +805,7 @@ fn parse_create_table_with_variant_default_expressions() { " b DATETIME EPHEMERAL now(),", " c DATETIME EPHEMERAL,", " d STRING ALIAS toString(c)", - ") ENGINE=MergeTree" + ") ENGINE = MergeTree" ); match clickhouse_and_generic().verified_stmt(sql) { Statement::CreateTable(CreateTable { columns, .. }) => { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 1ddf3f92e..7a8b8bdaa 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -3657,7 +3657,7 @@ fn parse_create_table() { name, columns, constraints, - with_options, + table_options, if_not_exists: false, external: false, file_format: None, @@ -3795,7 +3795,7 @@ fn parse_create_table() { }, ] ); - assert_eq!(with_options, vec![]); + assert_eq!(table_options, CreateTableOptions::None); } _ => unreachable!(), } @@ -3840,7 +3840,7 @@ fn parse_create_table_with_constraint_characteristics() { name, columns, constraints, - with_options, + table_options, if_not_exists: false, external: false, file_format: None, @@ -3934,7 +3934,7 @@ fn parse_create_table_with_constraint_characteristics() { }, ] ); - assert_eq!(with_options, vec![]); + assert_eq!(table_options, CreateTableOptions::None); } _ => unreachable!(), } @@ -4421,7 +4421,11 @@ fn parse_create_table_with_options() { let sql = "CREATE TABLE t (c INT) WITH (foo = 'bar', a = 123)"; match generic.verified_stmt(sql) { - Statement::CreateTable(CreateTable { with_options, .. }) => { + Statement::CreateTable(CreateTable { table_options, .. }) => { + let with_options = match table_options { + CreateTableOptions::With(options) => options, + _ => unreachable!(), + }; assert_eq!( vec![ SqlOption::KeyValue { @@ -4482,7 +4486,7 @@ fn parse_create_external_table() { name, columns, constraints, - with_options, + table_options, if_not_exists, external, file_format, @@ -4525,7 +4529,7 @@ fn parse_create_external_table() { assert_eq!(FileFormat::TEXTFILE, file_format.unwrap()); assert_eq!("/tmp/example.csv", location.unwrap()); - assert_eq!(with_options, vec![]); + assert_eq!(table_options, CreateTableOptions::None); assert!(!if_not_exists); } _ => unreachable!(), @@ -4550,7 +4554,7 @@ fn parse_create_or_replace_external_table() { name, columns, constraints, - with_options, + table_options, if_not_exists, external, file_format, @@ -4579,7 +4583,7 @@ fn parse_create_or_replace_external_table() { assert_eq!(FileFormat::TEXTFILE, file_format.unwrap()); assert_eq!("/tmp/example.csv", location.unwrap()); - assert_eq!(with_options, vec![]); + assert_eq!(table_options, CreateTableOptions::None); assert!(!if_not_exists); assert!(or_replace); } @@ -11420,7 +11424,9 @@ fn test_parse_inline_comment() { // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) match all_dialects_except(|d| d.is::()).verified_stmt(sql) { Statement::CreateTable(CreateTable { - columns, comment, .. + columns, + table_options, + .. }) => { assert_eq!( columns, @@ -11434,8 +11440,10 @@ fn test_parse_inline_comment() { }] ); assert_eq!( - comment.unwrap(), - CommentDef::WithEq("comment with equal".to_string()) + table_options, + CreateTableOptions::Plain(vec![SqlOption::Comment(CommentDef::WithEq( + "comment with equal".to_string() + ))]) ); } _ => unreachable!(), @@ -12460,21 +12468,6 @@ fn parse_select_wildcard_with_except() { ); } -#[test] -fn parse_auto_increment_too_large() { - let dialect = GenericDialect {}; - let u64_max = u64::MAX; - let sql = - format!("CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) AUTO_INCREMENT=1{u64_max}"); - - let res = Parser::new(&dialect) - .try_with_sql(&sql) - .expect("tokenize to work") - .parse_statements(); - - assert!(res.is_err(), "{res:?}"); -} - #[test] fn test_group_by_nothing() { let Select { group_by, .. } = all_dialects_where(|d| d.supports_group_by_expr()) diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index d1a87e0cf..1f417c6ab 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -735,43 +735,13 @@ fn test_duckdb_union_datatype() { storage: Default::default(), location: Default::default() }), - table_properties: Default::default(), - with_options: Default::default(), file_format: Default::default(), location: Default::default(), query: Default::default(), without_rowid: Default::default(), like: Default::default(), clone: Default::default(), - engine: Default::default(), - comment: Default::default(), - auto_increment_offset: Default::default(), - default_charset: Default::default(), - insert_method: Default::default(), - data_directory: Default::default(), - key_block_size: Default::default(), - pack_keys: Default::default(), - row_format: Default::default(), - stats_auto_recalc: Default::default(), - stats_persistent: Default::default(), - stats_sample_pages: Default::default(), - max_rows: Default::default(), - min_rows: Default::default(), - autoextend_size: Default::default(), - avg_row_length: Default::default(), - checksum: Default::default(), - connection: Default::default(), - engine_attribute: Default::default(), - password: Default::default(), - secondary_engine_attribute: Default::default(), - tablespace_option: Default::default(), - compression: Default::default(), - delay_key_write: Default::default(), - encryption: Default::default(), - start_transaction: Default::default(), - union_tables: Default::default(), - index_directory: Default::default(), - collation: Default::default(), + comment_after_column_def: Default::default(), on_commit: Default::default(), on_cluster: Default::default(), primary_key: Default::default(), @@ -779,7 +749,6 @@ fn test_duckdb_union_datatype() { partition_by: Default::default(), cluster_by: Default::default(), clustered_by: Default::default(), - options: Default::default(), inherits: Default::default(), strict: Default::default(), copy_grants: Default::default(), @@ -796,6 +765,7 @@ fn test_duckdb_union_datatype() { catalog: Default::default(), catalog_sync: Default::default(), storage_serialization_policy: Default::default(), + table_options: CreateTableOptions::None }), stmt ); diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 9b0430947..6d6427932 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -130,12 +130,13 @@ fn create_table_with_comment() { " INTO 4 BUCKETS" ); match hive().verified_stmt(sql) { - Statement::CreateTable(CreateTable { comment, .. }) => { + Statement::CreateTable(CreateTable { + comment_after_column_def: comment, + .. + }) => { assert_eq!( comment, - Some(CommentDef::AfterColumnDefsWithoutEq( - "table comment".to_string() - )) + Some(CommentDef::WithoutEq("table comment".to_string())) ) } _ => unreachable!(), diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 65aae0812..7f1b4de82 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1725,7 +1725,6 @@ fn parse_create_table_with_valid_options() { span: Span::empty(), }, data_type: Int(None,), - options: vec![], }, ColumnDef { @@ -1735,7 +1734,6 @@ fn parse_create_table_with_valid_options() { span: Span::empty(), }, data_type: Int(None,), - options: vec![], }, ], @@ -1747,43 +1745,13 @@ fn parse_create_table_with_valid_options() { storage: None, location: None, },), - table_properties: vec![], - with_options, file_format: None, location: None, query: None, without_rowid: false, like: None, clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - default_charset: None, - insert_method: None, - data_directory: None, - key_block_size: None, - pack_keys: None, - row_format: None, - stats_auto_recalc: None, - stats_persistent: None, - stats_sample_pages: None, - max_rows: None, - min_rows: None, - autoextend_size: None, - avg_row_length: None, - checksum: None, - connection: None, - engine_attribute: None, - password: None, - secondary_engine_attribute: None, - tablespace_option: None, - compression: None, - delay_key_write: None, - encryption: None, - start_transaction: None, - union_tables: None, - index_directory: None, - collation: None, + comment_after_column_def: None, on_commit: None, on_cluster: None, primary_key: None, @@ -1791,7 +1759,6 @@ fn parse_create_table_with_valid_options() { partition_by: None, cluster_by: None, clustered_by: None, - options: None, inherits: None, strict: false, iceberg: false, @@ -1809,6 +1776,7 @@ fn parse_create_table_with_valid_options() { catalog: None, catalog_sync: None, storage_serialization_policy: None, + table_options: CreateTableOptions::With(with_options) }) ); } @@ -1942,43 +1910,13 @@ fn parse_create_table_with_identity_column() { storage: None, location: None, },), - table_properties: vec![], - with_options: vec![], file_format: None, location: None, query: None, without_rowid: false, like: None, clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - default_charset: None, - insert_method: None, - data_directory: None, - key_block_size: None, - pack_keys: None, - row_format: None, - stats_auto_recalc: None, - stats_persistent: None, - stats_sample_pages: None, - max_rows: None, - min_rows: None, - autoextend_size: None, - avg_row_length: None, - checksum: None, - connection: None, - engine_attribute: None, - password: None, - secondary_engine_attribute: None, - tablespace_option: None, - compression: None, - delay_key_write: None, - encryption: None, - start_transaction: None, - union_tables: None, - index_directory: None, - collation: None, + comment_after_column_def: None, on_commit: None, on_cluster: None, primary_key: None, @@ -1986,7 +1924,6 @@ fn parse_create_table_with_identity_column() { partition_by: None, cluster_by: None, clustered_by: None, - options: None, inherits: None, strict: false, copy_grants: false, @@ -2003,6 +1940,7 @@ fn parse_create_table_with_identity_column() { catalog: None, catalog_sync: None, storage_serialization_policy: None, + table_options: CreateTableOptions::None }), ); } diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index d5ab25e6b..b2bb88bc1 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -25,9 +25,9 @@ use matches::assert_matches; use sqlparser::ast::MysqlInsertPriority::{Delayed, HighPriority, LowPriority}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MySqlDialect}; -use sqlparser::parser::{ParserError, ParserOptions, RowFormat, StorageType, TablespaceOption}; -use sqlparser::tokenizer::Span; +use sqlparser::parser::{Parser, ParserError, ParserOptions}; use sqlparser::tokenizer::Token; +use sqlparser::tokenizer::{Location, Span}; use test_utils::*; #[macro_use] @@ -848,9 +848,23 @@ fn parse_create_table_comment() { for sql in [without_equal, with_equal] { match mysql().verified_stmt(sql) { - Statement::CreateTable(CreateTable { name, comment, .. }) => { + Statement::CreateTable(CreateTable { + name, + table_options, + .. + }) => { assert_eq!(name.to_string(), "foo"); - assert_eq!(comment.expect("Should exist").to_string(), "baz"); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + let comment = match plain_options.first().unwrap() { + SqlOption::Comment(CommentDef::WithEq(c)) + | SqlOption::Comment(CommentDef::WithoutEq(c)) => c, + _ => unreachable!(), + }; + assert_eq!(comment, "baz"); } _ => unreachable!(), } @@ -859,19 +873,26 @@ fn parse_create_table_comment() { #[test] fn parse_create_table_auto_increment_offset() { - let sql = "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123"; + let sql = + "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE = InnoDB AUTO_INCREMENT = 123"; match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, - auto_increment_offset, + table_options, .. }) => { assert_eq!(name.to_string(), "foo"); - assert_eq!( - auto_increment_offset.expect("Should exist").to_string(), - "123" - ); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("AUTO_INCREMENT"), + value: Expr::Value(Value::Number("123".to_owned(), false).into()) + })); } _ => unreachable!(), } @@ -879,29 +900,44 @@ fn parse_create_table_auto_increment_offset() { #[test] fn parse_create_table_multiple_options_order_independent() { - let sql1 = "CREATE TABLE mytable (id INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8"; - let sql2 = "CREATE TABLE mytable (id INT) KEY_BLOCK_SIZE=8 ENGINE=InnoDB ROW_FORMAT=DYNAMIC"; - let sql3 = "CREATE TABLE mytable (id INT) ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 ENGINE=InnoDB"; + let sql1 = "CREATE TABLE mytable (id INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 COMMENT='abc'"; + let sql2 = "CREATE TABLE mytable (id INT) KEY_BLOCK_SIZE=8 COMMENT='abc' ENGINE=InnoDB ROW_FORMAT=DYNAMIC"; + let sql3 = "CREATE TABLE mytable (id INT) ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 COMMENT='abc' ENGINE=InnoDB"; for sql in [sql1, sql2, sql3] { match mysql().parse_sql_statements(sql).unwrap().pop().unwrap() { Statement::CreateTable(CreateTable { name, - engine, - row_format, - key_block_size, + table_options, .. }) => { assert_eq!(name.to_string(), "mytable"); - assert_eq!( - engine, - Some(TableEngine { - name: "InnoDB".to_string(), - parameters: None - }) - ); - assert_eq!(row_format, Some(RowFormat::Dynamic)); - assert_eq!(key_block_size.expect("Should exist").to_string(), "8"); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("ENGINE"), + value: Some(Ident::new("InnoDB")), + parameters: vec![] + } + ))); + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("KEY_BLOCK_SIZE"), + value: Expr::Value(Value::Number("8".to_owned(), false).into()) + })); + + assert!(plain_options + .contains(&SqlOption::Comment(CommentDef::WithEq("abc".to_owned())))); + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("ROW_FORMAT"), + value: Expr::Identifier(Ident::new("DYNAMIC".to_owned())) + })); } _ => unreachable!(), } @@ -911,118 +947,149 @@ fn parse_create_table_multiple_options_order_independent() { #[test] fn parse_create_table_with_all_table_options() { let sql = - "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci INSERT_METHOD=FIRST KEY_BLOCK_SIZE=8 ROW_FORMAT=DYNAMIC DATA DIRECTORY='/var/lib/mysql/data' INDEX DIRECTORY='/var/lib/mysql/index' PACK_KEYS=1 STATS_AUTO_RECALC=1 STATS_PERSISTENT=0 STATS_SAMPLE_PAGES=128 DELAY_KEY_WRITE=1 COMPRESSION=ZLIB ENCRYPTION='Y' MAX_ROWS=10000 MIN_ROWS=10 AUTOEXTEND_SIZE=64 AVG_ROW_LENGTH=128 CHECKSUM=1 CONNECTION='mysql://localhost' ENGINE_ATTRIBUTE='primary' PASSWORD='secure_password' SECONDARY_ENGINE_ATTRIBUTE='secondary_attr' START TRANSACTION TABLESPACE my_tablespace STORAGE DISK UNION=(table1, table2, table3)"; + "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE = InnoDB AUTO_INCREMENT = 123 DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci INSERT_METHOD = FIRST KEY_BLOCK_SIZE = 8 ROW_FORMAT = DYNAMIC DATA DIRECTORY = '/var/lib/mysql/data' INDEX DIRECTORY = '/var/lib/mysql/index' PACK_KEYS = 1 STATS_AUTO_RECALC = 1 STATS_PERSISTENT = 0 STATS_SAMPLE_PAGES = 128 DELAY_KEY_WRITE = 1 COMPRESSION = 'ZLIB' ENCRYPTION = 'Y' MAX_ROWS = 10000 MIN_ROWS = 10 AUTOEXTEND_SIZE = 64 AVG_ROW_LENGTH = 128 CHECKSUM = 1 CONNECTION = 'mysql://localhost' ENGINE_ATTRIBUTE = 'primary' PASSWORD = 'secure_password' SECONDARY_ENGINE_ATTRIBUTE = 'secondary_attr' START TRANSACTION TABLESPACE my_tablespace STORAGE DISK UNION = (table1, table2, table3)"; + let x = mysql().verified_stmt(sql); + println!("{x:?}"); match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, - engine, - default_charset, - auto_increment_offset, - key_block_size, - row_format, - data_directory, - index_directory, - pack_keys, - stats_auto_recalc, - stats_persistent, - stats_sample_pages, - compression, - insert_method, - encryption, - max_rows, - min_rows, - collation, - autoextend_size, - avg_row_length, - checksum, - connection, - engine_attribute, - password, - secondary_engine_attribute, - start_transaction, - tablespace_option, - union_tables, + table_options, .. }) => { - assert_eq!(name.to_string(), "foo"); - assert_eq!( - engine, - Some(TableEngine { - name: "InnoDB".to_string(), - parameters: None - }) - ); - assert_eq!(default_charset, Some("utf8mb4".to_string())); - assert_eq!(collation, Some("utf8mb4_0900_ai_ci".to_string())); - assert_eq!( - auto_increment_offset.expect("Should exist").to_string(), - "123" - ); - assert_eq!(key_block_size.expect("Should exist").to_string(), "8"); - assert_eq!(row_format.expect("Should exist").to_string(), "DYNAMIC"); - assert_eq!(pack_keys.expect("Should exist").to_string(), "1"); - assert_eq!(stats_auto_recalc.expect("Should exist").to_string(), "1"); - assert_eq!(stats_persistent.expect("Should exist").to_string(), "0"); - assert_eq!(stats_sample_pages.expect("Should exist").to_string(), "128"); - assert_eq!(insert_method.expect("Should exist").to_string(), "FIRST"); - assert_eq!(compression.expect("Should exist").to_string(), "ZLIB"); - assert_eq!(encryption.expect("Should exist").to_string(), "Y"); - assert_eq!(max_rows.expect("Should exist").to_string(), "10000"); - assert_eq!(min_rows.expect("Should exist").to_string(), "10"); - assert_eq!(autoextend_size.expect("Should exist").to_string(), "64"); - assert_eq!(avg_row_length.expect("Should exist").to_string(), "128"); - assert_eq!( - if checksum.expect("Should exist") { - "1" - } else { - "0" - }, - "1" - ); - assert_eq!( - connection.expect("Should exist").to_string(), - "mysql://localhost" - ); - assert_eq!( - engine_attribute.expect("Should exist").to_string(), - "primary" - ); - assert_eq!( - password.expect("Should exist").to_string(), - "secure_password" - ); - assert_eq!( - secondary_engine_attribute - .expect("Should exist") - .to_string(), - "secondary_attr" - ); + assert_eq!(name, vec![Ident::new("foo".to_owned())].into()); - assert_eq!(start_transaction.expect("Should exist"), true); - assert_eq!( - tablespace_option.expect("Should exist"), - TablespaceOption { + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("ENGINE"), + value: Some(Ident::new("InnoDB")), + parameters: vec![] + } + ))); + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("COLLATE"), + value: Expr::Identifier(Ident::new("utf8mb4_0900_ai_ci".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("DEFAULT CHARSET"), + value: Expr::Identifier(Ident::new("utf8mb4".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("AUTO_INCREMENT"), + value: Expr::value(Value::Number("123".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("KEY_BLOCK_SIZE"), + value: Expr::value(Value::Number("8".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("ROW_FORMAT"), + value: Expr::Identifier(Ident::new("DYNAMIC".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("PACK_KEYS"), + value: Expr::value(Value::Number("1".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("STATS_AUTO_RECALC"), + value: Expr::value(Value::Number("1".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("STATS_PERSISTENT"), + value: Expr::value(Value::Number("0".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("STATS_SAMPLE_PAGES"), + value: Expr::value(Value::Number("128".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("STATS_SAMPLE_PAGES"), + value: Expr::value(Value::Number("128".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("INSERT_METHOD"), + value: Expr::Identifier(Ident::new("FIRST".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("COMPRESSION"), + value: Expr::value(Value::SingleQuotedString("ZLIB".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("ENCRYPTION"), + value: Expr::value(Value::SingleQuotedString("Y".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("MAX_ROWS"), + value: Expr::value(Value::Number("10000".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("MIN_ROWS"), + value: Expr::value(Value::Number("10".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("AUTOEXTEND_SIZE"), + value: Expr::value(Value::Number("64".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("AVG_ROW_LENGTH"), + value: Expr::value(Value::Number("128".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("CHECKSUM"), + value: Expr::value(Value::Number("1".to_owned(), false)) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("CONNECTION"), + value: Expr::value(Value::SingleQuotedString("mysql://localhost".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("ENGINE_ATTRIBUTE"), + value: Expr::value(Value::SingleQuotedString("primary".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("PASSWORD"), + value: Expr::value(Value::SingleQuotedString("secure_password".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("SECONDARY_ENGINE_ATTRIBUTE"), + value: Expr::value(Value::SingleQuotedString("secondary_attr".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::Ident(Ident::new( + "START TRANSACTION".to_owned() + )))); + assert!( + plain_options.contains(&SqlOption::TableSpace(TablespaceOption { name: "my_tablespace".to_string(), storage: Some(StorageType::Disk), - } - ); - assert_eq!( - union_tables.expect("Should exist"), - vec![ - "table1".to_string(), - "table2".to_string(), - "table3".to_string() - ] - ); - assert_eq!( - data_directory.expect("Should exist").path, - "/var/lib/mysql/data" - ); - assert_eq!( - index_directory.expect("Should exist").path, - "/var/lib/mysql/index" + })) ); + + assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("UNION"), + value: None, + parameters: vec![ + Ident::new("table1".to_string()), + Ident::new("table2".to_string()), + Ident::new("table3".to_string()) + ] + } + ))); + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("DATA DIRECTORY"), + value: Expr::value(Value::SingleQuotedString("/var/lib/mysql/data".to_owned())) + })); + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("INDEX DIRECTORY"), + value: Expr::value(Value::SingleQuotedString("/var/lib/mysql/index".to_owned())) + })); } _ => unreachable!(), } @@ -1062,13 +1129,12 @@ fn parse_create_table_set_enum() { #[test] fn parse_create_table_engine_default_charset() { - let sql = "CREATE TABLE foo (id INT(11)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3"; + let sql = "CREATE TABLE foo (id INT(11)) ENGINE = InnoDB DEFAULT CHARSET = utf8mb3"; match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, columns, - engine, - default_charset, + table_options, .. }) => { assert_eq!(name.to_string(), "foo"); @@ -1080,14 +1146,24 @@ fn parse_create_table_engine_default_charset() { },], columns ); - assert_eq!( - engine, - Some(TableEngine { - name: "InnoDB".to_string(), - parameters: None - }) - ); - assert_eq!(default_charset, Some("utf8mb3".to_string())); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("DEFAULT CHARSET"), + value: Expr::Identifier(Ident::new("utf8mb3".to_owned())) + })); + + assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( + NamedParenthesizedList { + key: Ident::new("ENGINE"), + value: Some(Ident::new("InnoDB")), + parameters: vec![] + } + ))); } _ => unreachable!(), } @@ -1095,12 +1171,12 @@ fn parse_create_table_engine_default_charset() { #[test] fn parse_create_table_collate() { - let sql = "CREATE TABLE foo (id INT(11)) COLLATE=utf8mb4_0900_ai_ci"; + let sql = "CREATE TABLE foo (id INT(11)) COLLATE = utf8mb4_0900_ai_ci"; match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, columns, - collation, + table_options, .. }) => { assert_eq!(name.to_string(), "foo"); @@ -1112,7 +1188,16 @@ fn parse_create_table_collate() { },], columns ); - assert_eq!(collation, Some("utf8mb4_0900_ai_ci".to_string())); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("COLLATE"), + value: Expr::Identifier(Ident::new("utf8mb4_0900_ai_ci".to_owned())) + })); } _ => unreachable!(), } @@ -1120,16 +1205,26 @@ fn parse_create_table_collate() { #[test] fn parse_create_table_both_options_and_as_query() { - let sql = "CREATE TABLE foo (id INT(11)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb4_0900_ai_ci AS SELECT 1"; + let sql = "CREATE TABLE foo (id INT(11)) ENGINE = InnoDB DEFAULT CHARSET = utf8mb3 COLLATE = utf8mb4_0900_ai_ci AS SELECT 1"; match mysql_and_generic().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, - collation, query, + table_options, .. }) => { assert_eq!(name.to_string(), "foo"); - assert_eq!(collation, Some("utf8mb4_0900_ai_ci".to_string())); + + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + + assert!(plain_options.contains(&SqlOption::KeyValue { + key: Ident::new("COLLATE"), + value: Expr::Identifier(Ident::new("utf8mb4_0900_ai_ci".to_owned())) + })); + assert_eq!( query.unwrap().body.as_select().unwrap().projection, vec![SelectItem::UnnamedExpr(Expr::Value( @@ -1140,7 +1235,8 @@ fn parse_create_table_both_options_and_as_query() { _ => unreachable!(), } - let sql = r"CREATE TABLE foo (id INT(11)) ENGINE=InnoDB AS SELECT 1 DEFAULT CHARSET=utf8mb3"; + let sql = + r"CREATE TABLE foo (id INT(11)) ENGINE = InnoDB AS SELECT 1 DEFAULT CHARSET = utf8mb3"; assert!(matches!( mysql_and_generic().parse_sql_statements(sql), Err(ParserError::ParserError(_)) diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index d7d874f5d..3b52ccd59 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -348,7 +348,7 @@ fn parse_create_table_with_defaults() { name, columns, constraints, - with_options, + table_options, if_not_exists: false, external: false, file_format: None, @@ -485,6 +485,11 @@ fn parse_create_table_with_defaults() { ] ); assert!(constraints.is_empty()); + + let with_options = match table_options { + CreateTableOptions::With(options) => options, + _ => unreachable!(), + }; assert_eq!( with_options, vec![ @@ -4668,7 +4673,6 @@ fn parse_create_table_with_alias() { name, columns, constraints, - with_options: _with_options, if_not_exists: false, external: false, file_format: None, @@ -5078,7 +5082,11 @@ fn parse_at_time_zone() { fn parse_create_table_with_options() { let sql = "CREATE TABLE t (c INT) WITH (foo = 'bar', a = 123)"; match pg().verified_stmt(sql) { - Statement::CreateTable(CreateTable { with_options, .. }) => { + Statement::CreateTable(CreateTable { table_options, .. }) => { + let with_options = match table_options { + CreateTableOptions::With(options) => options, + _ => unreachable!(), + }; assert_eq!( vec![ SqlOption::KeyValue { @@ -5506,43 +5514,13 @@ fn parse_trigger_related_functions() { storage: None, location: None }), - table_properties: vec![], - with_options: vec![], file_format: None, location: None, query: None, without_rowid: false, like: None, clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - default_charset: None, - insert_method: None, - data_directory: None, - key_block_size: None, - pack_keys: None, - row_format: None, - stats_auto_recalc: None, - stats_persistent: None, - stats_sample_pages: None, - max_rows: None, - min_rows: None, - autoextend_size: None, - avg_row_length: None, - checksum: None, - connection: None, - engine_attribute: None, - password: None, - secondary_engine_attribute: None, - tablespace_option: None, - compression: None, - delay_key_write: None, - encryption: None, - start_transaction: None, - union_tables: None, - index_directory: None, - collation: None, + comment_after_column_def: None, on_commit: None, on_cluster: None, primary_key: None, @@ -5550,7 +5528,6 @@ fn parse_trigger_related_functions() { partition_by: None, cluster_by: None, clustered_by: None, - options: None, inherits: None, strict: false, copy_grants: false, @@ -5567,6 +5544,7 @@ fn parse_trigger_related_functions() { catalog: None, catalog_sync: None, storage_serialization_policy: None, + table_options: CreateTableOptions::None } ); diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index aa974115d..52be31435 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -470,9 +470,22 @@ fn test_snowflake_create_table_cluster_by() { #[test] fn test_snowflake_create_table_comment() { match snowflake().verified_stmt("CREATE TABLE my_table (a INT) COMMENT = 'some comment'") { - Statement::CreateTable(CreateTable { name, comment, .. }) => { + Statement::CreateTable(CreateTable { + name, + table_options, + .. + }) => { assert_eq!("my_table", name.to_string()); - assert_eq!("some comment", comment.unwrap().to_string()); + let plain_options = match table_options { + CreateTableOptions::Plain(options) => options, + _ => unreachable!(), + }; + let comment = match plain_options.first().unwrap() { + SqlOption::Comment(CommentDef::WithEq(c)) + | SqlOption::Comment(CommentDef::WithoutEq(c)) => c, + _ => unreachable!(), + }; + assert_eq!("some comment", comment); } _ => unreachable!(), } From 32833cee53af650064c24292c44c84ef313804b0 Mon Sep 17 00:00:00 2001 From: Tomer Shani Date: Wed, 30 Apr 2025 11:08:05 +0300 Subject: [PATCH 3/3] create table parsing - review feeback --- src/ast/dml.rs | 6 ++-- src/ast/helpers/stmt_create_table.rs | 12 +++---- src/ast/mod.rs | 42 +++++++++++++---------- src/ast/spans.rs | 6 ++-- src/parser/mod.rs | 43 ++++++------------------ tests/sqlparser_clickhouse.rs | 4 +-- tests/sqlparser_duckdb.rs | 2 +- tests/sqlparser_hive.rs | 5 +-- tests/sqlparser_mssql.rs | 4 +-- tests/sqlparser_mysql.rs | 50 +++++++++++++--------------- tests/sqlparser_postgres.rs | 2 +- 11 files changed, 78 insertions(+), 98 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 94589e085..7ed17be9b 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -154,9 +154,9 @@ pub struct CreateTable { pub like: Option, pub clone: Option, // For Hive dialect, the table comment is after the column definitions without `=`, - // so we need to add an extra variant to allow to identify this case when displaying. + // so the `comment` field is optional and different than the comment field in the general options list. // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) - pub comment_after_column_def: Option, + pub comment: Option, pub on_commit: Option, /// ClickHouse "ON CLUSTER" clause: /// @@ -277,7 +277,7 @@ impl Display for CreateTable { // Hive table comment should be after column definitions, please refer to: // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) - if let Some(comment) = &self.comment_after_column_def { + if let Some(comment) = &self.comment { write!(f, " COMMENT '{comment}'")?; } diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index fe86007e5..542d30ea9 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -84,7 +84,7 @@ pub struct CreateTableBuilder { pub without_rowid: bool, pub like: Option, pub clone: Option, - pub comment_after_column_def: Option, + pub comment: Option, pub on_commit: Option, pub on_cluster: Option, pub primary_key: Option>, @@ -133,7 +133,7 @@ impl CreateTableBuilder { without_rowid: false, like: None, clone: None, - comment_after_column_def: None, + comment: None, on_commit: None, on_cluster: None, primary_key: None, @@ -250,7 +250,7 @@ impl CreateTableBuilder { } pub fn comment_after_column_def(mut self, comment: Option) -> Self { - self.comment_after_column_def = comment; + self.comment = comment; self } @@ -404,7 +404,7 @@ impl CreateTableBuilder { without_rowid: self.without_rowid, like: self.like, clone: self.clone, - comment_after_column_def: self.comment_after_column_def, + comment: self.comment, on_commit: self.on_commit, on_cluster: self.on_cluster, primary_key: self.primary_key, @@ -460,7 +460,7 @@ impl TryFrom for CreateTableBuilder { without_rowid, like, clone, - comment_after_column_def: comment, + comment, on_commit, on_cluster, primary_key, @@ -503,7 +503,7 @@ impl TryFrom for CreateTableBuilder { without_rowid, like, clone, - comment_after_column_def: comment, + comment, on_commit, on_cluster, primary_key, diff --git a/src/ast/mod.rs b/src/ast/mod.rs index b5e1a5ba4..d74d197e3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -7566,10 +7566,7 @@ pub enum SqlOption { /// Any option that consists of a key value pair where the value is an expression. e.g. /// /// WITH(DISTRIBUTION = ROUND_ROBIN) - KeyValue { - key: Ident, - value: Expr, - }, + KeyValue { key: Ident, value: Expr }, /// One or more table partitions and represents which partition the boundary values belong to, /// e.g. /// @@ -7581,15 +7578,17 @@ pub enum SqlOption { range_direction: Option, for_values: Vec, }, - + /// Comment parameter (supports `=` and no `=` syntax) Comment(CommentDef), - + /// MySQL TableSpace option + /// TableSpace(TablespaceOption), - - // Advanced parameter formations - // UNION = (tbl_name[,tbl_name]...) - // ENGINE = ReplicatedMergeTree('/table_name','{replica}', ver) - // ENGINE = SummingMergeTree([columns]) + /// An option representing a key value pair, where the value is a parenthesized list and with an optional name + /// e.g. + /// + /// UNION = (tbl_name\[,tbl_name\]...) + /// ENGINE = ReplicatedMergeTree('/table_name','{replica}', ver) + /// ENGINE = SummingMergeTree(\[columns\]) NamedParenthesizedList(NamedParenthesizedList), } @@ -7627,7 +7626,7 @@ impl fmt::Display for SqlOption { match tablespace_option.storage { Some(StorageType::Disk) => write!(f, " STORAGE DISK"), Some(StorageType::Memory) => write!(f, " STORAGE MEMORY"), - _ => Ok(()), + None => Ok(()), } } SqlOption::Comment(comment) => match comment { @@ -7640,11 +7639,11 @@ impl fmt::Display for SqlOption { }, SqlOption::NamedParenthesizedList(value) => { write!(f, "{} = ", value.key)?; - if let Some(key) = &value.value { + if let Some(key) = &value.name { write!(f, "{}", key)?; } - if !value.parameters.is_empty() { - write!(f, "({})", display_comma_separated(&value.parameters))? + if !value.values.is_empty() { + write!(f, "({})", display_comma_separated(&value.values))? } Ok(()) } @@ -7663,6 +7662,8 @@ pub enum StorageType { #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +/// MySql TableSpace option +/// pub struct TablespaceOption { pub name: String, pub storage: Option, @@ -8936,10 +8937,17 @@ impl Display for CreateViewParams { #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +/// Key/Value, where the value is a (optionally named) list of identifiers +/// +/// ```sql +/// UNION = (tbl_name[,tbl_name]...) +/// ENGINE = ReplicatedMergeTree('/table_name','{replica}', ver) +/// ENGINE = SummingMergeTree([columns]) +/// ``` pub struct NamedParenthesizedList { pub key: Ident, - pub value: Option, - pub parameters: Vec, + pub name: Option, + pub values: Vec, } /// Snowflake `WITH ROW ACCESS POLICY policy_name ON (identifier, ...)` diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 5eeed7e42..3f703ffaf 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -573,7 +573,7 @@ impl Spanned for CreateTable { without_rowid: _, // bool like, clone, - comment_after_column_def: _, // todo, no span + comment: _, // todo, no span on_commit: _, on_cluster: _, // todo, clickhouse specific primary_key: _, // todo, clickhouse specific @@ -1001,8 +1001,8 @@ impl Spanned for SqlOption { SqlOption::Comment(_) => Span::empty(), SqlOption::NamedParenthesizedList(NamedParenthesizedList { key: name, - value, - parameters: values, + name: value, + values, }) => union_spans(core::iter::once(name.span).chain(values.iter().map(|i| i.span))) .union_opt(&value.as_ref().map(|i| i.span)), } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index d6bca70c8..a347f3d4d 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -7222,11 +7222,13 @@ impl<'a> Parser<'a> { fn parse_plain_option(&mut self) -> Result, ParserError> { // Single parameter option + // if self.parse_keywords(&[Keyword::START, Keyword::TRANSACTION]) { return Ok(Some(SqlOption::Ident(Ident::new("START TRANSACTION")))); } // Custom option + // if self.parse_keywords(&[Keyword::COMMENT]) { let has_eq = self.consume_token(&Token::Eq); let value = self.next_token(); @@ -7245,6 +7247,8 @@ impl<'a> Parser<'a> { return comment; } + // + // if self.parse_keywords(&[Keyword::ENGINE]) { let _ = self.consume_token(&Token::Eq); let value = self.next_token(); @@ -7260,8 +7264,8 @@ impl<'a> Parser<'a> { Ok(Some(SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("ENGINE"), - value: Some(Ident::new(w.value)), - parameters, + name: Some(Ident::new(w.value)), + values: parameters, }, ))) } @@ -7273,12 +7277,12 @@ impl<'a> Parser<'a> { return engine; } + // if self.parse_keywords(&[Keyword::TABLESPACE]) { let _ = self.consume_token(&Token::Eq); let value = self.next_token(); let tablespace = match value.token { - // TABLESPACE tablespace_name [STORAGE DISK] | [TABLESPACE tablespace_name] STORAGE MEMORY Token::Word(Word { value: name, .. }) | Token::SingleQuotedString(name) => { let storage = match self.parse_keyword(Keyword::STORAGE) { true => { @@ -7310,12 +7314,12 @@ impl<'a> Parser<'a> { return tablespace; } + // if self.parse_keyword(Keyword::UNION) { let _ = self.consume_token(&Token::Eq); let value = self.next_token(); match value.token { - // UNION [=] (tbl_name[,tbl_name]...) Token::LParen => { let tables: Vec = self.parse_comma_separated0(Parser::parse_identifier, Token::RParen)?; @@ -7324,8 +7328,8 @@ impl<'a> Parser<'a> { return Ok(Some(SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("UNION"), - value: None, - parameters: tables, + name: None, + values: tables, }, ))); } @@ -7337,85 +7341,58 @@ impl<'a> Parser<'a> { // Key/Value parameter option let key = if self.parse_keywords(&[Keyword::DEFAULT, Keyword::CHARSET]) { - // [DEFAULT] CHARACTER SET [=] charset_name Ident::new("DEFAULT CHARSET") } else if self.parse_keyword(Keyword::CHARSET) { - // [DEFAULT] CHARACTER SET [=] charset_name Ident::new("CHARSET") } else if self.parse_keywords(&[Keyword::DEFAULT, Keyword::CHARACTER, Keyword::SET]) { - // [DEFAULT] CHARACTER SET [=] charset_name Ident::new("DEFAULT CHARACTER SET") } else if self.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) { - // [DEFAULT] CHARACTER SET [=] charset_name Ident::new("CHARACTER SET") } else if self.parse_keywords(&[Keyword::DEFAULT, Keyword::COLLATE]) { - // [DEFAULT] COLLATE [=] collation_name Ident::new("DEFAULT COLLATE") } else if self.parse_keyword(Keyword::COLLATE) { - // [DEFAULT] COLLATE [=] collation_name Ident::new("COLLATE") } else if self.parse_keywords(&[Keyword::DATA, Keyword::DIRECTORY]) { - // {DATA | INDEX} DIRECTORY [=] 'absolute path to directory' Ident::new("DATA DIRECTORY") } else if self.parse_keywords(&[Keyword::INDEX, Keyword::DIRECTORY]) { - // {DATA | INDEX} DIRECTORY [=] 'absolute path to directory' Ident::new("INDEX DIRECTORY") } else if self.parse_keyword(Keyword::KEY_BLOCK_SIZE) { - // KEY_BLOCK_SIZE [=] value Ident::new("KEY_BLOCK_SIZE") } else if self.parse_keyword(Keyword::ROW_FORMAT) { - // ROW_FORMAT [=] {DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT} Ident::new("ROW_FORMAT") } else if self.parse_keyword(Keyword::PACK_KEYS) { - // PACK_KEYS [=] {0 | 1 | DEFAULT} Ident::new("PACK_KEYS") } else if self.parse_keyword(Keyword::STATS_AUTO_RECALC) { - // STATS_AUTO_RECALC [=] {DEFAULT | 0 | 1} Ident::new("STATS_AUTO_RECALC") } else if self.parse_keyword(Keyword::STATS_PERSISTENT) { - //STATS_PERSISTENT [=] {DEFAULT | 0 | 1} Ident::new("STATS_PERSISTENT") } else if self.parse_keyword(Keyword::STATS_SAMPLE_PAGES) { - // STATS_SAMPLE_PAGES [=] value Ident::new("STATS_SAMPLE_PAGES") } else if self.parse_keyword(Keyword::DELAY_KEY_WRITE) { - // DELAY_KEY_WRITE [=] {0 | 1} Ident::new("DELAY_KEY_WRITE") } else if self.parse_keyword(Keyword::COMPRESSION) { - // COMPRESSION [=] {'ZLIB' | 'LZ4' | 'NONE'} Ident::new("COMPRESSION") } else if self.parse_keyword(Keyword::ENCRYPTION) { - // ENCRYPTION [=] {'Y' | 'N'} Ident::new("ENCRYPTION") } else if self.parse_keyword(Keyword::MAX_ROWS) { - // MAX_ROWS [=] value Ident::new("MAX_ROWS") } else if self.parse_keyword(Keyword::MIN_ROWS) { - // MIN_ROWS [=] value Ident::new("MIN_ROWS") } else if self.parse_keyword(Keyword::AUTOEXTEND_SIZE) { - // AUTOEXTEND_SIZE [=] value Ident::new("AUTOEXTEND_SIZE") } else if self.parse_keyword(Keyword::AVG_ROW_LENGTH) { - // AVG_ROW_LENGTH [=] value Ident::new("AVG_ROW_LENGTH") } else if self.parse_keyword(Keyword::CHECKSUM) { - // CHECKSUM [=] {0 | 1} Ident::new("CHECKSUM") } else if self.parse_keyword(Keyword::CONNECTION) { - // CONNECTION [=] 'connect_string' Ident::new("CONNECTION") } else if self.parse_keyword(Keyword::ENGINE_ATTRIBUTE) { - // ENGINE_ATTRIBUTE [=] 'string' Ident::new("ENGINE_ATTRIBUTE") } else if self.parse_keyword(Keyword::PASSWORD) { - // PASSWORD [=] 'string' Ident::new("PASSWORD") } else if self.parse_keyword(Keyword::SECONDARY_ENGINE_ATTRIBUTE) { - // SECONDARY_ENGINE_ATTRIBUTE [=] 'string' Ident::new("SECONDARY_ENGINE_ATTRIBUTE") } else if self.parse_keyword(Keyword::INSERT_METHOD) { - // INSERT_METHOD [=] { NO | FIRST | LAST } Ident::new("INSERT_METHOD") } else if self.parse_keyword(Keyword::AUTO_INCREMENT) { Ident::new("AUTO_INCREMENT") diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 38e10f1ea..d0218b6c3 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -751,8 +751,8 @@ fn parse_create_table_with_primary_key() { assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("ENGINE"), - value: Some(Ident::new("SharedMergeTree")), - parameters: vec![ + name: Some(Ident::new("SharedMergeTree")), + values: vec![ Ident::with_quote('\'', "/clickhouse/tables/{uuid}/{shard}"), Ident::with_quote('\'', "{replica}"), ] diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index 1f417c6ab..8e4983655 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -741,7 +741,7 @@ fn test_duckdb_union_datatype() { without_rowid: Default::default(), like: Default::default(), clone: Default::default(), - comment_after_column_def: Default::default(), + comment: Default::default(), on_commit: Default::default(), on_cluster: Default::default(), primary_key: Default::default(), diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 6d6427932..14dcbffd1 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -130,10 +130,7 @@ fn create_table_with_comment() { " INTO 4 BUCKETS" ); match hive().verified_stmt(sql) { - Statement::CreateTable(CreateTable { - comment_after_column_def: comment, - .. - }) => { + Statement::CreateTable(CreateTable { comment, .. }) => { assert_eq!( comment, Some(CommentDef::WithoutEq("table comment".to_string())) diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 7f1b4de82..8cc5758fd 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1751,7 +1751,7 @@ fn parse_create_table_with_valid_options() { without_rowid: false, like: None, clone: None, - comment_after_column_def: None, + comment: None, on_commit: None, on_cluster: None, primary_key: None, @@ -1916,7 +1916,7 @@ fn parse_create_table_with_identity_column() { without_rowid: false, like: None, clone: None, - comment_after_column_def: None, + comment: None, on_commit: None, on_cluster: None, primary_key: None, diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index b2bb88bc1..4bb1063dd 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -25,9 +25,9 @@ use matches::assert_matches; use sqlparser::ast::MysqlInsertPriority::{Delayed, HighPriority, LowPriority}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MySqlDialect}; -use sqlparser::parser::{Parser, ParserError, ParserOptions}; +use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use sqlparser::tokenizer::Token; -use sqlparser::tokenizer::{Location, Span}; use test_utils::*; #[macro_use] @@ -891,7 +891,7 @@ fn parse_create_table_auto_increment_offset() { assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("AUTO_INCREMENT"), - value: Expr::Value(Value::Number("123".to_owned(), false).into()) + value: Expr::Value(test_utils::number("123").with_empty_span()) })); } _ => unreachable!(), @@ -921,14 +921,14 @@ fn parse_create_table_multiple_options_order_independent() { assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("ENGINE"), - value: Some(Ident::new("InnoDB")), - parameters: vec![] + name: Some(Ident::new("InnoDB")), + values: vec![] } ))); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("KEY_BLOCK_SIZE"), - value: Expr::Value(Value::Number("8".to_owned(), false).into()) + value: Expr::Value(test_utils::number("8").with_empty_span()) })); assert!(plain_options @@ -949,8 +949,6 @@ fn parse_create_table_with_all_table_options() { let sql = "CREATE TABLE foo (bar INT NOT NULL AUTO_INCREMENT) ENGINE = InnoDB AUTO_INCREMENT = 123 DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci INSERT_METHOD = FIRST KEY_BLOCK_SIZE = 8 ROW_FORMAT = DYNAMIC DATA DIRECTORY = '/var/lib/mysql/data' INDEX DIRECTORY = '/var/lib/mysql/index' PACK_KEYS = 1 STATS_AUTO_RECALC = 1 STATS_PERSISTENT = 0 STATS_SAMPLE_PAGES = 128 DELAY_KEY_WRITE = 1 COMPRESSION = 'ZLIB' ENCRYPTION = 'Y' MAX_ROWS = 10000 MIN_ROWS = 10 AUTOEXTEND_SIZE = 64 AVG_ROW_LENGTH = 128 CHECKSUM = 1 CONNECTION = 'mysql://localhost' ENGINE_ATTRIBUTE = 'primary' PASSWORD = 'secure_password' SECONDARY_ENGINE_ATTRIBUTE = 'secondary_attr' START TRANSACTION TABLESPACE my_tablespace STORAGE DISK UNION = (table1, table2, table3)"; - let x = mysql().verified_stmt(sql); - println!("{x:?}"); match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, @@ -967,8 +965,8 @@ fn parse_create_table_with_all_table_options() { assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("ENGINE"), - value: Some(Ident::new("InnoDB")), - parameters: vec![] + name: Some(Ident::new("InnoDB")), + values: vec![] } ))); @@ -982,11 +980,11 @@ fn parse_create_table_with_all_table_options() { })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("AUTO_INCREMENT"), - value: Expr::value(Value::Number("123".to_owned(), false)) + value: Expr::value(test_utils::number("123")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("KEY_BLOCK_SIZE"), - value: Expr::value(Value::Number("8".to_owned(), false)) + value: Expr::value(test_utils::number("8")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("ROW_FORMAT"), @@ -994,23 +992,23 @@ fn parse_create_table_with_all_table_options() { })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("PACK_KEYS"), - value: Expr::value(Value::Number("1".to_owned(), false)) + value: Expr::value(test_utils::number("1")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("STATS_AUTO_RECALC"), - value: Expr::value(Value::Number("1".to_owned(), false)) + value: Expr::value(test_utils::number("1")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("STATS_PERSISTENT"), - value: Expr::value(Value::Number("0".to_owned(), false)) + value: Expr::value(test_utils::number("0")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("STATS_SAMPLE_PAGES"), - value: Expr::value(Value::Number("128".to_owned(), false)) + value: Expr::value(test_utils::number("128")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("STATS_SAMPLE_PAGES"), - value: Expr::value(Value::Number("128".to_owned(), false)) + value: Expr::value(test_utils::number("128")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("INSERT_METHOD"), @@ -1026,23 +1024,23 @@ fn parse_create_table_with_all_table_options() { })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("MAX_ROWS"), - value: Expr::value(Value::Number("10000".to_owned(), false)) + value: Expr::value(test_utils::number("10000")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("MIN_ROWS"), - value: Expr::value(Value::Number("10".to_owned(), false)) + value: Expr::value(test_utils::number("10")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("AUTOEXTEND_SIZE"), - value: Expr::value(Value::Number("64".to_owned(), false)) + value: Expr::value(test_utils::number("64")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("AVG_ROW_LENGTH"), - value: Expr::value(Value::Number("128".to_owned(), false)) + value: Expr::value(test_utils::number("128")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("CHECKSUM"), - value: Expr::value(Value::Number("1".to_owned(), false)) + value: Expr::value(test_utils::number("1")) })); assert!(plain_options.contains(&SqlOption::KeyValue { key: Ident::new("CONNECTION"), @@ -1073,8 +1071,8 @@ fn parse_create_table_with_all_table_options() { assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("UNION"), - value: None, - parameters: vec![ + name: None, + values: vec![ Ident::new("table1".to_string()), Ident::new("table2".to_string()), Ident::new("table3".to_string()) @@ -1160,8 +1158,8 @@ fn parse_create_table_engine_default_charset() { assert!(plain_options.contains(&SqlOption::NamedParenthesizedList( NamedParenthesizedList { key: Ident::new("ENGINE"), - value: Some(Ident::new("InnoDB")), - parameters: vec![] + name: Some(Ident::new("InnoDB")), + values: vec![] } ))); } diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 3b52ccd59..1fb7432a4 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -5520,7 +5520,7 @@ fn parse_trigger_related_functions() { without_rowid: false, like: None, clone: None, - comment_after_column_def: None, + comment: None, on_commit: None, on_cluster: None, primary_key: None,