diff --git a/src/formatter/ExpressionFormatter.ts b/src/formatter/ExpressionFormatter.ts index ce625761b1..9289b817ba 100644 --- a/src/formatter/ExpressionFormatter.ts +++ b/src/formatter/ExpressionFormatter.ts @@ -199,6 +199,7 @@ export default class ExpressionFormatter { return this.formatLogicalOperator(token); case TokenType.RESERVED_KEYWORD: case TokenType.RESERVED_FUNCTION_NAME: + case TokenType.RESERVED_PHRASE: return this.formatKeyword(token); case TokenType.RESERVED_CASE_START: return this.formatCaseStart(token); diff --git a/src/languages/bigquery/bigquery.formatter.ts b/src/languages/bigquery/bigquery.formatter.ts index a5a78ed86b..5c86f19627 100644 --- a/src/languages/bigquery/bigquery.formatter.ts +++ b/src/languages/bigquery/bigquery.formatter.ts @@ -116,6 +116,15 @@ const reservedJoins = expandPhrases([ '{INNER | CROSS} JOIN', ]); +const reservedPhrases = [ + // https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#tablesample_operator + 'TABLESAMPLE SYSTEM', + // From DDL: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language + 'ANY TYPE', + 'ALL COLUMNS', + 'NOT DETERMINISTIC', +]; + // https://cloud.google.com/bigquery/docs/reference/#standard-sql-reference export default class BigQueryFormatter extends Formatter { static operators = ['~', '>>', '<<', '||']; @@ -127,6 +136,7 @@ export default class BigQueryFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, openParens: ['(', '['], diff --git a/src/languages/bigquery/bigquery.keywords.ts b/src/languages/bigquery/bigquery.keywords.ts index c9a87243fd..e086fc633c 100644 --- a/src/languages/bigquery/bigquery.keywords.ts +++ b/src/languages/bigquery/bigquery.keywords.ts @@ -85,7 +85,7 @@ export const keywords = flatKeywordList({ 'SOME', 'STRUCT', 'TABLE', - 'TABLESAMPLE SYSTEM', + 'TABLESAMPLE', 'THEN', 'TO', 'TREAT', @@ -138,11 +138,8 @@ export const keywords = flatKeywordList({ 'INOUT', 'RETURNS', 'LANGUAGE', - 'ANY TYPE', - 'ALL COLUMNS', 'CASCADE', 'RESTRICT', 'DETERMINISTIC', - 'NOT DETERMINISTIC', ], }); diff --git a/src/languages/db2/db2.formatter.ts b/src/languages/db2/db2.formatter.ts index 5448697a6b..b8b7d62d19 100644 --- a/src/languages/db2/db2.formatter.ts +++ b/src/languages/db2/db2.formatter.ts @@ -147,6 +147,8 @@ const reservedJoins = expandPhrases([ '{INNER | CROSS} JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE']; + // https://www.ibm.com/support/knowledgecenter/en/ssw_ibm_i_72/db2/rbafzintro.htm export default class Db2Formatter extends Formatter { static operators = ['**', '¬=', '¬>', '¬<', '!>', '!<', '||']; @@ -157,6 +159,7 @@ export default class Db2Formatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, stringTypes: [{ quote: "''", prefixes: ['G', 'N', 'X', 'BX', 'GX', 'UX', 'U&'] }], diff --git a/src/languages/db2/db2.keywords.ts b/src/languages/db2/db2.keywords.ts index 756286848e..8df36b6cf5 100644 --- a/src/languages/db2/db2.keywords.ts +++ b/src/languages/db2/db2.keywords.ts @@ -236,5 +236,4 @@ export const keywords = flatKeywordList({ 'YEARS', 'ZONE', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/languages/mariadb/mariadb.formatter.ts b/src/languages/mariadb/mariadb.formatter.ts index 9786388f20..eff6c8d5ba 100644 --- a/src/languages/mariadb/mariadb.formatter.ts +++ b/src/languages/mariadb/mariadb.formatter.ts @@ -243,6 +243,8 @@ const reservedJoins = expandPhrases([ 'STRAIGHT_JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CHARACTER SET']; + // For reference: https://mariadb.com/kb/en/sql-statements-structure/ export default class MariaDbFormatter extends Formatter { static operators = [':=', '<<', '>>', '<=>', '&&', '||']; @@ -253,6 +255,7 @@ export default class MariaDbFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF', 'ELSIF'], + reservedPhrases, reservedLogicalOperators: ['AND', 'OR', 'XOR'], reservedKeywords: keywords, reservedFunctionNames: functions, diff --git a/src/languages/mariadb/mariadb.keywords.ts b/src/languages/mariadb/mariadb.keywords.ts index 5fc25d4e72..e067d5da33 100644 --- a/src/languages/mariadb/mariadb.keywords.ts +++ b/src/languages/mariadb/mariadb.keywords.ts @@ -689,6 +689,4 @@ export const keywords = flatKeywordList({ 'YEAR_MONTH', 'ZEROFILL', ], - constraints: ['ON DELETE', 'ON UPDATE'], - charset: ['CHARACTER SET'], }); diff --git a/src/languages/mysql/mysql.formatter.ts b/src/languages/mysql/mysql.formatter.ts index 6a93288335..32a42dbf2b 100644 --- a/src/languages/mysql/mysql.formatter.ts +++ b/src/languages/mysql/mysql.formatter.ts @@ -210,6 +210,8 @@ const reservedJoins = expandPhrases([ 'STRAIGHT_JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CHARACTER SET']; + // https://dev.mysql.com/doc/refman/8.0/en/ export default class MySqlFormatter extends Formatter { static operators = ['~', ':=', '<<', '>>', '<=>', '&&', '||', '->', '->>']; @@ -220,6 +222,7 @@ export default class MySqlFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'], + reservedPhrases, reservedLogicalOperators: ['AND', 'OR', 'XOR'], reservedKeywords: keywords, reservedFunctionNames: functions, diff --git a/src/languages/mysql/mysql.keywords.ts b/src/languages/mysql/mysql.keywords.ts index 5f9cb2864c..c27177e666 100644 --- a/src/languages/mysql/mysql.keywords.ts +++ b/src/languages/mysql/mysql.keywords.ts @@ -750,6 +750,4 @@ export const keywords = flatKeywordList({ 'ZEROFILL', // (R) 'ZONE', ], - constraints: ['ON DELETE', 'ON UPDATE'], - charset: ['CHARACTER SET'], }); diff --git a/src/languages/plsql/plsql.formatter.ts b/src/languages/plsql/plsql.formatter.ts index 7343220d1c..21f91fea28 100644 --- a/src/languages/plsql/plsql.formatter.ts +++ b/src/languages/plsql/plsql.formatter.ts @@ -54,6 +54,8 @@ const reservedJoins = expandPhrases([ '{CROSS | OUTER} APPLY', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'ON COMMIT']; + export default class PlSqlFormatter extends Formatter { static operators = [ '||', @@ -73,6 +75,7 @@ export default class PlSqlFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedLogicalOperators: ['AND', 'OR', 'XOR'], reservedKeywords: keywords, reservedFunctionNames: functions, diff --git a/src/languages/plsql/plsql.keywords.ts b/src/languages/plsql/plsql.keywords.ts index 84139fdf09..6d889b3f3e 100644 --- a/src/languages/plsql/plsql.keywords.ts +++ b/src/languages/plsql/plsql.keywords.ts @@ -328,5 +328,4 @@ export const keywords = flatKeywordList({ 'YEAR', 'ZONE', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/languages/postgresql/postgresql.formatter.ts b/src/languages/postgresql/postgresql.formatter.ts index 82aa9a2c56..27dc51a44e 100644 --- a/src/languages/postgresql/postgresql.formatter.ts +++ b/src/languages/postgresql/postgresql.formatter.ts @@ -223,6 +223,8 @@ const reservedJoins = expandPhrases([ 'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE']; + const binaryOperators = [ // Math Operators '<<', @@ -306,6 +308,7 @@ export default class PostgreSqlFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, openParens: ['(', '['], diff --git a/src/languages/postgresql/postgresql.keywords.ts b/src/languages/postgresql/postgresql.keywords.ts index 54e804e9aa..7962a4e5c1 100644 --- a/src/languages/postgresql/postgresql.keywords.ts +++ b/src/languages/postgresql/postgresql.keywords.ts @@ -461,5 +461,4 @@ export const keywords = flatKeywordList({ 'YES', 'ZONE', ], - constraints: ['ON DELETE', 'ON UPDATE', 'ON COMMIT'], }); diff --git a/src/languages/redshift/redshift.formatter.ts b/src/languages/redshift/redshift.formatter.ts index 34d56fc662..cf5b879ab4 100644 --- a/src/languages/redshift/redshift.formatter.ts +++ b/src/languages/redshift/redshift.formatter.ts @@ -117,6 +117,14 @@ const reservedJoins = expandPhrases([ 'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN', ]); +const reservedPhrases = [ + // https://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-data-conversion.html + 'NULL AS', + // https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html + 'DATA CATALOG', + 'HIVE METASTORE', +]; + // https://docs.aws.amazon.com/redshift/latest/dg/cm_chap_SQLCommandRef.html export default class RedshiftFormatter extends Formatter { static operators = ['~', '|/', '||/', '<<', '>>', '||']; @@ -127,6 +135,7 @@ export default class RedshiftFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, stringTypes: ["''"], diff --git a/src/languages/redshift/redshift.keywords.ts b/src/languages/redshift/redshift.keywords.ts index 6ffd8b5a73..6a7f3d69e2 100644 --- a/src/languages/redshift/redshift.keywords.ts +++ b/src/languages/redshift/redshift.keywords.ts @@ -122,7 +122,6 @@ export const keywords = flatKeywordList({ 'FILLRECORD', 'IGNOREBLANKLINES', 'IGNOREHEADER', - 'NULL AS', 'REMOVEQUOTES', 'ROUNDEC', 'TIMEFORMAT', @@ -177,7 +176,6 @@ export const keywords = flatKeywordList({ 'CATALOG_ROLE', 'SECRET_ARN', 'EXTERNAL', - 'HIVE METASTORE', // https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-external-schemas.html // https://docs.aws.amazon.com/redshift/latest/dg/c_choosing_dist_sort.html 'AUTO', 'EVEN', @@ -185,7 +183,6 @@ export const keywords = flatKeywordList({ 'PREDICATE', // ANALYZE | ANALYSE (https://docs.aws.amazon.com/redshift/latest/dg/r_ANALYZE.html) // unknown 'COMPRESSION', - 'DATA CATALOG', ], /** * Other keywords not included: @@ -194,14 +191,6 @@ export const keywords = flatKeywordList({ * SVL: https://docs.aws.amazon.com/redshift/latest/dg/svl_views.html * SVV: https://docs.aws.amazon.com/redshift/latest/dg/svv_views.html */ - dataTypes: [ - 'CHAR', - 'CHARACTER', - 'NCHAR', - 'VARCHAR', - 'CHARACTER VARYING', - 'NVARCHAR', - 'BPCHAR', - 'TEXT', - ], + // https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html#r_Character_types-text-and-bpchar-types + dataTypes: ['BPCHAR', 'TEXT'], }); diff --git a/src/languages/spark/spark.formatter.ts b/src/languages/spark/spark.formatter.ts index 5281b53cc2..8301d29f81 100644 --- a/src/languages/spark/spark.formatter.ts +++ b/src/languages/spark/spark.formatter.ts @@ -98,6 +98,8 @@ const reservedJoins = expandPhrases([ 'NATURAL [LEFT] {ANTI | SEMI} JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CURRENT ROW']; + // http://spark.apache.org/docs/latest/sql-programming-guide.html export default class SparkFormatter extends Formatter { static operators = ['~', '<=>', '&&', '||', '==', '->']; @@ -108,6 +110,7 @@ export default class SparkFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedLogicalOperators: ['AND', 'OR', 'XOR'], reservedKeywords: keywords, reservedFunctionNames: functions, diff --git a/src/languages/spark/spark.keywords.ts b/src/languages/spark/spark.keywords.ts index 2da63c9217..dfdad5d14e 100644 --- a/src/languages/spark/spark.keywords.ts +++ b/src/languages/spark/spark.keywords.ts @@ -240,7 +240,6 @@ export const keywords = flatKeywordList({ 'COALESCE', 'CONTAINS', 'CONVERT', - 'CURRENT ROW', 'DAYS', 'DAY_HOUR', 'DAY_MINUTE', @@ -275,5 +274,4 @@ export const keywords = flatKeywordList({ 'VARIABLES', 'YEAR_MONTH', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/languages/sql/sql.formatter.ts b/src/languages/sql/sql.formatter.ts index 8042611722..3a680fcc78 100644 --- a/src/languages/sql/sql.formatter.ts +++ b/src/languages/sql/sql.formatter.ts @@ -46,6 +46,8 @@ const reservedJoins = expandPhrases([ 'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE']; + export default class SqlFormatter extends Formatter { static operators = []; @@ -55,6 +57,7 @@ export default class SqlFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, stringTypes: [{ quote: "''", prefixes: ['N', 'X', 'U&'] }], diff --git a/src/languages/sql/sql.keywords.ts b/src/languages/sql/sql.keywords.ts index f28974ac40..a94a6919e7 100644 --- a/src/languages/sql/sql.keywords.ts +++ b/src/languages/sql/sql.keywords.ts @@ -225,5 +225,4 @@ export const keywords = flatKeywordList({ 'WITHOUT', 'YEAR', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/languages/sqlite/sqlite.formatter.ts b/src/languages/sqlite/sqlite.formatter.ts index 63ecb1780a..aa72b7516d 100644 --- a/src/languages/sqlite/sqlite.formatter.ts +++ b/src/languages/sqlite/sqlite.formatter.ts @@ -43,6 +43,8 @@ const reservedJoins = expandPhrases([ 'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE']; + export default class SqliteFormatter extends Formatter { // https://www.sqlite.org/lang_expr.html static operators = ['~', '->', '->>', '||', '<<', '>>', '==']; @@ -53,6 +55,7 @@ export default class SqliteFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, stringTypes: [ diff --git a/src/languages/sqlite/sqlite.keywords.ts b/src/languages/sqlite/sqlite.keywords.ts index cd4c5d3762..67c96fb9b4 100644 --- a/src/languages/sqlite/sqlite.keywords.ts +++ b/src/languages/sqlite/sqlite.keywords.ts @@ -156,5 +156,4 @@ export const keywords = flatKeywordList({ 'WITH', 'WITHOUT', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/languages/tsql/tsql.formatter.ts b/src/languages/tsql/tsql.formatter.ts index 0ff0a54953..bdab395310 100644 --- a/src/languages/tsql/tsql.formatter.ts +++ b/src/languages/tsql/tsql.formatter.ts @@ -200,6 +200,8 @@ const reservedJoins = expandPhrases([ '{CROSS | OUTER} APPLY', ]); +const reservedPhrases = ['ON DELETE', 'ON UPDATE']; + // https://docs.microsoft.com/en-us/sql/t-sql/language-reference?view=sql-server-ver15 export default class TSqlFormatter extends Formatter { static operators = ['~', '!<', '!>', '+=', '-=', '*=', '/=', '%=', '|=', '&=', '^=', '::']; @@ -210,6 +212,7 @@ export default class TSqlFormatter extends Formatter { reservedSetOperations, reservedJoins, reservedDependentClauses: ['WHEN', 'ELSE'], + reservedPhrases, reservedKeywords: keywords, reservedFunctionNames: functions, stringTypes: [{ quote: "''", prefixes: ['N'] }], diff --git a/src/languages/tsql/tsql.keywords.ts b/src/languages/tsql/tsql.keywords.ts index f5838b0cf8..1b057a416e 100644 --- a/src/languages/tsql/tsql.keywords.ts +++ b/src/languages/tsql/tsql.keywords.ts @@ -413,5 +413,4 @@ export const keywords = flatKeywordList({ 'YEAR', 'ZONE', ], - constraints: ['ON DELETE', 'ON UPDATE'], }); diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts index 4f98af5bb1..70ccc7c7f2 100644 --- a/src/lexer/Tokenizer.ts +++ b/src/lexer/Tokenizer.ts @@ -18,6 +18,9 @@ interface TokenizerOptions { reservedSetOperations: string[]; // Various joins like LEFT OUTER JOIN reservedJoins: string[]; + // These are essentially multi-word sequences of keywords, + // that we prioritize over normal keywords + reservedPhrases?: string[]; // built in function names reservedFunctionNames: string[]; // all other reserved words (not included to any of the above lists) @@ -99,6 +102,10 @@ export default class Tokenizer { regex: regex.reservedWord(cfg.reservedJoins, cfg.identChars), value: v => equalizeWhitespace(v.toUpperCase()), }, + [TokenType.RESERVED_PHRASE]: { + regex: regex.reservedWord(cfg.reservedPhrases ?? [], cfg.identChars), + value: v => equalizeWhitespace(v.toUpperCase()), + }, [TokenType.RESERVED_LOGICAL_OPERATOR]: { regex: regex.reservedWord(cfg.reservedLogicalOperators ?? ['AND', 'OR'], cfg.identChars), value: v => equalizeWhitespace(v.toUpperCase()), diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts index 253ed2095e..19a9a108bc 100644 --- a/src/lexer/TokenizerEngine.ts +++ b/src/lexer/TokenizerEngine.ts @@ -110,6 +110,7 @@ export default class TokenizerEngine { this.matchToken(TokenType.RESERVED_SET_OPERATION) || this.matchToken(TokenType.RESERVED_DEPENDENT_CLAUSE) || this.matchToken(TokenType.RESERVED_JOIN) || + this.matchToken(TokenType.RESERVED_PHRASE) || this.matchToken(TokenType.RESERVED_LOGICAL_OPERATOR) || this.matchToken(TokenType.RESERVED_FUNCTION_NAME) || this.matchToken(TokenType.RESERVED_KEYWORD) diff --git a/src/lexer/token.ts b/src/lexer/token.ts index 39b362444b..b697419ab0 100644 --- a/src/lexer/token.ts +++ b/src/lexer/token.ts @@ -7,6 +7,7 @@ export enum TokenType { RESERVED_KEYWORD = 'RESERVED_KEYWORD', RESERVED_FUNCTION_NAME = 'RESERVED_FUNCTION_NAME', RESERVED_LOGICAL_OPERATOR = 'RESERVED_LOGICAL_OPERATOR', + RESERVED_PHRASE = 'RESERVED_PHRASE', RESERVED_DEPENDENT_CLAUSE = 'RESERVED_DEPENDENT_CLAUSE', RESERVED_SET_OPERATION = 'RESERVED_SET_OPERATION', RESERVED_COMMAND = 'RESERVED_COMMAND', @@ -73,6 +74,7 @@ export const isReserved = (token: Token): boolean => token.type === TokenType.RESERVED_KEYWORD || token.type === TokenType.RESERVED_FUNCTION_NAME || token.type === TokenType.RESERVED_LOGICAL_OPERATOR || + token.type === TokenType.RESERVED_PHRASE || token.type === TokenType.RESERVED_DEPENDENT_CLAUSE || token.type === TokenType.RESERVED_COMMAND || token.type === TokenType.RESERVED_SET_OPERATION ||