Separate multi-word keywords from the rest

src/formatter/ExpressionFormatter.ts

@@ -199,6 +199,7 @@ export default class ExpressionFormatter {
         return this.formatLogicalOperator(token);
       case TokenType.RESERVED_KEYWORD:
       case TokenType.RESERVED_FUNCTION_NAME:
+      case TokenType.RESERVED_PHRASE:
         return this.formatKeyword(token);
       case TokenType.RESERVED_CASE_START:
         return this.formatCaseStart(token);

src/languages/bigquery/bigquery.formatter.ts

@@ -116,6 +116,15 @@ const reservedJoins = expandPhrases([
   '{INNER | CROSS} JOIN',
 ]);
 
+const reservedPhrases = [
+  // https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#tablesample_operator
+  'TABLESAMPLE SYSTEM',
+  // From DDL: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language
+  'ANY TYPE',
+  'ALL COLUMNS',
+  'NOT DETERMINISTIC',
+];
+
 // https://cloud.google.com/bigquery/docs/reference/#standard-sql-reference
 export default class BigQueryFormatter extends Formatter {
   static operators = ['~', '>>', '<<', '||'];
@@ -127,6 +136,7 @@ export default class BigQueryFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       openParens: ['(', '['],

src/languages/bigquery/bigquery.keywords.ts

@@ -85,7 +85,7 @@ export const keywords = flatKeywordList({
     'SOME',
     'STRUCT',
     'TABLE',
-    'TABLESAMPLE SYSTEM',
+    'TABLESAMPLE',
     'THEN',
     'TO',
     'TREAT',
@@ -138,11 +138,8 @@ export const keywords = flatKeywordList({
     'INOUT',
     'RETURNS',
     'LANGUAGE',
-    'ANY TYPE',
-    'ALL COLUMNS',
     'CASCADE',
     'RESTRICT',
     'DETERMINISTIC',
-    'NOT DETERMINISTIC',
   ],
 });

src/languages/db2/db2.formatter.ts

@@ -147,6 +147,8 @@ const reservedJoins = expandPhrases([
   '{INNER | CROSS} JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE'];
+
 // https://www.ibm.com/support/knowledgecenter/en/ssw_ibm_i_72/db2/rbafzintro.htm
 export default class Db2Formatter extends Formatter {
   static operators = ['**', '¬=', '¬>', '¬<', '!>', '!<', '||'];
@@ -157,6 +159,7 @@ export default class Db2Formatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: [{ quote: "''", prefixes: ['G', 'N', 'X', 'BX', 'GX', 'UX', 'U&'] }],

src/languages/db2/db2.keywords.ts

@@ -236,5 +236,4 @@ export const keywords = flatKeywordList({
     'YEARS',
     'ZONE',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/languages/mariadb/mariadb.formatter.ts

@@ -243,6 +243,8 @@ const reservedJoins = expandPhrases([
   'STRAIGHT_JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CHARACTER SET'];
+
 // For reference: https://mariadb.com/kb/en/sql-statements-structure/
 export default class MariaDbFormatter extends Formatter {
   static operators = [':=', '<<', '>>', '<=>', '&&', '||'];
@@ -253,6 +255,7 @@ export default class MariaDbFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF', 'ELSIF'],
+      reservedPhrases,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,

src/languages/mariadb/mariadb.keywords.ts

@@ -689,6 +689,4 @@ export const keywords = flatKeywordList({
     'YEAR_MONTH',
     'ZEROFILL',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
-  charset: ['CHARACTER SET'],
 });

src/languages/mysql/mysql.formatter.ts

@@ -210,6 +210,8 @@ const reservedJoins = expandPhrases([
   'STRAIGHT_JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CHARACTER SET'];
+
 // https://dev.mysql.com/doc/refman/8.0/en/
 export default class MySqlFormatter extends Formatter {
   static operators = ['~', ':=', '<<', '>>', '<=>', '&&', '||', '->', '->>'];
@@ -220,6 +222,7 @@ export default class MySqlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'],
+      reservedPhrases,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,

src/languages/mysql/mysql.keywords.ts

@@ -750,6 +750,4 @@ export const keywords = flatKeywordList({
     'ZEROFILL', // (R)
     'ZONE',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
-  charset: ['CHARACTER SET'],
 });

src/languages/plsql/plsql.formatter.ts

@@ -54,6 +54,8 @@ const reservedJoins = expandPhrases([
   '{CROSS | OUTER} APPLY',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'ON COMMIT'];
+
 export default class PlSqlFormatter extends Formatter {
   static operators = [
     '||',
@@ -73,6 +75,7 @@ export default class PlSqlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,

src/languages/plsql/plsql.keywords.ts

@@ -328,5 +328,4 @@ export const keywords = flatKeywordList({
     'YEAR',
     'ZONE',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/languages/postgresql/postgresql.formatter.ts

@@ -223,6 +223,8 @@ const reservedJoins = expandPhrases([
   'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE'];
+
 const binaryOperators = [
   // Math Operators
   '<<',
@@ -306,6 +308,7 @@ export default class PostgreSqlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       openParens: ['(', '['],

src/languages/postgresql/postgresql.keywords.ts

@@ -461,5 +461,4 @@ export const keywords = flatKeywordList({
     'YES',
     'ZONE',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE', 'ON COMMIT'],
 });

src/languages/redshift/redshift.formatter.ts

@@ -117,6 +117,14 @@ const reservedJoins = expandPhrases([
   'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN',
 ]);
 
+const reservedPhrases = [
+  // https://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-data-conversion.html
+  'NULL AS',
+  // https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_SCHEMA.html
+  'DATA CATALOG',
+  'HIVE METASTORE',
+];
+
 // https://docs.aws.amazon.com/redshift/latest/dg/cm_chap_SQLCommandRef.html
 export default class RedshiftFormatter extends Formatter {
   static operators = ['~', '|/', '||/', '<<', '>>', '||'];
@@ -127,6 +135,7 @@ export default class RedshiftFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: ["''"],

src/languages/redshift/redshift.keywords.ts

@@ -122,7 +122,6 @@ export const keywords = flatKeywordList({
     'FILLRECORD',
     'IGNOREBLANKLINES',
     'IGNOREHEADER',
-    'NULL AS',
     'REMOVEQUOTES',
     'ROUNDEC',
     'TIMEFORMAT',
@@ -177,15 +176,13 @@ export const keywords = flatKeywordList({
     'CATALOG_ROLE',
     'SECRET_ARN',
     'EXTERNAL',
-    'HIVE METASTORE', // https://docs.aws.amazon.com/redshift/latest/dg/c-spectrum-external-schemas.html
     // https://docs.aws.amazon.com/redshift/latest/dg/c_choosing_dist_sort.html
     'AUTO',
     'EVEN',
     'KEY',
     'PREDICATE', // ANALYZE | ANALYSE (https://docs.aws.amazon.com/redshift/latest/dg/r_ANALYZE.html)
     // unknown
     'COMPRESSION',
-    'DATA CATALOG',
   ],
   /**
    * Other keywords not included:
@@ -194,14 +191,6 @@ export const keywords = flatKeywordList({
    * SVL: https://docs.aws.amazon.com/redshift/latest/dg/svl_views.html
    * SVV: https://docs.aws.amazon.com/redshift/latest/dg/svv_views.html
    */
-  dataTypes: [
-    'CHAR',
-    'CHARACTER',
-    'NCHAR',
-    'VARCHAR',
-    'CHARACTER VARYING',
-    'NVARCHAR',
-    'BPCHAR',
-    'TEXT',
-  ],
+  // https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html#r_Character_types-text-and-bpchar-types
+  dataTypes: ['BPCHAR', 'TEXT'],
 });

src/languages/spark/spark.formatter.ts

@@ -98,6 +98,8 @@ const reservedJoins = expandPhrases([
   'NATURAL [LEFT] {ANTI | SEMI} JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE', 'CURRENT ROW'];
+
 // http://spark.apache.org/docs/latest/sql-programming-guide.html
 export default class SparkFormatter extends Formatter {
   static operators = ['~', '<=>', '&&', '||', '==', '->'];
@@ -108,6 +110,7 @@ export default class SparkFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,

src/languages/spark/spark.keywords.ts

@@ -240,7 +240,6 @@ export const keywords = flatKeywordList({
     'COALESCE',
     'CONTAINS',
     'CONVERT',
-    'CURRENT ROW',
     'DAYS',
     'DAY_HOUR',
     'DAY_MINUTE',
@@ -275,5 +274,4 @@ export const keywords = flatKeywordList({
     'VARIABLES',
     'YEAR_MONTH',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/languages/sql/sql.formatter.ts

@@ -46,6 +46,8 @@ const reservedJoins = expandPhrases([
   'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE'];
+
 export default class SqlFormatter extends Formatter {
   static operators = [];
 
@@ -55,6 +57,7 @@ export default class SqlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: [{ quote: "''", prefixes: ['N', 'X', 'U&'] }],

src/languages/sql/sql.keywords.ts

@@ -225,5 +225,4 @@ export const keywords = flatKeywordList({
     'WITHOUT',
     'YEAR',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/languages/sqlite/sqlite.formatter.ts

@@ -43,6 +43,8 @@ const reservedJoins = expandPhrases([
   'NATURAL {LEFT | RIGHT | FULL} [OUTER] JOIN',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE'];
+
 export default class SqliteFormatter extends Formatter {
   // https://www.sqlite.org/lang_expr.html
   static operators = ['~', '->', '->>', '||', '<<', '>>', '=='];
@@ -53,6 +55,7 @@ export default class SqliteFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: [

src/languages/sqlite/sqlite.keywords.ts

@@ -156,5 +156,4 @@ export const keywords = flatKeywordList({
     'WITH',
     'WITHOUT',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/languages/tsql/tsql.formatter.ts

@@ -200,6 +200,8 @@ const reservedJoins = expandPhrases([
   '{CROSS | OUTER} APPLY',
 ]);
 
+const reservedPhrases = ['ON DELETE', 'ON UPDATE'];
+
 // https://docs.microsoft.com/en-us/sql/t-sql/language-reference?view=sql-server-ver15
 export default class TSqlFormatter extends Formatter {
   static operators = ['~', '!<', '!>', '+=', '-=', '*=', '/=', '%=', '|=', '&=', '^=', '::'];
@@ -210,6 +212,7 @@ export default class TSqlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: [{ quote: "''", prefixes: ['N'] }],

src/languages/tsql/tsql.keywords.ts

@@ -413,5 +413,4 @@ export const keywords = flatKeywordList({
     'YEAR',
     'ZONE',
   ],
-  constraints: ['ON DELETE', 'ON UPDATE'],
 });

src/lexer/Tokenizer.ts

@@ -18,6 +18,9 @@ interface TokenizerOptions {
   reservedSetOperations: string[];
   // Various joins like LEFT OUTER JOIN
   reservedJoins: string[];
+  // These are essentially multi-word sequences of keywords,
+  // that we prioritize over normal keywords
+  reservedPhrases?: string[];
   // built in function names
   reservedFunctionNames: string[];
   // all other reserved words (not included to any of the above lists)
@@ -99,6 +102,10 @@ export default class Tokenizer {
         regex: regex.reservedWord(cfg.reservedJoins, cfg.identChars),
         value: v => equalizeWhitespace(v.toUpperCase()),
       },
+      [TokenType.RESERVED_PHRASE]: {
+        regex: regex.reservedWord(cfg.reservedPhrases ?? [], cfg.identChars),
+        value: v => equalizeWhitespace(v.toUpperCase()),
+      },
       [TokenType.RESERVED_LOGICAL_OPERATOR]: {
         regex: regex.reservedWord(cfg.reservedLogicalOperators ?? ['AND', 'OR'], cfg.identChars),
         value: v => equalizeWhitespace(v.toUpperCase()),

src/lexer/TokenizerEngine.ts

@@ -110,6 +110,7 @@ export default class TokenizerEngine {
       this.matchToken(TokenType.RESERVED_SET_OPERATION) ||
       this.matchToken(TokenType.RESERVED_DEPENDENT_CLAUSE) ||
       this.matchToken(TokenType.RESERVED_JOIN) ||
+      this.matchToken(TokenType.RESERVED_PHRASE) ||
       this.matchToken(TokenType.RESERVED_LOGICAL_OPERATOR) ||
       this.matchToken(TokenType.RESERVED_FUNCTION_NAME) ||
       this.matchToken(TokenType.RESERVED_KEYWORD)

src/lexer/token.ts

@@ -7,6 +7,7 @@ export enum TokenType {
   RESERVED_KEYWORD = 'RESERVED_KEYWORD',
   RESERVED_FUNCTION_NAME = 'RESERVED_FUNCTION_NAME',
   RESERVED_LOGICAL_OPERATOR = 'RESERVED_LOGICAL_OPERATOR',
+  RESERVED_PHRASE = 'RESERVED_PHRASE',
   RESERVED_DEPENDENT_CLAUSE = 'RESERVED_DEPENDENT_CLAUSE',
   RESERVED_SET_OPERATION = 'RESERVED_SET_OPERATION',
   RESERVED_COMMAND = 'RESERVED_COMMAND',
@@ -73,6 +74,7 @@ export const isReserved = (token: Token): boolean =>
   token.type === TokenType.RESERVED_KEYWORD ||
   token.type === TokenType.RESERVED_FUNCTION_NAME ||
   token.type === TokenType.RESERVED_LOGICAL_OPERATOR ||
+  token.type === TokenType.RESERVED_PHRASE ||
   token.type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
   token.type === TokenType.RESERVED_COMMAND ||
   token.type === TokenType.RESERVED_SET_OPERATION ||