Separate data types from generic keywords (#677)

Part of preparations for PR #673 

---

Copied from [my
comment](#673 (comment)):

I would want to then do a separate PR to separate out all of the data
type information in `src/languages/*/*.keywords.ts` in these flat arrays
in into a separate (optional) export in each file called `export const
dataTypes`, which would temporarily merged together with `keywords` to
keep compatibility before `reservedDataTypes` in the next step is
implemented

Author: nene

src/languages/bigquery/bigquery.formatter.ts

@@ -1,8 +1,8 @@
 import { DialectOptions } from '../../dialect.js';
-import { EOF_TOKEN, isToken, TokenType, Token } from '../../lexer/token.js';
 import { expandPhrases } from '../../expandPhrases.js';
-import { keywords } from './bigquery.keywords.js';
+import { EOF_TOKEN, isToken, Token, TokenType } from '../../lexer/token.js';
 import { functions } from './bigquery.functions.js';
+import { dataTypes, keywords } from './bigquery.keywords.js';
 
 const reservedSelect = expandPhrases(['SELECT [ALL | DISTINCT] [AS STRUCT | AS VALUE]']);
 
@@ -163,7 +163,9 @@ export const bigquery: DialectOptions = {
     reservedSetOperations,
     reservedJoins,
     reservedPhrases,
-    reservedKeywords: keywords,
+    reservedKeywords:
+      // Temporary, will be replaced by reservedDataTypes
+      [...new Set(keywords.concat(dataTypes))],
     reservedFunctionNames: functions,
     extraParens: ['[]'],
     stringTypes: [

src/languages/bigquery/bigquery.keywords.ts

@@ -3,7 +3,6 @@ export const keywords: string[] = [
   'ALL',
   'AND',
   'ANY',
-  'ARRAY',
   'AS',
   'ASC',
   'ASSERT_ROWS_MODIFIED',
@@ -46,13 +45,11 @@ export const keywords: string[] = [
   'IN',
   'INNER',
   'INTERSECT',
-  'INTERVAL',
   'INTO',
   'IS',
   'JOIN',
   'LATERAL',
   'LEFT',
-  'LIKE',
   'LIMIT',
   'LOOKUP',
   'MERGE',
@@ -80,7 +77,6 @@ export const keywords: string[] = [
   'SELECT',
   'SET',
   'SOME',
-  'STRUCT',
   'TABLE',
   'TABLESAMPLE',
   'THEN',
@@ -97,7 +93,24 @@ export const keywords: string[] = [
   'WITH',
   'WITHIN',
 
-  // datatypes
+  // misc
+  'SAFE',
+
+  // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language
+  'LIKE', // CREATE TABLE LIKE
+  'COPY', // CREATE TABLE COPY
+  'CLONE', // CREATE TABLE CLONE
+  'IN',
+  'OUT',
+  'INOUT',
+  'RETURNS',
+  'LANGUAGE',
+  'CASCADE',
+  'RESTRICT',
+  'DETERMINISTIC',
+];
+
+export const dataTypes: string[] = [
   'ARRAY', // parametric, ARRAY<T>
   'BOOL',
   'BYTES', // parameterised, BYTES(Length)
@@ -129,20 +142,4 @@ export const keywords: string[] = [
   'ASCII',
   'UTF-8',
   'UTF8',
-
-  // misc
-  'SAFE',
-
-  // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language
-  'LIKE', // CREATE TABLE LIKE
-  'COPY', // CREATE TABLE COPY
-  'CLONE', // CREATE TABLE CLONE
-  'IN',
-  'OUT',
-  'INOUT',
-  'RETURNS',
-  'LANGUAGE',
-  'CASCADE',
-  'RESTRICT',
-  'DETERMINISTIC',
 ];

src/languages/db2/db2.formatter.ts

@@ -1,7 +1,7 @@
 import { DialectOptions } from '../../dialect.js';
 import { expandPhrases } from '../../expandPhrases.js';
 import { functions } from './db2.functions.js';
-import { keywords } from './db2.keywords.js';
+import { dataTypes, keywords } from './db2.keywords.js';
 
 const reservedSelect = expandPhrases(['SELECT [ALL | DISTINCT]']);
 
@@ -265,7 +265,9 @@ export const db2: DialectOptions = {
     reservedSetOperations,
     reservedJoins,
     reservedPhrases,
-    reservedKeywords: keywords,
+    reservedKeywords:
+      // Temporary, will be replaced by reservedDataTypes
+      [...new Set(keywords.concat(dataTypes))],
     reservedFunctionNames: functions,
     extraParens: ['[]'],
     stringTypes: [

src/languages/db2/db2.keywords.ts

@@ -34,9 +34,6 @@ export const keywords: string[] = [
   'CASCADED',
   'CASE',
   'CAST',
-  'CCSID',
-  'CHAR',
-  'CHARACTER',
   'CHECK',
   'CLONE',
   'CLOSE',
@@ -73,7 +70,6 @@ export const keywords: string[] = [
   'DATABASE',
   'DATAPARTITIONNAME',
   'DATAPARTITIONNUM',
-  'DATE',
   'DAY',
   'DAYS',
   'DB2GENERAL',
@@ -100,7 +96,6 @@ export const keywords: string[] = [
   'DISTINCT',
   'DO',
   'DOCUMENT',
-  'DOUBLE',
   'DROP',
   'DSSIZE',
   'DYNAMIC',
@@ -208,7 +203,6 @@ export const keywords: string[] = [
   'LOCK',
   'LOCKMAX',
   'LOCKSIZE',
-  'LONG',
   'LOOP',
   'MAINTAINED',
   'MATERIALIZED',
@@ -368,8 +362,6 @@ export const keywords: string[] = [
   'TABLE',
   'TABLESPACE',
   'THEN',
-  'TIME',
-  'TIMESTAMP',
   'TO',
   'TRANSACTION',
   'TRIGGER',
@@ -408,3 +400,15 @@ export const keywords: string[] = [
   'YEAR',
   'YEARS',
 ];
+
+export const dataTypes: string[] = [
+  // https://www.ibm.com/docs/en/db2-for-zos/12?topic=columns-data-types
+  'CCSID',
+  'CHAR',
+  'CHARACTER',
+  'DATE',
+  'DOUBLE',
+  'LONG',
+  'TIME',
+  'TIMESTAMP',
+];

src/languages/db2i/db2i.formatter.ts

@@ -1,7 +1,7 @@
 import { DialectOptions } from '../../dialect.js';
 import { expandPhrases } from '../../expandPhrases.js';
 import { functions } from './db2i.functions.js';
-import { keywords } from './db2i.keywords.js';
+import { dataTypes, keywords } from './db2i.keywords.js';
 
 const reservedSelect = expandPhrases(['SELECT [ALL | DISTINCT]']);
 
@@ -161,7 +161,9 @@ export const db2i: DialectOptions = {
     reservedSetOperations,
     reservedJoins,
     reservedPhrases,
-    reservedKeywords: keywords,
+    reservedKeywords:
+      // Temporary, will be replaced by reservedDataTypes
+      [...new Set(keywords.concat(dataTypes))],
     reservedFunctionNames: functions,
     nestedBlockComments: true,
     extraParens: ['[]'],

src/languages/db2i/db2i.keywords.ts

@@ -32,10 +32,7 @@ export const keywords: string[] = [
   'BEFORE',
   'BEGIN',
   'BETWEEN',
-  'BINARY',
   'BIND',
-  'BIT',
-  'BOOLEAN',
   'BSON',
   'BUFFERPOOL',
   'BY',
@@ -45,9 +42,6 @@ export const keywords: string[] = [
   'CARDINALITY',
   'CASE',
   'CAST',
-  'CCSID',
-  'CHAR',
-  'CHARACTER',
   'CHECK',
   'CL',
   'CLOSE',
@@ -90,11 +84,9 @@ export const keywords: string[] = [
   'CURRENT_USER',
   'CURSOR',
   'CYCLE',
-  'DATA',
   'DATABASE',
   'DATAPARTITIONNAME',
   'DATAPARTITIONNUM',
-  'DATE',
   'DAY',
   'DAYS',
   'DB2GENERAL',
@@ -127,7 +119,6 @@ export const keywords: string[] = [
   'DISTINCT',
   'DO',
   'DOCUMENT',
-  'DOUBLE',
   'DROP',
   'DYNAMIC',
   'EACH',
@@ -177,7 +168,6 @@ export const keywords: string[] = [
   'GO',
   'GOTO',
   'GRANT',
-  'GRAPHIC',
   'GROUP',
   'HANDLER',
   'HASH',
@@ -256,7 +246,6 @@ export const keywords: string[] = [
   'LOCKSIZE',
   'LOG',
   'LOGGED',
-  'LONG',
   'LOOP',
   'MAINTAINED',
   'MASK',
@@ -436,8 +425,6 @@ export const keywords: string[] = [
   'TAG',
   'THEN',
   'THREADSAFE',
-  'TIME',
-  'TIMESTAMP',
   'TO',
   'TRANSACTION',
   'TRANSFER',
@@ -508,3 +495,37 @@ export const keywords: string[] = [
   'YES',
   'ZONE',
 ];
+
+export const dataTypes: string[] = [
+  // https://www.ibm.com/docs/en/i/7.2?topic=iaodsd-odbc-data-types-how-they-correspond-db2-i-database-types
+  'BIGINT',
+  'BINARY',
+  'BIT',
+  'BLOB',
+  'BOOLEAN',
+  'CCSID',
+  'CHAR',
+  'CHARACTER',
+  'CLOB',
+  'DATA',
+  'DATALINK',
+  'DATE',
+  'DBCLOB',
+  'DECFLOAT',
+  'DECIMAL',
+  'DOUBLE',
+  'FLOAT',
+  'GRAPHIC',
+  'INTEGER',
+  'LONG',
+  'NUMERIC',
+  'REAL',
+  'ROWID',
+  'SMALLINT',
+  'TIME',
+  'TIMESTAMP',
+  'VARBINARY',
+  'VARCHAR',
+  'VARGRAPHIC',
+  'XML',
+];

src/languages/hive/hive.formatter.ts

@@ -1,7 +1,7 @@
 import { DialectOptions } from '../../dialect.js';
 import { expandPhrases } from '../../expandPhrases.js';
 import { functions } from './hive.functions.js';
-import { keywords } from './hive.keywords.js';
+import { dataTypes, keywords } from './hive.keywords.js';
 
 const reservedSelect = expandPhrases(['SELECT [ALL | DISTINCT]']);
 
@@ -90,7 +90,9 @@ export const hive: DialectOptions = {
     reservedSetOperations,
     reservedJoins,
     reservedPhrases,
-    reservedKeywords: keywords,
+    reservedKeywords:
+      // Temporary, will be replaced by reservedDataTypes
+      [...new Set(keywords.concat(dataTypes))],
     reservedFunctionNames: functions,
     extraParens: ['[]'],
     stringTypes: ['""-bs', "''-bs"],