From 99e29f3f9f17eb004f13931d8858c2e790c5095a Mon Sep 17 00:00:00 2001 From: Nicky Abela Date: Mon, 11 Oct 2021 16:04:35 +0200 Subject: [PATCH 1/4] Only include field as key if it is a MySQL primary key --- clickhouse_mysql/tablesqlbuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_mysql/tablesqlbuilder.py b/clickhouse_mysql/tablesqlbuilder.py index 77d10f2..d766f5c 100644 --- a/clickhouse_mysql/tablesqlbuilder.py +++ b/clickhouse_mysql/tablesqlbuilder.py @@ -204,7 +204,7 @@ def create_table_columns_description(self, db=None, table=None, ): 'clickhouse_type': self.map_type(mysql_type=_type), 'clickhouse_type_nullable': self.map_type_nullable(mysql_type=_type, nullable=self.is_field_nullable(_null)), 'nullable': self.is_field_nullable(_null), - 'key': _key, + 'key': _key if _key == "PRI" else None, 'default': _default, 'extra': _extra, }) From 85409ca5eaf1212835c80c3bf44853d1618538ed Mon Sep 17 00:00:00 2001 From: Nicky Abela Date: Mon, 11 Oct 2021 17:08:28 +0200 Subject: [PATCH 2/4] Created dst-use-src-primary-key flag --- clickhouse_mysql/clioptions.py | 11 +++++++++++ clickhouse_mysql/config.py | 2 ++ clickhouse_mysql/tableprocessor.py | 2 ++ clickhouse_mysql/tablesqlbuilder.py | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/clickhouse_mysql/clioptions.py b/clickhouse_mysql/clioptions.py index 4be23a2..04b05ea 100644 --- a/clickhouse_mysql/clioptions.py +++ b/clickhouse_mysql/clioptions.py @@ -148,6 +148,7 @@ class CLIOptions(Options): 'dst_table': None, 'dst_table_prefix': None, 'dst_create_table': False, + 'dst_use_src_primary_key': False, # # converters section @@ -451,6 +452,15 @@ def options(self): default=self.default_options['dst_cluster'], help='Cluster to be used when writing to dst. Ex.: cluster1' ) + argparser.add_argument( + '--dst-use-src-primary-key', + action='store_true', + default=self.default_options['dst_src_primary_key'], + help='Whether the destination primary key should match the source primary key. ' + 'This flag will exclude columns that are used by MySQL indexes from being ' + 'used in the ClickHouse primary key. Only columns that are part of the primary ' + 'key in the MySQL table will be included in the ClickHouse table primary key.' + ) argparser.add_argument( '--dst-table', type=str, @@ -560,6 +570,7 @@ def options(self): 'dst_schema': args.dst_schema, 'dst_distribute': args.dst_distribute, 'dst_cluster': args.dst_cluster, + 'dst_use_src_primary_key': args.dst_use_src_primary_key, 'dst_table': args.dst_table, 'dst_table_prefix': args.dst_table_prefix, 'dst_create_table': args.dst_create_table, diff --git a/clickhouse_mysql/config.py b/clickhouse_mysql/config.py index e4551c8..947915c 100644 --- a/clickhouse_mysql/config.py +++ b/clickhouse_mysql/config.py @@ -122,6 +122,7 @@ def __init__(self): 'dst_schema': self.options['dst_schema'], 'dst_distribute': self.options['dst_distribute'], 'dst_cluster': self.options['dst_cluster'], + 'dst_use_src_primary_key': self.options.get_bool('dst_use_src_primary_key'), 'dst_table': self.options['dst_table'], 'dst_table_prefix': self.options['dst_table_prefix'], 'dst_create_table': self.options.get_bool('dst_create_table'), @@ -266,6 +267,7 @@ def table_sql_builder(self): dst_table_prefix=self.config['table_builder']['clickhouse']['dst_table_prefix'], distribute=self.config['table_builder']['clickhouse']['dst_distribute'], cluster=self.config['table_builder']['clickhouse']['dst_cluster'], + use_src_primary_key=self.config['table_builder']['clickhouse']['dst_use_src_primary_key'], tables=self.config['table_builder']['mysql']['tables'], tables_prefixes=self.config['table_builder']['mysql']['tables_prefixes'], column_skip=self.config['converter']['clickhouse']['column_skip'], diff --git a/clickhouse_mysql/tableprocessor.py b/clickhouse_mysql/tableprocessor.py index 7bb96b8..d6b644b 100644 --- a/clickhouse_mysql/tableprocessor.py +++ b/clickhouse_mysql/tableprocessor.py @@ -33,6 +33,7 @@ def __init__( tables=None, tables_prefixes=None, column_skip=[], + use_src_primary_key=False, ): """ :param host: string MySQL host @@ -57,6 +58,7 @@ def __init__( self.dst_table_prefix = dst_table_prefix self.cluster = cluster self.distribute = distribute + self.use_src_primary_key = use_src_primary_key self.column_skip = column_skip def dbs_tables_lists(self): diff --git a/clickhouse_mysql/tablesqlbuilder.py b/clickhouse_mysql/tablesqlbuilder.py index d766f5c..bfc6ab3 100644 --- a/clickhouse_mysql/tablesqlbuilder.py +++ b/clickhouse_mysql/tablesqlbuilder.py @@ -204,7 +204,7 @@ def create_table_columns_description(self, db=None, table=None, ): 'clickhouse_type': self.map_type(mysql_type=_type), 'clickhouse_type_nullable': self.map_type_nullable(mysql_type=_type, nullable=self.is_field_nullable(_null)), 'nullable': self.is_field_nullable(_null), - 'key': _key if _key == "PRI" else None, + 'key': (_key if _key == "PRI" else None) if self.use_src_primary_key else _key, 'default': _default, 'extra': _extra, }) From 2be0e5982f3707ab0e8a152ea37219b18d1e9d11 Mon Sep 17 00:00:00 2001 From: Nicky Abela Date: Mon, 11 Oct 2021 17:14:10 +0200 Subject: [PATCH 3/4] Fixed default value for dst_use_src_primary_key --- clickhouse_mysql/clioptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_mysql/clioptions.py b/clickhouse_mysql/clioptions.py index 04b05ea..f87a6e7 100644 --- a/clickhouse_mysql/clioptions.py +++ b/clickhouse_mysql/clioptions.py @@ -455,7 +455,7 @@ def options(self): argparser.add_argument( '--dst-use-src-primary-key', action='store_true', - default=self.default_options['dst_src_primary_key'], + default=self.default_options['dst_use_src_primary_key'], help='Whether the destination primary key should match the source primary key. ' 'This flag will exclude columns that are used by MySQL indexes from being ' 'used in the ClickHouse primary key. Only columns that are part of the primary ' From 39b6619ac57d2d921b2d0091bdbdd7b73cfff5fd Mon Sep 17 00:00:00 2001 From: Nicky Abela Date: Mon, 11 Oct 2021 17:49:44 +0200 Subject: [PATCH 4/4] Added dst_use_src_primary_key in TableMigrator instantiation --- clickhouse_mysql/config.py | 1 + clickhouse_mysql/tablemigrator.py | 2 ++ clickhouse_mysql/tableprocessor.py | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clickhouse_mysql/config.py b/clickhouse_mysql/config.py index 947915c..39f94f7 100644 --- a/clickhouse_mysql/config.py +++ b/clickhouse_mysql/config.py @@ -292,6 +292,7 @@ def table_migrator(self): dst_schema=self.config['table_migrator']['clickhouse']['dst_schema'], dst_table=self.config['table_builder']['clickhouse']['dst_table'], dst_table_prefix=self.config['table_builder']['clickhouse']['dst_table_prefix'], + use_src_primary_key=self.config['table_builder']['clickhouse']['dst_use_src_primary_key'], distribute=self.config['table_migrator']['clickhouse']['dst_distribute'], cluster=self.config['table_migrator']['clickhouse']['dst_cluster'], tables=self.config['table_migrator']['mysql']['tables'], diff --git a/clickhouse_mysql/tablemigrator.py b/clickhouse_mysql/tablemigrator.py index 07e0986..6ff7a38 100644 --- a/clickhouse_mysql/tablemigrator.py +++ b/clickhouse_mysql/tablemigrator.py @@ -43,6 +43,7 @@ def __init__( dst_table_prefix=None, distribute=None, cluster=None, + use_src_primary_key=None, tables=None, tables_prefixes=None, tables_where_clauses=None, @@ -59,6 +60,7 @@ def __init__( dst_table_prefix=dst_table_prefix, distribute=distribute, cluster=cluster, + use_src_primary_key=use_src_primary_key, tables=tables, tables_prefixes=tables_prefixes, column_skip=column_skip diff --git a/clickhouse_mysql/tableprocessor.py b/clickhouse_mysql/tableprocessor.py index d6b644b..a4bef2f 100644 --- a/clickhouse_mysql/tableprocessor.py +++ b/clickhouse_mysql/tableprocessor.py @@ -33,7 +33,7 @@ def __init__( tables=None, tables_prefixes=None, column_skip=[], - use_src_primary_key=False, + use_src_primary_key=None, ): """ :param host: string MySQL host @@ -58,8 +58,8 @@ def __init__( self.dst_table_prefix = dst_table_prefix self.cluster = cluster self.distribute = distribute - self.use_src_primary_key = use_src_primary_key self.column_skip = column_skip + self.use_src_primary_key = use_src_primary_key def dbs_tables_lists(self): """