diff --git a/clickhouse_mysql/clioptions.py b/clickhouse_mysql/clioptions.py index 4be23a2..f87a6e7 100644 --- a/clickhouse_mysql/clioptions.py +++ b/clickhouse_mysql/clioptions.py @@ -148,6 +148,7 @@ class CLIOptions(Options): 'dst_table': None, 'dst_table_prefix': None, 'dst_create_table': False, + 'dst_use_src_primary_key': False, # # converters section @@ -451,6 +452,15 @@ def options(self): default=self.default_options['dst_cluster'], help='Cluster to be used when writing to dst. Ex.: cluster1' ) + argparser.add_argument( + '--dst-use-src-primary-key', + action='store_true', + default=self.default_options['dst_use_src_primary_key'], + help='Whether the destination primary key should match the source primary key. ' + 'This flag will exclude columns that are used by MySQL indexes from being ' + 'used in the ClickHouse primary key. Only columns that are part of the primary ' + 'key in the MySQL table will be included in the ClickHouse table primary key.' + ) argparser.add_argument( '--dst-table', type=str, @@ -560,6 +570,7 @@ def options(self): 'dst_schema': args.dst_schema, 'dst_distribute': args.dst_distribute, 'dst_cluster': args.dst_cluster, + 'dst_use_src_primary_key': args.dst_use_src_primary_key, 'dst_table': args.dst_table, 'dst_table_prefix': args.dst_table_prefix, 'dst_create_table': args.dst_create_table, diff --git a/clickhouse_mysql/config.py b/clickhouse_mysql/config.py index e4551c8..39f94f7 100644 --- a/clickhouse_mysql/config.py +++ b/clickhouse_mysql/config.py @@ -122,6 +122,7 @@ def __init__(self): 'dst_schema': self.options['dst_schema'], 'dst_distribute': self.options['dst_distribute'], 'dst_cluster': self.options['dst_cluster'], + 'dst_use_src_primary_key': self.options.get_bool('dst_use_src_primary_key'), 'dst_table': self.options['dst_table'], 'dst_table_prefix': self.options['dst_table_prefix'], 'dst_create_table': self.options.get_bool('dst_create_table'), @@ -266,6 +267,7 @@ def table_sql_builder(self): dst_table_prefix=self.config['table_builder']['clickhouse']['dst_table_prefix'], distribute=self.config['table_builder']['clickhouse']['dst_distribute'], cluster=self.config['table_builder']['clickhouse']['dst_cluster'], + use_src_primary_key=self.config['table_builder']['clickhouse']['dst_use_src_primary_key'], tables=self.config['table_builder']['mysql']['tables'], tables_prefixes=self.config['table_builder']['mysql']['tables_prefixes'], column_skip=self.config['converter']['clickhouse']['column_skip'], @@ -290,6 +292,7 @@ def table_migrator(self): dst_schema=self.config['table_migrator']['clickhouse']['dst_schema'], dst_table=self.config['table_builder']['clickhouse']['dst_table'], dst_table_prefix=self.config['table_builder']['clickhouse']['dst_table_prefix'], + use_src_primary_key=self.config['table_builder']['clickhouse']['dst_use_src_primary_key'], distribute=self.config['table_migrator']['clickhouse']['dst_distribute'], cluster=self.config['table_migrator']['clickhouse']['dst_cluster'], tables=self.config['table_migrator']['mysql']['tables'], diff --git a/clickhouse_mysql/tablemigrator.py b/clickhouse_mysql/tablemigrator.py index 07e0986..6ff7a38 100644 --- a/clickhouse_mysql/tablemigrator.py +++ b/clickhouse_mysql/tablemigrator.py @@ -43,6 +43,7 @@ def __init__( dst_table_prefix=None, distribute=None, cluster=None, + use_src_primary_key=None, tables=None, tables_prefixes=None, tables_where_clauses=None, @@ -59,6 +60,7 @@ def __init__( dst_table_prefix=dst_table_prefix, distribute=distribute, cluster=cluster, + use_src_primary_key=use_src_primary_key, tables=tables, tables_prefixes=tables_prefixes, column_skip=column_skip diff --git a/clickhouse_mysql/tableprocessor.py b/clickhouse_mysql/tableprocessor.py index 7bb96b8..a4bef2f 100644 --- a/clickhouse_mysql/tableprocessor.py +++ b/clickhouse_mysql/tableprocessor.py @@ -33,6 +33,7 @@ def __init__( tables=None, tables_prefixes=None, column_skip=[], + use_src_primary_key=None, ): """ :param host: string MySQL host @@ -58,6 +59,7 @@ def __init__( self.cluster = cluster self.distribute = distribute self.column_skip = column_skip + self.use_src_primary_key = use_src_primary_key def dbs_tables_lists(self): """ diff --git a/clickhouse_mysql/tablesqlbuilder.py b/clickhouse_mysql/tablesqlbuilder.py index 77d10f2..bfc6ab3 100644 --- a/clickhouse_mysql/tablesqlbuilder.py +++ b/clickhouse_mysql/tablesqlbuilder.py @@ -204,7 +204,7 @@ def create_table_columns_description(self, db=None, table=None, ): 'clickhouse_type': self.map_type(mysql_type=_type), 'clickhouse_type_nullable': self.map_type_nullable(mysql_type=_type, nullable=self.is_field_nullable(_null)), 'nullable': self.is_field_nullable(_null), - 'key': _key, + 'key': (_key if _key == "PRI" else None) if self.use_src_primary_key else _key, 'default': _default, 'extra': _extra, })