Merge branch 'master' into support-top-operator

Author: dlawin

.github/workflows/formatter.yml

@@ -21,3 +21,6 @@ jobs:
         uses: reviewdog/action-suggester@v1
         with:
           tool_name: blackfmt
+      - name: Fail if there are formatting suggestions
+        if: steps.black_formatter.outputs.is_formatted == 'true'
+        run: exit 1

data_diff/__main__.py

@@ -267,6 +267,20 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
     metavar="PATH",
     help="Specify manifest to utilize for 'prod' comparison paths instead of using configuration.",
 )
+@click.option(
+    "-pd",
+    "--prod-database",
+    "prod_database",
+    default=None,
+    help="Override the dbt production database configuration within dbt_project.yml",
+)
+@click.option(
+    "-ps",
+    "--prod-schema",
+    "prod_schema",
+    default=None,
+    help="Override the dbt production schema configuration within dbt_project.yml",
+)
 def main(conf, run, **kw):
     log_handlers = _get_log_handlers(kw["dbt"])
     if kw["table2"] is None and kw["database2"]:
@@ -323,6 +337,8 @@ def main(conf, run, **kw):
                 where_flag=kw["where"],
                 stats_flag=kw["stats"],
                 columns_flag=kw["columns"],
+                production_database_flag=kw["prod_database"],
+                production_schema_flag=kw["prod_schema"],
             )
         else:
             return _data_diff(
@@ -366,6 +382,8 @@ def _data_diff(
     cloud,
     dbt_profiles_dir,
     dbt_project_dir,
+    prod_database,
+    prod_schema,
     select,
     state,
     threads1=None,

data_diff/databases/bigquery.py

@@ -193,17 +193,18 @@ def __init__(self, project, *, dataset, bigquery_credentials=None, **kw):
         bigquery = import_bigquery()
 
         keyfile = kw.pop("keyfile", None)
+        impersonate_service_account = kw.pop("impersonate_service_account", None)
         if keyfile:
             bigquery_service_account = import_bigquery_service_account()
             credentials = bigquery_service_account.Credentials.from_service_account_file(
                 keyfile,
                 scopes=["https://www.googleapis.com/auth/cloud-platform"],
             )
-        elif kw.get("impersonate_service_account"):
+        elif impersonate_service_account:
             bigquery_service_account_impersonation = import_bigquery_service_account_impersonation()
             credentials = bigquery_service_account_impersonation.Credentials(
                 source_credentials=credentials,
-                target_principal=kw["impersonate_service_account"],
+                target_principal=impersonate_service_account,
                 target_scopes=["https://www.googleapis.com/auth/cloud-platform"],
             )
 

data_diff/databases/databricks.py

@@ -56,6 +56,12 @@ class Dialect(BaseDialect):
         "BOOLEAN": Boolean,
     }
 
+    def type_repr(self, t) -> str:
+        try:
+            return {str: "STRING"}[t]
+        except KeyError:
+            return super().type_repr(t)
+
     def quote(self, s: str):
         return f"`{s}`"
 

data_diff/databases/mssql.py

@@ -98,7 +98,7 @@ def to_string(self, s: str):
 
     def type_repr(self, t) -> str:
         try:
-            return {bool: "bit"}[t]
+            return {bool: "bit", str: "text"}[t]
         except KeyError:
             return super().type_repr(t)
 

data_diff/dbt.py

@@ -76,6 +76,8 @@ def dbt_diff(
     where_flag: Optional[str] = None,
     stats_flag: bool = False,
     columns_flag: Optional[Tuple[str]] = None,
+    production_database_flag: Optional[str] = None,
+    production_schema_flag: Optional[str] = None,
 ) -> None:
     print_version_info()
     diff_threads = []
@@ -115,7 +117,16 @@ def dbt_diff(
             if log_status_handler:
                 log_status_handler.set_prefix(f"Diffing {model.alias} \n")
 
-            diff_vars = _get_diff_vars(dbt_parser, config, model, where_flag, stats_flag, columns_flag)
+            diff_vars = _get_diff_vars(
+                dbt_parser,
+                config,
+                model,
+                where_flag,
+                stats_flag,
+                columns_flag,
+                production_database_flag,
+                production_schema_flag,
+            )
 
             # we won't always have a prod path when using state
             # when the model DNE in prod manifest, skip the model diff
@@ -169,6 +180,8 @@ def _get_diff_vars(
     where_flag: Optional[str] = None,
     stats_flag: bool = False,
     columns_flag: Optional[Tuple[str]] = None,
+    production_database_flag: Optional[str] = None,
+    production_schema_flag: Optional[str] = None,
 ) -> TDiffVars:
     cli_columns = list(columns_flag) if columns_flag else []
     dev_database = model.database
@@ -182,6 +195,10 @@ def _get_diff_vars(
     else:
         prod_database, prod_schema = _get_prod_path_from_config(config, model, dev_database, dev_schema)
 
+    # cli flags take precedence over any project level config
+    prod_database = production_database_flag or prod_database
+    prod_schema = production_schema_flag or prod_schema
+
     if dbt_parser.requires_upper:
         dev_qualified_list = [x.upper() for x in [dev_database, dev_schema, dev_alias] if x]
         prod_qualified_list = [x.upper() for x in [prod_database, prod_schema, prod_alias] if x]

data_diff/dbt_config_validators.py

@@ -6,8 +6,8 @@
 class ManifestJsonConfig(BaseModel):
     class Metadata(BaseModel):
         dbt_version: str = Field(..., regex=r"^\d+\.\d+\.\d+([a-zA-Z0-9]+)?$")
-        project_id: str
-        user_id: str
+        project_id: Optional[str]
+        user_id: Optional[str]
 
     class Nodes(BaseModel):
         class Config(BaseModel):

data_diff/dbt_parser.py

@@ -64,7 +64,7 @@ def try_set_dbt_flags():
 PROJECT_FILE = "dbt_project.yml"
 PROFILES_FILE = "profiles.yml"
 LOWER_DBT_V = "1.0.0"
-UPPER_DBT_V = "1.7.0"
+UPPER_DBT_V = "1.8.0"
 
 
 # https://github.com/dbt-labs/dbt-core/blob/c952d44ec5c2506995fbad75320acbae49125d3d/core/dbt/cli/resolvers.py#L6

data_diff/version.py

@@ -1 +1 @@
-__version__ = "0.9.8"
+__version__ = "0.9.12"

poetry.lock

@@ -1,12 +1,11 @@
 [tool.poetry]
 name = "data-diff"
-version = "0.9.8"
+version = "0.9.12"
 description = "Command-line tool and Python library to efficiently diff rows across two different databases."
 authors = ["Datafold <data-diff@datafold.com>"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/datafold/data-diff"
-documentation = ""
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",
@@ -46,7 +45,7 @@ oracledb = {version = "*", optional=true}
 pyodbc = {version="^4.0.39", optional=true}
 typing-extensions = ">=4.0.1"
 attrs = "^23.1.0"
-mashumaro = {version = ">=3.8.1,<3.9.0", extras = ["msgpack"]}
+mashumaro = {version = ">=2.9,<3.11.0", extras = ["msgpack"]}
 
 [tool.poetry.dev-dependencies]
 parameterized = "*"