Add data diff testing for MSSQL

.github/workflows/ci.yml

@@ -33,8 +33,18 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install MSSQL packages
+        run: |
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc
+          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
+          curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg
+          sudo apt-get update
+          ACCEPT_EULA=Y sudo apt-get install -y msodbcsql18
+          ACCEPT_EULA=Y sudo apt-get install -y mssql-tools18
+          sudo apt-get install -y unixodbc-dev
+
       - name: Build the stack
-        run: docker-compose up -d mysql postgres presto trino clickhouse vertica
+        run: docker-compose up -d mysql postgres presto trino clickhouse vertica mssql
 
       - name: Install Poetry
         run: pip install poetry
@@ -66,6 +76,7 @@ jobs:
             DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}'
             # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
+            DATADIFF_MSSQL_URI: 'mssql://SA:Password123mssql@localhost:1433/master/dbo'
             DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
             DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}'
             MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}'

.github/workflows/ci_full.yml

@@ -33,8 +33,18 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install MSSQL packages
+        run: |
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc
+          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
+          curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg
+          sudo apt-get update
+          ACCEPT_EULA=Y sudo apt-get install -y msodbcsql18
+          ACCEPT_EULA=Y sudo apt-get install -y mssql-tools18
+          sudo apt-get install -y unixodbc-dev
+
       - name: Build the stack
-        run: docker-compose up -d mysql postgres presto trino clickhouse vertica
+        run: docker-compose up -d mysql postgres presto trino clickhouse vertica mssql
 
       - name: Install Poetry
         run: pip install poetry
@@ -62,6 +72,7 @@ jobs:
             DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
             DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
+            DATADIFF_MSSQL_URI: 'mssql://SA:Password123mssql@localhost:1433/master/dbo'
             # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
             DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}'
             MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}'

dev/dev.env

@@ -21,3 +21,7 @@ VERTICA_DB_NAME=vertica
 # leave VMART_DIR and VMART_ETL_SCRIPT empty.
 VMART_DIR=
 VMART_ETL_SCRIPT=
+
+# MSSQL credentials
+ACCEPT_EULA=Y
+MSSQL_SA_PASSWORD=Password123mssql

docker-compose.yml

@@ -117,13 +117,27 @@ services:
       networks:
       - local
 
+    mssql:
+      container_name: dd-mssql
+      image: mcr.microsoft.com/mssql/server
+      restart: always
+      volumes:
+        - mssql-data:/var/opt/mssql
+      ports:
+      - '1433:1433'
+      env_file:
+        - dev/dev.env
+      tty: true
+      networks:
+      - local
 
 
 volumes:
   postgresql-data:
   mysql-data:
   clickhouse-data:
   vertica-data:
+  mssql-data:
 
 networks:
   local:

pyproject.toml

@@ -62,6 +62,7 @@ vertica-python = "*"
 duckdb = "^0.9.0"
 dbt-core = "^1.0.0"
 ruff = "^0.1.4"
+pyodbc = "*"
 # google-cloud-bigquery = "*"
 # databricks-sql-connector = "*"