diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acaaa597..aeada059 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,8 +33,18 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install MSSQL packages + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg + sudo apt-get update + ACCEPT_EULA=Y sudo apt-get install -y msodbcsql18 + ACCEPT_EULA=Y sudo apt-get install -y mssql-tools18 + sudo apt-get install -y unixodbc-dev + - name: Build the stack - run: docker-compose up -d mysql postgres presto trino clickhouse vertica + run: docker-compose up -d mysql postgres presto trino clickhouse vertica mssql - name: Install Poetry run: pip install poetry @@ -66,6 +76,7 @@ jobs: DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}' # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}' DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' + DATADIFF_MSSQL_URI: 'mssql://SA:Password123mssql@localhost:1433/master/dbo' DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}' MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}' diff --git a/.github/workflows/ci_full.yml b/.github/workflows/ci_full.yml index 3e17af0e..f9819281 100644 --- a/.github/workflows/ci_full.yml +++ b/.github/workflows/ci_full.yml @@ -33,8 +33,18 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install MSSQL packages + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg + sudo apt-get update + ACCEPT_EULA=Y sudo apt-get install -y msodbcsql18 + ACCEPT_EULA=Y sudo apt-get install -y mssql-tools18 + sudo apt-get install -y unixodbc-dev + - name: Build the stack - run: docker-compose up -d mysql postgres presto trino clickhouse vertica + run: docker-compose up -d mysql postgres presto trino clickhouse vertica mssql - name: Install Poetry run: pip install poetry @@ -62,6 +72,7 @@ jobs: DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}' DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' + DATADIFF_MSSQL_URI: 'mssql://SA:Password123mssql@localhost:1433/master/dbo' # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}' DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}' MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}' diff --git a/dev/dev.env b/dev/dev.env index cb5eea12..07310bbb 100644 --- a/dev/dev.env +++ b/dev/dev.env @@ -21,3 +21,7 @@ VERTICA_DB_NAME=vertica # leave VMART_DIR and VMART_ETL_SCRIPT empty. VMART_DIR= VMART_ETL_SCRIPT= + +# MSSQL credentials +ACCEPT_EULA=Y +MSSQL_SA_PASSWORD=Password123mssql diff --git a/docker-compose.yml b/docker-compose.yml index 60bab061..f91322d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,6 +117,19 @@ services: networks: - local + mssql: + container_name: dd-mssql + image: mcr.microsoft.com/mssql/server + restart: always + volumes: + - mssql-data:/var/opt/mssql + ports: + - '1433:1433' + env_file: + - dev/dev.env + tty: true + networks: + - local volumes: @@ -124,6 +137,7 @@ volumes: mysql-data: clickhouse-data: vertica-data: + mssql-data: networks: local: diff --git a/poetry.lock b/poetry.lock index 073367c0..635a3605 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "agate" @@ -1584,7 +1584,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pyodbc" version = "4.0.39" description = "DB API Module for ODBC" -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ {file = "pyodbc-4.0.39-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74af348dbaee4885998858daf50c8964e767629ecf6c195868b016367b0bb861"}, @@ -2240,13 +2240,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "vertica-python" -version = "1.3.7" +version = "1.3.8" description = "Official native Python client for the Vertica database." optional = false python-versions = ">=3.7" files = [ - {file = "vertica-python-1.3.7.tar.gz", hash = "sha256:2a5f0d0111d3cc466b87923ffd3c262132a9fa0e38964e43eef90a3ccda57452"}, - {file = "vertica_python-1.3.7-py3-none-any.whl", hash = "sha256:e8d3ceaa2b7a61a861590c8b5d822d2d71afad9ff3399fc67ee9888c5565ff74"}, + {file = "vertica-python-1.3.8.tar.gz", hash = "sha256:e52b894fc32eff83270264d66fd4cbe5bd1fd07ba0da7ef45f906164c136bebc"}, + {file = "vertica_python-1.3.8-py3-none-any.whl", hash = "sha256:a52d05251b25adbadd8fc3d80d5a602f0982ad5e7e5fb84cb0ca26147419a0e8"}, ] [package.dependencies] @@ -2316,4 +2316,4 @@ vertica = ["vertica-python"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "54f5248145c8920e27de3e781bb01461b3f68bc6335d75d8793b19dffd5f4163" +content-hash = "78dabb1929a40e305c651df208dad73552e77af2ee22d77219ee598026dae4b4" diff --git a/pyproject.toml b/pyproject.toml index 0fe15b4c..3900b1c2 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ vertica-python = "*" duckdb = "^0.9.0" dbt-core = "^1.0.0" ruff = "^0.1.4" +pyodbc = "*" # google-cloud-bigquery = "*" # databricks-sql-connector = "*"