diff --git a/.github/workflows/formatter.yml b/.github/workflows/formatter.yml index 96e14cfb..52ee9919 100644 --- a/.github/workflows/formatter.yml +++ b/.github/workflows/formatter.yml @@ -7,20 +7,16 @@ on: jobs: linter_name: - name: runner / black + name: runner / ruff runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Check files using the black formatter - uses: rickstaa/action-black@v1 - id: black_formatter + - uses: actions/checkout@v3 + - name: Check files using the ruff formatter + uses: chartboost/ruff-action@v1 + id: ruff_formatter with: - black_args: ". -l 120" - - name: Annotate diff changes using reviewdog - if: steps.black_formatter.outputs.is_formatted == 'true' - uses: reviewdog/action-suggester@v1 + args: format + - name: Auto commit ruff formatting + uses: stefanzweifel/git-auto-commit-action@v5 with: - tool_name: blackfmt - - name: Fail if there are formatting suggestions - if: steps.black_formatter.outputs.is_formatted == 'true' - run: exit 1 + commit_message: 'style fixes by ruff' \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a17b4eff --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +default_language_version: + python: python3 + +repos: +# Run the Ruff linter. +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.2 + hooks: + # Run the Ruff formatter. + - id: ruff-format \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34440a27..03eaed22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,8 @@ Once inside, you can install the dependencies. - Option 2: Run `pip install -e .` to install them, and data-diff, in the global context. +- Run `pre-commit install` to automatically format your code before committing. + At the bare minimum, you need MySQL to run the tests. You can create a local MySQL instance using `docker-compose up mysql`. The URI for it will be `mysql://mysql:Password1@localhost/mysql`. If you're using a different server, make sure to update `TEST_MYSQL_CONN_STRING` in `tests/common.py`. For your convenience, we recommend creating `tests/local_settings.py`, and to override the value there. diff --git a/data_diff/databases/redshift.py b/data_diff/databases/redshift.py index b617f6d0..968f57bb 100644 --- a/data_diff/databases/redshift.py +++ b/data_diff/databases/redshift.py @@ -126,9 +126,7 @@ def select_view_columns(self, path: DbPath) -> str: return """select * from pg_get_cols('{}.{}') cols(view_schema name, view_name name, col_name name, col_type varchar, col_num int) - """.format( - schema, table - ) + """.format(schema, table) def query_pg_get_cols(self, path: DbPath) -> Dict[str, tuple]: rows = self.query(self.select_view_columns(path), list) diff --git a/data_diff/queries/ast_classes.py b/data_diff/queries/ast_classes.py index 9c140a5f..3bf14fd7 100644 --- a/data_diff/queries/ast_classes.py +++ b/data_diff/queries/ast_classes.py @@ -337,6 +337,7 @@ def else_(self, then: Expr) -> Self: @attrs.define(frozen=True, eq=False) class QB_When: "Partial case-when, used for query-building" + casewhen: CaseWhen when: Expr diff --git a/poetry.lock b/poetry.lock index 8fa89af1..16e4495c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -195,6 +195,17 @@ files = [ [package.dependencies] pycparser = "*" +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + [[package]] name = "charset-normalizer" version = "2.0.12" @@ -545,6 +556,17 @@ python-dateutil = ">=2.0,<3.0" pyyaml = ">=6.0,<7.0" typing-extensions = ">=4.0,<5.0" +[[package]] +name = "distlib" +version = "0.3.7" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, + {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, +] + [[package]] name = "dsnparse" version = "0.1.15" @@ -651,6 +673,20 @@ files = [ jsonschema = ">=3.0" python-dateutil = ">=2.8,<2.9" +[[package]] +name = "identify" +version = "2.5.31" +description = "File identification library for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "identify-2.5.31-py2.py3-none-any.whl", hash = "sha256:90199cb9e7bd3c5407a9b7e81b4abec4bb9d249991c79439ec8af740afc6293d"}, + {file = "identify-2.5.31.tar.gz", hash = "sha256:7736b3c7a28233637e3c36550646fc6389bedd74ae84cb788200cc8e2dd60b75"}, +] + +[package.extras] +license = ["ukkonen"] + [[package]] name = "idna" version = "3.4" @@ -1104,6 +1140,20 @@ doc = ["nb2plots (>=0.6)", "numpydoc (>=1.1)", "pillow (>=8.2)", "pydata-sphinx- extra = ["lxml (>=4.5)", "pydot (>=1.4.1)", "pygraphviz (>=1.7)"] test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] +[[package]] +name = "nodeenv" +version = "1.8.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] + +[package.dependencies] +setuptools = "*" + [[package]] name = "oracledb" version = "1.3.2" @@ -1211,6 +1261,39 @@ files = [ {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pre-commit" +version = "3.5.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pre_commit-3.5.0-py2.py3-none-any.whl", hash = "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660"}, + {file = "pre_commit-3.5.0.tar.gz", hash = "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + [[package]] name = "preql" version = "0.2.19" @@ -1724,6 +1807,32 @@ pygments = ">=2.6.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +[[package]] +name = "ruff" +version = "0.1.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.4-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:864958706b669cce31d629902175138ad8a069d99ca53514611521f532d91495"}, + {file = "ruff-0.1.4-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9fdd61883bb34317c788af87f4cd75dfee3a73f5ded714b77ba928e418d6e39e"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4eaca8c9cc39aa7f0f0d7b8fe24ecb51232d1bb620fc4441a61161be4a17539"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a9a1301dc43cbf633fb603242bccd0aaa34834750a14a4c1817e2e5c8d60de17"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78e8db8ab6f100f02e28b3d713270c857d370b8d61871d5c7d1702ae411df683"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:80fea754eaae06335784b8ea053d6eb8e9aac75359ebddd6fee0858e87c8d510"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bc02a480d4bfffd163a723698da15d1a9aec2fced4c06f2a753f87f4ce6969c"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862811b403063765b03e716dac0fda8fdbe78b675cd947ed5873506448acea4"}, + {file = "ruff-0.1.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58826efb8b3efbb59bb306f4b19640b7e366967a31c049d49311d9eb3a4c60cb"}, + {file = "ruff-0.1.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fdfd453fc91d9d86d6aaa33b1bafa69d114cf7421057868f0b79104079d3e66e"}, + {file = "ruff-0.1.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e8791482d508bd0b36c76481ad3117987301b86072158bdb69d796503e1c84a8"}, + {file = "ruff-0.1.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01206e361021426e3c1b7fba06ddcb20dbc5037d64f6841e5f2b21084dc51800"}, + {file = "ruff-0.1.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:645591a613a42cb7e5c2b667cbefd3877b21e0252b59272ba7212c3d35a5819f"}, + {file = "ruff-0.1.4-py3-none-win32.whl", hash = "sha256:99908ca2b3b85bffe7e1414275d004917d1e0dfc99d497ccd2ecd19ad115fd0d"}, + {file = "ruff-0.1.4-py3-none-win_amd64.whl", hash = "sha256:1dfd6bf8f6ad0a4ac99333f437e0ec168989adc5d837ecd38ddb2cc4a2e3db8a"}, + {file = "ruff-0.1.4-py3-none-win_arm64.whl", hash = "sha256:d98ae9ebf56444e18a3e3652b3383204748f73e247dea6caaf8b52d37e6b32da"}, + {file = "ruff-0.1.4.tar.gz", hash = "sha256:21520ecca4cc555162068d87c747b8f95e1e95f8ecfcbbe59e8dd00710586315"}, +] + [[package]] name = "runtype" version = "0.2.7" @@ -2002,6 +2111,26 @@ files = [ python-dateutil = ">=1.5" six = ">=1.10.0" +[[package]] +name = "virtualenv" +version = "20.24.6" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.24.6-py3-none-any.whl", hash = "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"}, + {file = "virtualenv-20.24.6.tar.gz", hash = "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<4" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + [[package]] name = "wcwidth" version = "0.2.5" @@ -2045,4 +2174,4 @@ vertica = ["vertica-python"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "925ab3b5dc0ef5a9ddb125e7a3bcf4800bee5d5d3ef67d73a48cfae587fa4505" +content-hash = "b7f8880be9658fa523ff4737d1fdefd09d124ab09fac22fa9ec6a83454940c1d" diff --git a/pyproject.toml b/pyproject.toml index 091d8fb2..f0544346 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ clickhouse-driver = "*" vertica-python = "*" duckdb = "^0.7.0" dbt-core = "^1.0.0" +ruff = "^0.1.4" # google-cloud-bigquery = "*" # databricks-sql-connector = "*" @@ -80,6 +81,9 @@ clickhouse = ["clickhouse-driver"] vertica = ["vertica-python"] duckdb = ["duckdb"] +[tool.poetry.group.dev.dependencies] +pre-commit = "^3.5.0" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..870cba20 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,2 @@ +# Allow lines to be as long as 120. +line-length = 120 \ No newline at end of file diff --git a/tests/common.py b/tests/common.py index 018f2e32..2fc6be19 100644 --- a/tests/common.py +++ b/tests/common.py @@ -135,6 +135,7 @@ def str_to_checksum(str: str): class DiffTestCase(unittest.TestCase): "Sets up two tables for diffing" + db_cls = None src_schema = None dst_schema = None