From cfe370dad0b738be13e3bf00860d732acda05945 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Fri, 28 Jul 2023 11:09:49 +0200 Subject: [PATCH] chore(ci): Add lint, test, regen workflows --- .github/workflows/lint.yml | 21 ++++++++++++++++++++ .github/workflows/publish.yml | 1 - .github/workflows/regen.yml | 36 ++++++++++++++++++++++++++++++++++ .github/workflows/unittest.yml | 23 ++++++++++++++++++++++ Makefile | 10 +++++++++- requirements.txt | 5 ++++- samples/plugin_client.py | 6 ++++-- samples/plugin_server.py | 7 +++---- setup.py | 11 +++++++++-- tests/plugin_v3/arrow.py | 16 --------------- tests/plugin_v3/arrow_test.py | 27 +++++++++++++++++++++++++ 11 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/regen.yml create mode 100644 .github/workflows/unittest.yml delete mode 100644 tests/plugin_v3/arrow.py create mode 100644 tests/plugin_v3/arrow_test.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7ce2828 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +name: Lint with Black + +on: + pull_request: + push: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Check formatting + run: make fmt-check diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c0dd026..858503f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,6 @@ jobs: python-version: '3.x' - name: Install dependencies run: | - python -m pip install --upgrade pip pip install -r requirements.txt pip install build - name: Build package diff --git a/.github/workflows/regen.yml b/.github/workflows/regen.yml new file mode 100644 index 0000000..84e285a --- /dev/null +++ b/.github/workflows/regen.yml @@ -0,0 +1,36 @@ +name: Generate Python Code from plugin-pb +on: + schedule: + - cron: "0 8 * * *" + workflow_dispatch: + +jobs: + regen: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: | + pip install -r requirements.txt + - name: Generate code + run: | + make clone-proto + make gen-proto + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + # required so the PR triggers workflow runs + token: ${{ secrets.GH_CQ_BOT }} + branch: fix/gen_proto + base: main + title: "fix: Generate Python Code from `plugin-pb`" + commit-message: "fix: Generate Python Code from `plugin-pb`" + body: This PR was created by a scheduled workflow to regenerate the Python code from `plugin-pb`. + author: cq-bot + labels: automerge diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 0000000..fd91639 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,23 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run tests + run: make test diff --git a/Makefile b/Makefile index 79c1a70..a4f0c40 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,16 @@ +test: + pytest . + +fmt: + black . --exclude=cloudquery + +fmt-check: + black --check . --exclude=cloudquery clone-proto: git clone https://github.com/cloudquery/plugin-pb -gen: +gen-proto: cd plugin-pb && git pull && cd .. mkdir -p ./protos/cloudquery/plugin_v3 diff --git a/requirements.txt b/requirements.txt index 6314f5c..bf86f81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -grpcio==1.56.0 +black==23.7.0 grpcio-tools==1.56.0 +grpcio==1.56.0 protobuf==4.23.4 +pyarrow==12.0.1 +pytest==7.4.0 \ No newline at end of file diff --git a/samples/plugin_client.py b/samples/plugin_client.py index c85e440..820db2a 100644 --- a/samples/plugin_client.py +++ b/samples/plugin_client.py @@ -2,15 +2,17 @@ import grpc from cloudquery.plugin_v3 import plugin_pb2, plugin_pb2_grpc + def run(): # NOTE(gRPC Python Team): .close() is possible on a channel and should be # used in circumstances in which the with statement does not fit the needs # of the code. - with grpc.insecure_channel('localhost:50051') as channel: + with grpc.insecure_channel("localhost:50051") as channel: stub = plugin_pb2_grpc.PluginStub(channel) response = stub.GetName(plugin_pb2.GetName.Request()) print(response.name) -if __name__ == '__main__': + +if __name__ == "__main__": logging.basicConfig() run() diff --git a/samples/plugin_server.py b/samples/plugin_server.py index db03900..8b82a6b 100644 --- a/samples/plugin_server.py +++ b/samples/plugin_server.py @@ -39,14 +39,13 @@ def Close(self, request, context): def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - plugin_pb2_grpc.add_PluginServicer_to_server( - PluginServicer(), server) - server.add_insecure_port('[::]:50051') + plugin_pb2_grpc.add_PluginServicer_to_server(PluginServicer(), server) + server.add_insecure_port("[::]:50051") print("Starting server. Listening on port 50051") server.start() server.wait_for_termination() -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig() serve() diff --git a/setup.py b/setup.py index 349144d..1f08d46 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "grpcio >= 1.56.0", "grpcio-tools >= 1.56.0", "protobuf >= 4.23.4", - "pyarrow >= 12.0.1" + "pyarrow >= 12.0.1", ] url = "https://github.com/cloudquery/plugin-pb-python" @@ -65,6 +65,13 @@ # namespace_packages=namespaces, install_requires=dependencies, include_package_data=True, - package_data={"cloudquery": ["plugin_v3/py.typed", "plugin_v3/*.pyi", "discovery_v1/py.typed", "discovery_v1/*.pyi"]}, + package_data={ + "cloudquery": [ + "plugin_v3/py.typed", + "plugin_v3/*.pyi", + "discovery_v1/py.typed", + "discovery_v1/*.pyi", + ] + }, zip_safe=False, ) diff --git a/tests/plugin_v3/arrow.py b/tests/plugin_v3/arrow.py deleted file mode 100644 index 513207a..0000000 --- a/tests/plugin_v3/arrow.py +++ /dev/null @@ -1,16 +0,0 @@ -import pyarrow as pa -from cloudquery.plugin_v3.arrow import schemas_to_bytes, new_schemas_from_bytes, record_to_bytes, new_record_from_bytes - -def test_schema_round_trip(): - sc = pa.schema(fields=[pa.field("a", pa.int64())], metadata={"foo":"bar", "baz":"quux"}) - b = schemas_to_bytes([sc]) - schemas = new_schemas_from_bytes(b) - assert len(schemas) == 1 - assert schemas[0].equals(sc) - -def test_record_round_trip(): - sc = pa.schema(fields=[pa.field("a", pa.int64())], metadata={"foo":"bar", "baz":"quux"}) - rec = pa.RecordBatch.from_arrays([pa.array([1,2,3])], schema=sc) - b = record_to_bytes(rec) - rec2 = new_record_from_bytes(b) - assert rec.equals(rec2) diff --git a/tests/plugin_v3/arrow_test.py b/tests/plugin_v3/arrow_test.py new file mode 100644 index 0000000..c137c77 --- /dev/null +++ b/tests/plugin_v3/arrow_test.py @@ -0,0 +1,27 @@ +import pyarrow as pa +from cloudquery.plugin_v3.arrow import ( + schemas_to_bytes, + new_schemas_from_bytes, + record_to_bytes, + new_record_from_bytes, +) + + +def test_schema_round_trip(): + sc = pa.schema( + fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"} + ) + b = schemas_to_bytes([sc]) + schemas = new_schemas_from_bytes(b) + assert len(schemas) == 1 + assert schemas[0].equals(sc) + + +def test_record_round_trip(): + sc = pa.schema( + fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"} + ) + rec = pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], schema=sc) + b = record_to_bytes(rec) + rec2 = new_record_from_bytes(b) + assert rec.equals(rec2)