chore(ci): Add lint, test, regen workflows

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint with Black
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Check formatting
+        run: make fmt-check

.github/workflows/publish.yml

@@ -20,7 +20,6 @@ jobs:
            python-version: '3.x'
        - name: Install dependencies
          run: |
-           python -m pip install --upgrade pip
            pip install -r requirements.txt
            pip install build
        - name: Build package

.github/workflows/regen.yml

@@ -0,0 +1,36 @@
+name: Generate Python Code from plugin-pb
+on:
+  schedule:
+    - cron: "0 8 * * *"
+  workflow_dispatch:
+
+jobs:
+  regen:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+      - name: Generate code
+        run: |
+          make clone-proto
+          make gen-proto
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          # required so the PR triggers workflow runs
+          token: ${{ secrets.GH_CQ_BOT }}
+          branch: fix/gen_proto
+          base: main
+          title: "fix: Generate Python Code from `plugin-pb`"
+          commit-message: "fix: Generate Python Code from `plugin-pb`"
+          body: This PR was created by a scheduled workflow to regenerate the Python code from `plugin-pb`.
+          author: cq-bot <cq-bot@users.noreply.github.com>
+          labels: automerge

.github/workflows/unittest.yml

@@ -0,0 +1,23 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Run tests
+        run: make test

Makefile

@@ -1,8 +1,16 @@
+test:
+	pytest .
+
+fmt:
+	black . --exclude=cloudquery
+
+fmt-check:
+	black --check . --exclude=cloudquery
 
 clone-proto:
 	git clone https://github.com/cloudquery/plugin-pb
 
-gen:
+gen-proto:
 	cd plugin-pb && git pull && cd ..
 
 	mkdir -p ./protos/cloudquery/plugin_v3

requirements.txt

@@ -1,3 +1,6 @@
-grpcio==1.56.0
+black==23.7.0
 grpcio-tools==1.56.0
+grpcio==1.56.0
 protobuf==4.23.4
+pyarrow==12.0.1
+pytest==7.4.0

samples/plugin_client.py

@@ -2,15 +2,17 @@
 import grpc
 from cloudquery.plugin_v3 import plugin_pb2, plugin_pb2_grpc
 
+
 def run():
     # NOTE(gRPC Python Team): .close() is possible on a channel and should be
     # used in circumstances in which the with statement does not fit the needs
     # of the code.
-    with grpc.insecure_channel('localhost:50051') as channel:
+    with grpc.insecure_channel("localhost:50051") as channel:
         stub = plugin_pb2_grpc.PluginStub(channel)
         response = stub.GetName(plugin_pb2.GetName.Request())
         print(response.name)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     logging.basicConfig()
     run()

samples/plugin_server.py

@@ -39,14 +39,13 @@ def Close(self, request, context):
 
 def serve():
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-    plugin_pb2_grpc.add_PluginServicer_to_server(
-        PluginServicer(), server)
-    server.add_insecure_port('[::]:50051')
+    plugin_pb2_grpc.add_PluginServicer_to_server(PluginServicer(), server)
+    server.add_insecure_port("[::]:50051")
     print("Starting server. Listening on port 50051")
     server.start()
     server.wait_for_termination()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     logging.basicConfig()
     serve()

setup.py

@@ -14,7 +14,7 @@
     "grpcio >= 1.56.0",
     "grpcio-tools >= 1.56.0",
     "protobuf >= 4.23.4",
-    "pyarrow >= 12.0.1"
+    "pyarrow >= 12.0.1",
 ]
 url = "https://github.com/cloudquery/plugin-pb-python"
 
@@ -65,6 +65,13 @@
     # namespace_packages=namespaces,
     install_requires=dependencies,
     include_package_data=True,
-    package_data={"cloudquery": ["plugin_v3/py.typed", "plugin_v3/*.pyi", "discovery_v1/py.typed", "discovery_v1/*.pyi"]},
+    package_data={
+        "cloudquery": [
+            "plugin_v3/py.typed",
+            "plugin_v3/*.pyi",
+            "discovery_v1/py.typed",
+            "discovery_v1/*.pyi",
+        ]
+    },
     zip_safe=False,
 )

tests/plugin_v3/arrow_test.py

@@ -0,0 +1,27 @@
+import pyarrow as pa
+from cloudquery.plugin_v3.arrow import (
+    schemas_to_bytes,
+    new_schemas_from_bytes,
+    record_to_bytes,
+    new_record_from_bytes,
+)
+
+
+def test_schema_round_trip():
+    sc = pa.schema(
+        fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"}
+    )
+    b = schemas_to_bytes([sc])
+    schemas = new_schemas_from_bytes(b)
+    assert len(schemas) == 1
+    assert schemas[0].equals(sc)
+
+
+def test_record_round_trip():
+    sc = pa.schema(
+        fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"}
+    )
+    rec = pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], schema=sc)
+    b = record_to_bytes(rec)
+    rec2 = new_record_from_bytes(b)
+    assert rec.equals(rec2)