Replace dependency on protoc with grpcio-tools

This change removes the dependency on platform provided protobuf tools
in favour of `grpcio-tools` dependency. This makes both development and
compiler use independent from platform dependencies.

Author: abn

.github/workflows/ci.yml

@@ -49,7 +49,6 @@ jobs:
             ${{ runner.os }}-poetry-
       - name: Install dependencies
         run: |
-          sudo apt install protobuf-compiler libprotobuf-dev
           poetry install
       - name: Run tests
         run: |

Makefile

@@ -31,7 +31,7 @@ clean:              ## - Clean out generated files from the workspace
 o=output
 plugin:             ## - Execute the protoc plugin, with output write to `output` or the value passed to `-o`
 	mkdir -p $(o)
-	protoc --plugin=protoc-gen-custom=betterproto/plugin.py $(i) --custom_out=$(o)
+	poetry run python -m grpc.tools.protoc $(i) --python_betterproto_out=$(o)
 
 # CI tasks
 

betterproto/tests/generate.py

@@ -11,8 +11,7 @@
     inputs_path,
     output_path_betterproto,
     output_path_reference,
-    protoc_plugin,
-    protoc_reference,
+    protoc,
 )
 
 # Force pure-python implementation instead of C++, otherwise imports
@@ -88,8 +87,8 @@ async def generate_test_case_output(
         (ref_out, ref_err, ref_code),
         (plg_out, plg_err, plg_code),
     ) = await asyncio.gather(
-        protoc_reference(test_case_input_path, test_case_output_path_reference),
-        protoc_plugin(test_case_input_path, test_case_output_path_betterproto),
+        protoc(test_case_input_path, test_case_output_path_reference, True),
+        protoc(test_case_input_path, test_case_output_path_betterproto, False),
     )
 
     message = f"Generated output for {test_case_name!r}"

betterproto/tests/util.py

@@ -2,9 +2,10 @@
 import importlib
 import os
 import pathlib
+import sys
 from pathlib import Path
 from types import ModuleType
-from typing import Callable, Generator, Optional
+from typing import Callable, Generator, Optional, Union
 
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 
@@ -13,11 +14,6 @@
 output_path_reference = root_path.joinpath("output_reference")
 output_path_betterproto = root_path.joinpath("output_betterproto")
 
-if os.name == "nt":
-    plugin_path = root_path.joinpath("..", "plugin.bat").resolve()
-else:
-    plugin_path = root_path.joinpath("..", "plugin.py").resolve()
-
 
 def get_files(path, suffix: str) -> Generator[str, None, None]:
     for r, dirs, files in os.walk(path):
@@ -31,22 +27,25 @@ def get_directories(path):
             yield directory
 
 
-async def protoc_plugin(path: str, output_dir: str):
-    proc = await asyncio.create_subprocess_shell(
-        f"protoc --plugin=protoc-gen-custom={plugin_path} --custom_out={output_dir} --proto_path={path} {path}/*.proto",
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    return (*(await proc.communicate()), proc.returncode)
-
-
-async def protoc_reference(path: str, output_dir: str):
-    proc = await asyncio.create_subprocess_shell(
-        f"protoc --python_out={output_dir} --proto_path={path} {path}/*.proto",
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
+async def protoc(
+    path: Union[str, Path], output_dir: Union[str, Path], reference: bool = False
+):
+    path: Path = Path(path).resolve()
+    output_dir: Path = Path(output_dir).resolve()
+    python_out_option: str = "python_betterproto_out" if not reference else "python_out"
+    command = [
+        sys.executable,
+        "-m",
+        "grpc.tools.protoc",
+        f"--proto_path={path.as_posix()}",
+        f"--{python_out_option}={output_dir.as_posix()}",
+        *[p.as_posix() for p in path.glob("*.proto")],
+    ]
+    proc = await asyncio.create_subprocess_exec(
+        *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
     )
-    return (*(await proc.communicate()), proc.returncode)
+    stdout, stderr = await proc.communicate()
+    return stdout, stderr, proc.returncode
 
 
 def get_test_case_json_data(test_case_name: str, json_file_name: Optional[str] = None):