[skip-changelog] add integration testing infrastructure

Author: umbynos

.github/workflows/test.yaml

@@ -48,5 +48,13 @@ jobs:
           GOARCH: "arm"
         run: task build
 
-      - name: Run unit tests
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Run unit and integration tests
         run: task test

.gitignore

@@ -4,6 +4,7 @@
 .idea
 coverage_*.txt
 /dist
+__pycache__/
 
 # Misc.
 .DS_Store

Taskfile.yml

@@ -33,9 +33,21 @@ tasks:
 
   test:
     desc: Run tests
+    cmds:
+      - task: test-unit
+      - task: test-integration
+
+  test-unit:
+    desc: Run unit tests
     cmds:
       - go test -short -race -run '.*' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt ./... {{.TEST_LDFLAGS}}
 
+  test-integration:
+    desc: Run integration tests
+    cmds:
+      - poetry install --no-root
+      - poetry run pytest test
+
   check:
     desc: Check fmt and lint
     cmds:

poetry.lock

@@ -0,0 +1,11 @@
+[tool.poetry]
+name = "FirmwareUploader"
+version = "0.0.0"
+description = "FirmwareUploader"
+authors = ["Arduino <info@arduino.cc>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pytest = "6.1.2"
+invoke = "^1.5.0"
+semver = "^2.13.0"

pyproject.toml

@@ -0,0 +1,59 @@
+#   FirmwareUploader
+#   Copyright (c) 2021 Arduino LLC.  All right reserved.
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+import pathlib
+import platform
+import invoke.context 
+import pytest
+
+@pytest.fixture(scope="function")
+def run_command(pytestconfig, working_dir):
+    """Provide a wrapper around invoke's `run` API so that every test will work in the same temporary folder.
+    Useful reference:
+        http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Result
+    """
+
+    fwuploader_path = pathlib.Path(pytestconfig.rootdir).parent / "FirmwareUploader"
+
+    def _run(cmd_string, custom_working_dir=None, custom_env=None):
+        if not custom_working_dir:
+            custom_working_dir = working_dir
+        cli_full_line = '"{}" {}'.format(fwuploader_path, cmd_string)
+        run_context = invoke.context.Context()
+        # It might happen that we need to change directories between drives on Windows,
+        # in that case the "/d" flag must be used otherwise directory wouldn't change
+        cd_command = "cd"
+        if platform.system() == "Windows":
+            cd_command += " /d"
+        # Context.cd() is not used since it doesn't work correctly on Windows.
+        # It escapes spaces in the path using "\ " but it doesn't always work,
+        # wrapping the path in quotation marks is the safest approach
+        with run_context.prefix(f'{cd_command} "{custom_working_dir}"'):
+            return run_context.run(
+                command=cli_full_line, echo=False, hide=True, warn=True, env=custom_env, encoding="utf-8"
+            )
+
+    return _run
+
+
+@pytest.fixture(scope="function")
+def working_dir(tmpdir_factory):
+    """Create a temporary folder for the test to run in. It will be created before running each test and deleted at the
+    end. This way all the tests work in isolation.
+    """
+    work_dir = tmpdir_factory.mktemp(basename="FirmwareUploaderTestWork")
+    yield str(work_dir)

test/conftest.py

@@ -0,0 +1,9 @@
+[pytest]
+filterwarnings =
+    error
+    ignore::DeprecationWarning
+    ignore::ResourceWarning
+
+# --capture=no - disable per-test capture
+# --tb=long sets the length of the traceback in case of failures
+addopts = --capture=no --tb=long --verbose

test/pytest.ini

@@ -0,0 +1,43 @@
+#   FirmwareUploader
+#   Copyright (c) 2021 Arduino LLC.  All right reserved.
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+import json
+import semver
+
+def test_help(run_command):
+    result = run_command("help")
+    assert result.ok
+    assert result.stderr == ""
+    assert "Usage" in result.stdout
+
+
+def test_version(run_command):
+    result = run_command("version")
+    assert result.ok
+    assert "Version:" in result.stdout
+    assert "Commit:" in result.stdout
+    assert "Date:" in result.stdout
+    assert "" == result.stderr
+
+    result = run_command("version --format json")
+    assert result.ok
+    parsed_out = json.loads(result.stdout)
+    assert parsed_out.get("Application", False) == "FirmwareUploader"
+    version = parsed_out.get("VersionString", False)
+    assert semver.VersionInfo.isvalid(version=version) or "git-snapshot" in version or "nightly" in version
+    assert isinstance(parsed_out.get("Commit", False), str)
+    assert isinstance(parsed_out.get("Date", False), str)