From 38a21e0502e8e87a9ac897279641c0ea9c21f51f Mon Sep 17 00:00:00 2001 From: per1234 Date: Sat, 14 Aug 2021 22:51:17 -0700 Subject: [PATCH] Sync testing assets with templates We have assembled a collection of reusable project assets: https://github.com/arduino/tooling-project-assets These assets will be used in the repositories of all Arduino tooling projects. Some improvements and standardizations have been made in the upstream "template" testing assets, and those are introduced to this repository here. Notable: - Split into separate workflows for each distinct testing type - Use Codecov to track code coverage - Make integration test work directory path canonical - Do clean up of integration test work directory - Update integration test dependencies Some task names have changed: - `test-unit` -> `go:test` - `test-integration` -> `go:test-integration` --- ...est.yaml => check-go-cross-build-task.yml} | 47 ++++++----- .../workflows/test-go-integration-task.yml | 74 +++++++++++++++++ .github/workflows/test-go-task.yml | 79 +++++++++++++++++++ README.md | 5 +- Taskfile.yml | 37 ++++++--- poetry.lock | 37 +++++---- pyproject.toml | 4 +- test/conftest.py | 28 +++++-- test/pytest.ini | 1 + 9 files changed, 251 insertions(+), 61 deletions(-) rename .github/workflows/{test.yaml => check-go-cross-build-task.yml} (50%) create mode 100644 .github/workflows/test-go-integration-task.yml create mode 100644 .github/workflows/test-go-task.yml diff --git a/.github/workflows/test.yaml b/.github/workflows/check-go-cross-build-task.yml similarity index 50% rename from .github/workflows/test.yaml rename to .github/workflows/check-go-cross-build-task.yml index c036de0e..02cd7ebe 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/check-go-cross-build-task.yml @@ -1,30 +1,47 @@ -name: Run tests +name: Check Cross Build +env: + # See: https://github.com/actions/setup-go/tree/v2#readme + GO_VERSION: "1.15" + +# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows on: push: - branches: - - main + paths: + - ".github/workflows/check-go-cross-build-task.ya?ml" + - "go.mod" + - "go.sum" + - "Taskfile.ya?ml" + - "**.go" pull_request: + paths: + - ".github/workflows/check-go-cross-build-task.ya?ml" + - "go.mod" + - "go.sum" + - "Taskfile.ya?ml" + - "**.go" + workflow_dispatch: + repository_dispatch: jobs: - native-os-build: + build: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: + - ubuntu-latest + - windows-latest + - macos-latest runs-on: ${{ matrix.os }} steps: - - name: Disable EOL conversions - run: git config --global core.autocrlf false - - name: Checkout uses: actions/checkout@v2 - name: Install Go uses: actions/setup-go@v2 with: - go-version: "1.15" + go-version: ${{ env.GO_VERSION }} - name: Install Taskfile uses: arduino/setup-task@v1 @@ -33,7 +50,6 @@ jobs: version: 3.x - name: Build native - shell: bash run: task build - name: Cross-build for 386 @@ -47,14 +63,3 @@ jobs: env: GOARCH: "arm" run: task build - - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Install Poetry - run: pip install poetry - - - name: Run unit and integration tests - run: task test diff --git a/.github/workflows/test-go-integration-task.yml b/.github/workflows/test-go-integration-task.yml new file mode 100644 index 00000000..60bde5c4 --- /dev/null +++ b/.github/workflows/test-go-integration-task.yml @@ -0,0 +1,74 @@ +# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/test-go-integration-task.md +name: Test Integration + +env: + # See: https://github.com/actions/setup-go/tree/v2#readme + GO_VERSION: "1.15" + # See: https://github.com/actions/setup-python/tree/v2#available-versions-of-python + PYTHON_VERSION: "3.9" + +# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows +on: + push: + paths: + - ".github/workflows/test-go-integration-task.ya?ml" + - "Taskfile.ya?ml" + - "**.go" + - "go.mod" + - "go.sum" + - "poetry.lock" + - "pyproject.toml" + - "tests/**" + pull_request: + paths: + - ".github/workflows/test-go-integration-task.ya?ml" + - "Taskfile.ya?ml" + - "**.go" + - "go.mod" + - "go.sum" + - "poetry.lock" + - "pyproject.toml" + - "tests/**" + workflow_dispatch: + repository_dispatch: + +jobs: + test: + strategy: + matrix: + operating-system: + - ubuntu-latest + - windows-latest + - macos-latest + + runs-on: ${{ matrix.operating-system }} + + steps: + # By default, actions/checkout converts the repo's LF line endings to CRLF on the Windows runner. + - name: Disable EOL conversions + run: git config --global core.autocrlf false + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: pip install poetry + + - name: Install Task + uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + + - name: Run integration tests + run: task go:test-integration diff --git a/.github/workflows/test-go-task.yml b/.github/workflows/test-go-task.yml new file mode 100644 index 00000000..912c1c0c --- /dev/null +++ b/.github/workflows/test-go-task.yml @@ -0,0 +1,79 @@ +# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/test-go-task.md +name: Test Go + +env: + # See: https://github.com/actions/setup-go/tree/v2#readme + GO_VERSION: "1.15" + +# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows +on: + push: + paths: + - ".github/workflows/test-go-task.ya?ml" + - "codecov.ya?ml" + - "**/go.mod" + - "**/go.sum" + - "Taskfile.ya?ml" + - "**.go" + - "**/testdata/**" + pull_request: + paths: + - ".github/workflows/test-go-task.ya?ml" + - "codecov.ya?ml" + - "**/go.mod" + - "**/go.sum" + - "Taskfile.ya?ml" + - "**.go" + - "**/testdata/**" + workflow_dispatch: + repository_dispatch: + +jobs: + test: + name: test (${{ matrix.module.path }} - ${{ matrix.operating-system }}) + + strategy: + fail-fast: false + + matrix: + operating-system: + - ubuntu-latest + - windows-latest + - macos-latest + module: + - path: ./ + codecov-flags: unit + + runs-on: ${{ matrix.operating-system }} + + steps: + # By default, actions/checkout converts the repo's LF line endings to CRLF on the Windows runner. + - name: Disable EOL conversions + run: git config --global core.autocrlf false + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Task + uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + + - name: Run tests + env: + GO_MODULE_PATH: ${{ matrix.module.path }} + run: task go:test + + - name: Send unit tests coverage to Codecov + if: runner.os == 'Linux' + uses: codecov/codecov-action@v2 + with: + file: ${{ matrix.module.path }}coverage_unit.txt + flags: ${{ matrix.module.codecov-flags }} + fail_ci_if_error: ${{ github.repository == 'arduino/arduino-fwuploader' }} diff --git a/README.md b/README.md index b234b8c2..683a2ffb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,9 @@ The Arduino Firmware Uploader is a tool made to update the firmware and/or add SSL certificates for any Arduino board equipped with WINC or NINA Wi-Fi module. -[![tests-badge]](https://github.com/arduino/arduino-fwuploader/actions/workflows/test.yaml) +[![Test Go status](https://github.com/arduino/arduino-fwuploader/actions/workflows/test-go-task.yml/badge.svg)](https://github.com/arduino/arduino-fwuploader/actions/workflows/test-go-task.yml) +[![Codecov](https://codecov.io/gh/arduino/arduino-fwuploader/branch/main/graph/badge.svg)](https://codecov.io/gh/arduino/arduino-fwuploader) +[![Test Integration status](https://github.com/arduino/arduino-fwuploader/actions/workflows/test-go-integration-task.yml/badge.svg)](https://github.com/arduino/arduino-fwuploader/actions/workflows/test-go-integration-task.yml) [![Deploy Website status](https://github.com/arduino/arduino-fwuploader/actions/workflows/deploy-cobra-mkdocs-versioned-poetry.yml/badge.svg)](https://github.com/arduino/arduino-fwuploader/actions/workflows/deploy-cobra-mkdocs-versioned-poetry.yml) ## ❗❗❗Notice regarding versions before 2.0.0 ❗❗❗ @@ -28,7 +30,6 @@ and report the bug to our Security Team 🛡️ Thank you! e-mail contact: security@arduino.cc -[tests-badge]: https://github.com/arduino/arduino-fwuploader/actions/workflows/test.yaml/badge.svg [security policy]: https://github.com/arduino/arduino-fwuploader/security/policy [user documentation]: https://arduino.github.io/arduino-fwuploader/ [install]: https://arduino.github.io/arduino-fwuploader/latest/installation diff --git a/Taskfile.yml b/Taskfile.yml index 4edafe9a..1c4ff1e2 100755 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -131,27 +131,47 @@ tasks: cmds: - npx markdownlint-cli "**/*.md" - build: - desc: Build the project + # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/go-task/Taskfile.yml + go:build: + desc: Build the Go code + dir: "{{.DEFAULT_GO_MODULE_PATH}}" cmds: - go build -v {{.LDFLAGS}} + build: + desc: Build the project + deps: + - task: go:build + test: desc: Run tests cmds: - task: test-unit - - task: test-integration + - task: go:test-integration - test-unit: + # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/test-go-task/Taskfile.yml + go:test: desc: Run unit tests + dir: "{{default .DEFAULT_GO_MODULE_PATH .GO_MODULE_PATH}}" cmds: - - go test -short -race -run '.*' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt ./... {{.TEST_LDFLAGS}} + - | + go test \ + -v \ + -short \ + -race \ + -run '{{default ".*" .GO_TEST_REGEX}}' \ + {{default "-timeout 10m -coverpkg=./... -covermode=atomic" .GO_TEST_FLAGS}} \ + -coverprofile=coverage_unit.txt \ + {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}} \ + {{.TEST_LDFLAGS}} - test-integration: + # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/test-go-integration-task/Taskfile.yml + go:test-integration: desc: Run integration tests - cmds: - - task: build + deps: + - task: go:build - task: poetry:install-deps + cmds: - poetry run pytest test check: @@ -235,7 +255,6 @@ vars: -X {{.CONFIGURATION_PACKAGE}}.date={{.TIMESTAMP}} ' # test vars - GOFLAGS: "-timeout 10m -v -coverpkg=./... -covermode=atomic" TEST_VERSION: "0.0.0-test.preview" TEST_COMMIT: "deadbeef" TEST_LDFLAGS: > diff --git a/poetry.lock b/poetry.lock index c36d554e..104e5ac6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -10,7 +10,7 @@ python-versions = "*" name = "atomicwrites" version = "1.4.0" description = "Atomic file writes." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -18,7 +18,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "21.2.0" description = "Classes Without Boilerplate" -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -62,7 +62,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -146,7 +146,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes name = "iniconfig" version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" -category = "main" +category = "dev" optional = false python-versions = "*" @@ -154,7 +154,7 @@ python-versions = "*" name = "invoke" version = "1.5.0" description = "Pythonic task execution" -category = "main" +category = "dev" optional = false python-versions = "*" @@ -297,7 +297,7 @@ python-versions = "*" name = "packaging" version = "20.9" description = "Core utilities for Python packages" -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -328,7 +328,7 @@ flake8-polyfill = ">=1.0.2,<2" name = "pluggy" version = "0.13.1" description = "plugin and hook calling mechanisms for python" -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -339,7 +339,7 @@ dev = ["pre-commit", "tox"] name = "py" version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -382,30 +382,29 @@ Markdown = ">=3.2" name = "pyparsing" version = "2.4.7" description = "Python parsing module" -category = "main" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.2" +version = "6.2.4" description = "pytest: simple powerful testing with Python" -category = "main" +category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" +pluggy = ">=0.12,<1.0.0a1" py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -474,7 +473,7 @@ python-versions = ">=3.5" name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -531,7 +530,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8cdb16153f023963c24d5500ce2cb73a213f57cc238c45cb8edee06d81703316" +content-hash = "f67945001c73d3dbfa0ff0e5971c4cbb2b1b38a36f6534dcd7e395224889f73e" [metadata.files] appdirs = [ @@ -707,8 +706,8 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, - {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, + {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"}, + {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"}, ] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, diff --git a/pyproject.toml b/pyproject.toml index 4f2439ce..73032237 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,6 @@ authors = ["Arduino "] [tool.poetry.dependencies] python = "^3.8" -pytest = "6.1.2" -invoke = "^1.5.0" semver = "^2.13.0" python-dateutil = "^2.8.1" @@ -20,6 +18,8 @@ mdx-truly-sane-lists = "^1.2" GitPython = "^3.1.20" mike = "^1.0.1" pep8-naming = "^0.12.1" +invoke = "^1.5.0" +pytest = "^6.2.4" [tool.black] line-length = 120 diff --git a/test/conftest.py b/test/conftest.py index 28b4cf24..dd1bcb85 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,6 @@ -# arduino-fwuploader +# Source: +# https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/test-integration/test_all.py + # Copyright (c) 2021 Arduino LLC. All right reserved. # This library is free software; you can redistribute it and/or @@ -15,8 +17,10 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os import pathlib import platform +import shutil import typing import invoke.context import pytest @@ -30,10 +34,12 @@ def run_command(pytestconfig, working_dir) -> typing.Callable[..., invoke.runner http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Result """ - fwuploader_path = pathlib.Path(pytestconfig.rootdir).parent / "arduino-fwuploader" + executable_path = pathlib.Path(pytestconfig.rootdir).parent / "arduino-fwuploader" def _run( - cmd: list, custom_working_dir: typing.Optional[str] = None, custom_env: typing.Optional[dict] = None + cmd: list, + custom_working_dir: typing.Optional[str] = None, + custom_env: typing.Optional[dict] = None, ) -> invoke.runners.Result: if cmd is None: cmd = [] @@ -42,7 +48,7 @@ def _run( quoted_cmd = [] for token in cmd: quoted_cmd.append(f'"{token}"') - cli_full_line = '"{}" {}'.format(fwuploader_path, " ".join(quoted_cmd)) + cli_full_line = '"{}" {}'.format(executable_path, " ".join(quoted_cmd)) run_context = invoke.context.Context() # It might happen that we need to change directories between drives on Windows, # in that case the "/d" flag must be used otherwise directory wouldn't change @@ -54,16 +60,22 @@ def _run( # wrapping the path in quotation marks is the safest approach with run_context.prefix(f'{cd_command} "{custom_working_dir}"'): return run_context.run( - command=cli_full_line, echo=False, hide=True, warn=True, env=custom_env, encoding="utf-8" + command=cli_full_line, + echo=False, + hide=True, + warn=True, + env=custom_env, + encoding="utf-8", ) return _run @pytest.fixture(scope="function") -def working_dir(tmpdir_factory): +def working_dir(tmpdir_factory) -> str: """Create a temporary folder for the test to run in. It will be created before running each test and deleted at the end. This way all the tests work in isolation. """ - work_dir = tmpdir_factory.mktemp(basename="FirmwareUploaderTestWork") - yield str(work_dir) + work_dir = tmpdir_factory.mktemp(basename="IntegrationTestWorkingDir") + yield os.path.realpath(work_dir) + shutil.rmtree(work_dir, ignore_errors=True) diff --git a/test/pytest.ini b/test/pytest.ini index d3f2009f..b8beed3f 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,4 @@ +# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/test-python/pytest.ini [pytest] filterwarnings = error