diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a343697f0..9784df590 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,7 +113,7 @@ jobs: - name: Install dependencies run: mix deps.get - name: Run tests - run: mix test + run: mix test --trace --exclude spectest lint: name: Lint @@ -139,3 +139,60 @@ jobs: run: mix format --check-formatted - name: Run credo run: mix credo --strict + + download-spectests: + name: Download spectests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Cache compressed spectests + id: output-cache + uses: actions/cache@v3 + with: + path: ./*.tar.gz + key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }} + lookup-only: true + - name: Download spectests + if: steps.output-cache.outputs.cache-hit != 'true' + run: make download-vectors + + spectests: + name: Run spec-tests + needs: [compile-native, download-spectests] + strategy: + matrix: + config: ["minimal"] #[ "general", "mainnet", "minimal" ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + version-type: strict + version-file: .tool-versions + env: + ImageOS: ubuntu20 + - name: Fetch native libraries + uses: actions/cache/restore@v3 + with: + path: priv/native/*.so + key: ${{ runner.os }}-native-${{ hashFiles('native/**') }} + fail-on-cache-miss: true + - name: Fetch spectest vectors + uses: actions/cache/restore@v3 + with: + path: ./*.tar.gz + key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }} + fail-on-cache-miss: true + - name: Restore dependencies cache + uses: actions/cache@v3 + with: + path: deps + key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-mix- + - name: Install dependencies + run: mix deps.get + - name: Uncompress vectors + run: make tests/${{ matrix.config }} + - name: Run tests + run: mix test --trace --only spectest diff --git a/.gitignore b/.gitignore index 15d0f6808..297c3aa21 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ native/libp2p_nif/main.h .vscode/ !libp2p/utils.h + +tests +*.tar.gz diff --git a/.spectest_version b/.spectest_version new file mode 100644 index 000000000..18fa8e74f --- /dev/null +++ b/.spectest_version @@ -0,0 +1 @@ +v1.3.0 diff --git a/Makefile b/Makefile index e06d8bcc8..941dc21e6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,10 @@ -.PHONY: iex deps test clean compile-native +.PHONY: iex deps test clean compile-native \ + clean-vectors download-vectors uncompress-vectors + +# Delete current file when command fails +.DELETE_ON_ERROR: + +##### NATIVE COMPILATION ##### # magic from sym_num https://elixirforum.com/t/where-is-erl-nif-h-header-file-required-for-nif/27142/5 ERLANG_INCLUDES := $(shell erl -eval 'io:format("~s", \ @@ -30,6 +36,32 @@ $(OUTPUT_DIR)/libp2p_nif.so: $(GO_ARCHIVES) $(GO_HEADERS) $(LIBP2P_DIR)/libp2p.c gcc $(CFLAGS) -o $@ \ $(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c $(GO_ARCHIVES) + +##### SPEC TEST VECTORS ##### + +SPECTEST_VERSION = $(shell cat .spectest_version) +SPECTEST_CONFIGS = general minimal mainnet + +SPECTEST_DIRS = $(patsubst %,tests/%,$(SPECTEST_CONFIGS)) +SPECTEST_TARS = $(patsubst %,%_${SPECTEST_VERSION}.tar.gz,$(SPECTEST_CONFIGS)) + +%_${SPECTEST_VERSION}.tar.gz: + curl -L -o "$@" \ + "https://github.com/ethereum/consensus-spec-tests/releases/download/${SPECTEST_VERSION}/$*.tar.gz" + +tests/%: %_${SPECTEST_VERSION}.tar.gz + -rm -rf $@ + tar -xzmf "$<" + +download-vectors: $(SPECTEST_TARS) + +clean-vectors: + -rm -rf tests + -rm *.tar.gz + + +##### TARGETS ##### + clean: -rm $(GO_ARCHIVES) $(GO_HEADERS) $(OUTPUT_DIR)/* @@ -46,7 +78,10 @@ deps: # Run tests test: compile-native - mix test + mix test --exclude spectest + +spec-test: compile-native tests/minimal #$(SPECTEST_DIRS) + mix test --only implemented_spectest lint: mix format --check-formatted diff --git a/mix.exs b/mix.exs index 3173cceab..84beb3c1e 100644 --- a/mix.exs +++ b/mix.exs @@ -9,6 +9,8 @@ defmodule LambdaEthereumConsensus.MixProject do start_permanent: Mix.env() == :prod, deps: deps(), dialyzer: dialyzer(), + elixirc_paths: compiler_paths(Mix.env()), + warn_test_pattern: "_remove_warning.exs", preferred_cli_env: [ dialyzer: :test ] @@ -27,15 +29,22 @@ defmodule LambdaEthereumConsensus.MixProject do defp deps do [ {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false}, + {:yaml_elixir, "~> 2.8", only: [:test]}, + {:snappyer, "~> 1.2", only: [:test]}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:stream_data, "~> 0.6", only: :test}, {:rustler, "~> 0.29.1"} ] end - defp dialyzer() do + defp dialyzer do [ + # https://elixirforum.com/t/help-with-dialyzer-output/15202/5 + plt_add_apps: [:ex_unit], plt_file: {:no_warn, "priv/plts/project.plt"} ] end + + defp compiler_paths(:test), do: ["test/spec"] ++ compiler_paths(:prod) + defp compiler_paths(_), do: ["lib"] end diff --git a/mix.lock b/mix.lock index 027972fc2..71e09af48 100644 --- a/mix.lock +++ b/mix.lock @@ -6,6 +6,9 @@ "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"}, + "snappyer": {:hex, :snappyer, "1.2.9", "9cc58470798648ce34c662ca0aa6daae31367667714c9a543384430a3586e5d3", [:rebar3], [], "hexpm", "18d00ca218ae613416e6eecafe1078db86342a66f86277bd45c95f05bf1c8b29"}, "stream_data": {:hex, :stream_data, "0.6.0", "e87a9a79d7ec23d10ff83eb025141ef4915eeb09d4491f79e52f2562b73e5f47", [:mix], [], "hexpm", "b92b5031b650ca480ced047578f1d57ea6dd563f5b57464ad274718c9c29501c"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, } diff --git a/test/libp2p_test.exs b/test/libp2p_test.exs index 63543ea8b..a45c3203f 100644 --- a/test/libp2p_test.exs +++ b/test/libp2p_test.exs @@ -147,7 +147,7 @@ defmodule Libp2pTest do end end - defp read_stream(_) do + defp read_stream(_, _writing_fun) do :ok end diff --git a/test/spec/general.ex b/test/spec/general.ex new file mode 100644 index 000000000..d98db601d --- /dev/null +++ b/test/spec/general.ex @@ -0,0 +1,9 @@ +defmodule GeneralSpecTest do + @moduledoc """ + "general" config spec tests + """ + use ExUnit.Case, async: true + require SpecTestUtils + + SpecTestUtils.generate_tests("general") +end diff --git a/test/spec/mainnet.ex b/test/spec/mainnet.ex new file mode 100644 index 000000000..43714bb1f --- /dev/null +++ b/test/spec/mainnet.ex @@ -0,0 +1,9 @@ +defmodule MainnetSpecTest do + @moduledoc """ + "mainnet" config spec tests + """ + use ExUnit.Case, async: true + require SpecTestUtils + + SpecTestUtils.generate_tests("mainnet") +end diff --git a/test/spec/minimal_cappela.ex b/test/spec/minimal_cappela.ex new file mode 100644 index 000000000..1ff5db457 --- /dev/null +++ b/test/spec/minimal_cappela.ex @@ -0,0 +1,9 @@ +defmodule MinimalCapellaSpecTest do + @moduledoc """ + "minimal" config spec tests for the "capella" fork + """ + use ExUnit.Case, async: true + require SpecTestUtils + + SpecTestUtils.generate_tests("minimal", "capella") +end diff --git a/test/spec/runners/ssz.ex b/test/spec/runners/ssz.ex new file mode 100644 index 000000000..d00bfcf32 --- /dev/null +++ b/test/spec/runners/ssz.ex @@ -0,0 +1,35 @@ +defmodule SSZTestRunner do + use ExUnit.CaseTemplate + + @moduledoc """ + Runner for SSZ test cases. `run_test_case/1` is the main entrypoint. + """ + + @doc """ + Returns true if the given testcase should be skipped + """ + def skip?(_testcase) do + # add SSZ test case skipping here + true + end + + @doc """ + Runs the given test case. + """ + def run_test_case(testcase = %SpecTestCase{}) do + case_dir = SpecTestCase.dir(testcase) + + compressed = File.read!(case_dir <> "/serialized.ssz_snappy") + assert {:ok, decompressed} = :snappyer.decompress(compressed) + + expected = YamlElixir.read_from_file!(case_dir <> "/value.yaml") + expected_root = YamlElixir.read_from_file!(case_dir <> "/roots.yaml") + + assert_ssz(decompressed, expected, expected_root) + end + + defp assert_ssz(_serialized, _expected, _expected_root) do + # add SSZ comparison here + assert false + end +end diff --git a/test/spec/utils/testcase.ex b/test/spec/utils/testcase.ex new file mode 100644 index 000000000..7e7ba605f --- /dev/null +++ b/test/spec/utils/testcase.ex @@ -0,0 +1,40 @@ +defmodule SpecTestCase do + @moduledoc """ + Helper methods for deriving test case metadata. + """ + @enforce_keys [:config, :fork, :runner, :handler, :suite, :case] + defstruct [:config, :fork, :runner, :handler, :suite, :case] + + def new([config, fork, runner, handler, suite, cse]) do + %SpecTestCase{ + config: config, + fork: fork, + runner: runner, + handler: handler, + suite: suite, + case: cse + } + end + + def name(%SpecTestCase{ + config: config, + fork: fork, + runner: runner, + handler: handler, + suite: suite, + case: cse + }) do + "c:#{config} f:#{fork} r:#{runner} h:#{handler} s:#{suite} -> #{cse}" + end + + def dir(%SpecTestCase{ + config: config, + fork: fork, + runner: runner, + handler: handler, + suite: suite, + case: cse + }) do + "tests/#{config}/#{fork}/#{runner}/#{handler}/#{suite}/#{cse}" + end +end diff --git a/test/spec/utils/utils.ex b/test/spec/utils/utils.ex new file mode 100644 index 000000000..b3f5ae40d --- /dev/null +++ b/test/spec/utils/utils.ex @@ -0,0 +1,79 @@ +defmodule SpecTestUtils do + @moduledoc """ + Utilities for running the spec tests. + """ + @all_cases ["tests"] + |> Stream.concat(["*"] |> Stream.cycle() |> Stream.take(6)) + |> Enum.join("/") + |> Path.wildcard() + |> Stream.map(&Path.relative_to(&1, "tests")) + |> Stream.map(&Path.split/1) + |> Enum.map(&SpecTestCase.new/1) + + @runner_map %{ + "ssz_static" => SSZTestRunner + } + + def all_cases, do: @all_cases + def runner_map, do: @runner_map + + # To filter tests, use: + # (only spectests) -> + # mix test --only spectest + # (only general) -> + # mix test --only config:general + # (only ssz_generic) -> + # mix test --only runner:ssz_generic + # (one specific test) -> + # mix test --only test:"test c:`config` f:`fork` r:`runner h:`handler` s:suite` -> `case`" + # + # Tests are too many to run all at the same time. We should pin a + # `config` (and `fork` in the case of `minimal`). + defmacro generate_tests(pinned_config, pinned_fork \\ "") do + quote bind_quoted: [ + pinned_config: pinned_config, + pinned_fork: pinned_fork + ] do + paths = Path.wildcard("tests/#{pinned_config}/#{pinned_fork}/**") + paths_hash = :erlang.md5(paths) + + # Recompile module only if corresponding dir layout changed + def __mix_recompile__? do + Path.wildcard(unquote("tests/#{pinned_config}/#{pinned_fork}/**")) + |> :erlang.md5() + end + + for testcase <- SpecTestUtils.all_cases(), + pinned_fork in [testcase.fork, ""], + testcase.config == pinned_config do + test_name = SpecTestCase.name(testcase) + + test_runner = Map.get(SpecTestUtils.runner_map(), testcase.runner) + + @tag :spectest + @tag config: testcase.config, fork: testcase.fork + @tag runner: testcase.runner, suite: testcase.suite + if test_runner == nil do + @tag :skip + test test_name + else + if test_runner.skip?(testcase) do + @tag :skip + end + + # Register the test case as ran in the runner + runner_file = + test_runner.__info__(:compile) + |> Keyword.get(:source) + |> to_string() + + @file runner_file + @tag :implemented_spectest + test test_name do + unquote(test_runner).run_test_case(unquote(Macro.escape(testcase))) + end + end + end + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e70..b2a1775b9 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,13 @@ ExUnit.start() + +# Load all modules as ExUnit tests (needed because we use .ex files) +# Copied from https://github.com/elixir-lang/elixir/issues/10983#issuecomment-1133554155 +for module <- Application.spec(Mix.Project.config()[:app], :modules) do + ex_unit = Keyword.get(module.module_info(:attributes), :ex_unit_async, []) + + cond do + true in ex_unit -> ExUnit.Server.add_async_module(module) + false in ex_unit -> ExUnit.Server.add_sync_module(module) + true -> :ok + end +end