test: add ability to run EF spec tests

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
     - name: Install dependencies
       run: mix deps.get
     - name: Run tests
-      run: mix test
+      run: mix test --trace --exclude spectest
 
   lint:
     name: Lint
@@ -139,3 +139,60 @@ jobs:
       run: mix format --check-formatted
     - name: Run credo
       run: mix credo --strict
+
+  download-spectests:
+    name: Download spectests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Cache compressed spectests
+        id: output-cache
+        uses: actions/cache@v3
+        with:
+          path: ./*.tar.gz
+          key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }}
+          lookup-only: true
+      - name: Download spectests
+        if: steps.output-cache.outputs.cache-hit != 'true'
+        run: make download-vectors
+
+  spectests:
+    name: Run spec-tests
+    needs: [compile-native, download-spectests]
+    strategy:
+      matrix:
+        config: ["minimal"] #[ "general", "mainnet", "minimal" ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          version-type: strict
+          version-file: .tool-versions
+        env:
+          ImageOS: ubuntu20
+      - name: Fetch native libraries
+        uses: actions/cache/restore@v3
+        with:
+          path: priv/native/*.so
+          key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
+          fail-on-cache-miss: true
+      - name: Fetch spectest vectors
+        uses: actions/cache/restore@v3
+        with:
+          path: ./*.tar.gz
+          key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }}
+          fail-on-cache-miss: true
+      - name: Restore dependencies cache
+        uses: actions/cache@v3
+        with:
+          path: deps
+          key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
+          restore-keys: ${{ runner.os }}-mix-
+      - name: Install dependencies
+        run: mix deps.get
+      - name: Uncompress vectors
+        run: make tests/${{ matrix.config }}
+      - name: Run tests
+        run: mix test --trace --only spectest

.gitignore

@@ -43,3 +43,6 @@ native/libp2p_nif/main.h
 .vscode/
 
 !libp2p/utils.h
+
+tests
+*.tar.gz

.spectest_version

@@ -0,0 +1 @@
+v1.3.0

Makefile

@@ -1,4 +1,10 @@
-.PHONY: iex deps test clean compile-native
+.PHONY: iex deps test clean compile-native \
+		clean-vectors download-vectors uncompress-vectors
+
+# Delete current file when command fails
+.DELETE_ON_ERROR:
+
+##### NATIVE COMPILATION #####
 
 # magic from sym_num https://elixirforum.com/t/where-is-erl-nif-h-header-file-required-for-nif/27142/5
 ERLANG_INCLUDES := $(shell erl -eval 'io:format("~s", \
@@ -30,6 +36,32 @@ $(OUTPUT_DIR)/libp2p_nif.so: $(GO_ARCHIVES) $(GO_HEADERS) $(LIBP2P_DIR)/libp2p.c
 	gcc $(CFLAGS) -o $@ \
 		$(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c $(GO_ARCHIVES)
 
+
+##### SPEC TEST VECTORS #####
+
+SPECTEST_VERSION = $(shell cat .spectest_version)
+SPECTEST_CONFIGS = general minimal mainnet
+
+SPECTEST_DIRS = $(patsubst %,tests/%,$(SPECTEST_CONFIGS))
+SPECTEST_TARS = $(patsubst %,%_${SPECTEST_VERSION}.tar.gz,$(SPECTEST_CONFIGS))
+
+%_${SPECTEST_VERSION}.tar.gz:
+	curl -L -o "$@" \
+		"https://github.com/ethereum/consensus-spec-tests/releases/download/${SPECTEST_VERSION}/$*.tar.gz"
+
+tests/%: %_${SPECTEST_VERSION}.tar.gz
+	-rm -rf $@
+	tar -xzmf  "$<"
+
+download-vectors: $(SPECTEST_TARS)
+
+clean-vectors:
+	-rm -rf tests
+	-rm *.tar.gz
+
+
+##### TARGETS #####
+
 clean:
 	-rm $(GO_ARCHIVES) $(GO_HEADERS) $(OUTPUT_DIR)/*
 
@@ -46,7 +78,10 @@ deps:
 
 # Run tests
 test: compile-native
-	mix test
+	mix test --exclude spectest
+
+spec-test: compile-native tests/minimal #$(SPECTEST_DIRS)
+	mix test --only implemented_spectest
 
 lint:
 	mix format --check-formatted

mix.exs

@@ -9,6 +9,8 @@ defmodule LambdaEthereumConsensus.MixProject do
       start_permanent: Mix.env() == :prod,
       deps: deps(),
       dialyzer: dialyzer(),
+      elixirc_paths: compiler_paths(Mix.env()),
+      warn_test_pattern: "_remove_warning.exs",
       preferred_cli_env: [
         dialyzer: :test
       ]
@@ -27,15 +29,22 @@ defmodule LambdaEthereumConsensus.MixProject do
   defp deps do
     [
       {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false},
+      {:yaml_elixir, "~> 2.8", only: [:test]},
+      {:snappyer, "~> 1.2", only: [:test]},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
       {:stream_data, "~> 0.6", only: :test},
       {:rustler, "~> 0.29.1"}
     ]
   end
 
-  defp dialyzer() do
+  defp dialyzer do
     [
+      # https://elixirforum.com/t/help-with-dialyzer-output/15202/5
+      plt_add_apps: [:ex_unit],
       plt_file: {:no_warn, "priv/plts/project.plt"}
     ]
   end
+
+  defp compiler_paths(:test), do: ["test/spec"] ++ compiler_paths(:prod)
+  defp compiler_paths(_), do: ["lib"]
 end

mix.lock

@@ -6,6 +6,9 @@
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"},
+  "snappyer": {:hex, :snappyer, "1.2.9", "9cc58470798648ce34c662ca0aa6daae31367667714c9a543384430a3586e5d3", [:rebar3], [], "hexpm", "18d00ca218ae613416e6eecafe1078db86342a66f86277bd45c95f05bf1c8b29"},
   "stream_data": {:hex, :stream_data, "0.6.0", "e87a9a79d7ec23d10ff83eb025141ef4915eeb09d4491f79e52f2562b73e5f47", [:mix], [], "hexpm", "b92b5031b650ca480ced047578f1d57ea6dd563f5b57464ad274718c9c29501c"},
   "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
+  "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
+  "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"},
 }

test/libp2p_test.exs

@@ -147,7 +147,7 @@ defmodule Libp2pTest do
     end
   end
 
-  defp read_stream(_) do
+  defp read_stream(_, _writing_fun) do
     :ok
   end
 

test/spec/general.ex

@@ -0,0 +1,9 @@
+defmodule GeneralSpecTest do
+  @moduledoc """
+  "general" config spec tests
+  """
+  use ExUnit.Case, async: true
+  require SpecTestUtils
+
+  SpecTestUtils.generate_tests("general")
+end

test/spec/mainnet.ex

@@ -0,0 +1,9 @@
+defmodule MainnetSpecTest do
+  @moduledoc """
+  "mainnet" config spec tests
+  """
+  use ExUnit.Case, async: true
+  require SpecTestUtils
+
+  SpecTestUtils.generate_tests("mainnet")
+end

test/spec/minimal_cappela.ex

@@ -0,0 +1,9 @@
+defmodule MinimalCapellaSpecTest do
+  @moduledoc """
+  "minimal" config spec tests for the "capella" fork
+  """
+  use ExUnit.Case, async: true
+  require SpecTestUtils
+
+  SpecTestUtils.generate_tests("minimal", "capella")
+end

test/spec/runners/ssz.ex

@@ -0,0 +1,35 @@
+defmodule SSZTestRunner do
+  use ExUnit.CaseTemplate
+
+  @moduledoc """
+  Runner for SSZ test cases. `run_test_case/1` is the main entrypoint.
+  """
+
+  @doc """
+  Returns true if the given testcase should be skipped
+  """
+  def skip?(_testcase) do
+    # add SSZ test case skipping here
+    true
+  end
+
+  @doc """
+  Runs the given test case.
+  """
+  def run_test_case(testcase = %SpecTestCase{}) do
+    case_dir = SpecTestCase.dir(testcase)
+
+    compressed = File.read!(case_dir <> "/serialized.ssz_snappy")
+    assert {:ok, decompressed} = :snappyer.decompress(compressed)
+
+    expected = YamlElixir.read_from_file!(case_dir <> "/value.yaml")
+    expected_root = YamlElixir.read_from_file!(case_dir <> "/roots.yaml")
+
+    assert_ssz(decompressed, expected, expected_root)
+  end
+
+  defp assert_ssz(_serialized, _expected, _expected_root) do
+    # add SSZ comparison here
+    assert false
+  end
+end

test/spec/utils/testcase.ex

@@ -0,0 +1,40 @@
+defmodule SpecTestCase do
+  @moduledoc """
+  Helper methods for deriving test case metadata.
+  """
+  @enforce_keys [:config, :fork, :runner, :handler, :suite, :case]
+  defstruct [:config, :fork, :runner, :handler, :suite, :case]
+
+  def new([config, fork, runner, handler, suite, cse]) do
+    %SpecTestCase{
+      config: config,
+      fork: fork,
+      runner: runner,
+      handler: handler,
+      suite: suite,
+      case: cse
+    }
+  end
+
+  def name(%SpecTestCase{
+        config: config,
+        fork: fork,
+        runner: runner,
+        handler: handler,
+        suite: suite,
+        case: cse
+      }) do
+    "c:#{config} f:#{fork} r:#{runner} h:#{handler} s:#{suite} -> #{cse}"
+  end
+
+  def dir(%SpecTestCase{
+        config: config,
+        fork: fork,
+        runner: runner,
+        handler: handler,
+        suite: suite,
+        case: cse
+      }) do
+    "tests/#{config}/#{fork}/#{runner}/#{handler}/#{suite}/#{cse}"
+  end
+end

test/spec/utils/utils.ex

@@ -0,0 +1,79 @@
+defmodule SpecTestUtils do
+  @moduledoc """
+  Utilities for running the spec tests.
+  """
+  @all_cases ["tests"]
+             |> Stream.concat(["*"] |> Stream.cycle() |> Stream.take(6))
+             |> Enum.join("/")
+             |> Path.wildcard()
+             |> Stream.map(&Path.relative_to(&1, "tests"))
+             |> Stream.map(&Path.split/1)
+             |> Enum.map(&SpecTestCase.new/1)
+
+  @runner_map %{
+    "ssz_static" => SSZTestRunner
+  }
+
+  def all_cases, do: @all_cases
+  def runner_map, do: @runner_map
+
+  # To filter tests, use:
+  #  (only spectests) ->
+  #   mix test --only spectest
+  #  (only general) ->
+  #   mix test --only config:general
+  #  (only ssz_generic) ->
+  #   mix test --only runner:ssz_generic
+  #  (one specific test) ->
+  #   mix test --only test:"test c:`config` f:`fork` r:`runner h:`handler` s:suite` -> `case`"
+  #
+  # Tests are too many to run all at the same time. We should pin a
+  # `config` (and `fork` in the case of `minimal`).
+  defmacro generate_tests(pinned_config, pinned_fork \\ "") do
+    quote bind_quoted: [
+            pinned_config: pinned_config,
+            pinned_fork: pinned_fork
+          ] do
+      paths = Path.wildcard("tests/#{pinned_config}/#{pinned_fork}/**")
+      paths_hash = :erlang.md5(paths)
+
+      # Recompile module only if corresponding dir layout changed
+      def __mix_recompile__? do
+        Path.wildcard(unquote("tests/#{pinned_config}/#{pinned_fork}/**"))
+        |> :erlang.md5()
+      end
+
+      for testcase <- SpecTestUtils.all_cases(),
+          pinned_fork in [testcase.fork, ""],
+          testcase.config == pinned_config do
+        test_name = SpecTestCase.name(testcase)
+
+        test_runner = Map.get(SpecTestUtils.runner_map(), testcase.runner)
+
+        @tag :spectest
+        @tag config: testcase.config, fork: testcase.fork
+        @tag runner: testcase.runner, suite: testcase.suite
+        if test_runner == nil do
+          @tag :skip
+          test test_name
+        else
+          if test_runner.skip?(testcase) do
+            @tag :skip
+          end
+
+          # Register the test case as ran in the runner
+          runner_file =
+            test_runner.__info__(:compile)
+            |> Keyword.get(:source)
+            |> to_string()
+
+          @file runner_file
+          @tag :implemented_spectest
+          test test_name do
+            unquote(test_runner).run_test_case(unquote(Macro.escape(testcase)))
+          end
+        end
+      end
+    end
+  end
+end

test/test_helper.exs

@@ -1 +1,13 @@
 ExUnit.start()
+
+# Load all modules as ExUnit tests (needed because we use .ex files)
+# Copied from https://github.com/elixir-lang/elixir/issues/10983#issuecomment-1133554155
+for module <- Application.spec(Mix.Project.config()[:app], :modules) do
+  ex_unit = Keyword.get(module.module_info(:attributes), :ex_unit_async, [])
+
+  cond do
+    true in ex_unit -> ExUnit.Server.add_async_module(module)
+    false in ex_unit -> ExUnit.Server.add_sync_module(module)
+    true -> :ok
+  end
+end