From eac1b66e73da2f72972964baf1d5e358a74b78bc Mon Sep 17 00:00:00 2001 From: Godspower Eze <61994334+Godspower-Eze@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:22:45 +0100 Subject: [PATCH 01/31] Fix spec-test runner command --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 424a827b8..fdea9d514 100644 --- a/README.md +++ b/README.md @@ -127,9 +127,9 @@ Or by a single runner in all configs, with: make spec-test-runner-`runner` # Some examples -make spec-test-config-ssz_static -make spec-test-config-bls -make spec-test-config-operations +make spec-test-runner-ssz_static +make spec-test-runner-bls +make spec-test-runner-operations ``` The complete list of test runners can be found [here](https://github.com/ethereum/consensus-specs/tree/dev/tests/formats). From 16d24cc983bcd1c0cc1877dd286f2b5d1dcde641 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Fri, 15 Dec 2023 00:40:55 +0100 Subject: [PATCH 02/31] feat: added packing for merklelization of list --- lib/ssz_ex.ex | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 8ff625942..2eb96450f 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -407,9 +407,21 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.any?() end + defp size_of(value) when is_integer(value) and value >= 0 do + value |> :binary.encode_unsigned() |> byte_size() + end + + # NOTE: + # - When the elements of the list is an uint then it is a basic list or basic vector + # - When the elements of the list is a boolean then it is a bit vector + defp chunk_count([head | _tail] = list) when is_integer(head) do + size = size_of(head) + length = length(list) + div(length * size + 31, 32) + end + defp pack(value, size) when is_integer(value) and value >= 0 do - pad = @bits_per_chunk - size - <> + <> |> pack_bytes() end defp pack(value) when is_boolean(value) do @@ -418,4 +430,22 @@ defmodule LambdaEthereumConsensus.SszEx do false -> <<0::@bits_per_chunk>> end end + + defp pack([head | _tail] = list) when is_integer(head) do + list + |> Enum.map(fn x -> + x |> pack(size_of(x)) + end) + end + + defp pack_bytes(value) when is_binary(value) do + incomplete_chunk_len = value |> bit_size() |> rem(@bits_per_chunk) + + if incomplete_chunk_len != 0 do + pad = @bits_per_chunk - incomplete_chunk_len + <> + else + value + end + end end From fcdc6ce4e980f0077ee4064380626dfe66f8a926 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Sun, 17 Dec 2023 22:39:29 +0100 Subject: [PATCH 03/31] feat: added next_pow_of_two --- lib/ssz_ex.ex | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 2eb96450f..b37467e73 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -6,8 +6,7 @@ defmodule LambdaEthereumConsensus.SszEx do ################# ### Public API ################# - - @bits_per_chunk 256 + import Bitwise @spec hash(iodata()) :: binary() def hash(data), do: :crypto.hash(:sha256, data) @@ -44,9 +43,12 @@ defmodule LambdaEthereumConsensus.SszEx do @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() def hash_tree_root!(value, {:int, size}), do: pack(value, size) + def hash_tree_root!(value) + ################# ### Private functions ################# + @bits_per_chunk 256 @bytes_per_length_offset 4 @bits_per_byte 8 @offset_bits 32 @@ -406,6 +408,13 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.map(fn {_, schema} -> variable_size?(schema) end) |> Enum.any?() end + + # NOTE: + # - chunks is a list of bytes + # - limit is the max size of the list + defp merklelize(chunks, limit // nil) do + + end defp size_of(value) when is_integer(value) and value >= 0 do value |> :binary.encode_unsigned() |> byte_size() @@ -448,4 +457,18 @@ defmodule LambdaEthereumConsensus.SszEx do value end end + + defp next_pow_of_two(len) when is_integer(len) and len >= 0 do + cond do + len == 0 or len == 1 -> + 1 + + len == 2 -> + 2 + + true -> + n = ((len <<< 1) - 1) |> :math.log2() |> trunc() + 2 ** n + end + end end From ac52732d77dd66333bc0c8a383303431b12fa66b Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 19 Dec 2023 00:20:01 +0100 Subject: [PATCH 04/31] feat: implemented list packing --- lib/spec/runners/ssz_generic.ex | 4 ++++ lib/ssz_ex.ex | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex index 28cb8a00b..236bc0594 100644 --- a/lib/spec/runners/ssz_generic.ex +++ b/lib/spec/runners/ssz_generic.ex @@ -89,6 +89,10 @@ defmodule SszGenericTestRunner do actual_hash_tree_root = SszEx.hash_tree_root!(real_deserialized, schema) assert actual_hash_tree_root == expected_hash_tree_root + + list = [{1, 8}, {2, 8}, {3, 8}, {3, 8}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}] + + SszEx.pack(list) |> IO.inspect(limit: :infinity) end defp assert_ssz("invalid", schema, real_serialized) do diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index b37467e73..f3999859f 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -43,12 +43,11 @@ defmodule LambdaEthereumConsensus.SszEx do @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() def hash_tree_root!(value, {:int, size}), do: pack(value, size) - def hash_tree_root!(value) - ################# ### Private functions ################# @bits_per_chunk 256 + @bytes_per_chunk 32 @bytes_per_length_offset 4 @bits_per_byte 8 @offset_bits 32 @@ -408,12 +407,18 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.map(fn {_, schema} -> variable_size?(schema) end) |> Enum.any?() end - + # NOTE: # - chunks is a list of bytes # - limit is the max size of the list - defp merklelize(chunks, limit // nil) do - + defp merklelize(chunks, limit \\ nil) when limit == nil do + size = next_pow_of_two(length(chunks)) + + if size == 1 do + [head | _tail] = chunks + head + else + end end defp size_of(value) when is_integer(value) and value >= 0 do @@ -429,22 +434,27 @@ defmodule LambdaEthereumConsensus.SszEx do div(length * size + 31, 32) end - defp pack(value, size) when is_integer(value) and value >= 0 do + def pack(value, size) when is_integer(value) and value >= 0 do <> |> pack_bytes() end - defp pack(value) when is_boolean(value) do + def pack(value) when is_boolean(value) do case value do true -> <<1::@bits_per_chunk-little>> false -> <<0::@bits_per_chunk>> end end - defp pack([head | _tail] = list) when is_integer(head) do + def pack([{value, _size} = head | _tail] = list) when is_integer(value) do list - |> Enum.map(fn x -> - x |> pack(size_of(x)) + |> Enum.reduce(<<>>, fn {x, size}, acc -> + {:ok, encoded} = encode_int(x, size) + acc <> encoded end) + |> pack_bytes() + |> :binary.bin_to_list() + |> Enum.chunk_every(@bytes_per_chunk) + |> Enum.map(fn x -> :binary.list_to_bin(x) end) end defp pack_bytes(value) when is_binary(value) do From dd6dfeafea1382a25ea761115fc98b3e7dfa8100 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 19 Dec 2023 01:40:27 +0100 Subject: [PATCH 05/31] feat: added unit tests for packing --- lib/spec/runners/ssz_generic.ex | 4 -- lib/ssz_ex.ex | 21 +++++-- test/unit/ssz_ex_test.exs | 104 ++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex index 236bc0594..28cb8a00b 100644 --- a/lib/spec/runners/ssz_generic.ex +++ b/lib/spec/runners/ssz_generic.ex @@ -89,10 +89,6 @@ defmodule SszGenericTestRunner do actual_hash_tree_root = SszEx.hash_tree_root!(real_deserialized, schema) assert actual_hash_tree_root == expected_hash_tree_root - - list = [{1, 8}, {2, 8}, {3, 8}, {3, 8}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}, {18446744073709551615, 64}] - - SszEx.pack(list) |> IO.inspect(limit: :infinity) end defp assert_ssz("invalid", schema, real_serialized) do diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index f3999859f..2f206b5f4 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -13,6 +13,7 @@ defmodule LambdaEthereumConsensus.SszEx do def encode(value, {:int, size}), do: encode_int(value, size) def encode(value, :bool), do: encode_bool(value) + def encode(value, {:bytes, _}), do: {:ok, value} def encode(list, {:list, basic_type, size}) do if variable_size?(basic_type), @@ -20,8 +21,6 @@ defmodule LambdaEthereumConsensus.SszEx do else: encode_fixed_size_list(list, basic_type, size) end - def encode(value, {:bytes, _}), do: {:ok, value} - def encode(container, module) when is_map(container), do: encode_container(container, module.schema()) @@ -445,10 +444,18 @@ defmodule LambdaEthereumConsensus.SszEx do end end - def pack([{value, _size} = head | _tail] = list) when is_integer(value) do + def pack(list, {:list, basic_type, _size}) do + if !variable_size?(basic_type) do + pack_basic_type_list(list) + else + pack_complex_type_list(list) + end + end + + defp pack_basic_type_list(list) do list - |> Enum.reduce(<<>>, fn {x, size}, acc -> - {:ok, encoded} = encode_int(x, size) + |> Enum.reduce(<<>>, fn {x, schema}, acc -> + {:ok, encoded} = encode(x, schema) acc <> encoded end) |> pack_bytes() @@ -457,6 +464,10 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.map(fn x -> :binary.list_to_bin(x) end) end + defp pack_complex_type_list(list) do + # TODO + end + defp pack_bytes(value) when is_binary(value) do incomplete_chunk_len = value |> bit_size() |> rem(@bits_per_chunk) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 44378e1f4..a33387ce4 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -12,6 +12,110 @@ defmodule Unit.SSZExTest do assert {:error, ^error_message} = SszEx.decode(serialized, schema) end + test "packing a list of uints" do + list_1 = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] + + expected_1 = [ + <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0>> + ] + + actual_1 = SszEx.pack(list_1, {:list, {:int, 8}, 5}) + assert expected_1 == actual_1 + + list_2 = [ + {1, {:int, 8}}, + {2, {:int, 8}}, + {3, {:int, 8}}, + {4, {:int, 8}}, + {5, {:int, 8}}, + {18_446_744_073_709_551_595, {:int, 64}}, + {18_446_744_073_709_551_596, {:int, 64}}, + {18_446_744_073_709_551_597, {:int, 64}}, + {18_446_744_073_709_551_598, {:int, 64}}, + {18_446_744_073_709_551_599, {:int, 64}}, + {18_446_744_073_709_551_600, {:int, 64}}, + {18_446_744_073_709_551_601, {:int, 64}}, + {18_446_744_073_709_551_603, {:int, 64}}, + {18_446_744_073_709_551_604, {:int, 64}}, + {18_446_744_073_709_551_605, {:int, 64}}, + {18_446_744_073_709_551_606, {:int, 64}}, + {18_446_744_073_709_551_607, {:int, 64}}, + {18_446_744_073_709_551_608, {:int, 64}}, + {18_446_744_073_709_551_609, {:int, 64}}, + {18_446_744_073_709_551_610, {:int, 64}}, + {18_446_744_073_709_551_611, {:int, 64}}, + {18_446_744_073_709_551_612, {:int, 64}}, + {18_446_744_073_709_551_613, {:int, 64}}, + {18_446_744_073_709_551_614, {:int, 64}}, + {18_446_744_073_709_551_615, {:int, 64}} + ] + + expected_2 = [ + <<1, 2, 3, 4, 5, 235, 255, 255, 255, 255, 255, 255, 255, 236, 255, 255, 255, 255, 255, 255, + 255, 237, 255, 255, 255, 255, 255, 255, 255, 238, 255, 255>>, + <<255, 255, 255, 255, 255, 239, 255, 255, 255, 255, 255, 255, 255, 240, 255, 255, 255, 255, + 255, 255, 255, 241, 255, 255, 255, 255, 255, 255, 255, 243, 255, 255>>, + <<255, 255, 255, 255, 255, 244, 255, 255, 255, 255, 255, 255, 255, 245, 255, 255, 255, 255, + 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, 255, 247, 255, 255>>, + <<255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, 255, 255, 255, 255, + 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255>>, + <<255, 255, 255, 255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, + 255, 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255>>, + <<255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0>> + ] + + actual_2 = SszEx.pack(list_2, {:list, {:int, 20}, 20}) + assert expected_2 == actual_2 + end + + test "packing a list of booleans" do + list = [{true, :bool}, {false, :bool}, {true, :bool}, {false, :bool}, {true, :bool}] + + expected = [ + <<1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0>> + ] + + actual = SszEx.pack(list, {:list, :bool, 5}) + assert expected == actual + end + + test "packing a list of bytes" do + list_1 = [ + {<<1>>, {:bytes, 8}}, + {<<2>>, {:bytes, 8}}, + {<<3>>, {:bytes, 8}}, + {<<4>>, {:bytes, 8}}, + {<<5>>, {:bytes, 8}} + ] + + expected_1 = [ + <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0>> + ] + + actual_1 = SszEx.pack(list_1, {:list, :bool, 5}) + assert expected_1 == actual_1 + + list_2 = [ + {<<255, 12>>, {:bytes, 16}}, + {<<2>>, {:bytes, 8}}, + {<<64, 78, 65, 90>>, {:bytes, 32}}, + {<<4>>, {:bytes, 8}}, + {<<5>>, {:bytes, 8}} + ] + + expected_2 = [ + <<255, 12, 2, 64, 78, 65, 90, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0>> + ] + + actual_2 = SszEx.pack(list_2, {:list, :bool, 5}) + assert expected_2 == actual_2 + end + test "serialize and deserialize uint" do assert_roundtrip(<<5>>, 5, {:int, 8}) assert_roundtrip(<<5, 0>>, 5, {:int, 16}) From cd58694bde547adadf98cf02ad72a10d70e5a736 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 19 Dec 2023 01:53:54 +0100 Subject: [PATCH 06/31] fix: refactor --- lib/ssz_ex.ex | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 2f206b5f4..9fb1c4e4b 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -8,6 +8,8 @@ defmodule LambdaEthereumConsensus.SszEx do ################# import Bitwise + @bits_per_chunk 256 + @spec hash(iodata()) :: binary() def hash(data), do: :crypto.hash(:sha256, data) @@ -37,15 +39,33 @@ defmodule LambdaEthereumConsensus.SszEx do def decode(binary, module) when is_atom(module), do: decode_container(binary, module) @spec hash_tree_root!(boolean, atom) :: SszTypes.root() - def hash_tree_root!(value, :bool), do: pack(value) + def hash_tree_root!(value, :bool), do: pack(value, :bool) @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() - def hash_tree_root!(value, {:int, size}), do: pack(value, size) + def hash_tree_root!(value, {:int, size}), do: pack(value, {:int, size}) + + def pack(value, {:int, size}) do + <> |> pack_bytes() + end + + def pack(value, :bool) do + case value do + true -> <<1::@bits_per_chunk-little>> + false -> <<0::@bits_per_chunk>> + end + end + + def pack(list, {:list, basic_type, _size}) do + if !variable_size?(basic_type) do + pack_basic_type_list(list) + else + pack_complex_type_list(list) + end + end ################# ### Private functions ################# - @bits_per_chunk 256 @bytes_per_chunk 32 @bytes_per_length_offset 4 @bits_per_byte 8 @@ -433,25 +453,6 @@ defmodule LambdaEthereumConsensus.SszEx do div(length * size + 31, 32) end - def pack(value, size) when is_integer(value) and value >= 0 do - <> |> pack_bytes() - end - - def pack(value) when is_boolean(value) do - case value do - true -> <<1::@bits_per_chunk-little>> - false -> <<0::@bits_per_chunk>> - end - end - - def pack(list, {:list, basic_type, _size}) do - if !variable_size?(basic_type) do - pack_basic_type_list(list) - else - pack_complex_type_list(list) - end - end - defp pack_basic_type_list(list) do list |> Enum.reduce(<<>>, fn {x, schema}, acc -> From 8d3e115c2ed9db8d07f0d00ed266cac4be81f74a Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Thu, 21 Dec 2023 11:51:08 +0100 Subject: [PATCH 07/31] update --- lib/ssz_ex.ex | 46 +++++++++++++++++++++++++++++++++++++-- lib/utils/merkle_trie.ex | 35 +++++++++++++++++++++++++++++ test/unit/ssz_ex_test.exs | 5 +++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 lib/utils/merkle_trie.ex diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 9fb1c4e4b..3eca7131f 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -8,6 +8,8 @@ defmodule LambdaEthereumConsensus.SszEx do ################# import Bitwise + alias LambdaEthereumConsensus.Utils + @bits_per_chunk 256 @spec hash(iodata()) :: binary() @@ -44,6 +46,46 @@ defmodule LambdaEthereumConsensus.SszEx do @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() def hash_tree_root!(value, {:int, size}), do: pack(value, {:int, size}) + @spec hash_tree_root(list(), {:list, any, non_neg_integer}, non_neg_integer | nil) :: + {:ok, SszTypes.root()} | {:error, String.t()} + def hash_tree_root(value, {:list, type, size}, limit \\ nil) + when limit != nil and length(value) > limit do + {:error, "chunk size exceeds limit"} + end + + def hash_tree_root(value, {:list, type, size}, limit) do + if !variable_size?(type) do + hash_tree_root_list_basic_type(value, {:list, type, size}, limit) + else + hash_tree_root_list_complex_type(value, {:list, type, size}, limit) + end + end + + def hash_tree_root_list_basic_type(list, {:list, type, size}, _limit) do + # len = length(chunks) + + # size = + # if limit do + # next_pow_of_two(limit) + # else + # next_pow_of_two(len) + # end + + # split = + # if size |> div(2) < len do + # size |> div(2) + # else + # len + # end + chunks = pack(list, {:list, type, size}) + chunks |> IO.inspect(limit: :infinity) + # MerkleTrie.create(chunks).hash + end + + def hash_tree_root_list_complex_type(value, {:list, type, size}, limit \\ nil) do + # TODO + end + def pack(value, {:int, size}) do <> |> pack_bytes() end @@ -55,8 +97,8 @@ defmodule LambdaEthereumConsensus.SszEx do end end - def pack(list, {:list, basic_type, _size}) do - if !variable_size?(basic_type) do + def pack(list, {:list, type, _size}) do + if !variable_size?(type) do pack_basic_type_list(list) else pack_complex_type_list(list) diff --git a/lib/utils/merkle_trie.ex b/lib/utils/merkle_trie.ex new file mode 100644 index 000000000..baba46574 --- /dev/null +++ b/lib/utils/merkle_trie.ex @@ -0,0 +1,35 @@ +defmodule LambdaEthereumConsensus.Utils.MerkleTrie do + @moduledoc """ + Simple Merkle Trie implementation in Elixir using SHA-256. + """ + + defstruct [:hash, :left, :right] + + # Function to create a leaf node + defp leaf(value) do + hash = :crypto.hash(:sha256, value) |> Base.encode16(case: :lower) + %{hash: hash} + end + + # Function to create an internal node + defp internal(left, right) do + combined_hash = :crypto.hash(:sha256, left.hash <> right.hash) |> Base.encode16(case: :lower) + %{hash: combined_hash, left: left, right: right} + end + + # Function to build the Merkle Trie recursively + defp build_merkle_tree([]), do: %{} + defp build_merkle_tree([value]), do: leaf(value) + + defp build_merkle_tree(values) do + {left_values, right_values} = Enum.split(values, div(length(values), 2)) + left = build_merkle_tree(left_values) + right = build_merkle_tree(right_values) + internal(left, right) + end + + # Public function to create a Merkle Trie from a list of values + def create(values) do + build_merkle_tree(values) + end +end diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index a33387ce4..a7ea7a94a 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -116,6 +116,11 @@ defmodule Unit.SSZExTest do assert expected_2 == actual_2 end + test "hash tree of list of basic type" do + list = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] + list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) + end + test "serialize and deserialize uint" do assert_roundtrip(<<5>>, 5, {:int, 8}) assert_roundtrip(<<5, 0>>, 5, {:int, 16}) From 959b158f4f8a62eaeb4d91db5ee194c03bd8f0d9 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Thu, 21 Dec 2023 11:58:35 +0100 Subject: [PATCH 08/31] fic --- lib/ssz_ex.ex | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index a96f03606..3eca7131f 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -104,12 +104,6 @@ defmodule LambdaEthereumConsensus.SszEx do pack_complex_type_list(list) end end - - @spec hash_tree_root!(boolean, atom) :: Types.root() - def hash_tree_root!(value, :bool), do: pack(value) - - @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: Types.root() - def hash_tree_root!(value, {:int, size}), do: pack(value, size) ################# ### Private functions From 3661057a4ac03eb1fec48ad40c0912b7dee0a42c Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Fri, 22 Dec 2023 11:31:26 +0100 Subject: [PATCH 09/31] update --- lib/ssz_ex.ex | 129 ++++++++++++++++++++------------------ lib/utils/merkle_trie.ex | 4 +- test/unit/ssz_ex_test.exs | 4 +- 3 files changed, 73 insertions(+), 64 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 3eca7131f..e21fceda2 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -8,9 +8,10 @@ defmodule LambdaEthereumConsensus.SszEx do ################# import Bitwise - alias LambdaEthereumConsensus.Utils + alias LambdaEthereumConsensus.Utils.MerkleTrie @bits_per_chunk 256 + @zero_chunk <<0::size(@bits_per_chunk)>> @spec hash(iodata()) :: binary() def hash(data), do: :crypto.hash(:sha256, data) @@ -46,44 +47,56 @@ defmodule LambdaEthereumConsensus.SszEx do @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() def hash_tree_root!(value, {:int, size}), do: pack(value, {:int, size}) - @spec hash_tree_root(list(), {:list, any, non_neg_integer}, non_neg_integer | nil) :: + @spec hash_tree_root(list(), {:list, any, non_neg_integer}) :: {:ok, SszTypes.root()} | {:error, String.t()} - def hash_tree_root(value, {:list, type, size}, limit \\ nil) - when limit != nil and length(value) > limit do - {:error, "chunk size exceeds limit"} - end - - def hash_tree_root(value, {:list, type, size}, limit) do + def hash_tree_root(list, {:list, type, size}) do if !variable_size?(type) do - hash_tree_root_list_basic_type(value, {:list, type, size}, limit) + packed_chunks = pack(list, {:list, type, size}) + limit = chunk_count(list) + hash_tree_root_list_basic_type(packed_chunks, limit, length(list)) else - hash_tree_root_list_complex_type(value, {:list, type, size}, limit) + # TODO + # hash_tree_root_list_complex_type(list, {:list, type, size}, limit) end end - def hash_tree_root_list_basic_type(list, {:list, type, size}, _limit) do - # len = length(chunks) + def hash_tree_root_list_basic_type(chunks, limit \\ nil, _len) + when limit and length(chunks) > limit do + {:error, "chunk size exceeds limit"} + end + + def hash_tree_root_list_basic_type(chunks, limit, len) do + merklelize(chunks, limit) |> mix_in_length(len) + end - # size = - # if limit do - # next_pow_of_two(limit) - # else - # next_pow_of_two(len) - # end + # def hash_tree_root_list_complex_type(value, {:list, type, size}, limit \\ nil) do + # # TODO + # end - # split = - # if size |> div(2) < len do - # size |> div(2) - # else - # len - # end - chunks = pack(list, {:list, type, size}) - chunks |> IO.inspect(limit: :infinity) - # MerkleTrie.create(chunks).hash + def mix_in_length(root, len) do + hash(root <> <>) end - def hash_tree_root_list_complex_type(value, {:list, type, size}, limit \\ nil) do - # TODO + def merklelize(chunks, limit) do + size = + if limit != nil and limit >= length(chunks) do + limit + else + length(chunks) + end + + leaf_count = next_pow_of_two(size) + + chunks = + if leaf_count == size do + chunks + else + diff = leaf_count - size + zero_chunks = 0..(diff - 1) |> Enum.map(fn _ -> @zero_chunk end) + chunks ++ zero_chunks + end + + MerkleTrie.create(chunks).hash end def pack(value, {:int, size}) do @@ -93,7 +106,7 @@ defmodule LambdaEthereumConsensus.SszEx do def pack(value, :bool) do case value do true -> <<1::@bits_per_chunk-little>> - false -> <<0::@bits_per_chunk>> + false -> @zero_chunk end end @@ -101,13 +114,14 @@ defmodule LambdaEthereumConsensus.SszEx do if !variable_size?(type) do pack_basic_type_list(list) else - pack_complex_type_list(list) + # pack_complex_type_list(list) end end ################# ### Private functions ################# + @bytes_per_boolean 4 @bytes_per_chunk 32 @bytes_per_length_offset 4 @bits_per_byte 8 @@ -469,30 +483,28 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.any?() end - # NOTE: - # - chunks is a list of bytes - # - limit is the max size of the list - defp merklelize(chunks, limit \\ nil) when limit == nil do - size = next_pow_of_two(length(chunks)) - - if size == 1 do - [head | _tail] = chunks - head - else - end + defp size_of(value) when is_boolean(value) do + @bytes_per_boolean end - defp size_of(value) when is_integer(value) and value >= 0 do - value |> :binary.encode_unsigned() |> byte_size() + defp size_of(value, size) when is_integer(value) and value >= 0 do + {:ok, encoded} = value |> encode_int(size) + encoded |> byte_size() end # NOTE: # - When the elements of the list is an uint then it is a basic list or basic vector # - When the elements of the list is a boolean then it is a bit vector - defp chunk_count([head | _tail] = list) when is_integer(head) do - size = size_of(head) - length = length(list) - div(length * size + 31, 32) + defp chunk_count([{value, {:int, size}} = _head | _tail] = list) do + size = size_of(value, size) + len = length(list) + ((len * size) + 31) |> div(32) + end + + defp chunk_count([{value, :bool} = _head | _tail] = list) do + size = size_of(value) + len = length(list) + ((len * size) + 31) |> div(32) end defp pack_basic_type_list(list) do @@ -507,9 +519,9 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.map(fn x -> :binary.list_to_bin(x) end) end - defp pack_complex_type_list(list) do - # TODO - end + # defp pack_complex_type_list(list) do + # # TODO + # end defp pack_bytes(value) when is_binary(value) do incomplete_chunk_len = value |> bit_size() |> rem(@bits_per_chunk) @@ -523,16 +535,11 @@ defmodule LambdaEthereumConsensus.SszEx do end defp next_pow_of_two(len) when is_integer(len) and len >= 0 do - cond do - len == 0 or len == 1 -> - 1 - - len == 2 -> - 2 - - true -> - n = ((len <<< 1) - 1) |> :math.log2() |> trunc() - 2 ** n + if len == 0 do + 0 + else + n = ((len <<< 1) - 1) |> :math.log2() |> trunc() + 2 ** n end end end diff --git a/lib/utils/merkle_trie.ex b/lib/utils/merkle_trie.ex index baba46574..9a4c25844 100644 --- a/lib/utils/merkle_trie.ex +++ b/lib/utils/merkle_trie.ex @@ -7,13 +7,13 @@ defmodule LambdaEthereumConsensus.Utils.MerkleTrie do # Function to create a leaf node defp leaf(value) do - hash = :crypto.hash(:sha256, value) |> Base.encode16(case: :lower) + hash = :crypto.hash(:sha256, value) %{hash: hash} end # Function to create an internal node defp internal(left, right) do - combined_hash = :crypto.hash(:sha256, left.hash <> right.hash) |> Base.encode16(case: :lower) + combined_hash = :crypto.hash(:sha256, left.hash <> right.hash) %{hash: combined_hash, left: left, right: right} end diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index df77c8898..18afa97dc 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -118,7 +118,9 @@ defmodule Unit.SSZExTest do test "hash tree of list of basic type" do list = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] - list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) + root = list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) + root |> IO.inspect() + # root = list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) end test "serialize and deserialize uint" do From 4426b5fcbd719b2b06e01a07ca4f18641816f0ba Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Fri, 22 Dec 2023 12:55:44 +0100 Subject: [PATCH 10/31] fix --- lib/ssz_ex.ex | 6 +++--- test/unit/ssz_ex_test.exs | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index e21fceda2..390d7deb5 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -78,7 +78,7 @@ defmodule LambdaEthereumConsensus.SszEx do end def merklelize(chunks, limit) do - size = + size = if limit != nil and limit >= length(chunks) do limit else @@ -498,13 +498,13 @@ defmodule LambdaEthereumConsensus.SszEx do defp chunk_count([{value, {:int, size}} = _head | _tail] = list) do size = size_of(value, size) len = length(list) - ((len * size) + 31) |> div(32) + (len * size + 31) |> div(32) end defp chunk_count([{value, :bool} = _head | _tail] = list) do size = size_of(value) len = length(list) - ((len * size) + 31) |> div(32) + (len * size + 31) |> div(32) end defp pack_basic_type_list(list) do diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 18afa97dc..3fd352305 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -119,8 +119,6 @@ defmodule Unit.SSZExTest do test "hash tree of list of basic type" do list = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] root = list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) - root |> IO.inspect() - # root = list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) end test "serialize and deserialize uint" do From fe5360adfa5cb0a6a1b972fca94bfc13c01d2485 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Mon, 8 Jan 2024 12:48:24 +0100 Subject: [PATCH 11/31] update --- lib/ssz_ex.ex | 101 ++++++++++++++++++++++++-------------- lib/utils/merkle_trie.ex | 35 ------------- test/unit/ssz_ex_test.exs | 39 ++++++--------- 3 files changed, 81 insertions(+), 94 deletions(-) delete mode 100644 lib/utils/merkle_trie.ex diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 390d7deb5..b558b6732 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -8,14 +8,17 @@ defmodule LambdaEthereumConsensus.SszEx do ################# import Bitwise - alias LambdaEthereumConsensus.Utils.MerkleTrie - @bits_per_chunk 256 + @bytes_per_chunk 32 + @bits_per_byte 8 @zero_chunk <<0::size(@bits_per_chunk)>> @spec hash(iodata()) :: binary() def hash(data), do: :crypto.hash(:sha256, data) + @spec hash_nodes(binary(), binary()) :: binary() + def hash_nodes(left, right), do: :crypto.hash(:sha256, left <> right) + def encode(value, {:int, size}), do: encode_int(value, size) def encode(value, :bool), do: encode_bool(value) def encode(value, {:bytes, _}), do: {:ok, value} @@ -66,43 +69,65 @@ defmodule LambdaEthereumConsensus.SszEx do end def hash_tree_root_list_basic_type(chunks, limit, len) do - merklelize(chunks, limit) |> mix_in_length(len) + merklelize_chunks(chunks, limit) |> mix_in_length(len) end - # def hash_tree_root_list_complex_type(value, {:list, type, size}, limit \\ nil) do - # # TODO - # end - def mix_in_length(root, len) do hash(root <> <>) end - def merklelize(chunks, limit) do - size = - if limit != nil and limit >= length(chunks) do - limit - else - length(chunks) - end - - leaf_count = next_pow_of_two(size) - - chunks = - if leaf_count == size do - chunks - else - diff = leaf_count - size - zero_chunks = 0..(diff - 1) |> Enum.map(fn _ -> @zero_chunk end) - chunks ++ zero_chunks - end - - MerkleTrie.create(chunks).hash + def merklelize_chunks(chunks, leaf_count) do + "Leaf Count: #{leaf_count}" |> IO.inspect() + node_count = 2 * leaf_count - 1 + "Node Count: #{node_count}" |> IO.inspect() + interior_count = node_count - leaf_count + "Interior Count: #{interior_count}" |> IO.inspect() + leaf_start = interior_count * @bytes_per_chunk + "Leaf Start: #{leaf_start}" |> IO.inspect() + padded_chunks = chunks |> convert_to_next_pow_of_two() + padded_chunks |> IO.inspect(limit: :infinity) + buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::binary>> + buffer_len = buffer |> byte_size() + buffer |> IO.inspect(limit: :infinity) + buffer_len |> IO.inspect() + new_buffer = + 1..node_count + |> Enum.filter(fn x -> rem(x, 2) == 0 end) + |> Enum.reverse() + |> Enum.reduce(buffer, fn index, acc_buffer -> + parent_index = (index - 1) |> div(2) + "Parent Index: #{parent_index}" |> IO.inspect() + start = parent_index * @bytes_per_chunk + "Start: #{start}" |> IO.inspect() + stop = (index + 1) * @bytes_per_chunk + "Stop: #{stop}" |> IO.inspect() + focus = acc_buffer |> :binary.part(start, stop - start) + focus_len = focus |> byte_size() + focus |> IO.inspect(limit: :infinity) + focus_len |> IO.inspect() + children_index = focus_len - 2 * @bytes_per_chunk + "Children Index: #{children_index}" |> IO.inspect() + children = focus |> :binary.part(children_index, focus_len - children_index) + children |> IO.inspect(limit: :infinity) + left = children |> :binary.part(0, 32) + right = children |> :binary.part(32, 32) + parent = hash_nodes(left, right) + first = acc_buffer |> :binary.part(0, start) + middle = parent <> children + last = acc_buffer |> :binary.part(stop, focus_len - stop) + new_buffer = first <> middle <> last + new_buffer_len = new_buffer |> byte_size() + new_buffer |> IO.inspect(limit: :infinity) + new_buffer + end) end + @spec pack(non_neg_integer, {:int, non_neg_integer}) :: binary() def pack(value, {:int, size}) do <> |> pack_bytes() end + @spec pack(boolean, :bool) :: binary() def pack(value, :bool) do case value do true -> <<1::@bits_per_chunk-little>> @@ -110,6 +135,7 @@ defmodule LambdaEthereumConsensus.SszEx do end end + @spec pack(list(), {:list, any, non_neg_integer}) :: binary() def pack(list, {:list, type, _size}) do if !variable_size?(type) do pack_basic_type_list(list) @@ -122,9 +148,7 @@ defmodule LambdaEthereumConsensus.SszEx do ### Private functions ################# @bytes_per_boolean 4 - @bytes_per_chunk 32 @bytes_per_length_offset 4 - @bits_per_byte 8 @offset_bits 32 defp encode_int(value, size) when is_integer(value), do: {:ok, <>} @@ -514,15 +538,8 @@ defmodule LambdaEthereumConsensus.SszEx do acc <> encoded end) |> pack_bytes() - |> :binary.bin_to_list() - |> Enum.chunk_every(@bytes_per_chunk) - |> Enum.map(fn x -> :binary.list_to_bin(x) end) end - # defp pack_complex_type_list(list) do - # # TODO - # end - defp pack_bytes(value) when is_binary(value) do incomplete_chunk_len = value |> bit_size() |> rem(@bits_per_chunk) @@ -534,6 +551,18 @@ defmodule LambdaEthereumConsensus.SszEx do end end + defp convert_to_next_pow_of_two(chunks) do + size = chunks |> byte_size() |> div(@bytes_per_chunk) + next_pow = size |> next_pow_of_two() + if size == next_pow do + chunks + else + diff = next_pow - size + zero_chunks = 0..(diff - 1) |> Enum.reduce(<<>>, fn _, acc -> <<0::256>> <> acc end) + chunks <> zero_chunks + end + end + defp next_pow_of_two(len) when is_integer(len) and len >= 0 do if len == 0 do 0 diff --git a/lib/utils/merkle_trie.ex b/lib/utils/merkle_trie.ex deleted file mode 100644 index 9a4c25844..000000000 --- a/lib/utils/merkle_trie.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule LambdaEthereumConsensus.Utils.MerkleTrie do - @moduledoc """ - Simple Merkle Trie implementation in Elixir using SHA-256. - """ - - defstruct [:hash, :left, :right] - - # Function to create a leaf node - defp leaf(value) do - hash = :crypto.hash(:sha256, value) - %{hash: hash} - end - - # Function to create an internal node - defp internal(left, right) do - combined_hash = :crypto.hash(:sha256, left.hash <> right.hash) - %{hash: combined_hash, left: left, right: right} - end - - # Function to build the Merkle Trie recursively - defp build_merkle_tree([]), do: %{} - defp build_merkle_tree([value]), do: leaf(value) - - defp build_merkle_tree(values) do - {left_values, right_values} = Enum.split(values, div(length(values), 2)) - left = build_merkle_tree(left_values) - right = build_merkle_tree(right_values) - internal(left, right) - end - - # Public function to create a Merkle Trie from a list of values - def create(values) do - build_merkle_tree(values) - end -end diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 3fd352305..b8638c3b6 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -15,10 +15,9 @@ defmodule Unit.SSZExTest do test "packing a list of uints" do list_1 = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] - expected_1 = [ + expected_1 = <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - ] actual_1 = SszEx.pack(list_1, {:list, {:int, 8}, 5}) assert expected_1 == actual_1 @@ -51,20 +50,17 @@ defmodule Unit.SSZExTest do {18_446_744_073_709_551_615, {:int, 64}} ] - expected_2 = [ + expected_2 = <<1, 2, 3, 4, 5, 235, 255, 255, 255, 255, 255, 255, 255, 236, 255, 255, 255, 255, 255, 255, - 255, 237, 255, 255, 255, 255, 255, 255, 255, 238, 255, 255>>, - <<255, 255, 255, 255, 255, 239, 255, 255, 255, 255, 255, 255, 255, 240, 255, 255, 255, 255, - 255, 255, 255, 241, 255, 255, 255, 255, 255, 255, 255, 243, 255, 255>>, - <<255, 255, 255, 255, 255, 244, 255, 255, 255, 255, 255, 255, 255, 245, 255, 255, 255, 255, - 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, 255, 247, 255, 255>>, - <<255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, 255, 255, 255, 255, - 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255>>, - <<255, 255, 255, 255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, - 255, 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255>>, - <<255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0>> - ] + 255, 237, 255, 255, 255, 255, 255, 255, 255, 238, 255, 255, 255, 255, 255, 255, 255, 239, + 255, 255, 255, 255, 255, 255, 255, 240, 255, 255, 255, 255, 255, 255, 255, 241, 255, 255, + 255, 255, 255, 255, 255, 243, 255, 255, 255, 255, 255, 255, 255, 244, 255, 255, 255, 255, + 255, 255, 255, 245, 255, 255, 255, 255, 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, + 255, 247, 255, 255, 255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, + 255, 255, 255, 255, 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255, + 255, 255, 255, 255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, + 255, 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> actual_2 = SszEx.pack(list_2, {:list, {:int, 20}, 20}) assert expected_2 == actual_2 @@ -73,10 +69,9 @@ defmodule Unit.SSZExTest do test "packing a list of booleans" do list = [{true, :bool}, {false, :bool}, {true, :bool}, {false, :bool}, {true, :bool}] - expected = [ + expected = <<1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - ] actual = SszEx.pack(list, {:list, :bool, 5}) assert expected == actual @@ -91,10 +86,9 @@ defmodule Unit.SSZExTest do {<<5>>, {:bytes, 8}} ] - expected_1 = [ + expected_1 = <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - ] actual_1 = SszEx.pack(list_1, {:list, :bool, 5}) assert expected_1 == actual_1 @@ -107,18 +101,17 @@ defmodule Unit.SSZExTest do {<<5>>, {:bytes, 8}} ] - expected_2 = [ + expected_2 = <<255, 12, 2, 64, 78, 65, 90, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - ] actual_2 = SszEx.pack(list_2, {:list, :bool, 5}) assert expected_2 == actual_2 end test "hash tree of list of basic type" do - list = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] - root = list |> SszEx.hash_tree_root({:list, {:int, 8}, 5}) + chunks = <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> + root = chunks |> SszEx.merklelize_chunks(4) end test "serialize and deserialize uint" do From adde98653839f4acae736c8e713fe43dc5e11666 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Mon, 8 Jan 2024 12:52:17 +0100 Subject: [PATCH 12/31] fix --- lib/ssz_ex.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 488210092..3fe31415d 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -109,6 +109,7 @@ defmodule LambdaEthereumConsensus.SszEx do buffer_len = buffer |> byte_size() buffer |> IO.inspect(limit: :infinity) buffer_len |> IO.inspect() + new_buffer = 1..node_count |> Enum.filter(fn x -> rem(x, 2) == 0 end) @@ -570,9 +571,8 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.any?() end - defp size_of(value) when is_boolean(value) do - @bytes_per_boolean - + defp size_of(value) when is_boolean(value), do: @bytes_per_boolean + def length_of_bitlist(bitlist) when is_binary(bitlist) do bit_size = bit_size(bitlist) <<_::size(bit_size - 8), last_byte>> = bitlist @@ -648,6 +648,7 @@ defmodule LambdaEthereumConsensus.SszEx do defp convert_to_next_pow_of_two(chunks) do size = chunks |> byte_size() |> div(@bytes_per_chunk) next_pow = size |> next_pow_of_two() + if size == next_pow do chunks else From 8d4c6dbf7edb6ef89a55a964d5209074d3a93146 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 11:35:21 +0100 Subject: [PATCH 13/31] feat: added merklelization of chunks --- lib/ssz_ex.ex | 57 +++--- test/unit/ssz_ex_test.exs | 380 +++++++++++++++++++++++++++++++++++++- 2 files changed, 400 insertions(+), 37 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 3fe31415d..e72050548 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -31,8 +31,6 @@ defmodule LambdaEthereumConsensus.SszEx do else: encode_fixed_size_list(list, basic_type, size) end - def encode(value, {:bytes, _}), do: {:ok, value} - def encode(value, {:bitlist, max_size}) when is_bitstring(value), do: encode_bitlist(value, max_size) @@ -88,7 +86,7 @@ defmodule LambdaEthereumConsensus.SszEx do end def hash_tree_root_list_basic_type(chunks, limit, len) do - merklelize_chunks(chunks, limit) |> mix_in_length(len) + merklelize_chunks(chunks, limit) end def mix_in_length(root, len) do @@ -96,19 +94,11 @@ defmodule LambdaEthereumConsensus.SszEx do end def merklelize_chunks(chunks, leaf_count) do - "Leaf Count: #{leaf_count}" |> IO.inspect() node_count = 2 * leaf_count - 1 - "Node Count: #{node_count}" |> IO.inspect() interior_count = node_count - leaf_count - "Interior Count: #{interior_count}" |> IO.inspect() leaf_start = interior_count * @bytes_per_chunk - "Leaf Start: #{leaf_start}" |> IO.inspect() - padded_chunks = chunks |> convert_to_next_pow_of_two() - padded_chunks |> IO.inspect(limit: :infinity) - buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::binary>> - buffer_len = buffer |> byte_size() - buffer |> IO.inspect(limit: :infinity) - buffer_len |> IO.inspect() + padded_chunks = chunks |> convert_to_next_pow_of_two(leaf_count) + buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::bitstring>> new_buffer = 1..node_count @@ -116,30 +106,22 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.reverse() |> Enum.reduce(buffer, fn index, acc_buffer -> parent_index = (index - 1) |> div(2) - "Parent Index: #{parent_index}" |> IO.inspect() start = parent_index * @bytes_per_chunk - "Start: #{start}" |> IO.inspect() stop = (index + 1) * @bytes_per_chunk - "Stop: #{stop}" |> IO.inspect() focus = acc_buffer |> :binary.part(start, stop - start) focus_len = focus |> byte_size() - focus |> IO.inspect(limit: :infinity) - focus_len |> IO.inspect() children_index = focus_len - 2 * @bytes_per_chunk - "Children Index: #{children_index}" |> IO.inspect() children = focus |> :binary.part(children_index, focus_len - children_index) - children |> IO.inspect(limit: :infinity) - left = children |> :binary.part(0, 32) - right = children |> :binary.part(32, 32) + + <> = children + parent = hash_nodes(left, right) - first = acc_buffer |> :binary.part(0, start) - middle = parent <> children - last = acc_buffer |> :binary.part(stop, focus_len - stop) - new_buffer = first <> middle <> last - new_buffer_len = new_buffer |> byte_size() - new_buffer |> IO.inspect(limit: :infinity) - new_buffer + replace_chunk(acc_buffer, start, parent) end) + + <> = new_buffer + root end @spec pack(non_neg_integer, {:int, non_neg_integer}) :: binary() @@ -600,11 +582,6 @@ defmodule LambdaEthereumConsensus.SszEx do defp remove_trailing_bit(<<0::7, 1::1>>), do: <<0::0>> defp remove_trailing_bit(<<0::8>>), do: <<0::0>> - defp pack(value, size) when is_integer(value) and value >= 0 do - pad = @bits_per_chunk - size - <> - end - defp size_of(value, size) when is_integer(value) and value >= 0 do {:ok, encoded} = value |> encode_int(size) encoded |> byte_size() @@ -645,9 +622,9 @@ defmodule LambdaEthereumConsensus.SszEx do end end - defp convert_to_next_pow_of_two(chunks) do + defp convert_to_next_pow_of_two(chunks, leaf_count) do size = chunks |> byte_size() |> div(@bytes_per_chunk) - next_pow = size |> next_pow_of_two() + next_pow = leaf_count |> next_pow_of_two() if size == next_pow do chunks @@ -666,4 +643,12 @@ defmodule LambdaEthereumConsensus.SszEx do 2 ** n end end + + defp replace_chunk(chunks, start, new_chunk) do + <> = + chunks + + <> + end end diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index f7a0eb239..5fa87f75c 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -109,9 +109,387 @@ defmodule Unit.SSZExTest do assert expected_2 == actual_2 end - test "hash tree of list of basic type" do + test "merklelization of chunks" do + chunks = <<0::256>> <> <<0::256>> + root = chunks |> SszEx.merklelize_chunks(2) + expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" + assert root |> Base.encode16(case: :lower) == expected_value + + ones = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<1>> <> acc end) + + chunks = ones <> ones + root = chunks |> SszEx.merklelize_chunks(2) + expected_value = "7c8975e1e60a5c8337f28edf8c33c3b180360b7279644a9bc1af3c51e6220bf5" + assert root |> Base.encode16(case: :lower) == expected_value + chunks = <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> root = chunks |> SszEx.merklelize_chunks(4) + expected_value = "db56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71" + assert root |> Base.encode16(case: :lower) == expected_value + + chunks = ones + root = chunks |> SszEx.merklelize_chunks(4) + expected_value = "29797eded0e83376b70f2bf034cc0811ae7f1414653b1d720dfd18f74cf13309" + assert root |> Base.encode16(case: :lower) == expected_value + + twos = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<2>> <> acc end) + + chunks = twos + root = chunks |> SszEx.merklelize_chunks(8) + expected_value = "fa4cf775712aa8a2fe5dcb5a517d19b2e9effcf58ff311b9fd8e4a7d308e6d00" + assert root |> Base.encode16(case: :lower) == expected_value + + chunks = ones <> ones <> ones + root = chunks |> SszEx.merklelize_chunks(4) + expected_value = "65aa94f2b59e517abd400cab655f42821374e433e41b8fe599f6bb15484adcec" + assert root |> Base.encode16(case: :lower) == expected_value + + chunks = ones <> ones <> ones <> ones <> ones + root = chunks |> SszEx.merklelize_chunks(8) + expected_value = "0ae67e34cba4ad2bbfea5dc39e6679b444021522d861fab00f05063c54341289" + assert root |> Base.encode16(case: :lower) == expected_value + + chunks = ones <> ones <> ones <> ones <> ones <> ones + root = chunks |> SszEx.merklelize_chunks(8) + expected_value = "0ef7df63c204ef203d76145627b8083c49aa7c55ebdee2967556f55a4f65a238" + assert root |> Base.encode16(case: :lower) == expected_value + + ## Large Leaf Count + + chunks = ones <> ones <> ones <> ones <> ones + root = chunks |> SszEx.merklelize_chunks(2 ** 10) + expected_value = "2647cb9e26bd83eeb0982814b2ac4d6cc4a65d0d98637f1a73a4c06d3db0e6ce" + assert root |> Base.encode16(case: :lower) == expected_value + + ## TOO HEAVY COMPUTATION! + # chunks = 1..70 |> Enum.reduce(<<>>, fn _, acc -> acc <> ones end) + # leaf_count = 9_223_372_036_854_775_808 # 2 ** 63 + # root = chunks |> SszEx.merklelize_chunks(leaf_count) + # expected_value = "9317695d95b5a3b46e976b5a9cbfcfccb600accaddeda9ac867cc9669b862979" + # assert root |> Base.encode16(case: :lower) == expected_value + end + + test "hash tree of list of basic type" do + list = [ + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535, + 65_535 + ] + # root = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) + # root |> Base.encode16() |> IO.inspect() end test "serialize and deserialize uint" do From 02ac111576a383df0d72b336725818d513ac7127 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 11:50:12 +0100 Subject: [PATCH 14/31] fix --- lib/ssz_ex.ex | 18 +++++++++--------- test/unit/ssz_ex_test.exs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index e72050548..cf3ad8dd5 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -80,13 +80,13 @@ defmodule LambdaEthereumConsensus.SszEx do end end - def hash_tree_root_list_basic_type(chunks, limit \\ nil, _len) - when limit and length(chunks) > limit do - {:error, "chunk size exceeds limit"} - end - - def hash_tree_root_list_basic_type(chunks, limit, len) do - merklelize_chunks(chunks, limit) + def hash_tree_root_list_basic_type(chunks, limit \\ nil, _len \\ nil) do + if limit and length(chunks) > limit do + {:error, "chunk size exceeds limit"} + else + root = merklelize_chunks(chunks, limit) + {:ok, root} + end end def mix_in_length(root, len) do @@ -553,8 +553,6 @@ defmodule LambdaEthereumConsensus.SszEx do |> Enum.any?() end - defp size_of(value) when is_boolean(value), do: @bytes_per_boolean - def length_of_bitlist(bitlist) when is_binary(bitlist) do bit_size = bit_size(bitlist) <<_::size(bit_size - 8), last_byte>> = bitlist @@ -582,6 +580,8 @@ defmodule LambdaEthereumConsensus.SszEx do defp remove_trailing_bit(<<0::7, 1::1>>), do: <<0::0>> defp remove_trailing_bit(<<0::8>>), do: <<0::0>> + defp size_of(value) when is_boolean(value), do: @bytes_per_boolean + defp size_of(value, size) when is_integer(value) and value >= 0 do {:ok, encoded} = value |> encode_int(size) encoded |> byte_size() diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 5fa87f75c..a7d8a22a0 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -488,7 +488,7 @@ defmodule Unit.SSZExTest do 65_535, 65_535 ] - # root = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) + root = list |> SszEx.hash_tree_root({:list, {:int, 16}, 316}) # root |> Base.encode16() |> IO.inspect() end From 6b2ada181d33140d98df7e05949c8a5e0859fd5d Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 11:51:53 +0100 Subject: [PATCH 15/31] fix --- lib/ssz_ex.ex | 2 +- test/unit/ssz_ex_test.exs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index cf3ad8dd5..2549852d5 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -581,7 +581,7 @@ defmodule LambdaEthereumConsensus.SszEx do defp remove_trailing_bit(<<0::8>>), do: <<0::0>> defp size_of(value) when is_boolean(value), do: @bytes_per_boolean - + defp size_of(value, size) when is_integer(value) and value >= 0 do {:ok, encoded} = value |> encode_int(size) encoded |> byte_size() diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index a7d8a22a0..288892312 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -488,6 +488,7 @@ defmodule Unit.SSZExTest do 65_535, 65_535 ] + root = list |> SszEx.hash_tree_root({:list, {:int, 16}, 316}) # root |> Base.encode16() |> IO.inspect() end From 3f1c58fbfdc20a8c8c86066b901feb7a2b5fed6b Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 11:55:38 +0100 Subject: [PATCH 16/31] update --- test/unit/ssz_ex_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 288892312..fa9fb3b40 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -110,6 +110,9 @@ defmodule Unit.SSZExTest do end test "merklelization of chunks" do + ## Reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L371 + ## https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L416 + chunks = <<0::256>> <> <<0::256>> root = chunks |> SszEx.merklelize_chunks(2) expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" From f742abdd78434801daf3b63a9fbbc6009d711f5a Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 23:03:52 +0100 Subject: [PATCH 17/31] feat: hash tree root of list --- lib/ssz_ex.ex | 47 +++++++++-------- test/unit/ssz_ex_test.exs | 103 +++++++++++++++++--------------------- 2 files changed, 68 insertions(+), 82 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 2549852d5..23485dbcd 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -72,25 +72,29 @@ defmodule LambdaEthereumConsensus.SszEx do def hash_tree_root(list, {:list, type, size}) do if !variable_size?(type) do packed_chunks = pack(list, {:list, type, size}) - limit = chunk_count(list) - hash_tree_root_list_basic_type(packed_chunks, limit, length(list)) + limit = chunk_count(list, {:list, type, size}) + len = length(list) + hash_tree_root_list_basic_type(packed_chunks, limit, len) else # TODO # hash_tree_root_list_complex_type(list, {:list, type, size}, limit) end end - def hash_tree_root_list_basic_type(chunks, limit \\ nil, _len \\ nil) do - if limit and length(chunks) > limit do + def hash_tree_root_list_basic_type(chunks, limit, len) do + chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) + + if chunks_len > limit do {:error, "chunk size exceeds limit"} else - root = merklelize_chunks(chunks, limit) + root = merklelize_chunks(chunks, limit) |> mix_in_length(len) {:ok, root} end end def mix_in_length(root, len) do - hash(root <> <>) + {:ok, serialized_len} = encode_int(len, @bits_per_chunk) + root |> hash_nodes(serialized_len) end def merklelize_chunks(chunks, leaf_count) do @@ -138,9 +142,9 @@ defmodule LambdaEthereumConsensus.SszEx do end @spec pack(list(), {:list, any, non_neg_integer}) :: binary() - def pack(list, {:list, type, _size}) do - if !variable_size?(type) do - pack_basic_type_list(list) + def pack(list, {:list, schema, _size}) do + if !variable_size?(schema) do + pack_basic_type_list(list, schema) else # pack_complex_type_list(list) end @@ -580,31 +584,26 @@ defmodule LambdaEthereumConsensus.SszEx do defp remove_trailing_bit(<<0::7, 1::1>>), do: <<0::0>> defp remove_trailing_bit(<<0::8>>), do: <<0::0>> - defp size_of(value) when is_boolean(value), do: @bytes_per_boolean + defp size_of(_value, :bool), do: @bytes_per_boolean - defp size_of(value, size) when is_integer(value) and value >= 0 do + defp size_of(value, {:int, size}) do {:ok, encoded} = value |> encode_int(size) encoded |> byte_size() end - # NOTE: - # - When the elements of the list is an uint then it is a basic list or basic vector - # - When the elements of the list is a boolean then it is a bit vector - defp chunk_count([{value, {:int, size}} = _head | _tail] = list) do - size = size_of(value, size) - len = length(list) - (len * size + 31) |> div(32) + defp chunk_count([head | _tail] = _list, {:list, {:int, size}, max_size}) do + size = size_of(head, {:int, size}) + (max_size * size + 31) |> div(32) end - defp chunk_count([{value, :bool} = _head | _tail] = list) do - size = size_of(value) - len = length(list) - (len * size + 31) |> div(32) + defp chunk_count([head | _tail] = _list, {:list, :bool, max_size}) do + size = size_of(head, :bool) + (max_size * size + 31) |> div(32) end - defp pack_basic_type_list(list) do + defp pack_basic_type_list(list, schema) do list - |> Enum.reduce(<<>>, fn {x, schema}, acc -> + |> Enum.reduce(<<>>, fn x, acc -> {:ok, encoded} = encode(x, schema) acc <> encoded end) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index fa9fb3b40..51615aa3b 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -13,7 +13,7 @@ defmodule Unit.SSZExTest do end test "packing a list of uints" do - list_1 = [{1, {:int, 8}}, {2, {:int, 8}}, {3, {:int, 8}}, {4, {:int, 8}}, {5, {:int, 8}}] + list_1 = [1, 2, 3, 4, 5] expected_1 = <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -23,51 +23,45 @@ defmodule Unit.SSZExTest do assert expected_1 == actual_1 list_2 = [ - {1, {:int, 8}}, - {2, {:int, 8}}, - {3, {:int, 8}}, - {4, {:int, 8}}, - {5, {:int, 8}}, - {18_446_744_073_709_551_595, {:int, 64}}, - {18_446_744_073_709_551_596, {:int, 64}}, - {18_446_744_073_709_551_597, {:int, 64}}, - {18_446_744_073_709_551_598, {:int, 64}}, - {18_446_744_073_709_551_599, {:int, 64}}, - {18_446_744_073_709_551_600, {:int, 64}}, - {18_446_744_073_709_551_601, {:int, 64}}, - {18_446_744_073_709_551_603, {:int, 64}}, - {18_446_744_073_709_551_604, {:int, 64}}, - {18_446_744_073_709_551_605, {:int, 64}}, - {18_446_744_073_709_551_606, {:int, 64}}, - {18_446_744_073_709_551_607, {:int, 64}}, - {18_446_744_073_709_551_608, {:int, 64}}, - {18_446_744_073_709_551_609, {:int, 64}}, - {18_446_744_073_709_551_610, {:int, 64}}, - {18_446_744_073_709_551_611, {:int, 64}}, - {18_446_744_073_709_551_612, {:int, 64}}, - {18_446_744_073_709_551_613, {:int, 64}}, - {18_446_744_073_709_551_614, {:int, 64}}, - {18_446_744_073_709_551_615, {:int, 64}} + 18_446_744_073_709_551_595, + 18_446_744_073_709_551_596, + 18_446_744_073_709_551_597, + 18_446_744_073_709_551_598, + 18_446_744_073_709_551_599, + 18_446_744_073_709_551_600, + 18_446_744_073_709_551_601, + 18_446_744_073_709_551_603, + 18_446_744_073_709_551_604, + 18_446_744_073_709_551_605, + 18_446_744_073_709_551_606, + 18_446_744_073_709_551_607, + 18_446_744_073_709_551_608, + 18_446_744_073_709_551_609, + 18_446_744_073_709_551_610, + 18_446_744_073_709_551_611, + 18_446_744_073_709_551_612, + 18_446_744_073_709_551_613, + 18_446_744_073_709_551_614, + 18_446_744_073_709_551_615 ] expected_2 = - <<1, 2, 3, 4, 5, 235, 255, 255, 255, 255, 255, 255, 255, 236, 255, 255, 255, 255, 255, 255, - 255, 237, 255, 255, 255, 255, 255, 255, 255, 238, 255, 255, 255, 255, 255, 255, 255, 239, - 255, 255, 255, 255, 255, 255, 255, 240, 255, 255, 255, 255, 255, 255, 255, 241, 255, 255, - 255, 255, 255, 255, 255, 243, 255, 255, 255, 255, 255, 255, 255, 244, 255, 255, 255, 255, - 255, 255, 255, 245, 255, 255, 255, 255, 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, - 255, 247, 255, 255, 255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, - 255, 255, 255, 255, 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255, - 255, 255, 255, 255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, - 255, 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - - actual_2 = SszEx.pack(list_2, {:list, {:int, 20}, 20}) + <<235, 255, 255, 255, 255, 255, 255, 255, 236, 255, 255, 255, 255, 255, 255, 255, 237, 255, + 255, 255, 255, 255, 255, 255, 238, 255, 255, 255, 255, 255, 255, 255, 239, 255, 255, 255, + 255, 255, 255, 255, 240, 255, 255, 255, 255, 255, 255, 255, 241, 255, 255, 255, 255, 255, + 255, 255, 243, 255, 255, 255, 255, 255, 255, 255, 244, 255, 255, 255, 255, 255, 255, 255, + 245, 255, 255, 255, 255, 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, 255, 247, 255, + 255, 255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, 255, 255, 255, + 255, 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255, 255, 255, 255, + 255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255>> + + actual_2 = SszEx.pack(list_2, {:list, {:int, 64}, 15}) assert expected_2 == actual_2 end test "packing a list of booleans" do - list = [{true, :bool}, {false, :bool}, {true, :bool}, {false, :bool}, {true, :bool}] + list = [true, false, true, false, true] expected = <<1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -78,34 +72,24 @@ defmodule Unit.SSZExTest do end test "packing a list of bytes" do - list_1 = [ - {<<1>>, {:bytes, 8}}, - {<<2>>, {:bytes, 8}}, - {<<3>>, {:bytes, 8}}, - {<<4>>, {:bytes, 8}}, - {<<5>>, {:bytes, 8}} - ] + list_1 = [<<1>>, <<2>>, <<3>>, <<4>>, <<5>>] expected_1 = <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - actual_1 = SszEx.pack(list_1, {:list, :bool, 5}) + actual_1 = SszEx.pack(list_1, {:list, {:bytes, 8}, 5}) assert expected_1 == actual_1 - list_2 = [ - {<<255, 12>>, {:bytes, 16}}, - {<<2>>, {:bytes, 8}}, - {<<64, 78, 65, 90>>, {:bytes, 32}}, - {<<4>>, {:bytes, 8}}, - {<<5>>, {:bytes, 8}} - ] + list_2 = [<<255, 12>>, <<2>>, <<64, 78, 65, 90>>, <<4>>, <<5>>] expected_2 = <<255, 12, 2, 64, 78, 65, 90, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>> - actual_2 = SszEx.pack(list_2, {:list, :bool, 5}) + ## the size doesn't matter because the bytes are self describing + size = 0 + actual_2 = SszEx.pack(list_2, {:list, {:bytes, size}, 5}) assert expected_2 == actual_2 end @@ -172,7 +156,9 @@ defmodule Unit.SSZExTest do # assert root |> Base.encode16(case: :lower) == expected_value end - test "hash tree of list of basic type" do + test "hash tree root of list" do + ## reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L459 + list = [ 65_535, 65_535, @@ -492,8 +478,9 @@ defmodule Unit.SSZExTest do 65_535 ] - root = list |> SszEx.hash_tree_root({:list, {:int, 16}, 316}) - # root |> Base.encode16() |> IO.inspect() + {:ok, root} = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) + expected_value = "d20d2246e1438d88de46f6f41c7b041f92b673845e51f2de93b944bf599e63b1" + assert root |> Base.encode16(case: :lower) == expected_value end test "serialize and deserialize uint" do From 868fb071de324bf1af42d73691f7137d97d5edf6 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 23:37:10 +0100 Subject: [PATCH 18/31] feat: hash tree of empty list --- lib/ssz_ex.ex | 79 ++++++++++++++++++++------------------- test/unit/ssz_ex_test.exs | 9 +++++ 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 23485dbcd..c52cee805 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -72,7 +72,7 @@ defmodule LambdaEthereumConsensus.SszEx do def hash_tree_root(list, {:list, type, size}) do if !variable_size?(type) do packed_chunks = pack(list, {:list, type, size}) - limit = chunk_count(list, {:list, type, size}) + limit = chunk_count({:list, type, size}) len = length(list) hash_tree_root_list_basic_type(packed_chunks, limit, len) else @@ -97,35 +97,41 @@ defmodule LambdaEthereumConsensus.SszEx do root |> hash_nodes(serialized_len) end - def merklelize_chunks(chunks, leaf_count) do - node_count = 2 * leaf_count - 1 - interior_count = node_count - leaf_count - leaf_start = interior_count * @bytes_per_chunk - padded_chunks = chunks |> convert_to_next_pow_of_two(leaf_count) - buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::bitstring>> - - new_buffer = - 1..node_count - |> Enum.filter(fn x -> rem(x, 2) == 0 end) - |> Enum.reverse() - |> Enum.reduce(buffer, fn index, acc_buffer -> - parent_index = (index - 1) |> div(2) - start = parent_index * @bytes_per_chunk - stop = (index + 1) * @bytes_per_chunk - focus = acc_buffer |> :binary.part(start, stop - start) - focus_len = focus |> byte_size() - children_index = focus_len - 2 * @bytes_per_chunk - children = focus |> :binary.part(children_index, focus_len - children_index) - - <> = children - - parent = hash_nodes(left, right) - replace_chunk(acc_buffer, start, parent) - end) + def merklelize_chunks(chunks, leaf_count \\ nil) do + chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) - <> = new_buffer - root + if chunks_len == 1 and leaf_count == nil do + chunks + else + node_count = 2 * leaf_count - 1 + interior_count = node_count - leaf_count + leaf_start = interior_count * @bytes_per_chunk + padded_chunks = chunks |> convert_to_next_pow_of_two(leaf_count) + buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::bitstring>> + + new_buffer = + 1..node_count + |> Enum.filter(fn x -> rem(x, 2) == 0 end) + |> Enum.reverse() + |> Enum.reduce(buffer, fn index, acc_buffer -> + parent_index = (index - 1) |> div(2) + start = parent_index * @bytes_per_chunk + stop = (index + 1) * @bytes_per_chunk + focus = acc_buffer |> :binary.part(start, stop - start) + focus_len = focus |> byte_size() + children_index = focus_len - 2 * @bytes_per_chunk + children = focus |> :binary.part(children_index, focus_len - children_index) + + <> = children + + parent = hash_nodes(left, right) + replace_chunk(acc_buffer, start, parent) + end) + + <> = new_buffer + root + end end @spec pack(non_neg_integer, {:int, non_neg_integer}) :: binary() @@ -584,20 +590,17 @@ defmodule LambdaEthereumConsensus.SszEx do defp remove_trailing_bit(<<0::7, 1::1>>), do: <<0::0>> defp remove_trailing_bit(<<0::8>>), do: <<0::0>> - defp size_of(_value, :bool), do: @bytes_per_boolean + defp size_of(:bool), do: @bytes_per_boolean - defp size_of(value, {:int, size}) do - {:ok, encoded} = value |> encode_int(size) - encoded |> byte_size() - end + defp size_of({:int, size}), do: size |> div(@bits_per_byte) - defp chunk_count([head | _tail] = _list, {:list, {:int, size}, max_size}) do - size = size_of(head, {:int, size}) + defp chunk_count({:list, {:int, size}, max_size}) do + size = size_of({:int, size}) (max_size * size + 31) |> div(32) end - defp chunk_count([head | _tail] = _list, {:list, :bool, max_size}) do - size = size_of(head, :bool) + defp chunk_count({:list, :bool, max_size}) do + size = size_of(:bool) (max_size * size + 31) |> div(32) end diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 51615aa3b..9afbabf67 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -97,6 +97,11 @@ defmodule Unit.SSZExTest do ## Reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L371 ## https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L416 + chunks = <<0::256>> + root = SszEx.merklelize_chunks(chunks) + expected_value = "0000000000000000000000000000000000000000000000000000000000000000" + assert root |> Base.encode16(case: :lower) == expected_value + chunks = <<0::256>> <> <<0::256>> root = chunks |> SszEx.merklelize_chunks(2) expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" @@ -481,6 +486,10 @@ defmodule Unit.SSZExTest do {:ok, root} = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) expected_value = "d20d2246e1438d88de46f6f41c7b041f92b673845e51f2de93b944bf599e63b1" assert root |> Base.encode16(case: :lower) == expected_value + + {:ok, root} = [] |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) + expected_value = "c9eece3e14d3c3db45c38bbf69a4cb7464981e2506d8424a0ba450dad9b9af30" + assert root |> Base.encode16(case: :lower) == expected_value end test "serialize and deserialize uint" do From 1532ddd93020686f5f8e8cf536d77210fdfc86d6 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 23:44:08 +0100 Subject: [PATCH 19/31] fix: credo --- lib/ssz_ex.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index c52cee805..38f1bdf01 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -70,14 +70,14 @@ defmodule LambdaEthereumConsensus.SszEx do @spec hash_tree_root(list(), {:list, any, non_neg_integer}) :: {:ok, SszTypes.root()} | {:error, String.t()} def hash_tree_root(list, {:list, type, size}) do - if !variable_size?(type) do + if variable_size?(type) do + # TODO + # hash_tree_root_list_complex_type(list, {:list, type, size}, limit) + else packed_chunks = pack(list, {:list, type, size}) limit = chunk_count({:list, type, size}) len = length(list) hash_tree_root_list_basic_type(packed_chunks, limit, len) - else - # TODO - # hash_tree_root_list_complex_type(list, {:list, type, size}, limit) end end @@ -149,10 +149,10 @@ defmodule LambdaEthereumConsensus.SszEx do @spec pack(list(), {:list, any, non_neg_integer}) :: binary() def pack(list, {:list, schema, _size}) do - if !variable_size?(schema) do - pack_basic_type_list(list, schema) - else + if variable_size?(schema) do # pack_complex_type_list(list) + else + pack_basic_type_list(list, schema) end end From 922d7449c8f2e593d5950a812295750d2646c1f6 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Tue, 9 Jan 2024 23:55:04 +0100 Subject: [PATCH 20/31] fix: dialyzer --- lib/ssz_ex.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 38f1bdf01..bef94dcf4 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -61,14 +61,14 @@ defmodule LambdaEthereumConsensus.SszEx do def decode(binary, module) when is_atom(module), do: decode_container(binary, module) - @spec hash_tree_root!(boolean, atom) :: SszTypes.root() + @spec hash_tree_root!(boolean, atom) :: Types.root() def hash_tree_root!(value, :bool), do: pack(value, :bool) - @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root() + @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: Types.root() def hash_tree_root!(value, {:int, size}), do: pack(value, {:int, size}) @spec hash_tree_root(list(), {:list, any, non_neg_integer}) :: - {:ok, SszTypes.root()} | {:error, String.t()} + {:ok, Types.root()} | {:error, String.t()} def hash_tree_root(list, {:list, type, size}) do if variable_size?(type) do # TODO @@ -81,6 +81,8 @@ defmodule LambdaEthereumConsensus.SszEx do end end + @spec hash_tree_root_list_basic_type(binary(), non_neg_integer, non_neg_integer) :: + {:ok, Types.root()} | {:error, String.t()} def hash_tree_root_list_basic_type(chunks, limit, len) do chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) @@ -92,11 +94,13 @@ defmodule LambdaEthereumConsensus.SszEx do end end + @spec mix_in_length(Types.root(), non_neg_integer) :: Types.root() def mix_in_length(root, len) do {:ok, serialized_len} = encode_int(len, @bits_per_chunk) root |> hash_nodes(serialized_len) end + @spec merklelize_chunks(binary(), non_neg_integer) :: Types.root() def merklelize_chunks(chunks, leaf_count \\ nil) do chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) From d284b8f4e3e283ed0a5006bfe986ea825d56fe61 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 00:04:38 +0100 Subject: [PATCH 21/31] fix: dialyzer --- lib/ssz_ex.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index bef94dcf4..33cd3ee74 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -100,7 +100,6 @@ defmodule LambdaEthereumConsensus.SszEx do root |> hash_nodes(serialized_len) end - @spec merklelize_chunks(binary(), non_neg_integer) :: Types.root() def merklelize_chunks(chunks, leaf_count \\ nil) do chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) From 23e0e176afea66ce23a5a72b4f38949d460e7633 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 07:13:13 +0100 Subject: [PATCH 22/31] update --- test/unit/ssz_ex_test.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 9afbabf67..8d4831b4d 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -119,6 +119,15 @@ defmodule Unit.SSZExTest do expected_value = "db56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71" assert root |> Base.encode16(case: :lower) == expected_value + chunks = + <<0::256>> <> + <<0::256>> <> + <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> + + root = chunks |> SszEx.merklelize_chunks(8) + expected_value = "c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c" + assert root |> Base.encode16(case: :lower) == expected_value + chunks = ones root = chunks |> SszEx.merklelize_chunks(4) expected_value = "29797eded0e83376b70f2bf034cc0811ae7f1414653b1d720dfd18f74cf13309" From da946e4502f6944fb312e074e1ce0723b11bd10b Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 07:30:02 +0100 Subject: [PATCH 23/31] update --- test/unit/ssz_ex_test.exs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 8d4831b4d..01cecee9e 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -96,13 +96,14 @@ defmodule Unit.SSZExTest do test "merklelization of chunks" do ## Reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L371 ## https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L416 + zero = <<0::256>> - chunks = <<0::256>> + chunks = zero root = SszEx.merklelize_chunks(chunks) expected_value = "0000000000000000000000000000000000000000000000000000000000000000" assert root |> Base.encode16(case: :lower) == expected_value - chunks = <<0::256>> <> <<0::256>> + chunks = zero <> zero root = chunks |> SszEx.merklelize_chunks(2) expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" assert root |> Base.encode16(case: :lower) == expected_value @@ -114,16 +115,12 @@ defmodule Unit.SSZExTest do expected_value = "7c8975e1e60a5c8337f28edf8c33c3b180360b7279644a9bc1af3c51e6220bf5" assert root |> Base.encode16(case: :lower) == expected_value - chunks = <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> + chunks = zero <> zero <> zero <> zero root = chunks |> SszEx.merklelize_chunks(4) expected_value = "db56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71" assert root |> Base.encode16(case: :lower) == expected_value - chunks = - <<0::256>> <> - <<0::256>> <> - <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> <> <<0::256>> - + chunks = zero <> zero <> zero <> zero <> zero <> zero <> zero <> zero root = chunks |> SszEx.merklelize_chunks(8) expected_value = "c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c" assert root |> Base.encode16(case: :lower) == expected_value From 3fcc626a8f4906d596b47055ac1e7c97eaabcbb9 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 10:19:25 +0100 Subject: [PATCH 24/31] update --- test/unit/ssz_ex_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 01cecee9e..99128b606 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -493,6 +493,7 @@ defmodule Unit.SSZExTest do expected_value = "d20d2246e1438d88de46f6f41c7b041f92b673845e51f2de93b944bf599e63b1" assert root |> Base.encode16(case: :lower) == expected_value + ## hash tree root of empty list {:ok, root} = [] |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) expected_value = "c9eece3e14d3c3db45c38bbf69a4cb7464981e2506d8424a0ba450dad9b9af30" assert root |> Base.encode16(case: :lower) == expected_value From 227840f9844af7a1872e76b0c8475bc7774bc98c Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 19:16:41 +0100 Subject: [PATCH 25/31] fix --- lib/ssz_ex.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 33cd3ee74..a66ebcf39 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -10,9 +10,9 @@ defmodule LambdaEthereumConsensus.SszEx do ################# import Bitwise - @bits_per_chunk 256 @bytes_per_chunk 32 @bits_per_byte 8 + @bits_per_chunk @bytes_per_chunk * @bits_per_byte @zero_chunk <<0::size(@bits_per_chunk)>> @spec hash(iodata()) :: binary() @@ -73,6 +73,7 @@ defmodule LambdaEthereumConsensus.SszEx do if variable_size?(type) do # TODO # hash_tree_root_list_complex_type(list, {:list, type, size}, limit) + {:error, "Not implemented"} else packed_chunks = pack(list, {:list, type, size}) limit = chunk_count({:list, type, size}) From ab10d98ccc447a8ab862abb67cd37bcc8561b3c6 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 19:36:21 +0100 Subject: [PATCH 26/31] fix --- lib/ssz_ex.ex | 4 +- test/unit/ssz_ex_test.exs | 344 ++------------------------------------ 2 files changed, 15 insertions(+), 333 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index a66ebcf39..aa1cadea8 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -90,7 +90,7 @@ defmodule LambdaEthereumConsensus.SszEx do if chunks_len > limit do {:error, "chunk size exceeds limit"} else - root = merklelize_chunks(chunks, limit) |> mix_in_length(len) + root = merkleize_chunks(chunks, limit) |> mix_in_length(len) {:ok, root} end end @@ -101,7 +101,7 @@ defmodule LambdaEthereumConsensus.SszEx do root |> hash_nodes(serialized_len) end - def merklelize_chunks(chunks, leaf_count \\ nil) do + def merkleize_chunks(chunks, leaf_count \\ nil) do chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk) if chunks_len == 1 and leaf_count == nil do diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 99128b606..9fcdb9cc1 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -99,70 +99,70 @@ defmodule Unit.SSZExTest do zero = <<0::256>> chunks = zero - root = SszEx.merklelize_chunks(chunks) + root = SszEx.merkleize_chunks(chunks) expected_value = "0000000000000000000000000000000000000000000000000000000000000000" assert root |> Base.encode16(case: :lower) == expected_value chunks = zero <> zero - root = chunks |> SszEx.merklelize_chunks(2) + root = chunks |> SszEx.merkleize_chunks(2) expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" assert root |> Base.encode16(case: :lower) == expected_value ones = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<1>> <> acc end) chunks = ones <> ones - root = chunks |> SszEx.merklelize_chunks(2) + root = chunks |> SszEx.merkleize_chunks(2) expected_value = "7c8975e1e60a5c8337f28edf8c33c3b180360b7279644a9bc1af3c51e6220bf5" assert root |> Base.encode16(case: :lower) == expected_value chunks = zero <> zero <> zero <> zero - root = chunks |> SszEx.merklelize_chunks(4) + root = chunks |> SszEx.merkleize_chunks(4) expected_value = "db56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71" assert root |> Base.encode16(case: :lower) == expected_value chunks = zero <> zero <> zero <> zero <> zero <> zero <> zero <> zero - root = chunks |> SszEx.merklelize_chunks(8) + root = chunks |> SszEx.merkleize_chunks(8) expected_value = "c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c" assert root |> Base.encode16(case: :lower) == expected_value chunks = ones - root = chunks |> SszEx.merklelize_chunks(4) + root = chunks |> SszEx.merkleize_chunks(4) expected_value = "29797eded0e83376b70f2bf034cc0811ae7f1414653b1d720dfd18f74cf13309" assert root |> Base.encode16(case: :lower) == expected_value twos = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<2>> <> acc end) chunks = twos - root = chunks |> SszEx.merklelize_chunks(8) + root = chunks |> SszEx.merkleize_chunks(8) expected_value = "fa4cf775712aa8a2fe5dcb5a517d19b2e9effcf58ff311b9fd8e4a7d308e6d00" assert root |> Base.encode16(case: :lower) == expected_value chunks = ones <> ones <> ones - root = chunks |> SszEx.merklelize_chunks(4) + root = chunks |> SszEx.merkleize_chunks(4) expected_value = "65aa94f2b59e517abd400cab655f42821374e433e41b8fe599f6bb15484adcec" assert root |> Base.encode16(case: :lower) == expected_value chunks = ones <> ones <> ones <> ones <> ones - root = chunks |> SszEx.merklelize_chunks(8) + root = chunks |> SszEx.merkleize_chunks(8) expected_value = "0ae67e34cba4ad2bbfea5dc39e6679b444021522d861fab00f05063c54341289" assert root |> Base.encode16(case: :lower) == expected_value chunks = ones <> ones <> ones <> ones <> ones <> ones - root = chunks |> SszEx.merklelize_chunks(8) + root = chunks |> SszEx.merkleize_chunks(8) expected_value = "0ef7df63c204ef203d76145627b8083c49aa7c55ebdee2967556f55a4f65a238" assert root |> Base.encode16(case: :lower) == expected_value ## Large Leaf Count chunks = ones <> ones <> ones <> ones <> ones - root = chunks |> SszEx.merklelize_chunks(2 ** 10) + root = chunks |> SszEx.merkleize_chunks(2 ** 10) expected_value = "2647cb9e26bd83eeb0982814b2ac4d6cc4a65d0d98637f1a73a4c06d3db0e6ce" assert root |> Base.encode16(case: :lower) == expected_value ## TOO HEAVY COMPUTATION! # chunks = 1..70 |> Enum.reduce(<<>>, fn _, acc -> acc <> ones end) # leaf_count = 9_223_372_036_854_775_808 # 2 ** 63 - # root = chunks |> SszEx.merklelize_chunks(leaf_count) + # root = chunks |> SszEx.merkleize_chunks(leaf_count) # expected_value = "9317695d95b5a3b46e976b5a9cbfcfccb600accaddeda9ac867cc9669b862979" # assert root |> Base.encode16(case: :lower) == expected_value end @@ -170,325 +170,7 @@ defmodule Unit.SSZExTest do test "hash tree root of list" do ## reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L459 - list = [ - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535, - 65_535 - ] - + list = Stream.cycle([65_535]) |> Enum.take(316) {:ok, root} = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024}) expected_value = "d20d2246e1438d88de46f6f41c7b041f92b673845e51f2de93b944bf599e63b1" assert root |> Base.encode16(case: :lower) == expected_value From f6fc7c4ac97e606640d8ee31dee39c393f51a89b Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 19:42:32 +0100 Subject: [PATCH 27/31] fix --- lib/ssz_ex.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index aa1cadea8..0d9cea5c4 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -138,19 +138,15 @@ defmodule LambdaEthereumConsensus.SszEx do end end + @spec pack(boolean, :bool) :: binary() + def pack(true, :bool), do: <<1::@bits_per_chunk-little>> + def pack(false, :bool), do: @zero_chunk + @spec pack(non_neg_integer, {:int, non_neg_integer}) :: binary() def pack(value, {:int, size}) do <> |> pack_bytes() end - @spec pack(boolean, :bool) :: binary() - def pack(value, :bool) do - case value do - true -> <<1::@bits_per_chunk-little>> - false -> @zero_chunk - end - end - @spec pack(list(), {:list, any, non_neg_integer}) :: binary() def pack(list, {:list, schema, _size}) do if variable_size?(schema) do From e1a3b79dd564ff47c8458afe7534b434c1ef9f51 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Wed, 10 Jan 2024 19:46:19 +0100 Subject: [PATCH 28/31] fix --- lib/ssz_ex.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 0d9cea5c4..147995d0b 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -147,10 +147,12 @@ defmodule LambdaEthereumConsensus.SszEx do <> |> pack_bytes() end - @spec pack(list(), {:list, any, non_neg_integer}) :: binary() + @spec pack(list(), {:list, any, non_neg_integer}) :: binary() | :error def pack(list, {:list, schema, _size}) do if variable_size?(schema) do + # TODO # pack_complex_type_list(list) + :error else pack_basic_type_list(list, schema) end From b7b449f0b687492253fc066e2d0ac68759c497d0 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Thu, 11 Jan 2024 13:07:58 +0100 Subject: [PATCH 29/31] fix --- test/unit/ssz_ex_test.exs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test/unit/ssz_ex_test.exs b/test/unit/ssz_ex_test.exs index 9fcdb9cc1..2f9c5fec3 100644 --- a/test/unit/ssz_ex_test.exs +++ b/test/unit/ssz_ex_test.exs @@ -71,28 +71,6 @@ defmodule Unit.SSZExTest do assert expected == actual end - test "packing a list of bytes" do - list_1 = [<<1>>, <<2>>, <<3>>, <<4>>, <<5>>] - - expected_1 = - <<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0>> - - actual_1 = SszEx.pack(list_1, {:list, {:bytes, 8}, 5}) - assert expected_1 == actual_1 - - list_2 = [<<255, 12>>, <<2>>, <<64, 78, 65, 90>>, <<4>>, <<5>>] - - expected_2 = - <<255, 12, 2, 64, 78, 65, 90, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0>> - - ## the size doesn't matter because the bytes are self describing - size = 0 - actual_2 = SszEx.pack(list_2, {:list, {:bytes, size}, 5}) - assert expected_2 == actual_2 - end - test "merklelization of chunks" do ## Reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L371 ## https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L416 From f2f71ac35a8901664e908a014c5b29436adbee76 Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Fri, 12 Jan 2024 11:55:53 +0100 Subject: [PATCH 30/31] fix --- lib/ssz_ex.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 147995d0b..fd5aeda73 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -126,14 +126,14 @@ defmodule LambdaEthereumConsensus.SszEx do children_index = focus_len - 2 * @bytes_per_chunk children = focus |> :binary.part(children_index, focus_len - children_index) - <> = children + <> = children parent = hash_nodes(left, right) replace_chunk(acc_buffer, start, parent) end) - <> = new_buffer + <> = new_buffer root end end @@ -649,10 +649,10 @@ defmodule LambdaEthereumConsensus.SszEx do end defp replace_chunk(chunks, start, new_chunk) do - <> = + <> = chunks - <> + <> end end From f2011e445fb23a2679297d08af3abdd189f024eb Mon Sep 17 00:00:00 2001 From: Godspower Eze Date: Fri, 12 Jan 2024 12:00:05 +0100 Subject: [PATCH 31/31] fix link --- lib/ssz_ex.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index fd5aeda73..7f0add49a 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -126,8 +126,7 @@ defmodule LambdaEthereumConsensus.SszEx do children_index = focus_len - 2 * @bytes_per_chunk children = focus |> :binary.part(children_index, focus_len - children_index) - <> = children + <> = children parent = hash_nodes(left, right) replace_chunk(acc_buffer, start, parent) @@ -649,8 +648,7 @@ defmodule LambdaEthereumConsensus.SszEx do end defp replace_chunk(chunks, start, new_chunk) do - <> = + <> = chunks <>