Skip to content

Commit c146fec

Browse files
Godspower-EzeMegaRedHand
Godspower-Eze
and
MegaRedHand
authored
feat: merklelization of lists (#528)
Co-authored-by: Tomás Grüner <47506558+MegaRedHand@users.noreply.github.com>
1 parent 91469bc commit c146fec

File tree

2 files changed

+307
-13
lines changed

2 files changed

+307
-13
lines changed

‎lib/ssz_ex.ex

Lines changed: 160 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,29 @@ defmodule LambdaEthereumConsensus.SszEx do
88
#################
99
### Public API
1010
#################
11+
import Bitwise
1112

12-
@bits_per_chunk 256
13+
@bytes_per_chunk 32
14+
@bits_per_byte 8
15+
@bits_per_chunk @bytes_per_chunk * @bits_per_byte
16+
@zero_chunk <<0::size(@bits_per_chunk)>>
1317

1418
@spec hash(iodata()) :: binary()
1519
def hash(data), do: :crypto.hash(:sha256, data)
1620

21+
@spec hash_nodes(binary(), binary()) :: binary()
22+
def hash_nodes(left, right), do: :crypto.hash(:sha256, left <> right)
23+
1724
def encode(value, {:int, size}), do: encode_int(value, size)
1825
def encode(value, :bool), do: encode_bool(value)
26+
def encode(value, {:bytes, _}), do: {:ok, value}
1927

2028
def encode(list, {:list, basic_type, size}) do
2129
if variable_size?(basic_type),
2230
do: encode_variable_size_list(list, basic_type, size),
2331
else: encode_fixed_size_list(list, basic_type, size)
2432
end
2533

26-
def encode(value, {:bytes, _}), do: {:ok, value}
27-
2834
def encode(value, {:bitlist, max_size}) when is_bitstring(value),
2935
do: encode_bitlist(value, max_size)
3036

@@ -56,16 +62,106 @@ defmodule LambdaEthereumConsensus.SszEx do
5662
def decode(binary, module) when is_atom(module), do: decode_container(binary, module)
5763

5864
@spec hash_tree_root!(boolean, atom) :: Types.root()
59-
def hash_tree_root!(value, :bool), do: pack(value)
65+
def hash_tree_root!(value, :bool), do: pack(value, :bool)
6066

6167
@spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: Types.root()
62-
def hash_tree_root!(value, {:int, size}), do: pack(value, size)
68+
def hash_tree_root!(value, {:int, size}), do: pack(value, {:int, size})
69+
70+
@spec hash_tree_root(list(), {:list, any, non_neg_integer}) ::
71+
{:ok, Types.root()} | {:error, String.t()}
72+
def hash_tree_root(list, {:list, type, size}) do
73+
if variable_size?(type) do
74+
# TODO
75+
# hash_tree_root_list_complex_type(list, {:list, type, size}, limit)
76+
{:error, "Not implemented"}
77+
else
78+
packed_chunks = pack(list, {:list, type, size})
79+
limit = chunk_count({:list, type, size})
80+
len = length(list)
81+
hash_tree_root_list_basic_type(packed_chunks, limit, len)
82+
end
83+
end
84+
85+
@spec hash_tree_root_list_basic_type(binary(), non_neg_integer, non_neg_integer) ::
86+
{:ok, Types.root()} | {:error, String.t()}
87+
def hash_tree_root_list_basic_type(chunks, limit, len) do
88+
chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk)
89+
90+
if chunks_len > limit do
91+
{:error, "chunk size exceeds limit"}
92+
else
93+
root = merkleize_chunks(chunks, limit) |> mix_in_length(len)
94+
{:ok, root}
95+
end
96+
end
97+
98+
@spec mix_in_length(Types.root(), non_neg_integer) :: Types.root()
99+
def mix_in_length(root, len) do
100+
{:ok, serialized_len} = encode_int(len, @bits_per_chunk)
101+
root |> hash_nodes(serialized_len)
102+
end
103+
104+
def merkleize_chunks(chunks, leaf_count \\ nil) do
105+
chunks_len = chunks |> byte_size() |> div(@bytes_per_chunk)
106+
107+
if chunks_len == 1 and leaf_count == nil do
108+
chunks
109+
else
110+
node_count = 2 * leaf_count - 1
111+
interior_count = node_count - leaf_count
112+
leaf_start = interior_count * @bytes_per_chunk
113+
padded_chunks = chunks |> convert_to_next_pow_of_two(leaf_count)
114+
buffer = <<0::size(leaf_start * @bits_per_byte), padded_chunks::bitstring>>
115+
116+
new_buffer =
117+
1..node_count
118+
|> Enum.filter(fn x -> rem(x, 2) == 0 end)
119+
|> Enum.reverse()
120+
|> Enum.reduce(buffer, fn index, acc_buffer ->
121+
parent_index = (index - 1) |> div(2)
122+
start = parent_index * @bytes_per_chunk
123+
stop = (index + 1) * @bytes_per_chunk
124+
focus = acc_buffer |> :binary.part(start, stop - start)
125+
focus_len = focus |> byte_size()
126+
children_index = focus_len - 2 * @bytes_per_chunk
127+
children = focus |> :binary.part(children_index, focus_len - children_index)
128+
129+
<<left::binary-size(@bytes_per_chunk), right::binary-size(@bytes_per_chunk)>> = children
130+
131+
parent = hash_nodes(left, right)
132+
replace_chunk(acc_buffer, start, parent)
133+
end)
134+
135+
<<root::binary-size(@bytes_per_chunk), _::binary>> = new_buffer
136+
root
137+
end
138+
end
139+
140+
@spec pack(boolean, :bool) :: binary()
141+
def pack(true, :bool), do: <<1::@bits_per_chunk-little>>
142+
def pack(false, :bool), do: @zero_chunk
143+
144+
@spec pack(non_neg_integer, {:int, non_neg_integer}) :: binary()
145+
def pack(value, {:int, size}) do
146+
<<value::size(size)-little>> |> pack_bytes()
147+
end
148+
149+
@spec pack(list(), {:list, any, non_neg_integer}) :: binary() | :error
150+
def pack(list, {:list, schema, _size}) do
151+
if variable_size?(schema) do
152+
# TODO
153+
# pack_complex_type_list(list)
154+
:error
155+
else
156+
pack_basic_type_list(list, schema)
157+
end
158+
end
63159

64160
#################
65161
### Private functions
66162
#################
163+
@bytes_per_boolean 4
67164
@bytes_per_length_offset 4
68-
@bits_per_byte 8
69165
@offset_bits 32
70166

71167
defp encode_int(value, size) when is_integer(value), do: {:ok, <<value::size(size)-little>>}
@@ -495,15 +591,66 @@ defmodule LambdaEthereumConsensus.SszEx do
495591
defp remove_trailing_bit(<<0::7, 1::1>>), do: <<0::0>>
496592
defp remove_trailing_bit(<<0::8>>), do: <<0::0>>
497593

498-
defp pack(value, size) when is_integer(value) and value >= 0 do
499-
pad = @bits_per_chunk - size
500-
<<value::size(size)-little, 0::size(pad)>>
594+
defp size_of(:bool), do: @bytes_per_boolean
595+
596+
defp size_of({:int, size}), do: size |> div(@bits_per_byte)
597+
598+
defp chunk_count({:list, {:int, size}, max_size}) do
599+
size = size_of({:int, size})
600+
(max_size * size + 31) |> div(32)
601+
end
602+
603+
defp chunk_count({:list, :bool, max_size}) do
604+
size = size_of(:bool)
605+
(max_size * size + 31) |> div(32)
501606
end
502607

503-
defp pack(value) when is_boolean(value) do
504-
case value do
505-
true -> <<1::@bits_per_chunk-little>>
506-
false -> <<0::@bits_per_chunk>>
608+
defp pack_basic_type_list(list, schema) do
609+
list
610+
|> Enum.reduce(<<>>, fn x, acc ->
611+
{:ok, encoded} = encode(x, schema)
612+
acc <> encoded
613+
end)
614+
|> pack_bytes()
615+
end
616+
617+
defp pack_bytes(value) when is_binary(value) do
618+
incomplete_chunk_len = value |> bit_size() |> rem(@bits_per_chunk)
619+
620+
if incomplete_chunk_len != 0 do
621+
pad = @bits_per_chunk - incomplete_chunk_len
622+
<<value::binary, 0::size(pad)>>
623+
else
624+
value
507625
end
508626
end
627+
628+
defp convert_to_next_pow_of_two(chunks, leaf_count) do
629+
size = chunks |> byte_size() |> div(@bytes_per_chunk)
630+
next_pow = leaf_count |> next_pow_of_two()
631+
632+
if size == next_pow do
633+
chunks
634+
else
635+
diff = next_pow - size
636+
zero_chunks = 0..(diff - 1) |> Enum.reduce(<<>>, fn _, acc -> <<0::256>> <> acc end)
637+
chunks <> zero_chunks
638+
end
639+
end
640+
641+
defp next_pow_of_two(len) when is_integer(len) and len >= 0 do
642+
if len == 0 do
643+
0
644+
else
645+
n = ((len <<< 1) - 1) |> :math.log2() |> trunc()
646+
2 ** n
647+
end
648+
end
649+
650+
defp replace_chunk(chunks, start, new_chunk) do
651+
<<left::binary-size(start), _::size(@bits_per_chunk), right::binary>> =
652+
chunks
653+
654+
<<left::binary, new_chunk::binary, right::binary>>
655+
end
509656
end

‎test/unit/ssz_ex_test.exs

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,153 @@ defmodule Unit.SSZExTest do
1212
assert {:error, ^error_message} = SszEx.decode(serialized, schema)
1313
end
1414

15+
test "packing a list of uints" do
16+
list_1 = [1, 2, 3, 4, 5]
17+
18+
expected_1 =
19+
<<1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
20+
0, 0>>
21+
22+
actual_1 = SszEx.pack(list_1, {:list, {:int, 8}, 5})
23+
assert expected_1 == actual_1
24+
25+
list_2 = [
26+
18_446_744_073_709_551_595,
27+
18_446_744_073_709_551_596,
28+
18_446_744_073_709_551_597,
29+
18_446_744_073_709_551_598,
30+
18_446_744_073_709_551_599,
31+
18_446_744_073_709_551_600,
32+
18_446_744_073_709_551_601,
33+
18_446_744_073_709_551_603,
34+
18_446_744_073_709_551_604,
35+
18_446_744_073_709_551_605,
36+
18_446_744_073_709_551_606,
37+
18_446_744_073_709_551_607,
38+
18_446_744_073_709_551_608,
39+
18_446_744_073_709_551_609,
40+
18_446_744_073_709_551_610,
41+
18_446_744_073_709_551_611,
42+
18_446_744_073_709_551_612,
43+
18_446_744_073_709_551_613,
44+
18_446_744_073_709_551_614,
45+
18_446_744_073_709_551_615
46+
]
47+
48+
expected_2 =
49+
<<235, 255, 255, 255, 255, 255, 255, 255, 236, 255, 255, 255, 255, 255, 255, 255, 237, 255,
50+
255, 255, 255, 255, 255, 255, 238, 255, 255, 255, 255, 255, 255, 255, 239, 255, 255, 255,
51+
255, 255, 255, 255, 240, 255, 255, 255, 255, 255, 255, 255, 241, 255, 255, 255, 255, 255,
52+
255, 255, 243, 255, 255, 255, 255, 255, 255, 255, 244, 255, 255, 255, 255, 255, 255, 255,
53+
245, 255, 255, 255, 255, 255, 255, 255, 246, 255, 255, 255, 255, 255, 255, 255, 247, 255,
54+
255, 255, 255, 255, 255, 255, 248, 255, 255, 255, 255, 255, 255, 255, 249, 255, 255, 255,
55+
255, 255, 255, 255, 250, 255, 255, 255, 255, 255, 255, 255, 251, 255, 255, 255, 255, 255,
56+
255, 255, 252, 255, 255, 255, 255, 255, 255, 255, 253, 255, 255, 255, 255, 255, 255, 255,
57+
254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255>>
58+
59+
actual_2 = SszEx.pack(list_2, {:list, {:int, 64}, 15})
60+
assert expected_2 == actual_2
61+
end
62+
63+
test "packing a list of booleans" do
64+
list = [true, false, true, false, true]
65+
66+
expected =
67+
<<1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68+
0, 0>>
69+
70+
actual = SszEx.pack(list, {:list, :bool, 5})
71+
assert expected == actual
72+
end
73+
74+
test "merklelization of chunks" do
75+
## Reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L371
76+
## https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L416
77+
zero = <<0::256>>
78+
79+
chunks = zero
80+
root = SszEx.merkleize_chunks(chunks)
81+
expected_value = "0000000000000000000000000000000000000000000000000000000000000000"
82+
assert root |> Base.encode16(case: :lower) == expected_value
83+
84+
chunks = zero <> zero
85+
root = chunks |> SszEx.merkleize_chunks(2)
86+
expected_value = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
87+
assert root |> Base.encode16(case: :lower) == expected_value
88+
89+
ones = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<1>> <> acc end)
90+
91+
chunks = ones <> ones
92+
root = chunks |> SszEx.merkleize_chunks(2)
93+
expected_value = "7c8975e1e60a5c8337f28edf8c33c3b180360b7279644a9bc1af3c51e6220bf5"
94+
assert root |> Base.encode16(case: :lower) == expected_value
95+
96+
chunks = zero <> zero <> zero <> zero
97+
root = chunks |> SszEx.merkleize_chunks(4)
98+
expected_value = "db56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71"
99+
assert root |> Base.encode16(case: :lower) == expected_value
100+
101+
chunks = zero <> zero <> zero <> zero <> zero <> zero <> zero <> zero
102+
root = chunks |> SszEx.merkleize_chunks(8)
103+
expected_value = "c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c"
104+
assert root |> Base.encode16(case: :lower) == expected_value
105+
106+
chunks = ones
107+
root = chunks |> SszEx.merkleize_chunks(4)
108+
expected_value = "29797eded0e83376b70f2bf034cc0811ae7f1414653b1d720dfd18f74cf13309"
109+
assert root |> Base.encode16(case: :lower) == expected_value
110+
111+
twos = 0..31 |> Enum.reduce(<<>>, fn _, acc -> <<2>> <> acc end)
112+
113+
chunks = twos
114+
root = chunks |> SszEx.merkleize_chunks(8)
115+
expected_value = "fa4cf775712aa8a2fe5dcb5a517d19b2e9effcf58ff311b9fd8e4a7d308e6d00"
116+
assert root |> Base.encode16(case: :lower) == expected_value
117+
118+
chunks = ones <> ones <> ones
119+
root = chunks |> SszEx.merkleize_chunks(4)
120+
expected_value = "65aa94f2b59e517abd400cab655f42821374e433e41b8fe599f6bb15484adcec"
121+
assert root |> Base.encode16(case: :lower) == expected_value
122+
123+
chunks = ones <> ones <> ones <> ones <> ones
124+
root = chunks |> SszEx.merkleize_chunks(8)
125+
expected_value = "0ae67e34cba4ad2bbfea5dc39e6679b444021522d861fab00f05063c54341289"
126+
assert root |> Base.encode16(case: :lower) == expected_value
127+
128+
chunks = ones <> ones <> ones <> ones <> ones <> ones
129+
root = chunks |> SszEx.merkleize_chunks(8)
130+
expected_value = "0ef7df63c204ef203d76145627b8083c49aa7c55ebdee2967556f55a4f65a238"
131+
assert root |> Base.encode16(case: :lower) == expected_value
132+
133+
## Large Leaf Count
134+
135+
chunks = ones <> ones <> ones <> ones <> ones
136+
root = chunks |> SszEx.merkleize_chunks(2 ** 10)
137+
expected_value = "2647cb9e26bd83eeb0982814b2ac4d6cc4a65d0d98637f1a73a4c06d3db0e6ce"
138+
assert root |> Base.encode16(case: :lower) == expected_value
139+
140+
## TOO HEAVY COMPUTATION!
141+
# chunks = 1..70 |> Enum.reduce(<<>>, fn _, acc -> acc <> ones end)
142+
# leaf_count = 9_223_372_036_854_775_808 # 2 ** 63
143+
# root = chunks |> SszEx.merkleize_chunks(leaf_count)
144+
# expected_value = "9317695d95b5a3b46e976b5a9cbfcfccb600accaddeda9ac867cc9669b862979"
145+
# assert root |> Base.encode16(case: :lower) == expected_value
146+
end
147+
148+
test "hash tree root of list" do
149+
## reference: https://github.com/ralexstokes/ssz-rs/blob/1f94d5dfc70c86dab672e91ac46af04a5f96c342/ssz-rs/src/merkleization/mod.rs#L459
150+
151+
list = Stream.cycle([65_535]) |> Enum.take(316)
152+
{:ok, root} = list |> SszEx.hash_tree_root({:list, {:int, 16}, 1024})
153+
expected_value = "d20d2246e1438d88de46f6f41c7b041f92b673845e51f2de93b944bf599e63b1"
154+
assert root |> Base.encode16(case: :lower) == expected_value
155+
156+
## hash tree root of empty list
157+
{:ok, root} = [] |> SszEx.hash_tree_root({:list, {:int, 16}, 1024})
158+
expected_value = "c9eece3e14d3c3db45c38bbf69a4cb7464981e2506d8424a0ba450dad9b9af30"
159+
assert root |> Base.encode16(case: :lower) == expected_value
160+
end
161+
15162
test "serialize and deserialize uint" do
16163
assert_roundtrip(<<5>>, 5, {:int, 8})
17164
assert_roundtrip(<<5, 0>>, 5, {:int, 16})

0 commit comments

Comments
 (0)