From b3cf9a0f1705b8cec21025c01824bcb1ad4956e9 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 20 May 2024 22:22:42 -0300 Subject: [PATCH 1/4] fix: trigger pruning on tick --- .../fork_choice/fork_choice.ex | 20 +++++++++++++------ .../store/state_db.ex | 8 +++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex index 3f0ea89a0..e07fd0afa 100644 --- a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex +++ b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex @@ -89,12 +89,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do %Store{finalized_checkpoint: new_finalized_checkpoint} = new_store - if last_finalized_checkpoint.epoch < new_finalized_checkpoint.epoch do - new_finalized_slot = - new_finalized_checkpoint.epoch * ChainSpec.get("SLOTS_PER_EPOCH") - - Task.async(StateDb, :prune_states_older_than, [new_finalized_slot]) - end + prune_old_states(last_finalized_checkpoint.epoch, new_finalized_checkpoint.epoch) GenServer.cast(from, {:block_processed, block_root, true}) {:noreply, new_store} @@ -139,7 +134,11 @@ defmodule LambdaEthereumConsensus.ForkChoice do @impl GenServer def handle_cast({:on_tick, time}, store) do + %Store{finalized_checkpoint: last_finalized_checkpoint} = store + new_store = Handlers.on_tick(store, time) + %Store{finalized_checkpoint: new_finalized_checkpoint} = new_store + prune_old_states(last_finalized_checkpoint.epoch, new_finalized_checkpoint.epoch) {:noreply, new_store} end @@ -152,6 +151,15 @@ defmodule LambdaEthereumConsensus.ForkChoice do ### Private Functions ########################## + def prune_old_states(last_finalized_epoch, new_finalized_epoch) do + if last_finalized_epoch < new_finalized_epoch do + new_finalized_slot = + new_finalized_epoch * ChainSpec.get("SLOTS_PER_EPOCH") + + Task.async(StateDb, :prune_states_older_than, [new_finalized_slot]) + end + end + @spec apply_handler(any(), any(), any()) :: any() def apply_handler(iter, state, handler) do iter diff --git a/lib/lambda_ethereum_consensus/store/state_db.ex b/lib/lambda_ethereum_consensus/store/state_db.ex index f09c2400c..b35b43cf7 100644 --- a/lib/lambda_ethereum_consensus/store/state_db.ex +++ b/lib/lambda_ethereum_consensus/store/state_db.ex @@ -2,6 +2,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do @moduledoc """ Beacon node state storage. """ + require Logger alias LambdaEthereumConsensus.Store.Db alias LambdaEthereumConsensus.Store.Utils alias Types.BeaconState @@ -31,6 +32,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do end def prune_states_older_than(slot) do + Logger.info("[StateDb] Pruning started.", slot: slot) last_finalized_key = slot |> root_by_slot_key() with {:ok, it} <- Db.iterate(), @@ -39,21 +41,21 @@ defmodule LambdaEthereumConsensus.Store.StateDb do {:ok, slots_to_remove} <- get_slots_to_remove(it), :ok <- Exleveldb.iterator_close(it) do slots_to_remove |> Enum.map(&remove_by_slot/1) + Logger.info("[StateDb] Pruning finished. #{length(slots_to_remove)} slots removed.") end end defp get_slots_to_remove(slots_to_remove \\ [], iterator) do case Exleveldb.iterator_move(iterator, :prev) do {:ok, @stateslot_prefix <> slot, _root} -> - [slot | slots_to_remove] |> get_slots_to_remove(iterator) + [slot |> :binary.decode_unsigned() | slots_to_remove] |> get_slots_to_remove(iterator) _ -> {:ok, slots_to_remove} end end - defp remove_by_slot(binary_slot) do - slot = :binary.decode_unsigned(binary_slot) + def remove_by_slot(slot) do key_slot = root_by_slot_key(slot) with {:ok, block_root} <- Db.get(key_slot), From a0cfcea5519f945a9fbded51e975893029df85cb Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 20 May 2024 23:10:51 -0300 Subject: [PATCH 2/4] Fix linter --- lib/lambda_ethereum_consensus/store/state_db.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lambda_ethereum_consensus/store/state_db.ex b/lib/lambda_ethereum_consensus/store/state_db.ex index b35b43cf7..e2c4b7a42 100644 --- a/lib/lambda_ethereum_consensus/store/state_db.ex +++ b/lib/lambda_ethereum_consensus/store/state_db.ex @@ -40,7 +40,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do Exleveldb.iterator_move(it, last_finalized_key), {:ok, slots_to_remove} <- get_slots_to_remove(it), :ok <- Exleveldb.iterator_close(it) do - slots_to_remove |> Enum.map(&remove_by_slot/1) + slots_to_remove |> Enum.each(&remove_by_slot/1) Logger.info("[StateDb] Pruning finished. #{length(slots_to_remove)} slots removed.") end end @@ -55,7 +55,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do end end - def remove_by_slot(slot) do + defp remove_by_slot(slot) do key_slot = root_by_slot_key(slot) with {:ok, block_root} <- Db.get(key_slot), From db6c78bc456418a36a38747a9f759bf2a8def8a8 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 21 May 2024 10:45:31 -0300 Subject: [PATCH 3/4] Add typespec --- lib/lambda_ethereum_consensus/store/state_db.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/lambda_ethereum_consensus/store/state_db.ex b/lib/lambda_ethereum_consensus/store/state_db.ex index e2c4b7a42..63cda854f 100644 --- a/lib/lambda_ethereum_consensus/store/state_db.ex +++ b/lib/lambda_ethereum_consensus/store/state_db.ex @@ -31,6 +31,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do Db.put(slothash_key_block, block_root) end + @spec prune_states_older_than(non_neg_integer()) :: :ok | {:error, String.t()} | :not_found def prune_states_older_than(slot) do Logger.info("[StateDb] Pruning started.", slot: slot) last_finalized_key = slot |> root_by_slot_key() @@ -45,6 +46,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do end end + @spec get_slots_to_remove(list(), :eleveldb.itr_ref()) :: {:ok, list(non_neg_integer())} defp get_slots_to_remove(slots_to_remove \\ [], iterator) do case Exleveldb.iterator_move(iterator, :prev) do {:ok, @stateslot_prefix <> slot, _root} -> @@ -55,6 +57,7 @@ defmodule LambdaEthereumConsensus.Store.StateDb do end end + @spec remove_by_slot(non_neg_integer()) :: :ok | :not_found defp remove_by_slot(slot) do key_slot = root_by_slot_key(slot) From abb4f3ee9d727ffb5232a89000509ef4acb7c01b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 21 May 2024 12:17:59 -0300 Subject: [PATCH 4/4] Add typespec --- lib/lambda_ethereum_consensus/store/state_db.ex | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/lambda_ethereum_consensus/store/state_db.ex b/lib/lambda_ethereum_consensus/store/state_db.ex index 63cda854f..ca959befe 100644 --- a/lib/lambda_ethereum_consensus/store/state_db.ex +++ b/lib/lambda_ethereum_consensus/store/state_db.ex @@ -41,24 +41,25 @@ defmodule LambdaEthereumConsensus.Store.StateDb do Exleveldb.iterator_move(it, last_finalized_key), {:ok, slots_to_remove} <- get_slots_to_remove(it), :ok <- Exleveldb.iterator_close(it) do - slots_to_remove |> Enum.each(&remove_by_slot/1) + slots_to_remove |> Enum.each(&remove_state_by_slot/1) Logger.info("[StateDb] Pruning finished. #{length(slots_to_remove)} slots removed.") end end - @spec get_slots_to_remove(list(), :eleveldb.itr_ref()) :: {:ok, list(non_neg_integer())} + @spec get_slots_to_remove(list(non_neg_integer()), :eleveldb.itr_ref()) :: + {:ok, list(non_neg_integer())} defp get_slots_to_remove(slots_to_remove \\ [], iterator) do case Exleveldb.iterator_move(iterator, :prev) do - {:ok, @stateslot_prefix <> slot, _root} -> - [slot |> :binary.decode_unsigned() | slots_to_remove] |> get_slots_to_remove(iterator) + {:ok, @stateslot_prefix <> <>, _root} -> + [slot | slots_to_remove] |> get_slots_to_remove(iterator) _ -> {:ok, slots_to_remove} end end - @spec remove_by_slot(non_neg_integer()) :: :ok | :not_found - defp remove_by_slot(slot) do + @spec remove_state_by_slot(non_neg_integer()) :: :ok | :not_found + defp remove_state_by_slot(slot) do key_slot = root_by_slot_key(slot) with {:ok, block_root} <- Db.get(key_slot),