Merge branch 'main' of github.com:lambdaclass/lambda_ethereum_consensus into refactor-pending-blocks

Author: mpaulucci

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "mix"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "cargo"
+    directory: "/native/bls_nif"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/native/libp2p_port"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/" # Expands to `.github/workflows`
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
             native/libp2p_port/go.sum
       - name: Cache output artifacts
         id: output-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: priv/native/*
           key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
@@ -58,7 +58,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Cache Beacon Node OAPI
         id: output-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ./beacon-node-oapi.json
           key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
@@ -80,19 +80,19 @@ jobs:
         version-file: .tool-versions
     - name: Fetch native libraries
       id: output-cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: priv/native/*
         key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
         fail-on-cache-miss: true
     - name: Restore dependencies cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
     - name: Fetch beacon node oapi file
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: ./beacon-node-oapi.json
         key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
@@ -111,7 +111,7 @@ jobs:
     - name: Compile Elixir (Warnings as errors)
       run: mix compile --warnings-as-errors
     - name: Retrieve PLT Cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: plt-cache
       with:
         path: priv/plts
@@ -137,19 +137,19 @@ jobs:
         version-file: .tool-versions
     - name: Fetch native libraries
       id: output-cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: priv/native/*
         key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
         fail-on-cache-miss: true
     - name: Restore dependencies cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
     - name: Fetch beacon node oapi file
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: ./beacon-node-oapi.json
         key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
@@ -184,19 +184,19 @@ jobs:
         version-file: .tool-versions
     - name: Fetch native libraries
       id: output-cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: priv/native/*
         key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
         fail-on-cache-miss: true
     - name: Restore dependencies cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
     - name: Fetch beacon node oapi file
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: ./beacon-node-oapi.json
         key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
@@ -225,7 +225,7 @@ jobs:
         version-type: strict
         version-file: .tool-versions
     - name: Restore dependencies cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
@@ -246,7 +246,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Cache compressed spectests
         id: output-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ./*.tar.gz
           key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }}
@@ -270,25 +270,25 @@ jobs:
           version-type: strict
           version-file: .tool-versions
       - name: Fetch native libraries
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: priv/native/*
           key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
           fail-on-cache-miss: true
       - name: Fetch spectest vectors
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ./*.tar.gz
           key: ${{ runner.os }}-spectest-${{ hashFiles('.spectest_version') }}
           fail-on-cache-miss: true
       - name: Restore dependencies cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
           restore-keys: ${{ runner.os }}-mix-
       - name: Fetch beacon node oapi file
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ./beacon-node-oapi.json
           key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}

docs/concurrency_design.md

@@ -7,7 +7,7 @@ The following is a sequence diagram on the lifecycle of a block, from the moment
 ```mermaid
 sequenceDiagram
 
-participant port as LibP2P Port <br> (Genserver)
+participant port as LibP2P Port <br> (GenServer)
 participant sub as Subscriber <br> (GenStage)
 participant consumer as GossipConsumer <br> (broadway)
 participant pending as Pending Blocks <br> (GenServer)
@@ -66,7 +66,7 @@ Improvements:
 ### Other issues
 
 - States aren't ever stored in the DB. This is not a concurrency issue, but we should fix it.
-- Low priority, but we should evaluate dropping the Subscriber genserver and broadway, and have one task per message under a supervisor.
+- Low priority, but we should evaluate dropping the Subscriber GenServer and Broadway, and have one task per message under a supervisor.
 
 ## State Diagram
 
@@ -99,11 +99,11 @@ stateDiagram-v2
 
 ```mermaid
 sequenceDiagram
-  participant port as LibP2P Port <br> (Genserver)
+  participant port as LibP2P Port <br> (GenServer)
   participant decoder as Decoder <br> (Supervised task)
 	participant tracker as Block Tracker <br> (GenServer)
 	participant down as Downloader <br> (Supervised task)
-	participant store as Fork Choice Store <br> (Genserver)
+	participant store as Fork Choice Store <br> (GenServer)
 	participant state_t as State Transition Task <br> (Supervised task)
 	participant DB as KV Store
 	
@@ -148,7 +148,7 @@ With this strategy in mind, the block won’t need to be passed around during it
 
 There are some tasks that, of course, will require interaction with the outside world:
 
-- When a parent is missing, we need to download it. The block processor may pause until it receives a notification that the parent was downloaded and transitioned. After that, it may continue with its own state transition.
+- When a parent is missing, we need to download it. The block processor may exit and be re-spawned after the parent is downloaded and transitioned. After that, it may continue with its own state transition.
 - When validating it will need to interact with the global fork choice store. This just requires validating a few parameters like the epoch/slot, which is a fast process, so it’s a relatively simple call without passing the full block and without changing the store state.
 - The final step is updating the store/beacon chain state, which still doesn’t need to pass around big state and is fast. If additional validations are performed or repeated, it doesn’t represent a big performance hit.
 
@@ -157,9 +157,9 @@ There are some tasks that, of course, will require interaction with the outside
 ```mermaid
 sequenceDiagram
 
-participant port as LibP2P Port <br> (Genserver)
-participant bp as Block Processor (Genserver)
-participant store as Fork Choice <br> (Genserver)
+participant port as LibP2P Port <br> (GenServer)
+participant bp as Block Processor (GenServer)
+participant store as Fork Choice <br> (GenServer)
 participant db as Persistent Store <br> (DB/Cache)
 
 port ->> bp: gossip(block)
@@ -187,33 +187,32 @@ We can see here that:
 ```mermaid
 sequenceDiagram
 
-participant bp as Block Processor (Genserver)
+participant bp as Block Processor (GenServer)
 participant db as Persistent Store <br> (DB/Cache)
-participant dt as Download Task
-participant bpp as Parent Block Processor (Genserver)
+participant bpp as Parent Block Processor (GenServer)
 
 bp ->>+ db: parent_present?(block_root)
 activate bp
 db -->>- bp: false
-bp ->>+ dt: spawn(parent_root)
+bp ->>+ bpp: spawn(parent_root)
+bp ->> bp: exit()
 deactivate bp
-dt ->> dt: download(parent_root)
-dt ->>- bpp: spawn(parent)
-activate bpp
+bpp ->> bpp: download(parent_root) 
+bpp ->> bpp: decompress <br> ssz decode
 bpp ->> db: store_block_if_not_present(block)
 bpp ->> bpp: validation and state transition
-bpp ->> db: put_state(new_parent_state)
-deactivate bpp
-db ->>+ bp: parent_ready()
+bpp ->>- db: put_state(new_parent_state)
+bpp ->>+ bp: spawn(block_root)
 bp ->>- bp: validation and state transition
 ```
 
-This is a simplified sequence diagram highlighting the differences when the parent block is not present. To summarize:
 
-- The block processor spawns a download task under a download supervisor.
-- When the download finishes it will start a block processor for the parent, repeating the whole process for the parent.
+This is a simplified sequence diagram highlighting the differences when the parent block is not present, so it omits the interaction with fork choice GenServer. To summarize:
+
+- The block processor spawns another processor for the parent, at a `download stage` and exits.
+- When the download finishes, the parent processor will do as a normal one, decoding, deserializing, interacting with the db, etc.
 - When the parent finishes the state transition, it will have saved both the block and the state to the persistent store.
-- The store will then notify the child block processor that it can now proceed with its own state transition.
+- The parent process will then spawn a new block processor for the child, at a `transition stage`.
 
 Note that the persistent store here is a simplified structure. Internally, it will contain the fork tree and the cache. The fork tree will contain the relationship between blocks (the tree structure), which will enable to get a block’s children without iterating through the DB.
 

lib/beacon_api/utils.ex

@@ -4,14 +4,6 @@ defmodule BeaconApi.Utils do
   """
   alias LambdaEthereumConsensus.ForkChoice.Helpers
 
-  @doc """
-  Checks if the value is 32 bytes and starting with "0x"
-  """
-  @spec is_bytes32?(binary) :: boolean
-  def is_bytes32?(value) when is_binary(value) do
-    String.starts_with?(value, "0x") and byte_size(value) == 66
-  end
-
   @spec parse_id(binary) :: Helpers.block_id()
   def parse_id("genesis"), do: :genesis
   def parse_id("justified"), do: :justified

lib/lambda_ethereum_consensus/fork_choice/handlers.ex

@@ -116,7 +116,7 @@ defmodule LambdaEthereumConsensus.ForkChoice.Handlers do
   end
 
   defp check_valid_indexed_attestation(target_state, indexed_attestation) do
-    if Predicates.is_valid_indexed_attestation(target_state, indexed_attestation),
+    if Predicates.valid_indexed_attestation?(target_state, indexed_attestation),
       do: :ok,
       else: {:error, "invalid indexed attestation"}
   end
@@ -137,13 +137,13 @@ defmodule LambdaEthereumConsensus.ForkChoice.Handlers do
     state = BlockStates.get_state!(store.justified_checkpoint.root)
 
     cond do
-      not Predicates.is_slashable_attestation_data(attestation_1.data, attestation_2.data) ->
+      not Predicates.slashable_attestation_data?(attestation_1.data, attestation_2.data) ->
         {:error, "attestation is not slashable"}
 
-      not Predicates.is_valid_indexed_attestation(state, attestation_1) ->
+      not Predicates.valid_indexed_attestation?(state, attestation_1) ->
         {:error, "attestation 1 is not valid"}
 
-      not Predicates.is_valid_indexed_attestation(state, attestation_2) ->
+      not Predicates.valid_indexed_attestation?(state, attestation_2) ->
         {:error, "attestation 2 is not valid"}
 
       true ->

lib/lambda_ethereum_consensus/fork_choice/helpers.ex

@@ -128,7 +128,7 @@ defmodule LambdaEthereumConsensus.ForkChoice.Helpers do
     # If the previous epoch is justified, the block should be pulled-up. In this case, check that unrealized
     # justification is higher than the store and that the voting source is not more than two epochs ago
     correct_justified =
-      if not correct_justified and is_previous_epoch_justified(store) do
+      if not correct_justified and previous_epoch_justified?(store) do
         store.unrealized_justifications[block_root].epoch >= store.justified_checkpoint.epoch and
           voting_source.epoch + 2 >= current_epoch
       else
@@ -171,7 +171,7 @@ defmodule LambdaEthereumConsensus.ForkChoice.Helpers do
     end
   end
 
-  def is_previous_epoch_justified(%Store{} = store) do
+  def previous_epoch_justified?(%Store{} = store) do
     current_slot = Store.get_current_slot(store)
     current_epoch = Misc.compute_epoch_at_slot(current_slot)
     store.justified_checkpoint.epoch + 1 == current_epoch