feat: add go-libp2p Elixir bindings

.github/workflows/ci.yml

@@ -13,8 +13,38 @@ permissions:
   contents: read
 
 jobs:
+  compile-native:
+    name: Build native libraries
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Elixir
+        # NOTE: this is needed for the NIF header files
+        uses: erlef/setup-beam@v1
+        with:
+          version-type: strict
+          version-file: .tool-versions
+        env:
+          ImageOS: ubuntu20
+      - name: Set up Go
+        # NOTE: this action comes with caching by default
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+          cache-dependency-path: native/libp2p_nif/go.sum
+      - name: Cache output artifacts
+        id: output-cache
+        uses: actions/cache@v3
+        with:
+          path: priv/native/*.so
+          key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
+      - name: Compile native code
+        if: steps.output-cache.outputs.cache-hit != 'true'
+        run: make compile-native
+
   build:
     name: Build
+    needs: compile-native
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -24,7 +54,14 @@ jobs:
         version-type: strict
         version-file: .tool-versions
       env:
-        ImageOS: ubuntu20 
+        ImageOS: ubuntu20
+    - name: Fetch native libraries
+      id: output-cache
+      uses: actions/cache/restore@v3
+      with:
+        path: priv/native/*.so
+        key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
+        fail-on-cache-miss: true
     - name: Restore dependencies cache
       uses: actions/cache@v3
       with:
@@ -46,8 +83,10 @@ jobs:
         mix dialyzer --plt
     - name: Run dialyzer
       run: mix dialyzer --no-check
+
   test:
     name: Test
+    needs: compile-native
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -58,6 +97,13 @@ jobs:
         version-file: .tool-versions
       env:
         ImageOS: ubuntu20
+    - name: Fetch native libraries
+      id: output-cache
+      uses: actions/cache/restore@v3
+      with:
+        path: priv/native/*.so
+        key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
+        fail-on-cache-miss: true
     - name: Restore dependencies cache
       uses: actions/cache@v3
       with:

.gitignore

@@ -32,3 +32,14 @@ lambda_ethereum_consensus-*.tar
 *.beam
 /config/*.secret.exs
 .elixir_ls/
+
+# Compiled artifacts.
+*.o
+*.a
+*.h
+*.so
+
+# VSCode configuration dir.
+.vscode/
+
+!libp2p/utils.h

Makefile

@@ -1,4 +1,39 @@
-.PHONY: iex deps test
+.PHONY: iex deps test clean compile-native
+
+# magic from sym_num https://elixirforum.com/t/where-is-erl-nif-h-header-file-required-for-nif/27142/5
+ERLANG_INCLUDES := $(shell erl -eval 'io:format("~s", \
+		[lists:concat([code:root_dir(), "/erts-", erlang:system_info(version), "/include"])] \
+		)' -s init stop -noshell)
+
+LIBP2P_DIR = native/libp2p_nif
+OUTPUT_DIR = priv/native
+
+# create directories if they don't exist
+DIRS=$(OUTPUT_DIR)
+$(info $(shell mkdir -p $(DIRS)))
+
+GO_SOURCES = $(LIBP2P_DIR)/main.go
+GO_ARCHIVES := $(patsubst %.go,%.a,$(GO_SOURCES))
+GO_HEADERS := $(patsubst %.go,%.h,$(GO_SOURCES))
+
+CFLAGS = -Wall -Werror
+CFLAGS += -Wl,-undefined -Wl,dynamic_lookup -fPIC -shared
+CFLAGS += -I$(ERLANG_INCLUDES)
+
+$(LIBP2P_DIR)/%.a $(LIBP2P_DIR)/%.h: $(LIBP2P_DIR)/%.go
+	cd $(LIBP2P_DIR); \
+	go install; \
+	go build -buildmode=c-archive -tags only_go $*.go
+
+$(OUTPUT_DIR)/libp2p_nif.so: $(GO_ARCHIVES) $(GO_HEADERS) $(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c
+	gcc $(CFLAGS) -o $@ \
+		$(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c $(GO_ARCHIVES)
+
+clean:
+	-rm $(GO_ARCHIVES) $(GO_HEADERS) $(OUTPUT_DIR)/*
+
+# Compile C and Go artifacts.
+compile-native: $(OUTPUT_DIR)/libp2p_nif.so
 
 # Run an interactive terminal with the main supervisor setup.
 iex:
@@ -9,5 +44,5 @@ deps:
 	mix deps.get
 
 # Run tests
-test:
+test: compile-native
 	mix test

lib/lambda_ethereum_consensus/libp2p.ex

@@ -0,0 +1,144 @@
+defmodule Libp2p do
+  @moduledoc """
+  Documentation for `Libp2p`.
+  """
+
+  @on_load :load_nifs
+
+  def load_nifs do
+    dir = :code.priv_dir(:lambda_ethereum_consensus)
+    :erlang.load_nif(dir ++ ~c"/native/libp2p_nif", 0)
+  end
+
+  @typedoc """
+  A handle to a Go resource.
+  """
+  @type handle :: integer
+
+  @typedoc """
+  A handle to a host.Host.
+  """
+  @type host :: handle
+
+  @typedoc """
+  A handle to a peerstore.Peerstore.
+  """
+  @type peerstore :: handle
+
+  @typedoc """
+  A handle to a peer.ID.
+  """
+  @type peer_id :: handle
+
+  @typedoc """
+  A handle to a []multiaddr.MultiAddr.
+  """
+  @type addrs :: handle
+
+  @typedoc """
+  A handle to a stream.
+  """
+  @type stream :: handle
+
+  @typedoc """
+  A handle to an Option.
+  """
+  @type option :: handle
+
+  @typedoc """
+  An error returned by this module.
+  """
+  @type error :: {:error, binary}
+
+  @doc """
+  The ttl for a "permanent address" (e.g. bootstrap nodes).
+  """
+  @spec ttl_permanent_addr :: integer
+  def ttl_permanent_addr, do: 2 ** 63 - 1
+
+  @spec host_new(list(option)) :: {:ok, host} | error
+  def host_new(_option_list \\ []),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Deletes a Host.
+  """
+  @spec host_close(host) :: :ok | error
+  def host_close(_host),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Sets the stream handler associated to a protocol id.
+  """
+  @spec host_set_stream_handler(host, binary) :: :ok | error
+  def host_set_stream_handler(_host, _protocol_id),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Returns an `Option` that can be passed to `host_new`
+  as an argument to configures libp2p to listen on the
+  given (unparsed) addresses.
+  """
+  @spec listen_addr_strings(binary) :: {:ok, option} | error
+  def listen_addr_strings(_addr),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Creates a new `Stream` connected to the
+  peer with the given id, using the protocol with given id.
+  """
+  @spec host_new_stream(host, peer_id, binary) :: {:ok, stream} | error
+  def host_new_stream(_host, _peer_id, _protocol_id),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Gets the `Peerstore` of the given `Host`.
+  """
+  @spec host_peerstore(host) :: {:ok, peerstore} | error
+  def host_peerstore(_host),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Gets the `ID` of the given `Host`.
+  """
+  @spec host_id(host) :: {:ok, peer_id} | error
+  def host_id(_host),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Gets the addresses of the given `Host`.
+  """
+  @spec host_addrs(host) :: {:ok, addrs} | error
+  def host_addrs(_host),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Adds the addresses of the peer with the given ID to
+  the `Peerstore`. The addresses are valid for the given
+  TTL.
+  """
+  @spec peerstore_add_addrs(peerstore, peer_id, addrs, integer) :: :ok | error
+  def peerstore_add_addrs(_peerstore, _peer_id, _addrs, _ttl),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Reads bytes from the stream (up to a predefined maximum).
+  """
+  @spec stream_read(stream) :: {:ok, binary} | error
+  def stream_read(_stream),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Writes data into the stream.
+  """
+  @spec stream_write(stream, binary) :: :ok | error
+  def stream_write(_stream, _data),
+    do: :erlang.nif_error(:not_implemented)
+
+  @doc """
+  Closes the stream.
+  """
+  @spec stream_close(stream) :: :ok | error
+  def stream_close(_stream),
+    do: :erlang.nif_error(:not_implemented)
+end

native/libp2p_nif/go.mod

@@ -0,0 +1,92 @@
+module lambdaclass.com/libp2p
+
+go 1.20
+
+require github.com/libp2p/go-libp2p v0.29.0
+
+require (
+	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/huin/goupnp v1.2.0 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.2.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.3.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.55 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.10.1 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multistream v0.4.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
+	github.com/quic-go/quic-go v0.36.2 // indirect
+	github.com/quic-go/webtransport-go v0.5.3 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/fx v1.20.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/tools v0.11.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+)