Skip to content

Commit 3f203b9

Browse files
MegaRedHandMegaRedHand
authored
feat: add go-libp2p Elixir bindings (#12)
* Add example calling Go functions from Elixir * Add simple libp2p.New example * Add host_close, and use cgo handles * Use macro for C function signature * Fix host_close and tests * Add Host.SetStreamHandler binding * Add PoC for message sending Go -> Elixir * Fix test * Divide main.go in sections, and add more methods * Add Read+Write+Close methods for Stream * Add getters for Host * Port ListenAddrStrings and getters * Port NewStream method * Port Peerstore.AddAddrs * Port Stream methods * Add integration test for bindings * Fix test * Add disclaimers to docs * Remove dummy tests * Add disclaimer * Add constant, and clean up * Remove unneeded macro * Add specs * Use charlists and GoStrings * Allow receiving Options in host_new * Revert CString -> GoString * Use macros to reduce C boilerplate * Simplify errors raised in stubs * Use binaries instead of charlists * Change target name to `compile-native` * Fix makefile * Move files to mimic rustler's skeleton * Fix path to erlang headers * Add native compilation to CI * Install elixir for building step * Add go install step to makefile * Fix: yaml eating trailing 0 * Find the Erlang includes dynamically * Change C flags used * Create dir if it doesn't exist * Create output dir outside targets * Remove print from CI * Add dependency path for go action
1 parent 3c8a814 commit 3f203b9

File tree

11 files changed

+1628
-3
lines changed

11 files changed

+1628
-3
lines changed

‎.github/workflows/ci.yml

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,38 @@ permissions:
1313
contents: read
1414

1515
jobs:
16+
compile-native:
17+
name: Build native libraries
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v3
21+
- name: Set up Elixir
22+
# NOTE: this is needed for the NIF header files
23+
uses: erlef/setup-beam@v1
24+
with:
25+
version-type: strict
26+
version-file: .tool-versions
27+
env:
28+
ImageOS: ubuntu20
29+
- name: Set up Go
30+
# NOTE: this action comes with caching by default
31+
uses: actions/setup-go@v4
32+
with:
33+
go-version: "1.20"
34+
cache-dependency-path: native/libp2p_nif/go.sum
35+
- name: Cache output artifacts
36+
id: output-cache
37+
uses: actions/cache@v3
38+
with:
39+
path: priv/native/*.so
40+
key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
41+
- name: Compile native code
42+
if: steps.output-cache.outputs.cache-hit != 'true'
43+
run: make compile-native
44+
1645
build:
1746
name: Build
47+
needs: compile-native
1848
runs-on: ubuntu-latest
1949
steps:
2050
- uses: actions/checkout@v3
@@ -24,7 +54,14 @@ jobs:
2454
version-type: strict
2555
version-file: .tool-versions
2656
env:
27-
ImageOS: ubuntu20
57+
ImageOS: ubuntu20
58+
- name: Fetch native libraries
59+
id: output-cache
60+
uses: actions/cache/restore@v3
61+
with:
62+
path: priv/native/*.so
63+
key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
64+
fail-on-cache-miss: true
2865
- name: Restore dependencies cache
2966
uses: actions/cache@v3
3067
with:
@@ -46,8 +83,10 @@ jobs:
4683
mix dialyzer --plt
4784
- name: Run dialyzer
4885
run: mix dialyzer --no-check
86+
4987
test:
5088
name: Test
89+
needs: compile-native
5190
runs-on: ubuntu-latest
5291
steps:
5392
- uses: actions/checkout@v3
@@ -58,6 +97,13 @@ jobs:
5897
version-file: .tool-versions
5998
env:
6099
ImageOS: ubuntu20
100+
- name: Fetch native libraries
101+
id: output-cache
102+
uses: actions/cache/restore@v3
103+
with:
104+
path: priv/native/*.so
105+
key: ${{ runner.os }}-native-${{ hashFiles('native/**') }}
106+
fail-on-cache-miss: true
61107
- name: Restore dependencies cache
62108
uses: actions/cache@v3
63109
with:

‎.gitignore

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,14 @@ lambda_ethereum_consensus-*.tar
3232
*.beam
3333
/config/*.secret.exs
3434
.elixir_ls/
35+
36+
# Compiled artifacts.
37+
*.o
38+
*.a
39+
*.h
40+
*.so
41+
42+
# VSCode configuration dir.
43+
.vscode/
44+
45+
!libp2p/utils.h

‎Makefile

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,39 @@
1-
.PHONY: iex deps test
1+
.PHONY: iex deps test clean compile-native
2+
3+
# magic from sym_num https://elixirforum.com/t/where-is-erl-nif-h-header-file-required-for-nif/27142/5
4+
ERLANG_INCLUDES := $(shell erl -eval 'io:format("~s", \
5+
[lists:concat([code:root_dir(), "/erts-", erlang:system_info(version), "/include"])] \
6+
)' -s init stop -noshell)
7+
8+
LIBP2P_DIR = native/libp2p_nif
9+
OUTPUT_DIR = priv/native
10+
11+
# create directories if they don't exist
12+
DIRS=$(OUTPUT_DIR)
13+
$(info $(shell mkdir -p $(DIRS)))
14+
15+
GO_SOURCES = $(LIBP2P_DIR)/main.go
16+
GO_ARCHIVES := $(patsubst %.go,%.a,$(GO_SOURCES))
17+
GO_HEADERS := $(patsubst %.go,%.h,$(GO_SOURCES))
18+
19+
CFLAGS = -Wall -Werror
20+
CFLAGS += -Wl,-undefined -Wl,dynamic_lookup -fPIC -shared
21+
CFLAGS += -I$(ERLANG_INCLUDES)
22+
23+
$(LIBP2P_DIR)/%.a $(LIBP2P_DIR)/%.h: $(LIBP2P_DIR)/%.go
24+
cd $(LIBP2P_DIR); \
25+
go install; \
26+
go build -buildmode=c-archive -tags only_go $*.go
27+
28+
$(OUTPUT_DIR)/libp2p_nif.so: $(GO_ARCHIVES) $(GO_HEADERS) $(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c
29+
gcc $(CFLAGS) -o $@ \
30+
$(LIBP2P_DIR)/libp2p.c $(LIBP2P_DIR)/utils.c $(GO_ARCHIVES)
31+
32+
clean:
33+
-rm $(GO_ARCHIVES) $(GO_HEADERS) $(OUTPUT_DIR)/*
34+
35+
# Compile C and Go artifacts.
36+
compile-native: $(OUTPUT_DIR)/libp2p_nif.so
237

338
# Run an interactive terminal with the main supervisor setup.
439
iex:
@@ -9,5 +44,5 @@ deps:
944
mix deps.get
1045

1146
# Run tests
12-
test:
47+
test: compile-native
1348
mix test

‎lib/lambda_ethereum_consensus/libp2p.ex

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
defmodule Libp2p do
2+
@moduledoc """
3+
Documentation for `Libp2p`.
4+
"""
5+
6+
@on_load :load_nifs
7+
8+
def load_nifs do
9+
dir = :code.priv_dir(:lambda_ethereum_consensus)
10+
:erlang.load_nif(dir ++ ~c"/native/libp2p_nif", 0)
11+
end
12+
13+
@typedoc """
14+
A handle to a Go resource.
15+
"""
16+
@type handle :: integer
17+
18+
@typedoc """
19+
A handle to a host.Host.
20+
"""
21+
@type host :: handle
22+
23+
@typedoc """
24+
A handle to a peerstore.Peerstore.
25+
"""
26+
@type peerstore :: handle
27+
28+
@typedoc """
29+
A handle to a peer.ID.
30+
"""
31+
@type peer_id :: handle
32+
33+
@typedoc """
34+
A handle to a []multiaddr.MultiAddr.
35+
"""
36+
@type addrs :: handle
37+
38+
@typedoc """
39+
A handle to a stream.
40+
"""
41+
@type stream :: handle
42+
43+
@typedoc """
44+
A handle to an Option.
45+
"""
46+
@type option :: handle
47+
48+
@typedoc """
49+
An error returned by this module.
50+
"""
51+
@type error :: {:error, binary}
52+
53+
@doc """
54+
The ttl for a "permanent address" (e.g. bootstrap nodes).
55+
"""
56+
@spec ttl_permanent_addr :: integer
57+
def ttl_permanent_addr, do: 2 ** 63 - 1
58+
59+
@spec host_new(list(option)) :: {:ok, host} | error
60+
def host_new(_option_list \\ []),
61+
do: :erlang.nif_error(:not_implemented)
62+
63+
@doc """
64+
Deletes a Host.
65+
"""
66+
@spec host_close(host) :: :ok | error
67+
def host_close(_host),
68+
do: :erlang.nif_error(:not_implemented)
69+
70+
@doc """
71+
Sets the stream handler associated to a protocol id.
72+
"""
73+
@spec host_set_stream_handler(host, binary) :: :ok | error
74+
def host_set_stream_handler(_host, _protocol_id),
75+
do: :erlang.nif_error(:not_implemented)
76+
77+
@doc """
78+
Returns an `Option` that can be passed to `host_new`
79+
as an argument to configures libp2p to listen on the
80+
given (unparsed) addresses.
81+
"""
82+
@spec listen_addr_strings(binary) :: {:ok, option} | error
83+
def listen_addr_strings(_addr),
84+
do: :erlang.nif_error(:not_implemented)
85+
86+
@doc """
87+
Creates a new `Stream` connected to the
88+
peer with the given id, using the protocol with given id.
89+
"""
90+
@spec host_new_stream(host, peer_id, binary) :: {:ok, stream} | error
91+
def host_new_stream(_host, _peer_id, _protocol_id),
92+
do: :erlang.nif_error(:not_implemented)
93+
94+
@doc """
95+
Gets the `Peerstore` of the given `Host`.
96+
"""
97+
@spec host_peerstore(host) :: {:ok, peerstore} | error
98+
def host_peerstore(_host),
99+
do: :erlang.nif_error(:not_implemented)
100+
101+
@doc """
102+
Gets the `ID` of the given `Host`.
103+
"""
104+
@spec host_id(host) :: {:ok, peer_id} | error
105+
def host_id(_host),
106+
do: :erlang.nif_error(:not_implemented)
107+
108+
@doc """
109+
Gets the addresses of the given `Host`.
110+
"""
111+
@spec host_addrs(host) :: {:ok, addrs} | error
112+
def host_addrs(_host),
113+
do: :erlang.nif_error(:not_implemented)
114+
115+
@doc """
116+
Adds the addresses of the peer with the given ID to
117+
the `Peerstore`. The addresses are valid for the given
118+
TTL.
119+
"""
120+
@spec peerstore_add_addrs(peerstore, peer_id, addrs, integer) :: :ok | error
121+
def peerstore_add_addrs(_peerstore, _peer_id, _addrs, _ttl),
122+
do: :erlang.nif_error(:not_implemented)
123+
124+
@doc """
125+
Reads bytes from the stream (up to a predefined maximum).
126+
"""
127+
@spec stream_read(stream) :: {:ok, binary} | error
128+
def stream_read(_stream),
129+
do: :erlang.nif_error(:not_implemented)
130+
131+
@doc """
132+
Writes data into the stream.
133+
"""
134+
@spec stream_write(stream, binary) :: :ok | error
135+
def stream_write(_stream, _data),
136+
do: :erlang.nif_error(:not_implemented)
137+
138+
@doc """
139+
Closes the stream.
140+
"""
141+
@spec stream_close(stream) :: :ok | error
142+
def stream_close(_stream),
143+
do: :erlang.nif_error(:not_implemented)
144+
end

‎native/libp2p_nif/go.mod

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
module lambdaclass.com/libp2p
2+
3+
go 1.20
4+
5+
require github.com/libp2p/go-libp2p v0.29.0
6+
7+
require (
8+
github.com/benbjohnson/clock v1.3.5 // indirect
9+
github.com/beorn7/perks v1.0.1 // indirect
10+
github.com/cespare/xxhash/v2 v2.2.0 // indirect
11+
github.com/containerd/cgroups v1.1.0 // indirect
12+
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
13+
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
14+
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
15+
github.com/docker/go-units v0.5.0 // indirect
16+
github.com/elastic/gosigar v0.14.2 // indirect
17+
github.com/flynn/noise v1.0.0 // indirect
18+
github.com/francoispqt/gojay v1.2.13 // indirect
19+
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
20+
github.com/godbus/dbus/v5 v5.1.0 // indirect
21+
github.com/gogo/protobuf v1.3.2 // indirect
22+
github.com/golang/mock v1.6.0 // indirect
23+
github.com/golang/protobuf v1.5.3 // indirect
24+
github.com/google/gopacket v1.1.19 // indirect
25+
github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
26+
github.com/gorilla/websocket v1.5.0 // indirect
27+
github.com/huin/goupnp v1.2.0 // indirect
28+
github.com/ipfs/go-cid v0.4.1 // indirect
29+
github.com/ipfs/go-log/v2 v2.5.1 // indirect
30+
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
31+
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
32+
github.com/klauspost/compress v1.16.7 // indirect
33+
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
34+
github.com/koron/go-ssdp v0.0.4 // indirect
35+
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
36+
github.com/libp2p/go-cidranger v1.1.0 // indirect
37+
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
38+
github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
39+
github.com/libp2p/go-msgio v0.3.0 // indirect
40+
github.com/libp2p/go-nat v0.2.0 // indirect
41+
github.com/libp2p/go-netroute v0.2.1 // indirect
42+
github.com/libp2p/go-reuseport v0.3.0 // indirect
43+
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
44+
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
45+
github.com/mattn/go-isatty v0.0.19 // indirect
46+
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
47+
github.com/miekg/dns v1.1.55 // indirect
48+
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
49+
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
50+
github.com/minio/sha256-simd v1.0.1 // indirect
51+
github.com/mr-tron/base58 v1.2.0 // indirect
52+
github.com/multiformats/go-base32 v0.1.0 // indirect
53+
github.com/multiformats/go-base36 v0.2.0 // indirect
54+
github.com/multiformats/go-multiaddr v0.10.1 // indirect
55+
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
56+
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
57+
github.com/multiformats/go-multibase v0.2.0 // indirect
58+
github.com/multiformats/go-multicodec v0.9.0 // indirect
59+
github.com/multiformats/go-multihash v0.2.3 // indirect
60+
github.com/multiformats/go-multistream v0.4.1 // indirect
61+
github.com/multiformats/go-varint v0.0.7 // indirect
62+
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
63+
github.com/opencontainers/runtime-spec v1.0.2 // indirect
64+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
65+
github.com/pkg/errors v0.9.1 // indirect
66+
github.com/prometheus/client_golang v1.14.0 // indirect
67+
github.com/prometheus/client_model v0.4.0 // indirect
68+
github.com/prometheus/common v0.37.0 // indirect
69+
github.com/prometheus/procfs v0.8.0 // indirect
70+
github.com/quic-go/qpack v0.4.0 // indirect
71+
github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
72+
github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
73+
github.com/quic-go/quic-go v0.36.2 // indirect
74+
github.com/quic-go/webtransport-go v0.5.3 // indirect
75+
github.com/raulk/go-watchdog v1.3.0 // indirect
76+
github.com/spaolacci/murmur3 v1.1.0 // indirect
77+
go.uber.org/atomic v1.11.0 // indirect
78+
go.uber.org/dig v1.17.0 // indirect
79+
go.uber.org/fx v1.20.0 // indirect
80+
go.uber.org/multierr v1.11.0 // indirect
81+
go.uber.org/zap v1.24.0 // indirect
82+
golang.org/x/crypto v0.11.0 // indirect
83+
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
84+
golang.org/x/mod v0.12.0 // indirect
85+
golang.org/x/net v0.12.0 // indirect
86+
golang.org/x/sync v0.3.0 // indirect
87+
golang.org/x/sys v0.10.0 // indirect
88+
golang.org/x/text v0.11.0 // indirect
89+
golang.org/x/tools v0.11.0 // indirect
90+
google.golang.org/protobuf v1.30.0 // indirect
91+
lukechampine.com/blake3 v1.2.1 // indirect
92+
)

0 commit comments

Comments
 (0)