From 0bdc701753c9ebd9635a0b8805388e9e91ff3a37 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 22 Jul 2019 18:06:25 +0200 Subject: [PATCH 01/11] feat: gossipsub as default pubsub --- package.json | 4 +++- src/core/components/libp2p.js | 4 ++-- src/core/runtime/libp2p-browser.js | 8 +++++--- src/core/runtime/libp2p-nodejs.js | 8 +++++--- test/core/interface.spec.js | 17 +++++++++++++++++ test/core/libp2p.spec.js | 15 ++++++--------- 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index a53177f21f..b73a44e48f 100644 --- a/package.json +++ b/package.json @@ -121,11 +121,13 @@ "iso-url": "~0.4.6", "just-safe-set": "^2.1.0", "kind-of": "^6.0.2", - "libp2p": "~0.25.4", + "libp2p": "^0.26.0", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.16.0", "libp2p-delegated-content-routing": "^0.2.4", "libp2p-delegated-peer-routing": "^0.2.4", + "libp2p-floodsub": "^0.17.0", + "libp2p-gossipsub": "^0.4.0", "libp2p-kad-dht": "~0.15.3", "libp2p-keychain": "~0.4.2", "libp2p-mdns": "~0.12.0", diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 684bfe2714..3119fca06f 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -105,8 +105,8 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { ipns: ipnsUtils.selector } }, - EXPERIMENTAL: { - pubsub: get(options, 'EXPERIMENTAL.pubsub', false) + pubsub: { + enabled: get(options, 'EXPERIMENTAL.pubsub', false) } }, connectionManager: get(options, 'connectionManager', diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index f05f87cebc..a5b21ba021 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -7,6 +7,7 @@ const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') +const GossipSub = require('libp2p-gossipsub') const libp2p = require('libp2p') const mergeOptions = require('merge-options') const multiaddr = require('multiaddr') @@ -46,7 +47,8 @@ class Node extends libp2p { wsstar.discovery, Bootstrap ], - dht: KadDHT + dht: KadDHT, + pubsub: GossipSub }, config: { peerDiscovery: { @@ -64,8 +66,8 @@ class Node extends libp2p { dht: { enabled: false }, - EXPERIMENTAL: { - pubsub: false + pubsub: { + enabled: false } } } diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index eb281a034a..da69cc76e1 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -6,6 +6,7 @@ const WS = require('libp2p-websockets') const WebSocketStarMulti = require('libp2p-websocket-star-multi') const Bootstrap = require('libp2p-bootstrap') const KadDHT = require('libp2p-kad-dht') +const GossipSub = require('libp2p-gossipsub') const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') const libp2p = require('libp2p') @@ -45,7 +46,8 @@ class Node extends libp2p { Bootstrap, wsstar.discovery ], - dht: KadDHT + dht: KadDHT, + pubsub: GossipSub }, config: { peerDiscovery: { @@ -67,8 +69,8 @@ class Node extends libp2p { enabled: false } }, - EXPERIMENTAL: { - pubsub: false + pubsub: { + enabled: false } } } diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index f930641355..b855e1e00f 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -34,6 +34,23 @@ describe('interface-ipfs-core tests', function () { ] }) + tests.config(defaultCommonFactory, { + skip: [ + { + name: 'should set a number', + reason: 'Failing - needs to be fixed' + }, + { + name: 'should output changes but not save them for dry run', + reason: 'not implemented' + }, + { + name: 'should set a config profile', + reason: 'not implemented' + } + ] + }) + tests.dag(defaultCommonFactory) tests.dht(CommonFactory.create({ diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index 818902d523..14b41535ae 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -45,8 +45,8 @@ describe('libp2p customization', function () { Enabled: false } }, - EXPERIMENTAL: { - pubsub: false + pubsub: { + enabled: false } } datastore = new MemoryStore() @@ -149,8 +149,8 @@ describe('libp2p customization', function () { enabled: true } }, - EXPERIMENTAL: { - pubsub: false + pubsub: { + enabled: false } }) expect(_libp2p._transport).to.have.length(3) @@ -177,8 +177,8 @@ describe('libp2p customization', function () { } } }, - EXPERIMENTAL: { - pubsub: true + pubsub: { + enabled: true }, libp2p: { modules: { @@ -213,9 +213,6 @@ describe('libp2p customization', function () { websocketStar: { enabled: true } - }, - EXPERIMENTAL: { - pubsub: true } }) expect(_libp2p._transport).to.have.length(1) From 679c4092d12cd632f1f43940546f5d5f9975604a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 24 Jul 2019 21:02:27 +0200 Subject: [PATCH 02/11] chore: add config for pubsub router --- src/core/components/libp2p.js | 4 +- src/core/config.js | 3 ++ src/core/runtime/config-browser.js | 3 ++ src/core/runtime/config-nodejs.js | 3 ++ test/core/interface.spec.js | 17 --------- test/core/libp2p.spec.js | 59 ++++++++++++++++++++++++++++-- 6 files changed, 68 insertions(+), 21 deletions(-) diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 3119fca06f..4a5f7fd31f 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -64,7 +64,9 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { peerBook, modules: { contentRouting, - peerRouting + peerRouting, + pubsub: get(config, 'Pubsub.Router', 'gossipsub') === 'floodsub' + ? require('libp2p-floodsub') : require('libp2p-gossipsub') }, config: { peerDiscovery: { diff --git a/src/core/config.js b/src/core/config.js index 1d40e862c5..c259aa7451 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -68,6 +68,9 @@ const configSchema = s({ })) })), Bootstrap: optional(s(['multiaddr-ipfs'])), + Pubsub: optional(s({ + Router: 'string?' + })), Swarm: optional(s({ ConnMgr: optional(s({ LowWater: 'number?', diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js index 537316d431..13c803b011 100644 --- a/src/core/runtime/config-browser.js +++ b/src/core/runtime/config-browser.js @@ -27,6 +27,9 @@ module.exports = () => ({ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ], + Pubsub: { + Router: 'gossipsub' + }, Swarm: { ConnMgr: { LowWater: 200, diff --git a/src/core/runtime/config-nodejs.js b/src/core/runtime/config-nodejs.js index 60169ef562..1a9598e515 100644 --- a/src/core/runtime/config-nodejs.js +++ b/src/core/runtime/config-nodejs.js @@ -40,6 +40,9 @@ module.exports = () => ({ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ], + Pubsub: { + Router: 'gossipsub' + }, Swarm: { ConnMgr: { LowWater: 200, diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index b855e1e00f..f930641355 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -34,23 +34,6 @@ describe('interface-ipfs-core tests', function () { ] }) - tests.config(defaultCommonFactory, { - skip: [ - { - name: 'should set a number', - reason: 'Failing - needs to be fixed' - }, - { - name: 'should output changes but not save them for dry run', - reason: 'not implemented' - }, - { - name: 'should set a config profile', - reason: 'not implemented' - } - ] - }) - tests.dag(defaultCommonFactory) tests.dht(CommonFactory.create({ diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index 14b41535ae..9d7fc350e8 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -44,9 +44,6 @@ describe('libp2p customization', function () { webRTCStar: { Enabled: false } - }, - pubsub: { - enabled: false } } datastore = new MemoryStore() @@ -290,4 +287,60 @@ describe('libp2p customization', function () { }) }) }) + + describe('bundle via custom config for pubsub', () => { + it('select gossipsub as pubsub router', (done) => { + const ipfs = { + _repo: { + datastore + }, + _peerInfo: peerInfo, + _peerBook: peerBook, + // eslint-disable-next-line no-console + _print: console.log, + _options: {} + } + const customConfig = { + ...testConfig, + Pubsub: { + Router: 'gossipsub' + } + } + + _libp2p = libp2pComponent(ipfs, customConfig) + + _libp2p.start((err) => { + expect(err).to.not.exist() + expect(_libp2p._modules.pubsub).to.eql(require('libp2p-gossipsub')) + done() + }) + }) + + it('select floodsub as pubsub router', (done) => { + const ipfs = { + _repo: { + datastore + }, + _peerInfo: peerInfo, + _peerBook: peerBook, + // eslint-disable-next-line no-console + _print: console.log, + _options: {} + } + const customConfig = { + ...testConfig, + Pubsub: { + Router: 'floodsub' + } + } + + _libp2p = libp2pComponent(ipfs, customConfig) + + _libp2p.start((err) => { + expect(err).to.not.exist() + expect(_libp2p._modules.pubsub).to.eql(require('libp2p-floodsub')) + done() + }) + }) + }) }) From de89e4d9d345c85088cdfa4b4714fc0807bf95c8 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 24 Jul 2019 21:18:33 +0200 Subject: [PATCH 03/11] chore: increase max bundle size --- .aegir.js | 2 +- doc/config.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.aegir.js b/.aegir.js index 6d0ed4b2dd..97a84f2565 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,7 +8,7 @@ const ipfsdServer = IPFSFactory.createServer() const preloadNode = MockPreloadNode.createNode() module.exports = { - bundlesize: { maxSize: '689kB' }, + bundlesize: { maxSize: '691kB' }, webpack: { resolve: { mainFields: ['browser', 'main'], diff --git a/doc/config.md b/doc/config.md index b1df05db22..fa331c4d77 100644 --- a/doc/config.md +++ b/doc/config.md @@ -19,6 +19,8 @@ The js-ipfs config file is a JSON document located in the root directory of the - [`PeerID`](#peerid) - [`PrivKey`](#privkey) - [`Keychain`](#keychain) +- [`Pubsub`](#pubsub) + - [`Router`](#router) - [`Swarm`](#swarm) - [`ConnMgr`](#connmgr) @@ -168,6 +170,16 @@ Default: You can check the [parameter choice for pbkdf2](https://cryptosense.com/parameter-choice-for-pbkdf2/) for more information. +## `Pubsub` + +Options for configuring the pubsub subsystem. + +### `Router` + +A string value for specifying which pubsub routing protocol to use. You can either use `gossipsub` in order to use the [ChainSafe/gossipsub-js](https://github.com/ChainSafe/gossipsub-js) implementation, or `floodsub` to use the [libp2p/js-libp2p-floodsub](https://github.com/libp2p/js-libp2p-floodsub) implementation. You can read more about these implementations on the [libp2p/specs/pubsub](https://github.com/libp2p/specs/tree/master/pubsub) document. + +Default: `gossipsub` + ## `Swarm` Options for configuring the swarm. From 406563ec51cd9dfa811ef586e566569ecf6154c8 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Wed, 31 Jul 2019 10:03:59 +0200 Subject: [PATCH 04/11] fix: correct pubsub methods for next libp2p release --- src/core/components/pubsub.js | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/core/components/pubsub.js b/src/core/components/pubsub.js index 0171eea179..e847cafeb2 100644 --- a/src/core/components/pubsub.js +++ b/src/core/components/pubsub.js @@ -16,65 +16,56 @@ module.exports = function pubsub (self) { options = {} } - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { return callback ? setImmediate(() => callback(errPubsubDisabled())) : Promise.reject(errPubsubDisabled()) } if (!callback) { - return new Promise((resolve, reject) => { - self.libp2p.pubsub.subscribe(topic, options, handler, (err) => { - if (err) { - return reject(err) - } - resolve() - }) - }) + return self.libp2p.pubsub.subscribe(topic, handler, options) } self.libp2p.pubsub.subscribe(topic, options, handler, callback) }, unsubscribe: (topic, handler, callback) => { - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { return callback ? setImmediate(() => callback(errPubsubDisabled())) : Promise.reject(errPubsubDisabled()) } - self.libp2p.pubsub.unsubscribe(topic, handler) - if (!callback) { - return Promise.resolve() + return self.libp2p.pubsub.unsubscribe(topic, handler) } - setImmediate(() => callback()) + self.libp2p.pubsub.unsubscribe(topic, handler, callback) }, publish: promisify((topic, data, callback) => { - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { return setImmediate(() => callback(errPubsubDisabled())) } self.libp2p.pubsub.publish(topic, data, callback) }), ls: promisify((callback) => { - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { return setImmediate(() => callback(errPubsubDisabled())) } self.libp2p.pubsub.ls(callback) }), peers: promisify((topic, callback) => { - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { return setImmediate(() => callback(errPubsubDisabled())) } self.libp2p.pubsub.peers(topic, callback) }), setMaxListeners (n) { - if (!self._options.EXPERIMENTAL.pubsub) { + if (!self.libp2p.pubsub) { throw errPubsubDisabled() } self.libp2p.pubsub.setMaxListeners(n) From c80e942ddd68af199c0a40fa9cab51ae4fd43cc8 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Wed, 31 Jul 2019 11:48:30 +0200 Subject: [PATCH 05/11] fix(pubsub): subscribe call to libp2p --- src/core/components/pubsub.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/components/pubsub.js b/src/core/components/pubsub.js index e847cafeb2..ac6a0981cd 100644 --- a/src/core/components/pubsub.js +++ b/src/core/components/pubsub.js @@ -26,7 +26,7 @@ module.exports = function pubsub (self) { return self.libp2p.pubsub.subscribe(topic, handler, options) } - self.libp2p.pubsub.subscribe(topic, options, handler, callback) + self.libp2p.pubsub.subscribe(topic, handler, options, callback) }, unsubscribe: (topic, handler, callback) => { From a9b41e1213996f17fcc458eb9d1a3e497a13f1c1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 31 Jul 2019 10:26:37 +0200 Subject: [PATCH 06/11] chore: update gossipsub and libp2p --- .aegir.js | 2 +- package.json | 2 +- src/core/runtime/libp2p-browser.js | 4 ++-- src/core/runtime/libp2p-nodejs.js | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.aegir.js b/.aegir.js index 97a84f2565..2e535ada4b 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,7 +8,7 @@ const ipfsdServer = IPFSFactory.createServer() const preloadNode = MockPreloadNode.createNode() module.exports = { - bundlesize: { maxSize: '691kB' }, + bundlesize: { maxSize: '694kB' }, webpack: { resolve: { mainFields: ['browser', 'main'], diff --git a/package.json b/package.json index b73a44e48f..65cfb8b8b1 100644 --- a/package.json +++ b/package.json @@ -127,7 +127,7 @@ "libp2p-delegated-content-routing": "^0.2.4", "libp2p-delegated-peer-routing": "^0.2.4", "libp2p-floodsub": "^0.17.0", - "libp2p-gossipsub": "^0.4.0", + "libp2p-gossipsub": "~0.0.4", "libp2p-kad-dht": "~0.15.3", "libp2p-keychain": "~0.4.2", "libp2p-mdns": "~0.12.0", diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index a5b21ba021..4ac3621525 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -24,8 +24,8 @@ class Node extends libp2p { const defaults = { switch: { - blacklistTTL: 2 * 60 * 1e3, // 2 minute base - blackListAttempts: 5, // back off 5 times + denyTTL: 2 * 60 * 1e3, // 2 minute base + denyAttempts: 5, // back off 5 times maxParallelDials: 100, maxColdCalls: 25, dialTimeout: 20e3 diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index da69cc76e1..aeb5111640 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -23,8 +23,8 @@ class Node extends libp2p { const defaults = { switch: { - blacklistTTL: 2 * 60 * 1e3, // 2 minute base - blackListAttempts: 5, // back off 5 times + denyTTL: 2 * 60 * 1e3, // 2 minute base + denyAttempts: 5, // back off 5 times maxParallelDials: 150, maxColdCalls: 50, dialTimeout: 10e3 // Be strict with dial time From fa38df712e9c67f1b43e8805cc93410c7ad7a2f9 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Wed, 21 Aug 2019 18:53:17 +0200 Subject: [PATCH 07/11] test: fix libp2p config test --- test/core/libp2p.spec.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index 9d7fc350e8..bd80afde81 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -147,7 +147,10 @@ describe('libp2p customization', function () { } }, pubsub: { - enabled: false + enabled: false, + emitSelf: true, + signMessages: true, + strictSigning: true } }) expect(_libp2p._transport).to.have.length(3) From 85d66d712d91f8cb3ae38d1f24b83861d47c6bed Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Wed, 21 Aug 2019 19:57:40 +0200 Subject: [PATCH 08/11] chore: update libp2p --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 65cfb8b8b1..d98f59fad4 100644 --- a/package.json +++ b/package.json @@ -121,7 +121,7 @@ "iso-url": "~0.4.6", "just-safe-set": "^2.1.0", "kind-of": "^6.0.2", - "libp2p": "^0.26.0", + "libp2p": "~0.26.1", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.16.0", "libp2p-delegated-content-routing": "^0.2.4", From ee837cffec3d901c68928191631d9480ed1d7cfb Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 28 Aug 2019 14:25:46 +0200 Subject: [PATCH 09/11] chore: address review --- .aegir.js | 2 +- README.md | 9 ++++++++- doc/config.md | 2 +- package.json | 3 ++- src/core/components/libp2p.js | 15 +++++++++++++-- src/core/runtime/config-browser.js | 3 --- src/core/runtime/libp2p-browser.js | 3 ++- src/core/runtime/libp2p-nodejs.js | 3 ++- src/core/runtime/libp2p-pubsub-routers-browser.js | 5 +++++ src/core/runtime/libp2p-pubsub-routers-nodejs.js | 6 ++++++ test/core/libp2p.spec.js | 13 +++++++++++-- 11 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 src/core/runtime/libp2p-pubsub-routers-browser.js create mode 100644 src/core/runtime/libp2p-pubsub-routers-nodejs.js diff --git a/.aegir.js b/.aegir.js index 2e535ada4b..6d0ed4b2dd 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,7 +8,7 @@ const ipfsdServer = IPFSFactory.createServer() const preloadNode = MockPreloadNode.createNode() module.exports = { - bundlesize: { maxSize: '694kB' }, + bundlesize: { maxSize: '689kB' }, webpack: { resolve: { mainFields: ['browser', 'main'], diff --git a/README.md b/README.md index 706eff4073..4cc262124c 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ Configure remote preload nodes. The remote will preload content added on this no | Type | Default | |------|---------| -| object | `{ pubsub: false, ipnsPubsub: false, sharding: false }` | +| object | `{ ipnsPubsub: false, sharding: false }` | Enable and configure experimental features. @@ -495,6 +495,8 @@ You can see the bundle in action in the [custom libp2p example](examples/custom- - `modules` (object): - `transport` (Array<[libp2p.Transport](https://github.com/libp2p/interface-transport)>): An array of Libp2p transport classes/instances to use _instead_ of the defaults. See [libp2p/interface-transport](https://github.com/libp2p/interface-transport) for details. - `peerDiscovery` (Array<[libp2p.PeerDiscovery](https://github.com/libp2p/interface-peer-discovery)>): An array of Libp2p peer discovery classes/instances to use _instead_ of the defaults. See [libp2p/peer-discovery](https://github.com/libp2p/interface-peer-discovery) for details. If passing a class, configuration can be passed using the config section below under the key corresponding to you module's unique `tag` (a static property on the class) + - `dht` (object): a DHT implementation that enables PeerRouting and ContentRouting. Example [libp2p/js-libp2p-kad-dht](https://github.com/libp2p/js-libp2p-kad-dht) + - `pubsub` (object): a Pubsub implementation on top of [libp2p/js-libp2p-pubsub](https://github.com/libp2p/js-libp2p-pubsub) - `config` (object): - `peerDiscovery` (object): - `autoDial` (boolean): Dial to discovered peers when under the Connection Manager min peer count watermark. (default `true`) @@ -506,6 +508,11 @@ You can see the bundle in action in the [custom libp2p example](examples/custom- - `kBucketSize` (number): bucket size (default `20`) - `randomWalk` (object): configuration for random walk - `enabled` (boolean): whether random DHT walking is enabled (default `false`) + - `pubsub` (object): Configuration options for Pubsub + - `enabled` (boolean): if pubbsub subsystem should be enabled (default: `false`) + - `emitSelf` (boolean): whether the node should emit to self on publish, in the event of the topic being subscribed (default: `true`) + - `signMessages` (boolean): if messages should be signed (default: `true`) + - `strictSigning` (boolean): if message signing should be required (default: `true`) ##### `options.connectionManager` diff --git a/doc/config.md b/doc/config.md index fa331c4d77..08222f4a10 100644 --- a/doc/config.md +++ b/doc/config.md @@ -172,7 +172,7 @@ You can check the [parameter choice for pbkdf2](https://cryptosense.com/paramete ## `Pubsub` -Options for configuring the pubsub subsystem. +Options for configuring the pubsub subsystem. It is important pointing out that this is not supported in the browser. If you want to configure a different pubsub router in the browser you must configure `libp2p.modules.pubsub` options instead. ### `Router` diff --git a/package.json b/package.json index d98f59fad4..7635200276 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js", "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js", "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js", + "./src/core/runtime/libp2p-pubsub-routers-nodejs.js": "./src/core/runtime/libp2p-pubsub-routers-browser.js", "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js", "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js", "./src/core/runtime/ipld-nodejs.js": "./src/core/runtime/ipld-browser.js", @@ -193,7 +194,7 @@ "execa": "^2.0.4", "form-data": "^2.5.1", "hat": "0.0.3", - "interface-ipfs-core": "^0.110.0", + "interface-ipfs-core": "^0.111.0", "ipfsd-ctl": "^0.44.1", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 4a5f7fd31f..ef261f8c1c 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -2,10 +2,12 @@ const get = require('dlv') const mergeOptions = require('merge-options') +const errCode = require('err-code') const ipnsUtils = require('../ipns/routing/utils') const multiaddr = require('multiaddr') const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') const DelegatedContentRouter = require('libp2p-delegated-content-routing') +const PubsubRouters = require('../runtime/libp2p-pubsub-routers-nodejs') module.exports = function libp2p (self, config) { const options = self._options || {} @@ -58,6 +60,16 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { peerRouting = [new DelegatedPeerRouter(delegatedApiOptions)] } + const getPubsubRouter = () => { + const router = get(config, 'Pubsub.Router', 'gossipsub') + + if (!PubsubRouters[router]) { + throw errCode(new Error(`Router unavailable. Configure libp2p.modules.pubsub to use the ${router} router.`), 'ERR_NOT_SUPPORTED') + } + + return PubsubRouters[router] + } + const libp2pDefaults = { datastore, peerInfo, @@ -65,8 +77,7 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { modules: { contentRouting, peerRouting, - pubsub: get(config, 'Pubsub.Router', 'gossipsub') === 'floodsub' - ? require('libp2p-floodsub') : require('libp2p-gossipsub') + pubsub: getPubsubRouter() }, config: { peerDiscovery: { diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js index 13c803b011..537316d431 100644 --- a/src/core/runtime/config-browser.js +++ b/src/core/runtime/config-browser.js @@ -27,9 +27,6 @@ module.exports = () => ({ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ], - Pubsub: { - Router: 'gossipsub' - }, Swarm: { ConnMgr: { LowWater: 200, diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index 4ac3621525..2054461afd 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -67,7 +67,8 @@ class Node extends libp2p { enabled: false }, pubsub: { - enabled: false + enabled: false, + emitSelf: true } } } diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index aeb5111640..744d0adb9a 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -70,7 +70,8 @@ class Node extends libp2p { } }, pubsub: { - enabled: false + enabled: false, + emitSelf: true } } } diff --git a/src/core/runtime/libp2p-pubsub-routers-browser.js b/src/core/runtime/libp2p-pubsub-routers-browser.js new file mode 100644 index 0000000000..5b6aaedc0a --- /dev/null +++ b/src/core/runtime/libp2p-pubsub-routers-browser.js @@ -0,0 +1,5 @@ +'use strict' + +module.exports = { + gossipsub: require('libp2p-gossipsub') +} diff --git a/src/core/runtime/libp2p-pubsub-routers-nodejs.js b/src/core/runtime/libp2p-pubsub-routers-nodejs.js new file mode 100644 index 0000000000..0af111e881 --- /dev/null +++ b/src/core/runtime/libp2p-pubsub-routers-nodejs.js @@ -0,0 +1,6 @@ +'use strict' + +module.exports = { + gossipsub: require('libp2p-gossipsub'), + floodsub: require('libp2p-floodsub') +} diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index bd80afde81..b4f274996e 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -15,6 +15,7 @@ const Multiplex = require('pull-mplex') const SECIO = require('libp2p-secio') const KadDHT = require('libp2p-kad-dht') const Libp2p = require('libp2p') +const isNode = require('detect-node') const libp2pComponent = require('../../src/core/components/libp2p') @@ -319,7 +320,7 @@ describe('libp2p customization', function () { }) }) - it('select floodsub as pubsub router', (done) => { + it('select floodsub as pubsub router if node', (done) => { const ipfs = { _repo: { datastore @@ -337,7 +338,15 @@ describe('libp2p customization', function () { } } - _libp2p = libp2pComponent(ipfs, customConfig) + try { + _libp2p = libp2pComponent(ipfs, customConfig) + } catch (err) { + if (!isNode) { + expect(err).to.exist() + expect(err.code).to.eql('ERR_NOT_SUPPORTED') + done() + } + } _libp2p.start((err) => { expect(err).to.not.exist() From f02da5c857b67823e764f936098a0bb21ae8f371 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sat, 31 Aug 2019 14:43:08 +0200 Subject: [PATCH 10/11] chore: address review --- examples/circuit-relaying/src/app.js | 4 ++-- examples/custom-libp2p/index.js | 4 ++-- examples/custom-libp2p/package.json | 2 +- examples/exchange-files-in-browser/public/app.js | 4 ++-- package.json | 2 +- src/cli/commands/daemon.js | 5 +++-- src/cli/daemon.js | 2 +- src/cli/utils.js | 4 ++-- src/core/components/libp2p.js | 2 +- src/core/config.js | 3 +++ src/core/index.js | 15 ++++++++++----- test/cli/pubsub.js | 4 ++-- test/core/config.spec.js | 5 ++--- test/core/interface.spec.js | 2 +- test/core/pubsub.spec.js | 4 ++-- test/http-api/interface.js | 2 +- test/http-api/routes.js | 4 +--- 17 files changed, 37 insertions(+), 31 deletions(-) diff --git a/examples/circuit-relaying/src/app.js b/examples/circuit-relaying/src/app.js index 290a57cf26..dd34347c1c 100644 --- a/examples/circuit-relaying/src/app.js +++ b/examples/circuit-relaying/src/app.js @@ -36,8 +36,8 @@ document.addEventListener('DOMContentLoaded', async () => { enabled: true // make this node a relay (HOP) } }, - EXPERIMENTAL: { - pubsub: true // enable pubsub + pubsub: { + enabled: true }, config: { Bootstrap: [] diff --git a/examples/custom-libp2p/index.js b/examples/custom-libp2p/index.js index 4b85fa2280..fa1a097b86 100644 --- a/examples/custom-libp2p/index.js +++ b/examples/custom-libp2p/index.js @@ -96,8 +96,8 @@ const libp2pBundle = (opts) => { timeout: 2e3 // End the query quickly since we're running so frequently } }, - EXPERIMENTAL: { - pubsub: true + pubsub: { + enabled: true } } }) diff --git a/examples/custom-libp2p/package.json b/examples/custom-libp2p/package.json index 81209e705b..944b8a2b7f 100644 --- a/examples/custom-libp2p/package.json +++ b/examples/custom-libp2p/package.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "ipfs": "file:../../", - "libp2p": "~0.25.0", + "libp2p": "~0.26.1", "libp2p-bootstrap": "~0.9.7", "libp2p-kad-dht": "~0.15.0", "libp2p-mdns": "~0.12.2", diff --git a/examples/exchange-files-in-browser/public/app.js b/examples/exchange-files-in-browser/public/app.js index 4578bc588d..d45f4c9b6e 100644 --- a/examples/exchange-files-in-browser/public/app.js +++ b/examples/exchange-files-in-browser/public/app.js @@ -44,8 +44,8 @@ let info async function start () { if (!node) { const options = { - EXPERIMENTAL: { - pubsub: true + pubsub: { + enabled: true }, repo: 'ipfs-' + Math.random(), config: { diff --git a/package.json b/package.json index 7635200276..27cca15d4f 100644 --- a/package.json +++ b/package.json @@ -195,7 +195,7 @@ "form-data": "^2.5.1", "hat": "0.0.3", "interface-ipfs-core": "^0.111.0", - "ipfsd-ctl": "^0.44.1", + "ipfsd-ctl": "~0.45.0", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", "p-event": "^4.1.0", diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index 140ac2d224..f3f61f80e6 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -16,7 +16,8 @@ module.exports = { type: 'boolean', default: false }) - .option('enable-pubsub-experiment', { + .option('enable-pubsub', { + alias: 'enable-pubsub-experiment', type: 'boolean', default: false }) @@ -53,8 +54,8 @@ module.exports = { offline: argv.offline, pass: argv.pass, preload: { enabled: argv.enablePreload }, + pubsub: { enabled: argv.enablePubsub }, EXPERIMENTAL: { - pubsub: argv.enablePubsubExperiment, ipnsPubsub: argv.enableNamesysPubsub, dht: argv.enableDhtExperiment, sharding: argv.enableShardingExperiment diff --git a/src/cli/daemon.js b/src/cli/daemon.js index 879432018b..7181e8c1e7 100644 --- a/src/cli/daemon.js +++ b/src/cli/daemon.js @@ -30,7 +30,7 @@ class Daemon { async start () { this._log('starting') - const libp2p = { modules: {} } + const libp2p = { modules: {}, config: {} } // Attempt to use any of the WebRTC versions available globally let electronWebRTC diff --git a/src/cli/utils.js b/src/cli/utils.js index eed5cbdac6..0562ff6737 100644 --- a/src/cli/utils.js +++ b/src/cli/utils.js @@ -50,8 +50,8 @@ exports.getIPFS = (argv, callback) => { init: false, start: false, pass: argv.pass, - EXPERIMENTAL: { - pubsub: true + pubsub: { + enabled: true } }) diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index ef261f8c1c..ba420f120f 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -119,7 +119,7 @@ function defaultBundle ({ datastore, peerInfo, peerBook, options, config }) { } }, pubsub: { - enabled: get(options, 'EXPERIMENTAL.pubsub', false) + enabled: get(options, 'pubsub.enabled', false) } }, connectionManager: get(options, 'connectionManager', diff --git a/src/core/config.js b/src/core/config.js index c259aa7451..53231bcc73 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -32,6 +32,9 @@ const configSchema = s({ addresses: optional(s(['multiaddr'])), interval: 'number?' }, { enabled: true, interval: 30 * 1000 }), + pubsub: optional(s({ + enabled: 'boolean?' + })), init: optional(union(['boolean', s({ bits: 'number?', emptyRepo: 'boolean?', diff --git a/src/core/index.js b/src/core/index.js index 8b0b717716..4016aa47fc 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -15,6 +15,7 @@ const multihashing = require('multihashing-async') const CID = require('cids') const debug = require('debug') const mergeOptions = require('merge-options') +const get = require('dlv') const EventEmitter = require('events') const config = require('./config') @@ -46,6 +47,9 @@ class IPFS extends EventEmitter { init: true, start: true, EXPERIMENTAL: {}, + pubsub: { + enabled: false + }, preload: { enabled: true, addresses: [ @@ -131,13 +135,14 @@ class IPFS extends EventEmitter { this.stats = components.stats(this) this.resolve = components.resolve(this) - if (this._options.EXPERIMENTAL.pubsub) { - this.log('EXPERIMENTAL pubsub is enabled') + if (this._options.pubsub.enabled) { + this.log('pubsub is enabled') } if (this._options.EXPERIMENTAL.ipnsPubsub) { - if (!this._options.EXPERIMENTAL.pubsub) { - this.log('EXPERIMENTAL pubsub is enabled to use IPNS pubsub') - this._options.EXPERIMENTAL.pubsub = true + // if (!this._options.pubsub.enabled) { + if (!get(this._options, 'pubsub.enabled', false)) { + this.log('pubsub is enabled to use EXPERIMENTAL IPNS pubsub') + this._options.pubsub.enabled = true } this.log('EXPERIMENTAL IPNS pubsub is enabled') diff --git a/test/cli/pubsub.js b/test/cli/pubsub.js index 89ae432bcd..e4358e90f2 100644 --- a/test/cli/pubsub.js +++ b/test/cli/pubsub.js @@ -44,7 +44,7 @@ describe('pubsub', function () { exec: IPFS, initOptions: { bits: 512 }, config, - args: ['--enable-pubsub-experiment'] + args: ['--enable-pubsub'] }) node = ipfsdA.api }) @@ -59,7 +59,7 @@ describe('pubsub', function () { const df = DaemonFactory.create({ type: 'js' }) ipfsdB = await df.spawn({ initOptions: { bits: 512 }, - args: ['--enable-pubsub-experiment'], + args: ['--enable-pubsub'], exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config }) diff --git a/test/core/config.spec.js b/test/core/config.spec.js index 8f2e643e1c..e67b077209 100644 --- a/test/core/config.spec.js +++ b/test/core/config.spec.js @@ -98,8 +98,8 @@ describe('config', () => { it('should validate valid EXPERIMENTAL', () => { const cfgs = [ - { EXPERIMENTAL: { pubsub: true, dht: true, sharding: true } }, - { EXPERIMENTAL: { pubsub: false, dht: false, sharding: false } }, + { EXPERIMENTAL: { dht: true, sharding: true } }, + { EXPERIMENTAL: { dht: false, sharding: false } }, { EXPERIMENTAL: undefined } ] @@ -108,7 +108,6 @@ describe('config', () => { it('should validate invalid EXPERIMENTAL', () => { const cfgs = [ - { EXPERIMENTAL: { pubsub: 138 } }, { EXPERIMENTAL: { dht: 138 } }, { EXPERIMENTAL: { sharding: 138 } } ] diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index f930641355..f2151a88df 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -149,7 +149,7 @@ describe('interface-ipfs-core tests', function () { tests.pubsub(CommonFactory.create({ spawnOptions: { - args: ['--enable-pubsub-experiment'], + args: ['--enable-pubsub'], initOptions: { bits: 512 } } }), { diff --git a/test/core/pubsub.spec.js b/test/core/pubsub.spec.js index c1a8ec9092..de0f4404c6 100644 --- a/test/core/pubsub.spec.js +++ b/test/core/pubsub.spec.js @@ -29,8 +29,8 @@ describe('pubsub disabled', () => { preload: { enabled: false }, - EXPERIMENTAL: { - pubsub: false + pubsub: { + enabled: false } }) diff --git a/test/http-api/interface.js b/test/http-api/interface.js index 8cbc2423ef..23b5916624 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -148,7 +148,7 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { tests.pubsub(CommonFactory.create({ spawnOptions: { - args: ['--enable-pubsub-experiment'], + args: ['--enable-pubsub'], initOptions: { bits: 512 } } })) diff --git a/test/http-api/routes.js b/test/http-api/routes.js index ded737cc6a..8c090727a5 100644 --- a/test/http-api/routes.js +++ b/test/http-api/routes.js @@ -26,9 +26,7 @@ describe('HTTP API', () => { repo: repoTests, pass: hat(), config, - EXPERIMENTAL: { - pubsub: true - }, + pubsub: { enabled: true }, preload: { enabled: false } }) await ncp(repoExample, repoTests) From 60c732a5f003ded07eb1bf9a04b9100f281c48f6 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 2 Sep 2019 20:16:46 +0200 Subject: [PATCH 11/11] chore: skip dns tests --- package.json | 2 +- test/core/interface.spec.js | 49 ++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 27cca15d4f..5102b9508d 100644 --- a/package.json +++ b/package.json @@ -97,7 +97,7 @@ "ipfs-bitswap": "~0.25.1", "ipfs-block": "~0.8.1", "ipfs-block-service": "~0.15.2", - "ipfs-http-client": "^33.1.1", + "ipfs-http-client": "^34.0.0", "ipfs-http-response": "~0.3.1", "ipfs-mfs": "~0.12.0", "ipfs-multipart": "~0.1.1", diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index f2151a88df..e650743875 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -102,6 +102,18 @@ describe('interface-ipfs-core tests', function () { { name: 'should resolve IPNS link recursively', reason: 'TODO: IPNS resolve not yet implemented https://github.com/ipfs/js-ipfs/issues/1918' + }, + { + name: 'should non-recursively resolve ipfs.io', + reasone: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should recursively resolve ipfs.io', + reasone: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve subdomain docs.ipfs.io', + reasone: 'TODO: gateway issue on dns. To be enabled on a new PR' } ] }) @@ -110,7 +122,42 @@ describe('interface-ipfs-core tests', function () { spawnOptions: { args: ['--pass ipfs-is-awesome-software', '--offline'] } - })) + }), { + skip: [ + { + name: 'should resolve /ipns/ipfs.io', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve /ipns/ipfs.io recursive === false', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve /ipns/ipfs.io recursive === true', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve /ipns/ipfs.io with remainder', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve /ipns/ipfs.io with remainder recursive === false', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve /ipns/ipfs.io with remainder recursive === true', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should fail to resolve /ipns/ipfs.a', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + }, + { + name: 'should resolve ipns path with hamt-shard recursive === true', + reasones: 'TODO: gateway issue on dns. To be enabled on a new PR' + } + ] + }) tests.namePubsub(CommonFactory.create({ spawnOptions: {