feat: use async/await based ipfs-repo

This is WIP and IPFS doesn't even start properly.

Author: vmx

package.json

@@ -91,23 +91,23 @@
     "hapi-pino": "^6.0.2",
     "hashlru": "^2.3.0",
     "human-to-milliseconds": "^2.0.0",
-    "interface-datastore": "~0.6.0",
+    "interface-datastore": "~0.7.0",
     "ipfs-bitswap": "~0.25.1",
     "ipfs-block": "~0.8.1",
-    "ipfs-block-service": "~0.15.2",
+    "ipfs-block-service": "~0.16.0",
     "ipfs-http-client": "^33.1.0",
     "ipfs-http-response": "~0.3.1",
-    "ipfs-mfs": "~0.12.0",
+    "ipfs-mfs": "^0.13.0",
     "ipfs-multipart": "~0.1.1",
     "ipfs-repo": "~0.27.0",
     "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-exporter": "~0.37.7",
-    "ipfs-unixfs-importer": "~0.39.11",
+    "ipfs-unixfs-exporter": "^0.38.0",
+    "ipfs-unixfs-importer": "^0.40.0",
     "ipfs-utils": "~0.0.4",
-    "ipld": "~0.24.1",
+    "ipld": "~0.25.0",
     "ipld-bitcoin": "~0.3.0",
     "ipld-dag-cbor": "~0.15.0",
-    "ipld-dag-pb": "~0.17.4",
+    "ipld-dag-pb": "~0.18.0",
     "ipld-ethereum": "^4.0.0",
     "ipld-git": "~0.5.0",
     "ipld-raw": "^4.0.0",
@@ -120,7 +120,7 @@
     "iso-url": "~0.4.6",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
-    "libp2p": "~0.25.4",
+    "libp2p": "git+https://github.com/libp2p/js-libp2p.git",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",
     "libp2p-delegated-content-routing": "^0.2.4",
@@ -146,6 +146,7 @@
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.6.0",
     "node-fetch": "^2.3.0",
+    "p-iteration": "^1.1.8",
     "peer-book": "~0.9.0",
     "peer-id": "~0.12.3",
     "peer-info": "~0.15.0",

src/cli/daemon.js

@@ -9,7 +9,6 @@ const TCP = require('libp2p-tcp')
 const MulticastDNS = require('libp2p-mdns')
 const WS = require('libp2p-websockets')
 const Bootstrap = require('libp2p-bootstrap')
-const promisify = require('promisify-es6')
 
 class Daemon {
   constructor (options) {
@@ -75,7 +74,7 @@ class Daemon {
 
     // for the CLI to know the where abouts of the API
     if (this._httpApi._apiServers.length) {
-      await promisify(ipfs._repo.apiAddr.set)(this._httpApi._apiServers[0].info.ma)
+      await ipfs._repo.apiAddr.set(this._httpApi._apiServers[0].info.ma)
     }
 
     this._log('started')

src/cli/utils.js

@@ -10,6 +10,7 @@ log.error = debug('cli:error')
 const Progress = require('progress')
 const byteman = require('byteman')
 const promisify = require('promisify-es6')
+const callbackify = require('callbackify')
 
 exports.isDaemonOn = isDaemonOn
 function isDaemonOn () {
@@ -55,11 +56,9 @@ exports.getIPFS = (argv, callback) => {
     }
   })
 
-  const cleanup = promisify((cb) => {
+  const cleanup = callbackify(async () => {
     if (node && node._repo && !node._repo.closed) {
-      node._repo.close((err) => cb(err))
-    } else {
-      cb()
+      return node._repo.close()
     }
   })
 

src/core/boot.js

@@ -15,17 +15,21 @@ module.exports = (self) => {
     // Checks if a repo exists, and if so opens it
     // Will return callback with a bool indicating the existence
     // of the repo
-    (cb) => {
+    // TODO vmx 2019-08-05: THIS WON'T WORK IN THE BROWSER due to transpiling, this needs a proper fix. This is just a hack to keep things moving
+    async () => {
       // nothing to do
       if (!self._repo.closed) {
-        return cb(null, true)
+        return true
       }
 
-      self._repo.open((err, res) => {
-        if (isRepoUninitializedError(err)) return cb(null, false)
-        if (err) return cb(err)
-        cb(null, true)
-      })
+      try {
+        const res = await self._repo.open()
+      } catch (err) {
+        if (isRepoUninitializedError(err)) return false
+        if (err) throw err
+      }
+
+      return true
     },
     (repoOpened, cb) => {
       // Init with existing initialized, opened, repo

src/core/components/bootstrap.js

@@ -2,7 +2,7 @@
 
 const defaultConfig = require('../runtime/config-nodejs.js')
 const isMultiaddr = require('mafmt').IPFS.matches
-const promisify = require('promisify-es6')
+const callbackify = require('callbackify')
 
 function isValidMultiaddr (ma) {
   try {
@@ -18,76 +18,47 @@ function invalidMultiaddrError (ma) {
 
 module.exports = function bootstrap (self) {
   return {
-    list: promisify((callback) => {
-      self._repo.config.get((err, config) => {
-        if (err) {
-          return callback(err)
-        }
-        callback(null, { Peers: config.Bootstrap })
-      })
+    list: callbackify(async () => {
+      const config = await self._repo.config.get()
+      return { Peers: config.Bootstrap }
     }),
-    add: promisify((multiaddr, args, callback) => {
-      if (typeof args === 'function') {
-        callback = args
-        args = { default: false }
-      }
-
+    add: callbackify(async (multiaddr, args = { default: false }) => {
       if (multiaddr && !isValidMultiaddr(multiaddr)) {
-        return setImmediate(() => callback(invalidMultiaddrError(multiaddr)))
+        throw invalidMultiaddrError(multiaddr)
       }
 
-      self._repo.config.get((err, config) => {
-        if (err) {
-          return callback(err)
-        }
-        if (args.default) {
-          config.Bootstrap = defaultConfig().Bootstrap
-        } else if (multiaddr && config.Bootstrap.indexOf(multiaddr) === -1) {
-          config.Bootstrap.push(multiaddr)
-        }
-        self._repo.config.set(config, (err) => {
-          if (err) {
-            return callback(err)
-          }
+      const config = self._repo.config.get()
+      if (args.default) {
+        config.Bootstrap = defaultConfig().Bootstrap
+      } else if (multiaddr && config.Bootstrap.indexOf(multiaddr) === -1) {
+        config.Bootstrap.push(multiaddr)
+      }
+      await self._repo.config.set(config)
 
-          callback(null, {
-            Peers: args.default ? defaultConfig().Bootstrap : [multiaddr]
-          })
-        })
-      })
-    }),
-    rm: promisify((multiaddr, args, callback) => {
-      if (typeof args === 'function') {
-        callback = args
-        args = { all: false }
+      return {
+        Peers: args.default ? defaultConfig().Bootstrap : [multiaddr]
       }
+    }),
+    rm: callbackify(async (multiaddr, args = { all: false }) => {
       if (multiaddr && !isValidMultiaddr(multiaddr)) {
-        return setImmediate(() => callback(invalidMultiaddrError(multiaddr)))
+        throw invalidMultiaddrError(multiaddr)
       }
 
-      self._repo.config.get((err, config) => {
-        if (err) {
-          return callback(err)
-        }
-        if (args.all) {
-          config.Bootstrap = []
-        } else {
-          config.Bootstrap = config.Bootstrap.filter((mh) => mh !== multiaddr)
-        }
+      const config = await self._repo.config.get()
+      if (args.all) {
+        config.Bootstrap = []
+      } else {
+        config.Bootstrap = config.Bootstrap.filter((mh) => mh !== multiaddr)
+      }
 
-        self._repo.config.set(config, (err) => {
-          if (err) {
-            return callback(err)
-          }
+      await self._repo.config.set(config)
 
-          const res = []
-          if (!args.all && multiaddr) {
-            res.push(multiaddr)
-          }
+      const res = []
+      if (!args.all && multiaddr) {
+        res.push(multiaddr)
+      }
 
-          callback(null, { Peers: res })
-        })
-      })
+      return { Peers: res }
     })
   }
 }

src/core/components/config.js

@@ -1,22 +1,11 @@
 'use strict'
 
-const promisify = require('promisify-es6')
+const callbackify = require('callbackify')
 
 module.exports = function config (self) {
   return {
-    get: promisify((key, callback) => {
-      if (typeof key === 'function') {
-        callback = key
-        key = undefined
-      }
-
-      return self._repo.config.get(key, callback)
-    }),
-    set: promisify((key, value, callback) => {
-      self._repo.config.set(key, value, callback)
-    }),
-    replace: promisify((config, callback) => {
-      self._repo.config.set(config, callback)
-    })
+    get: callbackify(self._repo.config.get),
+    set: callbackify(self._repo.config.set),
+    replace: callbackify(self._repo.config.set)
   }
 }

src/core/components/dht.js

@@ -1,12 +1,13 @@
 'use strict'
 
 const promisify = require('promisify-es6')
-const every = require('async/every')
 const PeerId = require('peer-id')
 const PeerInfo = require('peer-info')
 const CID = require('cids')
 const each = require('async/each')
 const nextTick = require('async/nextTick')
+const { every, forEach } = require('p-iteration')
+const callbackify = require('callbackify')
 
 const errcode = require('err-code')
 
@@ -127,7 +128,7 @@ module.exports = (self) => {
      * @param {function(Error)} [callback]
      * @returns {Promise|void}
      */
-    provide: promisify((keys, options, callback) => {
+    provide: callbackify(async (keys, options) => {
       if (!Array.isArray(keys)) {
         keys = [keys]
       }
@@ -139,29 +140,25 @@ module.exports = (self) => {
       options = options || {}
 
       // ensure blocks are actually local
-      every(keys, (key, cb) => {
-        self._repo.blocks.has(key, cb)
-      }, (err, has) => {
-        if (err) {
-          return callback(err)
-        }
+      const has = await every(keys, async (key) => {
+        return self._repo.blocks.has(key)
+      })
 
-        if (!has) {
-          const errMsg = 'block(s) not found locally, cannot provide'
+      if (!has) {
+        const errMsg = 'block(s) not found locally, cannot provide'
 
-          log.error(errMsg)
-          return callback(errcode(errMsg, 'ERR_BLOCK_NOT_FOUND'))
-        }
+        log.error(errMsg)
+        throw errcode(errMsg, 'ERR_BLOCK_NOT_FOUND')
+      }
 
-        if (options.recursive) {
-          // TODO: Implement recursive providing
-          return callback(errcode('not implemented yet', 'ERR_NOT_IMPLEMENTED_YET'))
-        } else {
-          each(keys, (cid, cb) => {
-            self.libp2p.contentRouting.provide(cid, cb)
-          }, callback)
-        }
-      })
+      if (options.recursive) {
+        // TODO: Implement recursive providing
+        throw errcode('not implemented yet', 'ERR_NOT_IMPLEMENTED_YET')
+      } else {
+        forEach(keys, (cid) => {
+            self.libp2p.contentRouting.provide(cid)
+        })
+      }
     }),
 
     /**

src/core/components/files-regular/refs-local-pull-stream.js

@@ -9,14 +9,14 @@ module.exports = function (self) {
   return () => {
     const deferred = pullDefer.source()
 
-    self._repo.blocks.query({ keysOnly: true }, (err, blocks) => {
-      if (err) {
-        return deferred.resolve(pull.error(err))
+    self._repo.blocks.query({ keysOnly: true }).then(
+      (blocks) => {
+        const refs = blocks.map(b => dsKeyToRef(b.key))
+        deferred.resolve(pull.values(refs))
+      }, (err) => {
+        deferred.resolve(pull.error(err))
       }
-
-      const refs = blocks.map(b => dsKeyToRef(b.key))
-      deferred.resolve(pull.values(refs))
-    })
+    )
 
     return deferred
   }