big exporter overhaul: centralized dag reslving inside internal resolver stream

Author: pgte

package.json

@@ -70,7 +70,6 @@
     "pull-batch": "^1.0.0",
     "pull-block": "1.2.0",
     "pull-cat": "^1.1.11",
-    "pull-defer": "~0.2.2",
     "pull-pair": "^1.1.0",
     "pull-paramap": "^1.2.2",
     "pull-pause": "0.0.1",

src/exporter/dir-flat.js

@@ -1,14 +1,13 @@
 'use strict'
 
 const pull = require('pull-stream')
-const paramap = require('pull-paramap')
 const CID = require('cids')
 const cat = require('pull-cat')
 
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+function dirExporter (node, name, pathRest, resolve) {
   const accepts = pathRest[0]
 
   const dir = {
@@ -20,19 +19,13 @@ function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
     pull(
       pull.values(node.links),
       pull.map((link) => ({
-        linkName: link.name,
         path: name + '/' + link.name,
-        hash: link.multihash
+        multihash: link.multihash,
+        linkName: link.name,
+        pathRest: pathRest.slice(1)
       })),
       pull.filter((item) => accepts === undefined || item.linkName === accepts),
-      paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
-        if (err) {
-          return cb(err)
-        }
-
-        cb(null, resolve(n.value, accepts || item.path, pathRest, ipldResolver, name, parent))
-      })),
-      pull.flatten()
+      resolve
     )
   ]
 
@@ -41,7 +34,5 @@ function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
     streams.unshift(pull.values([dir]))
   }
 
-  pathRest.shift()
-
   return cat(streams)
 }

src/exporter/dir-hamt-sharded.js

@@ -1,15 +1,14 @@
 'use strict'
 
 const pull = require('pull-stream')
-const paramap = require('pull-paramap')
 const CID = require('cids')
 const cat = require('pull-cat')
 const cleanHash = require('./clean-multihash')
 
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+function shardedDirExporter (node, name, pathRest, resolve, dag, parent) {
   let dir
   if (!parent || parent.path !== name) {
     dir = [{
@@ -37,29 +36,16 @@ function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent
             fromPathRest: fromPathRest,
             name: p,
             path: pp,
-            hash: link.multihash,
-            pathRest: p ? pathRest.slice(1) : pathRest
+            multihash: link.multihash,
+            pathRest: p ? pathRest.slice(1) : pathRest,
+            parent: (dir && dir[0]) || parent
           }
         } else {
           return ''
         }
       }),
       pull.filter(Boolean),
-      paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
-        if (err) {
-          return cb(err)
-        }
-
-        cb(
-          null,
-          resolve(
-            n.value,
-            item.fromPathRest ? item.name : item.path,
-            item.pathRest,
-            ipldResolver,
-            (dir && dir[0]) || parent))
-      })),
-      pull.flatten()
+      resolve
     )
   ]
 

src/exporter/file.js

@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, pathRest, ipldResolver) => {
+module.exports = (node, name, pathRest, resolve, dag) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -20,12 +20,12 @@ module.exports = (node, name, pathRest, ipldResolver) => {
   function visitor (node) {
     return pull(
       pull.values(node.links),
-      paramap((link, cb) => ipldResolver.get(new CID(link.multihash), cb)),
+      paramap((link, cb) => dag.get(new CID(link.multihash), cb)),
       pull.map((result) => result.value)
     )
   }
 
-  const accepts = pathRest.shift()
+  const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== name) {
     return pull.empty()

src/exporter/index.js

@@ -2,9 +2,9 @@
 
 const pull = require('pull-stream')
 const CID = require('cids')
-const pullDefer = require('pull-defer')
+const b58Encode = require('bs58').encode
 
-const resolve = require('./resolve').resolve
+const createResolver = require('./resolve').createResolver
 
 function pathBaseAndRest (path) {
   // Buffer -> raw multihash or CID in buffer
@@ -36,28 +36,60 @@ function pathBaseAndRest (path) {
   }
 }
 
-module.exports = (path, dag) => {
+const defaultOptions = {
+  recurse: true
+}
+
+module.exports = (path, dag, _options) => {
+
+  const options = Object.assign({}, defaultOptions, _options)
+
+  let dPath
   try {
-    path = pathBaseAndRest(path)
+    dPath = pathBaseAndRest(path)
   } catch (err) {
     return pull.error(err)
   }
 
-  const d = pullDefer.source()
-
-  const cid = new CID(path.base)
+  return pull(
+    pull.values([{
+      multihash: new CID(dPath.base),
+      name: dPath.base,
+      path: dPath.base,
+      pathRest: dPath.rest
+    }]),
+    createResolver(dag, options),
+    pull.map((node) => ({
+      path: finalPathFor(node),
+      size: node.size,
+      hash: node.hash,
+      content: node.content
+    }))
+  )
 
-  dag.get(cid, (err, node) => {
-    if (err) {
-      return pull.error(err)
+  function finalPathFor(node) {
+    if (!dPath.rest.length) {
+      return node.path
     }
-    d.resolve(pull.values([node]))
-  })
 
-  return pull(
-    d,
-    pull.map((result) => result.value),
-    pull.map((node) => resolve(node, path.base, path.rest, dag)),
-    pull.flatten()
-  )
+    const cutElements = [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))
+    const lengthToCut = join(cutElements).length
+    let retPath = node.path.substring(lengthToCut)
+    if (retPath.charAt(0) === '/') {
+      retPath = retPath.substring(1)
+    }
+    if (!retPath) {
+      retPath = dPath.rest[dPath.rest.length - 1] || dPath.base
+    }
+    return retPath
+  }
 }
+
+function join(paths) {
+  return paths.reduce((acc, path) => {
+    if (acc.length) {
+      acc += '/'
+    }
+    return acc + path
+  }, '')
+}

src/exporter/object.js

@@ -2,29 +2,26 @@
 
 const CID = require('cids')
 const pull = require('pull-stream')
-const pullDefer = require('pull-defer')
 
-module.exports = (node, name, pathRest, ipldResolver, resolve) => {
+module.exports = (node, name, pathRest, resolve, dag, parent) => {
   let newNode
   if (pathRest.length) {
-    const pathElem = pathRest.shift()
+    const pathElem = pathRest[0]
     newNode = node[pathElem]
     const newName = name + '/' + pathElem
-    if (CID.isCID(newNode)) {
-      const d = pullDefer.source()
-      ipldResolver.get(sanitizeCID(newNode), (err, newNode) => {
-        if (err) {
-          d.resolve(pull.error(err))
-        } else {
-          d.resolve(resolve(newNode.value, newName, pathRest, ipldResolver, node))
-        }
-      })
-      return d
-    } else if (newNode !== undefined) {
-      return resolve(newNode, newName, pathRest, ipldResolver, node)
-    } else {
+    if (!newNode) {
       return pull.error('not found')
     }
+    const isCID = CID.isCID(newNode)
+    return pull(
+      pull.values([{
+        path: newName,
+        pathRest: pathRest.slice(1),
+        multihash: isCID && newNode,
+        object: !isCID && newNode,
+        parent: parent
+      }]),
+      resolve)
   } else {
     return pull.error(new Error('invalid node type'))
   }

src/exporter/resolve.js

@@ -2,6 +2,8 @@
 
 const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
+const paramap = require('pull-paramap')
+const CID = require('cids')
 
 const resolvers = {
   directory: require('./dir-flat'),
@@ -11,17 +13,44 @@ const resolvers = {
 }
 
 module.exports = Object.assign({
-  resolve: resolve,
+  createResolver: createResolver,
   typeOf: typeOf
 }, resolvers)
 
-function resolve (node, hash, pathRest, ipldResolver, parentNode) {
-  const type = typeOf(node)
-  const resolver = resolvers[type]
-  if (!resolver) {
-    return pull.error(new Error('Unkown node type ' + type))
+function createResolver (dag, options, depth, parent) {
+  if (! depth) {
+    depth = 0
+  }
+
+  if (!options.recurse && depth > 0) {
+    return pull.map(identity)
+  }
+
+  return pull(
+    paramap((item, cb) => {
+      if (item.object) {
+        return cb(null, resolve(item.object, item.path, item.pathRest, dag, item.parent || parent))
+      }
+      dag.get(new CID(item.multihash), (err, node) => {
+        if (err) {
+          return cb(err)
+        }
+        const name = item.fromPathRest ? item.name : item.path
+        cb(null, resolve(node.value, name, item.pathRest, dag, item.parent || parent))
+      })
+    }),
+    pull.flatten()
+  )
+
+  function resolve (node, hash, pathRest, parentNode) {
+    const type = typeOf(node)
+    const nodeResolver = resolvers[type]
+    if (!nodeResolver) {
+      return pull.error(new Error('Unkown node type ' + type))
+    }
+    const resolveDeep = createResolver(dag, options, depth + 1, node)
+    return nodeResolver(node, hash, pathRest, resolveDeep, dag, parentNode)
   }
-  return resolver(node, hash, pathRest, ipldResolver, resolve, parentNode)
 }
 
 function typeOf (node) {
@@ -31,3 +60,7 @@ function typeOf (node) {
     return 'object'
   }
 }
+
+function identity (o) {
+  return o
+}

src/importer/tree-builder.js

@@ -91,7 +91,7 @@ function createTreeBuilder (ipldResolver, _options) {
   // ---- Add to tree
 
   function addToTree (elem, callback) {
-    const pathElems = elem.path.split('/').filter(notEmpty)
+    const pathElems = (elem.path || '').split('/').filter(notEmpty)
     let parent = tree
     const lastIndex = pathElems.length - 1
 

test/exporter-subtree.js

@@ -16,7 +16,7 @@ const exporter = unixFSEngine.exporter
 const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
 
 module.exports = (repo) => {
-  describe('exporter', function () {
+  describe('exporter subtree', () => {
     this.timeout(10 * 1000)
 
     let ipldResolver

test/exporter.js

@@ -117,8 +117,8 @@ module.exports = (repo) => {
       pull(
         exporter(hash, ipldResolver),
         pull.collect((err, files) => {
-          files.forEach(file => expect(file).to.have.property('hash'))
           expect(err).to.not.exist()
+          files.forEach(file => expect(file).to.have.property('hash'))
 
           expect(
             files.map((file) => file.path)