feat: support for fetching subtree

Author: pgte

src/exporter/dir-flat.js

@@ -9,7 +9,9 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, ipldResolver, resolve, parent) {
+function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+  const accepts = pathRest.shift()
+
   const dir = {
     path: name,
     hash: node.multihash
@@ -20,15 +22,17 @@ function dirExporter (node, name, ipldResolver, resolve, parent) {
     pull(
       pull.values(node.links),
       pull.map((link) => ({
+        linkName: link.name,
         path: path.join(name, link.name),
         hash: link.multihash
       })),
+      pull.filter((item) => accepts === undefined || item.linkName === accepts),
       paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
         if (err) {
           return cb(err)
         }
 
-        cb(null, resolve(n.value, item.path, ipldResolver, name, parent))
+        cb(null, resolve(n.value, item.path, pathRest, ipldResolver, name, parent))
       })),
       pull.flatten()
     )

src/exporter/dir-hamt-sharded.js

@@ -10,7 +10,7 @@ const cleanHash = require('./clean-multihash')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, ipldResolver, resolve, parent) {
+function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
   let dir
   if (!parent || parent.path !== name) {
     dir = [{
@@ -25,22 +25,31 @@ function shardedDirExporter (node, name, ipldResolver, resolve, parent) {
       pull.values(node.links),
       pull.map((link) => {
         // remove the link prefix (2 chars for the bucket index)
-        let p = link.name.substring(2)
-        // another sharded dir or file?
-        p = p ? path.join(name, p) : name
+        const p = link.name.substring(2)
+        const pp = p ? path.join(name, p) : name
+        let accept = true
 
-        return {
-          name: link.name,
-          path: p,
-          hash: link.multihash
+        if (p && pathRest.length) {
+          accept = (p === pathRest[0])
+        }
+        if (accept) {
+          return {
+            name: p,
+            path: pp,
+            hash: link.multihash,
+            pathRest: p ? pathRest.slice(1) : pathRest
+          }
+        } else {
+          return ''
         }
       }),
+      pull.filter(Boolean),
       paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
         if (err) {
           return cb(err)
         }
 
-        cb(null, resolve(n.value, item.path, ipldResolver, (dir && dir[0]) || parent))
+        cb(null, resolve(n.value, item.path, item.pathRest, ipldResolver, (dir && dir[0]) || parent))
       })),
       pull.flatten()
     )

src/exporter/dir.js

@@ -1,5 +1,7 @@
 'use strict'
 
+// TODO: REMOVE??
+
 const path = require('path')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
@@ -12,7 +14,7 @@ const switchType = require('../util').switchType
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, ipldResolver) {
+function dirExporter (node, name, pathRest, ipldResolver) {
   // The algorithm below is as follows
   //
   // 1. Take all links from a given directory node
@@ -24,12 +26,16 @@ function dirExporter (node, name, ipldResolver) {
   //      - `file`: use the fileExporter to load and return the file
   // 4. Flatten
 
+  const accepts = pathRest.shift()
+
   return pull(
     pull.values(node.links),
     pull.map((link) => ({
+      linkName: link.name,
       path: path.join(name, link.name),
       hash: link.multihash
     })),
+    pull.filter((item) => accepts === undefined || item.linkName === accepts),
     paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, result) => {
       if (err) {
         return cb(err)
@@ -44,8 +50,8 @@ function dirExporter (node, name, ipldResolver) {
 
       cb(null, switchType(
         node,
-        () => cat([pull.values([dir]), dirExporter(node, item.path, ipldResolver)]),
-        () => fileExporter(node, item.path, ipldResolver)
+        () => cat([pull.values([dir]), dirExporter(node, item.path, pathRest, ipldResolver)]),
+        () => fileExporter(node, item.path, pathRest, ipldResolver)
       ))
     })),
     pull.flatten()

src/exporter/file.js

@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, ipldResolver) => {
+module.exports = (node, name, pathRest, ipldResolver) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -25,6 +25,12 @@ module.exports = (node, name, ipldResolver) => {
     )
   }
 
+  const accepts = pathRest.shift()
+
+  if (accepts !== undefined && accepts !== name) {
+    return pull.empty()
+  }
+
   let content = pull(
     traverse.depthFirst(node, visitor),
     pull.map(getData)

src/exporter/index.js

@@ -7,43 +7,46 @@ const pullDefer = require('pull-defer')
 
 const resolve = require('./resolve').resolve
 
-function sanitize (path) {
+function pathBaseAndRest (path) {
   // Buffer -> raw multihash or CID in buffer
-  if (Buffer.isBuffer(path)) {
-    return new CID(path).toBaseEncodedString()
-  }
+  let pathBase = path
+  let pathRest = '/'
 
-  if (CID.isCID(path)) {
-    return path.toBaseEncodedString()
+  if (Buffer.isBuffer(path)) {
+    pathBase = (new CID(path)).toBaseEncodedString()
   }
 
-  try {
-    const cid = new CID(path)
-    return cid.toBaseEncodedString()
-  } catch (err) {} // not an isolated CID, can be a path
-
-  if (v.ipfsPath(path)) {
-    // trim that ipfs prefix
+  if (typeof path === 'string') {
     if (path.indexOf('/ipfs/') === 0) {
       path = path.substring(6)
     }
+    const subtreeStart = path.indexOf('/')
+    if (subtreeStart > 0) {
+      pathBase = path.substring(0, subtreeStart)
+      pathRest = path.substring(subtreeStart)
+    }
+  } else if (CID.isCID(pathBase)) {
+    pathBase = pathBase.toBaseEncodedString()
+  }
+
+  pathBase = (new CID(pathBase)).toBaseEncodedString()
 
-    return path
-  } else {
-    throw new Error('not valid cid or path')
+  return {
+    base: pathBase,
+    rest: pathRest.split('/').filter(Boolean)
   }
 }
 
 module.exports = (path, dag) => {
   try {
-    path = sanitize(path)
+    path = pathBaseAndRest(path)
   } catch (err) {
     return pull.error(err)
   }
 
   const d = pullDefer.source()
 
-  const cid = new CID(path)
+  const cid = new CID(path.base)
 
   dag.get(cid, (err, node) => {
     if (err) {
@@ -55,7 +58,7 @@ module.exports = (path, dag) => {
   return pull(
     d,
     pull.map((result) => result.value),
-    pull.map((node) => resolve(node, path, dag)),
+    pull.map((node) => resolve(node, path.base, path.rest, dag)),
     pull.flatten()
   )
 }

src/exporter/resolve.js

@@ -14,13 +14,14 @@ module.exports = Object.assign({
   typeOf: typeOf
 }, resolvers)
 
-function resolve (node, name, ipldResolver, parentNode) {
+function resolve (node, hash, pathRest, ipldResolver, parentNode) {
   const type = typeOf(node)
   const resolver = resolvers[type]
   if (!resolver) {
     return pull.error(new Error('Unkown node type ' + type))
   }
-  let stream = resolver(node, name, ipldResolver, resolve, parentNode)
+  // TODO: pass remaining path to filter output
+  let stream = resolver(node, hash, pathRest, ipldResolver, resolve, parentNode)
   return stream
 }
 

test/node.js

@@ -44,6 +44,7 @@ describe('IPFS UnixFS Engine', () => {
   require('./test-consumable-hash')
   require('./test-hamt')
   require('./test-exporter')(repo)
+  require('./test-export-subtree')(repo)
   require('./test-importer')(repo)
   require('./test-importer-flush')(repo)
   require('./test-import-export')(repo)

test/test-dirbuilder-sharding.js

@@ -334,6 +334,36 @@ module.exports = (repo) => {
           }
         }
       })
+
+      it('exports a big dir with subpath', (done) => {
+        const exportHash = mh.toB58String(rootHash) + '/big/big/2000'
+        const entries = {}
+        pull(
+          exporter(exportHash, ipldResolver),
+          pull.collect(collected)
+        )
+
+        function collected (err, nodes) {
+          expect(err).to.not.exist()
+          const paths = Object.keys(entries).sort()
+          expect(nodes.length).to.equal(4)
+          expect(nodes.map((node) => node.path)).to.deep.equal([
+            'QmTm3ZdKxyDLvcEePEvGfB2QReXsiAF7f39yjRcWwWrA6d',
+            'QmTm3ZdKxyDLvcEePEvGfB2QReXsiAF7f39yjRcWwWrA6d/big',
+            'QmTm3ZdKxyDLvcEePEvGfB2QReXsiAF7f39yjRcWwWrA6d/big/big',
+            'QmTm3ZdKxyDLvcEePEvGfB2QReXsiAF7f39yjRcWwWrA6d/big/big/2000'
+          ])
+          pull(
+            nodes[3].content,
+            pull.collect((err, content) => {
+              expect(err).to.not.exist()
+              expect(content.toString()).to.equal('2000')
+              done()
+            })
+          )
+        }
+      })
+
     })
   })
 }

test/test-export-subtree.js

@@ -0,0 +1,87 @@
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+const expect = chai.expect
+const BlockService = require('ipfs-block-service')
+const IPLDResolver = require('ipld-resolver')
+const UnixFS = require('ipfs-unixfs')
+const bs58 = require('bs58')
+const pull = require('pull-stream')
+const zip = require('pull-zip')
+const CID = require('cids')
+const loadFixture = require('aegir/fixtures')
+const Buffer = require('safe-buffer').Buffer
+
+const unixFSEngine = require('./../src')
+const exporter = unixFSEngine.exporter
+
+const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
+
+module.exports = (repo) => {
+  describe('exporter', () => {
+    let ipldResolver
+
+    before(() => {
+      const bs = new BlockService(repo)
+      ipldResolver = new IPLDResolver(bs)
+    })
+
+
+    it('export a file 2 levels down', (done) => {
+      const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt'
+
+      pull(
+        exporter(hash, ipldResolver),
+        pull.collect((err, files) => {
+          expect(err).to.not.exist()
+          expect(files.length).to.equal(3)
+          expect(files[0].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN')
+          expect(files[0].content).to.not.exist()
+          expect(files[1].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1')
+          expect(files[1].content).to.not.exist()
+          expect(files[2].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt')
+          fileEql(files[2], smallFile, done)
+        })
+      )
+    })
+
+    it('export a non existing file', (done) => {
+      const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/doesnotexist'
+
+      pull(
+        exporter(hash, ipldResolver),
+        pull.collect((err, files) => {
+          expect(err).to.not.exist()
+          expect(files.length).to.equal(1)
+          expect(files[0].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN')
+          expect(files[0].content).to.not.exist()
+          done()
+        })
+      )
+    })
+  })
+}
+
+function fileEql (f1, f2, done) {
+  pull(
+    f1.content,
+    pull.collect((err, data) => {
+      if (err) {
+        return done(err)
+      }
+
+      try {
+        if (f2) {
+          expect(Buffer.concat(data)).to.eql(f2)
+        } else {
+          expect(data).to.exist()
+        }
+      } catch (err) {
+        return done(err)
+      }
+      done()
+    })
+  )
+}