Merge pull request #3 from ipfs/add-full-path-option

achingbrain · web-flow · commit d6c57be32d5b · 2018-11-23T17:00:12.000Z
feat: add `fullPath` option
diff --git a/README.md b/README.md
@@ -104,6 +104,91 @@ pull(
 )
 ```
 
+### `fullPath`
+
+If specified the exporter will emit an entry for every path component encountered.
+
+```javascript
+const exporter = require('ipfs-unixfs-exporter')
+const pull = require('pull-stream')
+const collect = require('pull-stream/sinks/collect')
+
+pull(
+  exporter('QmFoo.../bar/baz.txt', ipld, {
+    fullPath: true
+  })
+  collect((err, files) => {
+    console.info(files)
+
+    // [{
+    //   depth: 0,
+    //   name: 'QmFoo...',
+    //   path: 'QmFoo...',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 1,
+    //   name: 'bar',
+    //   path: 'QmFoo.../bar',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 2,
+    //   name: 'baz.txt',
+    //   path: 'QmFoo.../bar/baz.txt',
+    //   size: ...
+    //   hash: Buffer
+    //   content: <Pull stream>
+    //   type: 'file'
+    // }]
+    //
+  })
+)
+```
+
+### `maxDepth`
+
+If specified the exporter will only emit entries up to the specified depth.
+
+```javascript
+const exporter = require('ipfs-unixfs-exporter')
+const pull = require('pull-stream')
+const collect = require('pull-stream/sinks/collect')
+
+pull(
+  exporter('QmFoo.../bar/baz.txt', ipld, {
+    fullPath: true,
+    maxDepth: 1
+  })
+  collect((err, files) => {
+    console.info(files)
+
+    // [{
+    //   depth: 0,
+    //   name: 'QmFoo...',
+    //   path: 'QmFoo...',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 1,
+    //   name: 'bar',
+    //   path: 'QmFoo.../bar',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }]
+    //
+  })
+)
+```
+
 [dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md
 [ipld-resolver instance]: https://github.com/ipld/js-ipld-resolver
 [UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs
diff --git a/src/dir-flat.js b/src/dir-flat.js
@@ -6,7 +6,7 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {
   const accepts = pathRest[0]
 
   const dir = {
@@ -37,7 +37,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
   ]
 
   // place dir before if not specifying subtree
-  if (!pathRest.length) {
+  if (!pathRest.length || options.fullPath) {
     streams.unshift(pull.values([dir]))
   }
 
diff --git a/src/dir-hamt-sharded.js b/src/dir-hamt-sharded.js
@@ -6,7 +6,7 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {
   let dir
   if (!parent || (parent.path !== path)) {
     dir = {
@@ -49,7 +49,7 @@ function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag
     )
   ]
 
-  if (!pathRest.length) {
+  if (!pathRest.length || options.fullPath) {
     streams.unshift(pull.values([dir]))
   }
 
diff --git a/src/file.js b/src/file.js
@@ -7,7 +7,7 @@ const paramap = require('pull-paramap')
 const extractDataFromBlock = require('./extract-data-from-block')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) => {
   const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== path) {
@@ -24,6 +24,9 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
 
   const fileSize = size || file.fileSize()
 
+  let offset = options.offset
+  let length = options.length
+
   if (offset < 0) {
     return pull.error(new Error('Offset must be greater than or equal to 0'))
   }
diff --git a/src/index.js b/src/index.js
@@ -38,7 +38,8 @@ function pathBaseAndRest (path) {
 const defaultOptions = {
   maxDepth: Infinity,
   offset: undefined,
-  length: undefined
+  length: undefined,
+  fullPath: false
 }
 
 module.exports = (path, dag, options) => {
@@ -70,7 +71,7 @@ module.exports = (path, dag, options) => {
       return {
         depth: node.depth,
         name: node.name,
-        path: finalPathFor(node),
+        path: options.fullPath ? node.path : finalPathFor(node),
         size: node.size,
         hash: node.multihash,
         content: node.content,
diff --git a/src/raw.js b/src/raw.js
@@ -4,7 +4,7 @@ const pull = require('pull-stream')
 const extractDataFromBlock = require('./extract-data-from-block')
 
 // Logic to export a single raw block
-module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) => {
   const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== path) {
@@ -13,6 +13,9 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
 
   size = size || node.length
 
+  let offset = options.offset
+  let length = options.length
+
   if (offset < 0) {
     return pull.error(new Error('Offset must be greater than or equal to 0'))
   }
diff --git a/src/resolve.js b/src/resolve.js
@@ -4,6 +4,7 @@ const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 const CID = require('cids')
+const waterfall = require('async/waterfall')
 
 const resolvers = {
   directory: require('./dir-flat'),
@@ -34,30 +35,37 @@ function createResolver (dag, options, depth, parent) {
       }
 
       if (item.object) {
-        return cb(null, resolveItem(null, item.object, item, options.offset, options.length))
+        return cb(null, resolveItem(null, item.object, item, options))
       }
 
       const cid = new CID(item.multihash)
 
-      dag.get(cid, (err, node) => {
-        if (err) {
-          return cb(err)
-        }
-
-        // const name = item.fromPathRest ? item.name : item.path
-        cb(null, resolveItem(cid, node.value, item, options.offset, options.length))
-      })
+      waterfall([
+        (done) => dag.get(cid, done),
+        (node, done) => done(null, resolveItem(cid, node.value, item, options))
+      ], cb)
     }),
     pull.flatten(),
     pull.filter(Boolean),
     pull.filter((node) => node.depth <= options.maxDepth)
   )
 
-  function resolveItem (cid, node, item, offset, length) {
-    return resolve(cid, node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth, offset, length)
+  function resolveItem (cid, node, item, options) {
+    return resolve({
+      cid,
+      node,
+      name: item.name,
+      path: item.path,
+      pathRest: item.pathRest,
+      size: item.size,
+      dag,
+      parentNode: item.parent || parent,
+      depth: item.depth,
+      options
+    })
   }
 
-  function resolve (cid, node, name, path, pathRest, size, dag, parentNode, depth, offset, length) {
+  function resolve ({ cid, node, name, path, pathRest, size, dag, parentNode, depth, options }) {
     let type
 
     try {
@@ -67,11 +75,14 @@ function createResolver (dag, options, depth, parent) {
     }
 
     const nodeResolver = resolvers[type]
+
     if (!nodeResolver) {
       return pull.error(new Error('Unkown node type ' + type))
     }
+
     const resolveDeep = createResolver(dag, options, depth, node)
-    return nodeResolver(cid, node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, offset, length)
+
+    return nodeResolver(cid, node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, options)
   }
 }
 
diff --git a/test/exporter-sharded.spec.js b/test/exporter-sharded.spec.js
@@ -15,7 +15,7 @@ const randomBytes = require('./helpers/random-bytes')
 const exporter = require('../src')
 const importer = require('ipfs-unixfs-importer')
 
-const SHARD_SPLIT_THRESHOLD = 1000
+const SHARD_SPLIT_THRESHOLD = 10
 
 describe('exporter sharded', function () {
   this.timeout(30000)
@@ -51,7 +51,8 @@ describe('exporter sharded', function () {
           }))
         ),
         importer(ipld, {
-          wrap: true
+          wrap: true,
+          shardSplitThreshold: SHARD_SPLIT_THRESHOLD
         }),
         pull.collect(cb)
       ),
diff --git a/test/exporter-subtree.spec.js b/test/exporter-subtree.spec.js
@@ -143,6 +143,50 @@ describe('exporter subtree', () => {
       }
     ], done)
   })
+
+  it('exports all components of a path', (done) => {
+    const content = randomBytes(ONE_MEG)
+
+    waterfall([
+      (cb) => pull(
+        pull.values([{
+          path: './200Bytes.txt',
+          content: randomBytes(ONE_MEG)
+        }, {
+          path: './level-1/200Bytes.txt',
+          content
+        }, {
+          path: './level-1/level-2'
+        }, {
+          path: './level-1/level-2/200Bytes.txt',
+          content
+        }]),
+        importer(ipld),
+        pull.collect(cb)
+      ),
+      (files, cb) => cb(null, files.pop().multihash),
+      (buf, cb) => cb(null, new CID(buf)),
+      (cid, cb) => pull(
+        exporter(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld, {
+          fullPath: true
+        }),
+        pull.collect((err, files) => cb(err, { cid, files }))
+      ),
+      ({ cid, files }, cb) => {
+        expect(files.length).to.equal(4)
+        expect(files[0].path).to.equal(cid.toBaseEncodedString())
+        expect(files[0].name).to.equal(cid.toBaseEncodedString())
+        expect(files[1].path).to.equal(`${cid.toBaseEncodedString()}/level-1`)
+        expect(files[1].name).to.equal('level-1')
+        expect(files[2].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2`)
+        expect(files[2].name).to.equal('level-2')
+        expect(files[3].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`)
+        expect(files[3].name).to.equal('200Bytes.txt')
+
+        cb()
+      }
+    ], done)
+  })
 })
 
 function fileEql (f1, f2, done) {
diff --git a/test/exporter.spec.js b/test/exporter.spec.js
@@ -60,7 +60,7 @@ describe('exporter', () => {
     })
   }
 
-  function addTestFile ({file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves}, cb) {
+  function addTestFile ({ file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) {
     pull(
       pull.values([{
         path,
@@ -79,8 +79,8 @@ describe('exporter', () => {
     )
   }
 
-  function addAndReadTestFile ({file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves}, cb) {
-    addTestFile({file, strategy, path, maxChunkSize, rawLeaves}, (error, multihash) => {
+  function addAndReadTestFile ({ file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) {
+    addTestFile({ file, strategy, path, maxChunkSize, rawLeaves }, (error, multihash) => {
       if (error) {
         return cb(error)
       }
@@ -100,7 +100,7 @@ describe('exporter', () => {
     })
   }
 
-  function addTestDirectory ({directory, strategy = 'balanced', maxChunkSize}, callback) {
+  function addTestDirectory ({ directory, strategy = 'balanced', maxChunkSize }, callback) {
     const input = push()
     const dirName = path.basename(directory)
 
@@ -293,7 +293,8 @@ describe('exporter', () => {
         content: randomBytes(100),
         links: [
           new DAGLink('', file.node.size, file.cid)
-        ]}, cb),
+        ]
+      }, cb),
       (result, cb) => {
         pull(
           exporter(result.cid, ipld, {

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ const cat = require('pull-cat')`
`6`	`6`	`// Logic to export a unixfs directory.`
`7`	`7`	`module.exports = dirExporter`
`8`	`8`
`9`		`-function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {`
	`9`	`+function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {`
`10`	`10`	`const accepts = pathRest[0]`
`11`	`11`
`12`	`12`	`const dir = {`
`@@ -37,7 +37,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren`
`37`	`37`	`]`
`38`	`38`
`39`	`39`	`// place dir before if not specifying subtree`
`40`		`- if (!pathRest.length) {`
	`40`	`+ if (!pathRest.length \|\| options.fullPath) {`
`41`	`41`	`streams.unshift(pull.values([dir]))`
`42`	`42`	`}`
`43`	`43`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ const cat = require('pull-cat')`
`6`	`6`	`// Logic to export a unixfs directory.`
`7`	`7`	`module.exports = shardedDirExporter`
`8`	`8`
`9`		`-function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {`
	`9`	`+function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {`
`10`	`10`	`let dir`
`11`	`11`	`if (!parent \|\| (parent.path !== path)) {`
`12`	`12`	`dir = {`
`@@ -49,7 +49,7 @@ function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag`
`49`	`49`	`)`
`50`	`50`	`]`
`51`	`51`
`52`		`- if (!pathRest.length) {`
	`52`	`+ if (!pathRest.length \|\| options.fullPath) {`
`53`	`53`	`streams.unshift(pull.values([dir]))`
`54`	`54`	`}`
`55`	`55`