From 01e3d8f1ab74d0914c7e475a85cdf0d29df5be23 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 18 Nov 2019 09:47:28 -0600 Subject: [PATCH 1/6] feat: support storing metadata in unixfs nodes Adds `mtime` and `mode` properties to `{path, content}` import entries --- README.md | 27 ++++++++++++------------- package.json | 8 ++++---- src/dag-builder/file/index.js | 8 ++++++++ src/dag-builder/index.js | 12 +++++++++-- test/benchmark.spec.js | 2 +- test/importer.spec.js | 38 +++++++++++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 101f179..aab39d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ipfs-unixfs-importer +# ipfs-unixfs-importer [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) @@ -13,22 +13,19 @@ > JavaScript implementation of the layout and chunking mechanisms used by IPFS to handle Files -## Lead Maintainer +## Lead Maintainer [Alex Potsides](https://github.com/achingbrain) -## Table of Contents +## Table of Contents -- [ipfs-unixfs-importer](#ipfs-unixfs-importer) - - [Lead Maintainer](#lead-maintainer) - - [Table of Contents](#table-of-contents) - - [Install](#install) - - [Usage](#usage) - - [Example](#example) - - [API](#api) - - [const import = importer(source, ipld [, options])](#const-import--importersource-ipld--options) - - [Contribute](#contribute) - - [License](#license) +- [Install](#install) +- [Usage](#usage) + - [Example](#example) + - [API](#api) + - [const import = importer(source, ipld [, options])](#const-import--importersource-ipld--options) +- [Contribute](#contribute) +- [License](#license) ## Install @@ -108,7 +105,9 @@ The `import` function returns an async iterator takes a source async iterator th ```js { path: 'a name', - content: (Buffer or iterator emitting Buffers) + content: (Buffer or iterator emitting Buffers), + mtime: (Number representing seconds since (positive) or before (negative) the Unix Epoch), + mode: (Number representing ugo-rwx, setuid, setguid and sticky bit) } ``` diff --git a/package.json b/package.json index 43c5a4d..5935e5b 100644 --- a/package.json +++ b/package.json @@ -55,16 +55,16 @@ "async-iterator-all": "^1.0.0", "async-iterator-batch": "~0.0.1", "async-iterator-first": "^1.0.0", - "bl": "^3.0.0", + "bl": "^4.0.0", "deep-extend": "~0.6.0", "err-code": "^2.0.0", "hamt-sharding": "~0.0.2", - "ipfs-unixfs": "~0.1.16", + "ipfs-unixfs": "ipfs/js-ipfs-unixfs#add-metadata", "ipld-dag-pb": "^0.18.0", "multicodec": "~0.5.1", - "multihashing-async": "~0.7.0", + "multihashing-async": "^0.8.0", "rabin-wasm": "~0.0.8", - "superstruct": "~0.6.1" + "superstruct": "^0.8.2" }, "contributors": [ "Alan Shaw ", diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index a3fb565..fa49a92 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -81,6 +81,14 @@ const reduce = (file, ipld, options) => { // create a parent node and add all the leaves const f = new UnixFS('file') + if (options.mtime) { + f.mtime = options.mtime + } + + if (options.mode) { + f.mode = options.mode + } + const links = leaves .filter(leaf => { if (leaf.cid.codec === 'raw' && leaf.node.length) { diff --git a/src/dag-builder/index.js b/src/dag-builder/index.js index 013d498..6105e2d 100644 --- a/src/dag-builder/index.js +++ b/src/dag-builder/index.js @@ -33,10 +33,18 @@ async function * dagBuilder (source, ipld, options) { const chunker = createChunker(options.chunker, validateChunks(source), options.chunkerOptions) // item is a file - yield fileBuilder(entry, chunker, ipld, options) + yield fileBuilder(entry, chunker, ipld, { + ...options, + mode: entry.mode, + mtime: entry.mtime + }) } else { // item is a directory - yield dirBuilder(entry, ipld, options) + yield dirBuilder(entry, ipld, { + ...options, + mode: entry.mode, + mtime: entry.mtime + }) } } } diff --git a/test/benchmark.spec.js b/test/benchmark.spec.js index 1c96013..b0b1db2 100644 --- a/test/benchmark.spec.js +++ b/test/benchmark.spec.js @@ -24,7 +24,7 @@ describe.skip('benchmark', function () { const times = [] after(() => { - console.info(`Percent\tms`) // eslint-disable-line no-console + console.info('Percent\tms') // eslint-disable-line no-console times.forEach((time, index) => { console.info(`${index}\t${parseInt(time / REPEATS)}`) // eslint-disable-line no-console }) diff --git a/test/importer.spec.js b/test/importer.spec.js index ed618df..d5dd140 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -676,5 +676,43 @@ strategies.forEach((strategy) => { } } }) + + it('supports passing mtime', async () => { + this.timeout(60 * 1000) + + const options = { + rawLeaves: true + } + const now = parseInt(Date.now() / 1000) + + for await (const file of importer([{ + path: '1.2MiB.txt', + content: bigFile, + mtime: now + }], ipld, options)) { + const node = await exporter(file.cid, ipld) + + expect(node.unixfs.mtime).to.equal(now) + } + }) + + it('supports passing mode', async () => { + this.timeout(60 * 1000) + + const options = { + rawLeaves: true + } + const mode = parseInt('0111', 8) + + for await (const file of importer([{ + path: '1.2MiB.txt', + content: bigFile, + mode + }], ipld, options)) { + const node = await exporter(file.cid, ipld) + + expect(node.unixfs.mode).to.equal(mode) + } + }) }) }) From 2a89bc2410041541697796a9f36f201f19bb6803 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 18 Nov 2019 12:36:38 -0600 Subject: [PATCH 2/6] chore: remove gh url --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5935e5b..03bf3be 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "deep-extend": "~0.6.0", "err-code": "^2.0.0", "hamt-sharding": "~0.0.2", - "ipfs-unixfs": "ipfs/js-ipfs-unixfs#add-metadata", + "ipfs-unixfs": "^0.2.0", "ipld-dag-pb": "^0.18.0", "multicodec": "~0.5.1", "multihashing-async": "^0.8.0", From 639f95cbbd451acd30adcc06ee47c85fffbd313b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 18 Nov 2019 16:31:16 -0600 Subject: [PATCH 3/6] chore: upgrade node --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d0cf281..be3ad28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ stages: - cov node_js: - - '10' + - '12' os: - linux From 8a3aafa65940616f7ed3a8116744dde071ece3c6 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 18 Nov 2019 16:56:58 -0600 Subject: [PATCH 4/6] chore: update deps --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 03bf3be..a724e3e 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "cids": "~0.7.1", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", - "ipfs-unixfs-exporter": "~0.37.0", + "ipfs-unixfs-exporter": "^0.39.0", "ipld": "^0.25.0", "ipld-in-memory": "^3.0.0", "multihashes": "~0.4.14", From d2c5ec0804f53a68ec883628f761bd291a9786d9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 19 Nov 2019 18:02:19 -0600 Subject: [PATCH 5/6] fix: add metadata to directories too --- src/dag-builder/dir.js | 9 ++++ src/dag-builder/file/index.js | 21 ++++++--- src/dag-builder/index.js | 12 +---- src/dir-sharded.js | 8 ++++ test/importer.spec.js | 89 +++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 16 deletions(-) diff --git a/src/dag-builder/dir.js b/src/dag-builder/dir.js index 7c4d3b3..7ba8111 100644 --- a/src/dag-builder/dir.js +++ b/src/dag-builder/dir.js @@ -8,6 +8,15 @@ const { const dirBuilder = async (item, ipld, options) => { const unixfs = new UnixFS('directory') + + if (item.mtime) { + unixfs.mtime = item.mtime + } + + if (item.mode) { + unixfs.mode = item.mode + } + const node = new DAGNode(unixfs.marshal(), []) const cid = await persist(node, ipld, options) const path = item.path diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index fa49a92..f3be403 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -15,7 +15,7 @@ const dagBuilders = { trickle: require('./trickle') } -async function * buildFile (source, ipld, options) { +async function * buildFile (file, source, ipld, options) { let count = -1 let previous @@ -36,6 +36,15 @@ async function * buildFile (source, ipld, options) { opts.cidVersion = 1 } else { unixfs = new UnixFS(options.leafType, buffer) + + if (file.mtime) { + unixfs.mtime = file.mtime + } + + if (file.mode) { + unixfs.mode = file.mode + } + node = new DAGNode(unixfs.marshal()) } @@ -81,12 +90,12 @@ const reduce = (file, ipld, options) => { // create a parent node and add all the leaves const f = new UnixFS('file') - if (options.mtime) { - f.mtime = options.mtime + if (file.mtime) { + f.mtime = file.mtime } - if (options.mode) { - f.mode = options.mode + if (file.mode) { + f.mode = file.mode } const links = leaves @@ -140,7 +149,7 @@ const fileBuilder = async (file, source, ipld, options) => { throw errCode(new Error(`Unknown importer build strategy name: ${options.strategy}`), 'ERR_BAD_STRATEGY') } - const roots = await all(dagBuilder(buildFile(source, ipld, options), reduce(file, ipld, options), options.builderOptions)) + const roots = await all(dagBuilder(buildFile(file, source, ipld, options), reduce(file, ipld, options), options.builderOptions)) if (roots.length > 1) { throw errCode(new Error('expected a maximum of 1 roots and got ' + roots.length), 'ETOOMANYROOTS') diff --git a/src/dag-builder/index.js b/src/dag-builder/index.js index 6105e2d..013d498 100644 --- a/src/dag-builder/index.js +++ b/src/dag-builder/index.js @@ -33,18 +33,10 @@ async function * dagBuilder (source, ipld, options) { const chunker = createChunker(options.chunker, validateChunks(source), options.chunkerOptions) // item is a file - yield fileBuilder(entry, chunker, ipld, { - ...options, - mode: entry.mode, - mtime: entry.mtime - }) + yield fileBuilder(entry, chunker, ipld, options) } else { // item is a directory - yield dirBuilder(entry, ipld, { - ...options, - mode: entry.mode, - mtime: entry.mtime - }) + yield dirBuilder(entry, ipld, options) } } } diff --git a/src/dir-sharded.js b/src/dir-sharded.js index f27074f..dd1a78d 100644 --- a/src/dir-sharded.js +++ b/src/dir-sharded.js @@ -141,6 +141,14 @@ async function * flush (path, bucket, ipld, options) { dir.fanout = bucket.tableSize() dir.hashType = options.hashFn.code + if (options.mtime) { + dir.mtime = options.mtime + } + + if (options.mode) { + dir.mode = options.mode + } + const node = new DAGNode(dir.marshal(), links) const cid = await persist(node, ipld, options) diff --git a/test/importer.spec.js b/test/importer.spec.js index d5dd140..bcb283b 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -696,6 +696,20 @@ strategies.forEach((strategy) => { } }) + it('supports passing mtime for directories', async () => { + this.timeout(60 * 1000) + + const now = parseInt(Date.now() / 1000) + + const entries = await all(importer([{ + path: '/foo', + mtime: now + }], ipld)) + + const node = await exporter(entries[0].cid, ipld) + expect(node.unixfs.mtime).to.equal(now) + }) + it('supports passing mode', async () => { this.timeout(60 * 1000) @@ -714,5 +728,80 @@ strategies.forEach((strategy) => { expect(node.unixfs.mode).to.equal(mode) } }) + + it('supports passing mode for directories', async () => { + this.timeout(60 * 1000) + + const mode = parseInt('0111', 8) + + const entries = await all(importer([{ + path: '/foo', + mode + }], ipld)) + + const node = await exporter(entries[0].cid, ipld) + expect(node.unixfs.mode).to.equal(mode) + }) + + it('supports passing different modes for different files', async () => { + this.timeout(60 * 1000) + + const mode1 = parseInt('0111', 8) + const mode2 = parseInt('0222', 8) + + const entries = await all(importer([{ + path: '/foo/file1.txt', + content: bigFile, + mode: mode1 + }, { + path: '/foo/file2.txt', + content: bigFile, + mode: mode2 + }], ipld)) + + const node1 = await exporter(entries[0].cid, ipld) + expect(node1.unixfs.mode).to.equal(mode1) + + const node2 = await exporter(entries[1].cid, ipld) + expect(node2.unixfs.mode).to.equal(mode2) + }) + + it('supports deeply nested files do not inherit custom metadata', async () => { + this.timeout(60 * 1000) + + const mode = parseInt('0111', 8) + + const entries = await all(importer([{ + path: '/foo/file1.txt', + content: bigFile, + mode: mode + }, { + path: '/foo/bar/baz/file2.txt', + content: bigFile + }], ipld)) + + const node1 = await exporter(entries[0].cid, ipld) + expect(node1.unixfs.mode).to.equal(mode) + + const node2 = await exporter(entries[1].cid, ipld) + expect(node2.unixfs.mode).to.not.equal(mode) + }) + + it('files and directories get default metadata if not specified', async () => { + this.timeout(60 * 1000) + + const entries = await all(importer([{ + path: '/foo/file1.txt', + content: bigFile + }], ipld)) + + const node1 = await exporter(entries[0].cid, ipld) + expect(node1.unixfs.mode).to.equal(parseInt('0644', 8)) + expect(node1.unixfs.mtime).to.be.undefined() + + const node2 = await exporter(entries[1].cid, ipld) + expect(node2.unixfs.mode).to.equal(parseInt('0755', 8)) + expect(node2.unixfs.mtime).to.be.undefined() + }) }) }) From 11b592fd4038ccf55510f3a562c27b5f7e3e2140 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 21 Nov 2019 08:29:03 -0600 Subject: [PATCH 6/6] fix: add metadata to imported directories --- src/dir-flat.js | 10 ++++ src/dir-sharded.js | 14 +++--- src/flat-to-shard.js | 8 +-- src/tree-builder.js | 36 +++++++------- test/importer.spec.js | 113 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 29 deletions(-) diff --git a/src/dir-flat.js b/src/dir-flat.js index d2433be..cd8c10f 100644 --- a/src/dir-flat.js +++ b/src/dir-flat.js @@ -17,6 +17,7 @@ class DirFlat extends Dir { put (name, value) { this.cid = undefined this.size = undefined + this._children[name] = value } @@ -68,6 +69,15 @@ class DirFlat extends Dir { } const unixfs = new UnixFS('directory') + + if (this.mtime) { + unixfs.mtime = this.mtime + } + + if (this.mode) { + unixfs.mode = this.mode + } + const node = new DAGNode(unixfs.marshal(), links) const cid = await persist(node, ipld, this.options) diff --git a/src/dir-sharded.js b/src/dir-sharded.js index dd1a78d..e515b8a 100644 --- a/src/dir-sharded.js +++ b/src/dir-sharded.js @@ -73,7 +73,7 @@ class DirSharded extends Dir { } async * flush (path, ipld) { - for await (const entry of flush(path, this._bucket, ipld, this.options)) { + for await (const entry of flush(path, this._bucket, ipld, this, this.options)) { yield entry } } @@ -83,7 +83,7 @@ module.exports = DirSharded module.exports.hashFn = hashFn -async function * flush (path, bucket, ipld, options) { +async function * flush (path, bucket, ipld, shardRoot, options) { const children = bucket._children const links = [] @@ -99,7 +99,7 @@ async function * flush (path, bucket, ipld, options) { if (Bucket.isBucket(child)) { let shard - for await (const subShard of await flush('', child, ipld, options)) { + for await (const subShard of await flush('', child, ipld, null, options)) { shard = subShard } @@ -141,12 +141,12 @@ async function * flush (path, bucket, ipld, options) { dir.fanout = bucket.tableSize() dir.hashType = options.hashFn.code - if (options.mtime) { - dir.mtime = options.mtime + if (shardRoot && shardRoot.mtime) { + dir.mtime = shardRoot.mtime } - if (options.mode) { - dir.mode = options.mode + if (shardRoot && shardRoot.mode) { + dir.mode = shardRoot.mode } const node = new DAGNode(dir.marshal(), links) diff --git a/src/flat-to-shard.js b/src/flat-to-shard.js index aa7675e..1617b4d 100644 --- a/src/flat-to-shard.js +++ b/src/flat-to-shard.js @@ -20,9 +20,7 @@ module.exports = async function flatToShard (child, dir, threshold, options) { await parent.put(newDir.parentKey, newDir) } - if (parent) { - return flatToShard(newDir, parent, threshold, options) - } + return flatToShard(newDir, parent, threshold, options) } return newDir @@ -36,7 +34,9 @@ async function convertToShard (oldDir, options) { parentKey: oldDir.parentKey, path: oldDir.path, dirty: oldDir.dirty, - flat: false + flat: false, + mtime: oldDir.mtime, + mode: oldDir.mode }, options) for await (const { key, child } of oldDir.eachChildSeries()) { diff --git a/src/tree-builder.js b/src/tree-builder.js index 8f48595..55bab49 100644 --- a/src/tree-builder.js +++ b/src/tree-builder.js @@ -36,7 +36,9 @@ async function addToTree (elem, tree, options) { parentKey: pathElem, path: currentPath, dirty: true, - flat: true + flat: true, + mtime: dir && dir.unixfs && dir.unixfs.mtime, + mode: dir && dir.unixfs && dir.unixfs.mode }, options) } @@ -64,28 +66,26 @@ async function * treeBuilder (source, ipld, options) { yield entry } - if (tree) { - if (!options.wrapWithDirectory) { - if (tree.childCount() > 1) { - throw errCode(new Error('detected more than one root'), 'ERR_MORE_THAN_ONE_ROOT') - } - - const unwrapped = await first(tree.eachChildSeries()) - - if (!unwrapped) { - return - } - - tree = unwrapped.child + if (!options.wrapWithDirectory) { + if (tree.childCount() > 1) { + throw errCode(new Error('detected more than one root'), 'ERR_MORE_THAN_ONE_ROOT') } - if (!tree.dir) { + const unwrapped = await first(tree.eachChildSeries()) + + if (!unwrapped) { return } - for await (const entry of tree.flush(tree.path, ipld)) { - yield entry - } + tree = unwrapped.child + } + + if (!tree.dir) { + return + } + + for await (const entry of tree.flush(tree.path, ipld)) { + yield entry } } diff --git a/test/importer.spec.js b/test/importer.spec.js index bcb283b..6fab88e 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -710,6 +710,119 @@ strategies.forEach((strategy) => { expect(node.unixfs.mtime).to.equal(now) }) + it('supports passing metadata for wrapping directories', async () => { + this.timeout(60 * 1000) + + const now = parseInt(Date.now() / 1000) + const perms = parseInt('0777', 8) + + const entries = await all(importer([{ + path: '/foo', + mtime: now, + mode: perms + }, { + path: '/foo/bar.txt', + content: bigFile + }], ipld)) + + const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const node = nodes.filter(node => node.unixfs.type === 'directory').pop() + + if (!node) { + expect.fail('no directory found') + } + + expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mode).to.equal(perms) + }) + + it('supports passing metadata for intermediate directories', async () => { + this.timeout(60 * 1000) + + const now = parseInt(Date.now() / 1000) + const perms = parseInt('0777', 8) + + const entries = await all(importer([{ + path: '/foo/bar', + mtime: now, + mode: perms + }, { + path: '/foo/bar/baz.txt', + content: bigFile + }], ipld)) + + const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const node = nodes.filter(node => node.unixfs.type === 'directory').pop() + + if (!node) { + expect.fail('no directory found') + } + + expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mode).to.equal(perms) + }) + + it('supports passing metadata for out of order intermediate directories', async () => { + this.timeout(60 * 1000) + + const now = parseInt(Date.now() / 1000) + const perms = parseInt('0777', 8) + + const entries = await all(importer([{ + path: '/foo/bar/qux.txt', + content: bigFile + }, { + path: '/foo/bar', + mtime: now, + mode: perms + }, { + path: '/foo/quux' + }, { + path: '/foo/bar/baz.txt', + content: bigFile + }], ipld)) + + const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const node = nodes.filter(node => node.unixfs.type === 'directory' && node.name === 'bar').pop() + + if (!node) { + expect.fail('no directory found') + } + + expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mode).to.equal(perms) + }) + + it('supports passing mtime for hamt-sharded-directories', async () => { + this.timeout(60 * 1000) + + const now = parseInt(Date.now() / 1000) + + const entries = await all(importer([{ + path: '/foo', + mtime: now + }, { + path: '/foo/bar.txt', + content: bigFile + }, { + path: '/foo/baz.txt', + content: bigFile + }, { + path: '/foo/qux' + }], ipld, { + shardSplitThreshold: 0 + })) + + const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const node = nodes.filter(node => node.unixfs.type === 'hamt-sharded-directory').pop() + + if (!node) { + expect.fail('no hamt-sharded-directory found') + } + + expect(node.unixfs.mtime).to.equal(now) + }) + it('supports passing mode', async () => { this.timeout(60 * 1000)