From dc0132e779bd93e93d2fa9af4457f92f9f1b00c5 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 24 Apr 2019 13:40:03 +0800 Subject: [PATCH 1/9] feat: refs endpoint --- src/files-regular/index.js | 5 +- src/files-regular/refs-pull-stream.js | 14 + src/files-regular/refs-readable-stream.js | 15 ++ src/files-regular/refs-tests.js | 315 ++++++++++++++++++++++ src/files-regular/refs.js | 7 + 5 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 src/files-regular/refs-pull-stream.js create mode 100644 src/files-regular/refs-readable-stream.js create mode 100644 src/files-regular/refs-tests.js create mode 100644 src/files-regular/refs.js diff --git a/src/files-regular/index.js b/src/files-regular/index.js index 656fe371a..1fa53dda1 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -17,7 +17,10 @@ const tests = { getPullStream: require('./get-pull-stream'), ls: require('./ls'), lsReadableStream: require('./ls-readable-stream'), - lsPullStream: require('./ls-pull-stream') + lsPullStream: require('./ls-pull-stream'), + refs: require('./refs'), + refsReadableStream: require('./refs-readable-stream'), + refsPullStream: require('./refs-pull-stream') } module.exports = createSuite(tests) diff --git a/src/files-regular/refs-pull-stream.js b/src/files-regular/refs-pull-stream.js new file mode 100644 index 000000000..d26027371 --- /dev/null +++ b/src/files-regular/refs-pull-stream.js @@ -0,0 +1,14 @@ +/* eslint-env mocha */ +'use strict' + +const pull = require('pull-stream') + +module.exports = (createCommon, options) => { + const ipfsRefs = (ipfs) => { + return (path, params, cb) => { + const stream = ipfs.refsPullStream(path, params) + pull(stream, pull.collect(cb)) + } + } + require('./refs-tests')(createCommon, '.refsPullStream', ipfsRefs, options) +} diff --git a/src/files-regular/refs-readable-stream.js b/src/files-regular/refs-readable-stream.js new file mode 100644 index 000000000..23bc40065 --- /dev/null +++ b/src/files-regular/refs-readable-stream.js @@ -0,0 +1,15 @@ +/* eslint-env mocha */ +'use strict' + +const concat = require('concat-stream') + +module.exports = (createCommon, options) => { + const ipfsRefs = (ipfs) => { + return (path, params, cb) => { + const stream = ipfs.refsReadableStream(path, params) + stream.on('error', cb) + stream.pipe(concat((refs) => cb(null, refs))) + } + } + require('./refs-tests')(createCommon, '.refsReadableStream', ipfsRefs, options) +} diff --git a/src/files-regular/refs-tests.js b/src/files-regular/refs-tests.js new file mode 100644 index 000000000..5f82dbd13 --- /dev/null +++ b/src/files-regular/refs-tests.js @@ -0,0 +1,315 @@ +/* eslint-env mocha */ +'use strict' + +const async = require('async') +const { getDescribe, getIt, expect } = require('../utils/mocha') + +module.exports = (createCommon, suiteName, ipfsRefs, options) => { + const describe = getDescribe(options) + const it = getIt(options) + const common = createCommon() + + describe(suiteName, function () { + this.timeout(40 * 1000) + + let ipfs, rootCid + + before(function (done) { + // CI takes longer to instantiate the daemon, so we need to increase the + // timeout for the before step + this.timeout(60 * 1000) + + common.setup((err, factory) => { + expect(err).to.not.exist() + factory.spawnNode((err, node) => { + expect(err).to.not.exist() + ipfs = node + done() + }) + }) + }) + + before(function (done) { + loadContent(ipfs, getMockObjects(), (err, cid) => { + expect(err).to.not.exist() + rootCid = cid + done() + }) + }) + + after((done) => common.teardown(done)) + + for (const [name, options] of Object.entries(getRefsTests())) { + const { path, params, expected, expectError, expectTimeout } = options + // eslint-disable-next-line no-loop-func + it(name, function (done) { + this.timeout(20 * 1000) + + // If we're expecting a timeout, call done when it expires + let timeout + if (expectTimeout) { + timeout = setTimeout(() => { + done() + done = null + }, expectTimeout) + } + + // Call out to IPFS + const p = (path ? path(rootCid) : rootCid) + ipfsRefs(ipfs)(p, params, (err, refs) => { + if (!done) { + // Already timed out + return + } + + if (expectError) { + // Expected an error + expect(err).to.exist() + return done() + } + + if (expectTimeout && !err) { + // Expected a timeout but there wasn't one + return expect.fail('Expected timeout error') + } + + // Check there was no error and the refs match what was expected + expect(err).to.not.exist() + expect(refs.map(r => r.Ref)).to.eql(expected) + + // Clear any pending timeout + clearTimeout(timeout) + + done() + }) + }) + } + }) +} + +function getMockObjects () { + return { + animals: { + land: { + 'african.txt': ['elephant', 'rhinocerous'], + 'americas.txt': ['ñandu', 'tapir'], + 'australian.txt': ['emu', 'kangaroo'] + }, + sea: { + 'atlantic.txt': ['dolphin', 'whale'], + 'indian.txt': ['cuttlefish', 'octopus'] + } + }, + fruits: { + 'tropical.txt': ['banana', 'pineapple'] + }, + 'atlantic-animals': ['dolphin', 'whale'], + 'mushroom.txt': ['mushroom'] + } +} + +function getRefsTests () { + return { + 'prints added files': { + params: {}, + expected: [ + 'QmYEJ7qQNZUvBnv4SZ3rEbksagaan3sGvnUq948vSG8Z34', + 'QmUXzZKa3xhTauLektUiK4GiogHskuz1c57CnnoP4TgYJD', + 'QmYLvZrFn8KE2bcJ9UFhthScBVbbcXEgkJnnCBeKWYkpuQ', + 'QmRfqT4uTUgFXhWbfBZm6eZxi2FQ8pqYK5tcWRyTZ7RcgY' + ] + }, + + 'prints files in edges format': { + params: { e: true }, + expected: [ + 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmYEJ7qQNZUvBnv4SZ3rEbksagaan3sGvnUq948vSG8Z34', + 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmUXzZKa3xhTauLektUiK4GiogHskuz1c57CnnoP4TgYJD', + 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmYLvZrFn8KE2bcJ9UFhthScBVbbcXEgkJnnCBeKWYkpuQ', + 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmRfqT4uTUgFXhWbfBZm6eZxi2FQ8pqYK5tcWRyTZ7RcgY' + ] + }, + + 'prints files in custom format': { + params: { format: ': => ' }, + expected: [ + 'animals: Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s => QmYEJ7qQNZUvBnv4SZ3rEbksagaan3sGvnUq948vSG8Z34', + 'atlantic-animals: Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s => QmUXzZKa3xhTauLektUiK4GiogHskuz1c57CnnoP4TgYJD', + 'fruits: Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s => QmYLvZrFn8KE2bcJ9UFhthScBVbbcXEgkJnnCBeKWYkpuQ', + 'mushroom.txt: Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s => QmRfqT4uTUgFXhWbfBZm6eZxi2FQ8pqYK5tcWRyTZ7RcgY' + ] + }, + + 'follows a path, /': { + path: (cid) => `/ipfs/${cid}/animals`, + params: { format: '' }, + expected: [ + 'land', + 'sea' + ] + }, + + 'follows a path, //': { + path: (cid) => `/ipfs/${cid}/animals/land`, + params: { format: '' }, + expected: [ + 'african.txt', + 'americas.txt', + 'australian.txt' + ] + }, + + 'follows a path with recursion, /': { + path: (cid) => `/ipfs/${cid}/animals`, + params: { format: '', r: true }, + expected: [ + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt' + ] + }, + + 'recursively follows folders, -r': { + params: { format: '', r: true }, + expected: [ + 'animals', + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt', + 'atlantic-animals', + 'fruits', + 'tropical.txt', + 'mushroom.txt' + ] + }, + + 'recursive with unique option': { + params: { format: '', r: true, u: true }, + expected: [ + 'animals', + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt', + 'fruits', + 'tropical.txt', + 'mushroom.txt' + ] + }, + + 'max depth of 1': { + params: { format: '', r: true, 'max-depth': 1 }, + expected: [ + 'animals', + 'atlantic-animals', + 'fruits', + 'mushroom.txt' + ] + }, + + 'max depth of 2': { + params: { format: '', r: true, 'max-depth': 2 }, + expected: [ + 'animals', + 'land', + 'sea', + 'atlantic-animals', + 'fruits', + 'tropical.txt', + 'mushroom.txt' + ] + }, + + 'max depth of 3': { + params: { format: '', r: true, 'max-depth': 3 }, + expected: [ + 'animals', + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt', + 'atlantic-animals', + 'fruits', + 'tropical.txt', + 'mushroom.txt' + ] + }, + + 'max depth of 0': { + params: { r: true, 'max-depth': 0 }, + expected: [] + }, + + 'follows a path with max depth 1, /': { + path: (cid) => `/ipfs/${cid}/animals`, + params: { format: '', r: true, 'max-depth': 1 }, + expected: [ + 'land', + 'sea' + ] + }, + + 'follows a path with max depth 2, /': { + path: (cid) => `/ipfs/${cid}/animals`, + params: { format: '', r: true, 'max-depth': 2 }, + expected: [ + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt' + ] + }, + + 'cannot specify edges and format': { + params: { format: '', e: true }, + expectError: true + }, + + 'prints nothing for non-existent hashes': { + path: () => 'QmYmW4HiZhotsoSqnv2o1oSssvkRM8b9RweBoH7ao5nki2', + expectTimeout: 4000 + } + } +} + +function loadContent (ipfs, node, callback) { + if (Array.isArray(node)) { + ipfs.object.put({ Data: node.join('\n'), Links: [] }, callback) + } + + if (typeof node === 'object') { + const entries = Object.entries(node) + const sorted = entries.sort((a, b) => a[0] > b[0] ? 1 : a[0] < b[0] ? -1 : 0) + async.map(sorted, ([name, child], cb) => { + loadContent(ipfs, child, (err, cid) => { + cb(err, { name, cid: cid && cid.toString() }) + }) + }, (err, res) => { + if (err) { + return callback(err) + } + + ipfs.object.put({ + Data: '', + Links: res.map(({ name, cid }) => ({ Name: name, Hash: cid, Size: 8 })) + }, callback) + }) + } +} diff --git a/src/files-regular/refs.js b/src/files-regular/refs.js new file mode 100644 index 000000000..41dd8c03a --- /dev/null +++ b/src/files-regular/refs.js @@ -0,0 +1,7 @@ +/* eslint-env mocha */ +'use strict' + +module.exports = (createCommon, options) => { + const ipfsRefs = (ipfs) => ipfs.refs.bind(ipfs) + require('./refs-tests')(createCommon, '.refs', ipfsRefs, options) +} From 7c1fc4f95197ccc9e9fa5e1fdbfa1afbe43af347 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 30 Apr 2019 22:38:32 +0800 Subject: [PATCH 2/9] test: add tests for refs local --- src/files-regular/index.js | 3 +- src/files-regular/refs-local.js | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/files-regular/refs-local.js diff --git a/src/files-regular/index.js b/src/files-regular/index.js index 1fa53dda1..18c7dc598 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -20,7 +20,8 @@ const tests = { lsPullStream: require('./ls-pull-stream'), refs: require('./refs'), refsReadableStream: require('./refs-readable-stream'), - refsPullStream: require('./refs-pull-stream') + refsPullStream: require('./refs-pull-stream'), + refsLocal: require('./refs-local') } module.exports = createSuite(tests) diff --git a/src/files-regular/refs-local.js b/src/files-regular/refs-local.js new file mode 100644 index 000000000..ab6d10634 --- /dev/null +++ b/src/files-regular/refs-local.js @@ -0,0 +1,60 @@ +/* eslint-env mocha */ +'use strict' + +const { fixtures } = require('./utils') +const { getDescribe, getIt, expect } = require('../utils/mocha') + +module.exports = (createCommon, suiteName, ipfsRefs, options) => { + const describe = getDescribe(options) + const it = getIt(options) + const common = createCommon() + + describe('refs local', function () { + this.timeout(40 * 1000) + + let ipfs + + before(function (done) { + // CI takes longer to instantiate the daemon, so we need to increase the + // timeout for the before step + this.timeout(60 * 1000) + + common.setup((err, factory) => { + expect(err).to.not.exist() + factory.spawnNode((err, node) => { + expect(err).to.not.exist() + ipfs = node + done() + }) + }) + }) + + after((done) => common.teardown(done)) + + it('should get local refs', function (done) { + const content = (name) => ({ + path: `test-folder/${name}`, + content: fixtures.directory.files[name] + }) + + const dirs = [ + content('pp.txt'), + content('holmes.txt') + ] + + ipfs.add(dirs, (err, res) => { + expect(err).to.not.exist() + + ipfs.refs.local((err, refs) => { + expect(err).to.not.exist() + + const cids = refs.map(r => r.Ref) + expect(cids.includes('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')).to.eql(true) + expect(cids.includes('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')).to.eql(true) + + done() + }) + }) + }) + }) +} From 4da53baf227c38daa3c549791e2ff7a5f2a6006e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 1 May 2019 19:32:46 +0800 Subject: [PATCH 3/9] test: add tests for refs local pull / streamable --- src/files-regular/index.js | 4 +- src/files-regular/refs-local-pull-stream.js | 14 +++++ .../refs-local-readable-stream.js | 15 +++++ src/files-regular/refs-local-tests.js | 60 +++++++++++++++++++ src/files-regular/refs-local.js | 59 +----------------- 5 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 src/files-regular/refs-local-pull-stream.js create mode 100644 src/files-regular/refs-local-readable-stream.js create mode 100644 src/files-regular/refs-local-tests.js diff --git a/src/files-regular/index.js b/src/files-regular/index.js index 18c7dc598..d097ad9c9 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -21,7 +21,9 @@ const tests = { refs: require('./refs'), refsReadableStream: require('./refs-readable-stream'), refsPullStream: require('./refs-pull-stream'), - refsLocal: require('./refs-local') + refsLocal: require('./refs-local'), + refsLocalPullStream: require('./refs-local-pull-stream'), + refsLocalReadableStream: require('./refs-local-readable-stream') } module.exports = createSuite(tests) diff --git a/src/files-regular/refs-local-pull-stream.js b/src/files-regular/refs-local-pull-stream.js new file mode 100644 index 000000000..0f2b69774 --- /dev/null +++ b/src/files-regular/refs-local-pull-stream.js @@ -0,0 +1,14 @@ +/* eslint-env mocha */ +'use strict' + +const pull = require('pull-stream') + +module.exports = (createCommon, options) => { + const ipfsRefsLocal = (ipfs) => { + return (cb) => { + const stream = ipfs.refs.localPullStream() + pull(stream, pull.collect(cb)) + } + } + require('./refs-local-tests')(createCommon, '.refs.localPullStream', ipfsRefsLocal, options) +} diff --git a/src/files-regular/refs-local-readable-stream.js b/src/files-regular/refs-local-readable-stream.js new file mode 100644 index 000000000..9b1fbec7b --- /dev/null +++ b/src/files-regular/refs-local-readable-stream.js @@ -0,0 +1,15 @@ +/* eslint-env mocha */ +'use strict' + +const concat = require('concat-stream') + +module.exports = (createCommon, options) => { + const ipfsRefsLocal = (ipfs) => { + return (cb) => { + const stream = ipfs.refs.localReadableStream() + stream.on('error', cb) + stream.pipe(concat((refs) => cb(null, refs))) + } + } + require('./refs-local-tests')(createCommon, '.refs.localReadableStream', ipfsRefsLocal, options) +} diff --git a/src/files-regular/refs-local-tests.js b/src/files-regular/refs-local-tests.js new file mode 100644 index 000000000..d3347af73 --- /dev/null +++ b/src/files-regular/refs-local-tests.js @@ -0,0 +1,60 @@ +/* eslint-env mocha */ +'use strict' + +const { fixtures } = require('./utils') +const { getDescribe, getIt, expect } = require('../utils/mocha') + +module.exports = (createCommon, suiteName, ipfsRefsLocal, options) => { + const describe = getDescribe(options) + const it = getIt(options) + const common = createCommon() + + describe(suiteName, function () { + this.timeout(40 * 1000) + + let ipfs + + before(function (done) { + // CI takes longer to instantiate the daemon, so we need to increase the + // timeout for the before step + this.timeout(60 * 1000) + + common.setup((err, factory) => { + expect(err).to.not.exist() + factory.spawnNode((err, node) => { + expect(err).to.not.exist() + ipfs = node + done() + }) + }) + }) + + after((done) => common.teardown(done)) + + it('should get local refs', function (done) { + const content = (name) => ({ + path: `test-folder/${name}`, + content: fixtures.directory.files[name] + }) + + const dirs = [ + content('pp.txt'), + content('holmes.txt') + ] + + ipfs.add(dirs, (err, res) => { + expect(err).to.not.exist() + + ipfsRefsLocal(ipfs)((err, refs) => { + expect(err).to.not.exist() + + const cids = refs.map(r => r.Ref) + expect(cids.includes('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')).to.eql(true) + expect(cids.includes('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')).to.eql(true) + + done() + }) + }) + }) + }) +} diff --git a/src/files-regular/refs-local.js b/src/files-regular/refs-local.js index ab6d10634..d3f0b8150 100644 --- a/src/files-regular/refs-local.js +++ b/src/files-regular/refs-local.js @@ -1,60 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { fixtures } = require('./utils') -const { getDescribe, getIt, expect } = require('../utils/mocha') - -module.exports = (createCommon, suiteName, ipfsRefs, options) => { - const describe = getDescribe(options) - const it = getIt(options) - const common = createCommon() - - describe('refs local', function () { - this.timeout(40 * 1000) - - let ipfs - - before(function (done) { - // CI takes longer to instantiate the daemon, so we need to increase the - // timeout for the before step - this.timeout(60 * 1000) - - common.setup((err, factory) => { - expect(err).to.not.exist() - factory.spawnNode((err, node) => { - expect(err).to.not.exist() - ipfs = node - done() - }) - }) - }) - - after((done) => common.teardown(done)) - - it('should get local refs', function (done) { - const content = (name) => ({ - path: `test-folder/${name}`, - content: fixtures.directory.files[name] - }) - - const dirs = [ - content('pp.txt'), - content('holmes.txt') - ] - - ipfs.add(dirs, (err, res) => { - expect(err).to.not.exist() - - ipfs.refs.local((err, refs) => { - expect(err).to.not.exist() - - const cids = refs.map(r => r.Ref) - expect(cids.includes('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')).to.eql(true) - expect(cids.includes('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')).to.eql(true) - - done() - }) - }) - }) - }) +module.exports = (createCommon, options) => { + const ipfsRefsLocal = (ipfs) => (cb) => ipfs.refs.local(cb) + require('./refs-local-tests')(createCommon, '.refs.local', ipfsRefsLocal, options) } From 1386b656c5b71bbc0f6e7919a548c7a1e17ba533 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 2 May 2019 14:49:10 +0800 Subject: [PATCH 4/9] test: use expect.includes() --- src/files-regular/refs-local-tests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/files-regular/refs-local-tests.js b/src/files-regular/refs-local-tests.js index d3347af73..85e80d901 100644 --- a/src/files-regular/refs-local-tests.js +++ b/src/files-regular/refs-local-tests.js @@ -49,8 +49,8 @@ module.exports = (createCommon, suiteName, ipfsRefsLocal, options) => { expect(err).to.not.exist() const cids = refs.map(r => r.Ref) - expect(cids.includes('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')).to.eql(true) - expect(cids.includes('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')).to.eql(true) + expect(cids).to.include('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn') + expect(cids).to.include('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr') done() }) From 579ea56611f749c68dd71bd33e6e8b3ff2048f09 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 2 May 2019 14:51:25 +0800 Subject: [PATCH 5/9] fix: async.map -> map --- src/files-regular/refs-tests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/files-regular/refs-tests.js b/src/files-regular/refs-tests.js index 5f82dbd13..d448ed823 100644 --- a/src/files-regular/refs-tests.js +++ b/src/files-regular/refs-tests.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const async = require('async') +const map = require('async/map') const { getDescribe, getIt, expect } = require('../utils/mocha') module.exports = (createCommon, suiteName, ipfsRefs, options) => { @@ -297,7 +297,7 @@ function loadContent (ipfs, node, callback) { if (typeof node === 'object') { const entries = Object.entries(node) const sorted = entries.sort((a, b) => a[0] > b[0] ? 1 : a[0] < b[0] ? -1 : 0) - async.map(sorted, ([name, child], cb) => { + map(sorted, ([name, child], cb) => { loadContent(ipfs, child, (err, cid) => { cb(err, { name, cid: cid && cid.toString() }) }) From e595bdee9e5bfe48d13d7a9e4341a4997659d0b1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 2 May 2019 16:22:17 +0800 Subject: [PATCH 6/9] fix: ref -> Ref --- src/files-regular/refs-local-tests.js | 2 +- src/files-regular/refs-tests.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/files-regular/refs-local-tests.js b/src/files-regular/refs-local-tests.js index 85e80d901..af6f7fcb8 100644 --- a/src/files-regular/refs-local-tests.js +++ b/src/files-regular/refs-local-tests.js @@ -48,7 +48,7 @@ module.exports = (createCommon, suiteName, ipfsRefsLocal, options) => { ipfsRefsLocal(ipfs)((err, refs) => { expect(err).to.not.exist() - const cids = refs.map(r => r.Ref) + const cids = refs.map(r => r.ref) expect(cids).to.include('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn') expect(cids).to.include('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr') diff --git a/src/files-regular/refs-tests.js b/src/files-regular/refs-tests.js index d448ed823..c929dfad7 100644 --- a/src/files-regular/refs-tests.js +++ b/src/files-regular/refs-tests.js @@ -75,7 +75,7 @@ module.exports = (createCommon, suiteName, ipfsRefs, options) => { // Check there was no error and the refs match what was expected expect(err).to.not.exist() - expect(refs.map(r => r.Ref)).to.eql(expected) + expect(refs.map(r => r.ref)).to.eql(expected) // Clear any pending timeout clearTimeout(timeout) From ee304a9c6815d28f52885574c186864b2c8330b0 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 3 May 2019 18:11:09 +0800 Subject: [PATCH 7/9] chore: add tests for multiple refs, CBOR --- src/files-regular/refs-tests.js | 124 ++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 22 deletions(-) diff --git a/src/files-regular/refs-tests.js b/src/files-regular/refs-tests.js index c929dfad7..08414888f 100644 --- a/src/files-regular/refs-tests.js +++ b/src/files-regular/refs-tests.js @@ -12,7 +12,7 @@ module.exports = (createCommon, suiteName, ipfsRefs, options) => { describe(suiteName, function () { this.timeout(40 * 1000) - let ipfs, rootCid + let ipfs, pbRootCb, dagRootCid before(function (done) { // CI takes longer to instantiate the daemon, so we need to increase the @@ -30,9 +30,17 @@ module.exports = (createCommon, suiteName, ipfsRefs, options) => { }) before(function (done) { - loadContent(ipfs, getMockObjects(), (err, cid) => { + loadPbContent(ipfs, getMockObjects(), (err, cid) => { expect(err).to.not.exist() - rootCid = cid + pbRootCb = cid + done() + }) + }) + + before(function (done) { + loadDagContent(ipfs, getMockObjects(), (err, cid) => { + expect(err).to.not.exist() + dagRootCid = cid done() }) }) @@ -55,7 +63,7 @@ module.exports = (createCommon, suiteName, ipfsRefs, options) => { } // Call out to IPFS - const p = (path ? path(rootCid) : rootCid) + const p = (path ? path(pbRootCb) : pbRootCb) ipfsRefs(ipfs)(p, params, (err, refs) => { if (!done) { // Already timed out @@ -84,6 +92,32 @@ module.exports = (createCommon, suiteName, ipfsRefs, options) => { }) }) } + + it('dag refs test', function (done) { + this.timeout(20 * 1000) + + // Call out to IPFS + ipfsRefs(ipfs)(`/ipfs/${dagRootCid}`, { recursive: true }, (err, refs) => { + // Check there was no error and the refs match what was expected + expect(err).to.not.exist() + expect(refs.map(r => r.ref).sort()).to.eql([ + 'QmPDqvcuA4AkhBLBuh2y49yhUB98rCnxPxa3eVNC1kAbSC', + 'QmVwtsLUHurA6wUirPSdGeEW5tfBEqenXpeRaqr8XN7bNY', + 'QmXGL3ZdYV5rNLCfHe1QsFSQGekRFzgbBu1B3XGZ7DV9fd', + 'QmcSVZRN5E814KkPy4EHnftNAR7htbFvVhRKKqFs4FBwDG', + 'QmcSVZRN5E814KkPy4EHnftNAR7htbFvVhRKKqFs4FBwDG', + 'QmdBcHbK7uDQav8YrHsfKju3EKn48knxjd96KRMFs3gtS9', + 'QmeX96opBHZHLySMFoNiWS5msxjyX6rqtr3Rr1u7uxn7zJ', + 'Qmf8MwTnY7VdcnF8WcoJ3GB24NmNd1HsGzuEWCtUYDP38x', + 'zdpuAkqPgGuEFBFLcixZyFezWw3bsGUWVS6W7c8YhV5sdAc6E', + 'zdpuArVVBgigTbs6FdyqFFWUSsXymdruTtCVoboc91L3WTXi1', + 'zdpuAsrruPqzPDYs9c1FGNR5Wuyx8on64no6z62SRPv3viHGL', + 'zdpuAxTXSfaHaZNed3JG2WvcYNgd64v27ztB2zknrz5noPhz5' + ]) + + done() + }) + }) }) } @@ -121,7 +155,7 @@ function getRefsTests () { }, 'prints files in edges format': { - params: { e: true }, + params: { edges: true }, expected: [ 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmYEJ7qQNZUvBnv4SZ3rEbksagaan3sGvnUq948vSG8Z34', 'Qmd5MhNjx3NSZm3L2QKG1TFvqkTRbtZwGJinqEfqpfHH7s -> QmUXzZKa3xhTauLektUiK4GiogHskuz1c57CnnoP4TgYJD', @@ -161,7 +195,7 @@ function getRefsTests () { 'follows a path with recursion, /': { path: (cid) => `/ipfs/${cid}/animals`, - params: { format: '', r: true }, + params: { format: '', recursive: true }, expected: [ 'land', 'african.txt', @@ -174,7 +208,7 @@ function getRefsTests () { }, 'recursively follows folders, -r': { - params: { format: '', r: true }, + params: { format: '', recursive: true }, expected: [ 'animals', 'land', @@ -192,7 +226,7 @@ function getRefsTests () { }, 'recursive with unique option': { - params: { format: '', r: true, u: true }, + params: { format: '', recursive: true, unique: true }, expected: [ 'animals', 'land', @@ -209,7 +243,7 @@ function getRefsTests () { }, 'max depth of 1': { - params: { format: '', r: true, 'max-depth': 1 }, + params: { format: '', recursive: true, maxDepth: 1 }, expected: [ 'animals', 'atlantic-animals', @@ -219,7 +253,7 @@ function getRefsTests () { }, 'max depth of 2': { - params: { format: '', r: true, 'max-depth': 2 }, + params: { format: '', recursive: true, maxDepth: 2 }, expected: [ 'animals', 'land', @@ -232,7 +266,7 @@ function getRefsTests () { }, 'max depth of 3': { - params: { format: '', r: true, 'max-depth': 3 }, + params: { format: '', recursive: true, maxDepth: 3 }, expected: [ 'animals', 'land', @@ -250,13 +284,13 @@ function getRefsTests () { }, 'max depth of 0': { - params: { r: true, 'max-depth': 0 }, + params: { recursive: true, maxDepth: 0 }, expected: [] }, 'follows a path with max depth 1, /': { path: (cid) => `/ipfs/${cid}/animals`, - params: { format: '', r: true, 'max-depth': 1 }, + params: { format: '', recursive: true, maxDepth: 1 }, expected: [ 'land', 'sea' @@ -265,7 +299,7 @@ function getRefsTests () { 'follows a path with max depth 2, /': { path: (cid) => `/ipfs/${cid}/animals`, - params: { format: '', r: true, 'max-depth': 2 }, + params: { format: '', recursive: true, maxDepth: 2 }, expected: [ 'land', 'african.txt', @@ -277,8 +311,23 @@ function getRefsTests () { ] }, + 'prints refs for multiple paths': { + path: (cid) => [`/ipfs/${cid}/animals`, `/ipfs/${cid}/fruits`], + params: { format: '', recursive: true }, + expected: [ + 'land', + 'african.txt', + 'americas.txt', + 'australian.txt', + 'sea', + 'atlantic.txt', + 'indian.txt', + 'tropical.txt' + ] + }, + 'cannot specify edges and format': { - params: { format: '', e: true }, + params: { format: '', edges: true }, expectError: true }, @@ -289,16 +338,50 @@ function getRefsTests () { } } -function loadContent (ipfs, node, callback) { +function loadPbContent (ipfs, node, callback) { + const store = { + putData: (data, cb) => ipfs.object.put({ Data: data, Links: [] }, cb), + putLinks: (links, cb) => { + ipfs.object.put({ + Data: '', + Links: links.map(({ name, cid }) => ({ Name: name, Hash: cid, Size: 8 })) + }, cb) + } + } + loadContent(ipfs, store, node, callback) +} + +function loadDagContent (ipfs, node, callback) { + const store = { + putData: (data, cb) => { + ipfs.add(Buffer.from(data), (err, res) => { + if (err) { + return callback(err) + } + return cb(null, res[0].hash) + }) + }, + putLinks: (links, cb) => { + const obj = {} + for (const { name, cid } of links) { + obj[name] = { '/': cid } + } + ipfs.dag.put(obj, cb) + } + } + loadContent(ipfs, store, node, callback) +} + +function loadContent (ipfs, store, node, callback) { if (Array.isArray(node)) { - ipfs.object.put({ Data: node.join('\n'), Links: [] }, callback) + return store.putData(node.join('\n'), callback) } if (typeof node === 'object') { const entries = Object.entries(node) const sorted = entries.sort((a, b) => a[0] > b[0] ? 1 : a[0] < b[0] ? -1 : 0) map(sorted, ([name, child], cb) => { - loadContent(ipfs, child, (err, cid) => { + loadContent(ipfs, store, child, (err, cid) => { cb(err, { name, cid: cid && cid.toString() }) }) }, (err, res) => { @@ -306,10 +389,7 @@ function loadContent (ipfs, node, callback) { return callback(err) } - ipfs.object.put({ - Data: '', - Links: res.map(({ name, cid }) => ({ Name: name, Hash: cid, Size: 8 })) - }, callback) + store.putLinks(res, callback) }) } } From 50a4eda4e567f960271c5bde5b00151c36efc62a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 3 May 2019 22:08:46 +0800 Subject: [PATCH 8/9] chore: add REFS spec --- SPEC/REFS.md | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 SPEC/REFS.md diff --git a/SPEC/REFS.md b/SPEC/REFS.md new file mode 100644 index 000000000..e86fea704 --- /dev/null +++ b/SPEC/REFS.md @@ -0,0 +1,181 @@ +# Refs API + +* [refs](#refs) +* [refsReadableStream](#refsreadablestream) +* [refsPullStream](#refspullstream) +* [refs.local](#refslocal) +* [refs.localReadableStream](#refslocalreadablestream) +* [refs.localPullStream](#refslocalpullstream) + +#### `refs` + +> Get links (references) from an object. + +##### `ipfs.refs(ipfsPath, [options], [callback])` + +`ipfsPath` can be of type: + +- [`cid`][cid] of type: + - a [CID](https://github.com/ipfs/js-cid) instance + - [Buffer][b], the raw Buffer of the cid + - String, the base58 encoded version of the cid +- String, including the ipfs handler, a cid and a path to traverse to, ie: + - '/ipfs/QmXEmhrMpbVvTh61FNAxP9nU7ygVtyvZA8HZDUaqQCAb66' + - '/ipfs/QmXEmhrMpbVvTh61FNAxP9nU7ygVtyvZA8HZDUaqQCAb66/a.txt' + - 'QmXEmhrMpbVvTh61FNAxP9nU7ygVtyvZA8HZDUaqQCAb66/a.txt' + +`options` is an optional object that may contain the following keys: + - `recursive (false)`: recursively list references of child nodes + - `unique (false)`: omit duplicate references from output + - `format ("")`: output edges with given format. Available tokens: ``, ``, `` + - `edges (false)`: output references in edge format: `" -> "` + - `maxDepth (1)`: only for recursive refs, limits fetch and listing to the given depth + +`callback` must follow `function (err, refs) {}` signature, where `err` is an error if the operation was not successful and `refs` is an array of `{ ref: "myref", err: "error msg" }` + +If no `callback` is passed, a promise is returned. + +**Example:** + +```JavaScript +ipfs.refs(ipfsPath, { recursive: true }, function (err, refs) { + if (err) { + throw err + } + + for (const ref of refs) { + if (ref.err) { + console.error(ref.err) + } else { + console.log(ref.ref) + // output: "QmHash" + } + } +}) +``` + +#### `refsReadableStream` + +> Output references using a [Readable Stream][rs] + +##### `ipfs.refsReadableStream(ipfsPath, [options])` -> [Readable Stream][rs] + +`options` is an optional object argument identical to the options for [ipfs.refs](#refs) + +**Example:** + +```JavaScript +const stream = ipfs.refsReadableStream(ipfsPath, { recursive: true }) +stream.on('data', function (ref) { + // 'ref' will be of the form + // { + // ref: 'QmHash', + // err: 'err message' + // } +}) +``` + +#### `refsPullStream` + +> Output references using a [Pull Stream][ps]. + +##### `ipfs.refsReadableStream(ipfsPath, [options])` -> [Pull Stream][ps] + +`options` is an optional object argument identical to the options for [ipfs.refs](#refs) + +**Example:** + +```JavaScript +const stream = ipfs.refsPullStream(ipfsPath, { recursive: true }) + +pull( + stream, + pull.collect((err, values) => { + // values will be an array of objects, each one of the form + // { + // ref: 'QmHash', + // err: 'err message' + // } + }) +) +``` + +#### `refs.local` + +> Output all local references (CIDs of all blocks in the blockstore) + +##### `ipfs.refs.local([callback])` + +`callback` must follow `function (err, refs) {}` signature, where `err` is an error if the operation was not successful and `refs` is an array of `{ ref: "myref", err: "error msg" }` + +If no `callback` is passed, a promise is returned. + +**Example:** + +```JavaScript +ipfs.refs.local(function (err, refs) { + if (err) { + throw err + } + + for (const ref of refs) { + if (ref.err) { + console.error(ref.err) + } else { + console.log(ref.ref) + // output: "QmHash" + } + } +}) +``` + +#### `refs.localReadableStream` + +> Output all local references using a [Readable Stream][rs] + +##### `ipfs.localReadableStream()` -> [Readable Stream][rs] + +**Example:** + +```JavaScript +const stream = ipfs.refs.localReadableStream() +stream.on('data', function (ref) { + // 'ref' will be of the form + // { + // ref: 'QmHash', + // err: 'err message' + // } +}) +``` + +#### `refs.localPullStream` + +> Output all local references using a [Pull Stream][ps]. + +##### `ipfs.refs.localReadableStream()` -> [Pull Stream][ps] + +**Example:** + +```JavaScript +const stream = ipfs.refs.localPullStream() + +pull( + stream, + pull.collect((err, values) => { + // values will be an array of objects, each one of the form + // { + // ref: 'QmHash', + // err: 'err message' + // } + }) +) +``` + +A great source of [examples][] can be found in the tests for this API. + +[examples]: https://github.com/ipfs/interface-ipfs-core/blob/master/src/files-regular +[b]: https://www.npmjs.com/package/buffer +[rs]: https://www.npmjs.com/package/readable-stream +[ps]: https://www.npmjs.com/package/pull-stream +[cid]: https://www.npmjs.com/package/cids +[blob]: https://developer.mozilla.org/en-US/docs/Web/API/Blob From ba4f788978d5a2767ff5c41272541ee6be8b2d2e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 3 May 2019 22:58:19 +0800 Subject: [PATCH 9/9] test: add test for Object.links with CBOR --- src/object/links.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/object/links.js b/src/object/links.js index 04b757597..e9961a6e3 100644 --- a/src/object/links.js +++ b/src/object/links.js @@ -162,5 +162,42 @@ module.exports = (createCommon, options) => { }) }) }) + + it('should get links from CBOR object', (done) => { + const hashes = [] + ipfs.add(Buffer.from('test data'), (err, res1) => { + expect(err).to.not.exist() + hashes.push(res1[0].hash) + ipfs.add(Buffer.from('more test data'), (err, res2) => { + hashes.push(res2[0].hash) + expect(err).to.not.exist() + const obj = { + some: 'data', + mylink: { '/': hashes[0] }, + myobj: { + anotherLink: { '/': hashes[1] } + } + } + ipfs.dag.put(obj, (err, cid) => { + expect(err).to.not.exist() + ipfs.object.links(cid, (err, links) => { + expect(err).to.not.exist() + expect(links.length).to.eql(2) + + // TODO: js-ipfs succeeds but go returns empty strings for link name + // const names = [links[0].name, links[1].name] + // expect(names).includes('mylink') + // expect(names).includes('myobj/anotherLink') + + const cids = [links[0].cid.toString(), links[1].cid.toString()] + expect(cids).includes(hashes[0]) + expect(cids).includes(hashes[1]) + + done() + }) + }) + }) + }) + }) }) }