refactor: local tests

Author: Alan Shaw

README.md

@@ -384,8 +384,6 @@ Aside from the default export, `ipfs-http-client` exports various types and util
 
 - [`isIPFS`](https://www.npmjs.com/package/is-ipfs)
 - [`Buffer`](https://www.npmjs.com/package/buffer)
-- [`PeerId`](https://www.npmjs.com/package/peer-id)
-- [`PeerInfo`](https://www.npmjs.com/package/peer-info)
 - [`multiaddr`](https://www.npmjs.com/package/multiaddr)
 - [`multibase`](https://www.npmjs.com/package/multibase)
 - [`multicodec`](https://www.npmjs.com/package/multicodec)

package.json

@@ -48,7 +48,6 @@
     "cids": "~0.7.1",
     "debug": "^4.1.0",
     "err-code": "^2.0.0",
-    "explain-error": "^1.0.4",
     "form-data": "^3.0.0",
     "ipfs-block": "~0.8.1",
     "ipfs-utils": "github:ipfs/js-ipfs-utils#refactor/async-iterators",
@@ -78,11 +77,13 @@
     "go-ipfs-dep": "^0.4.22",
     "interface-ipfs-core": "^0.123.0",
     "ipfsd-ctl": "^0.47.1",
-    "ndjson": "^1.5.0",
+    "it-all": "^1.0.1",
+    "it-concat": "^1.0.0",
+    "it-pipe": "^1.1.0",
     "nock": "^11.4.0",
+    "pipe": "^1.1.0",
     "promisify-es6": "^1.0.3",
     "pull-stream": "^3.6.14",
-    "pump": "^3.0.0",
     "stream-equal": "^1.1.1"
   },
   "engines": {

src/pin/ls.js

@@ -1,9 +1,12 @@
 'use strict'
 
+const ndjson = require('iterable-ndjson')
 const configure = require('../lib/configure')
+const toIterable = require('../lib/stream-to-iterable')
+const toCamel = require('../lib/object-to-camel')
 
 module.exports = configure(({ ky }) => {
-  return async (path, options) => {
+  return async function * ls (path, options) {
     if (path && path.type) {
       options = path
       path = null
@@ -14,16 +17,26 @@ module.exports = configure(({ ky }) => {
     options = options || {}
 
     const searchParams = new URLSearchParams(options.searchParams)
+    searchParams.set('stream', true)
     path.forEach(p => searchParams.append('arg', `${p}`))
     if (options.type) searchParams.set('type', options.type)
 
-    const { Keys } = await ky.post('pin/ls', {
+    const res = await ky.post('pin/ls', {
       timeout: options.timeout,
       signal: options.signal,
       headers: options.headers,
       searchParams
-    }).json()
+    })
 
-    return Object.keys(Keys).map(hash => ({ hash, type: Keys[hash].Type }))
+    for await (const pin of ndjson(toIterable(res.body))) {
+      // For nodes that do not understand the `stream option`
+      if (pin.Keys) {
+        for (const hash of Object.keys(pin.Keys)) {
+          yield { hash, type: pin.Keys[hash].Type }
+        }
+        return
+      }
+      yield toCamel(pin)
+    }
   }
 })

src/pubsub/subscribe.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const ndjson = require('iterable-ndjson')
-const explain = require('explain-error')
 const bs58 = require('bs58')
 const { Buffer } = require('buffer')
 const log = require('debug')('ipfs-http-client:pubsub:subscribe')
@@ -71,7 +70,8 @@ async function readMessages (msgStream, { onMessage, onEnd, onError }) {
           topicIDs: msg.topicIDs
         })
       } catch (err) {
-        onError(explain(err, 'Failed to parse pubsub message'), false, msg) // Not fatal
+        err.message = `Failed to parse pubsub message: ${err.message}`
+        onError(err, false, msg) // Not fatal
       }
     }
   } catch (err) {

test/exports.spec.js

@@ -7,8 +7,6 @@ const multiaddr = require('multiaddr')
 const multibase = require('multibase')
 const multicodec = require('multicodec')
 const multihash = require('multihashes')
-const PeerId = require('peer-id')
-const PeerInfo = require('peer-info')
 const { expect } = require('interface-ipfs-core/src/utils/mocha')
 
 const IpfsHttpClient = require('../')
@@ -22,7 +20,5 @@ describe('exports', () => {
     expect(IpfsHttpClient.multibase).to.equal(multibase)
     expect(IpfsHttpClient.multicodec).to.equal(multicodec)
     expect(IpfsHttpClient.multihash).to.equal(multihash)
-    expect(IpfsHttpClient.PeerId).to.equal(PeerId)
-    expect(IpfsHttpClient.PeerInfo).to.equal(PeerInfo)
   })
 })

test/files-mfs.spec.js

@@ -6,13 +6,12 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha')
 const loadFixture = require('aegir/fixtures')
 const mh = require('multihashes')
 const CID = require('cids')
-const values = require('pull-stream/sources/values')
-const pull = require('pull-stream/pull')
-const collect = require('pull-stream/sinks/collect')
+const all = require('it-all')
+const pipe = require('it-pipe')
+const { TimeoutError } = require('ky-universal')
 
 const ipfsClient = require('../src')
 const f = require('./utils/factory')
-const expectTimeout = require('./utils/expect-timeout')
 
 const testfile = loadFixture('test/fixtures/testfile.txt')
 
@@ -53,7 +52,7 @@ describe('.files (the MFS API part)', function () {
   })
 
   it('.add file for testing', async () => {
-    const res = await ipfs.add(testfile)
+    const res = await all(ipfs.add(testfile))
 
     expect(res).to.have.length(1)
     expect(res[0].hash).to.equal(expectedMultihash)
@@ -66,7 +65,7 @@ describe('.files (the MFS API part)', function () {
     const expectedBufferMultihash = 'QmWfVY9y3xjsixTgbd9AorQxH7VtMpzfx2HaWtsoUYecaX'
     const file = Buffer.from('hello')
 
-    const res = await ipfs.add(file)
+    const res = await all(ipfs.add(file))
 
     expect(res).to.have.length(1)
     expect(res[0].hash).to.equal(expectedBufferMultihash)
@@ -77,7 +76,7 @@ describe('.files (the MFS API part)', function () {
     const expectedHash = 'QmWfVY9y3xjsixTgbd9AorQxH7VtMpzfx2HaWtsoUYecaX'
     const content = Buffer.from('hello')
 
-    const res = await ipfs.add([{ path: '', content }])
+    const res = await all(ipfs.add([{ path: '', content }]))
 
     expect(res).to.have.length(1)
     expect(res[0].hash).to.equal(expectedHash)
@@ -88,7 +87,7 @@ describe('.files (the MFS API part)', function () {
     const expectedCid = 'bafybeifogzovjqrcxvgt7g36y7g63hvwvoakledwk4b2fr2dl4wzawpnny'
     const options = { cidVersion: 1, rawLeaves: false }
 
-    const res = await ipfs.add(testfile, options)
+    const res = await all(ipfs.add(testfile, options))
 
     expect(res).to.have.length(1)
     expect(res[0].hash).to.equal(expectedCid)
@@ -98,15 +97,16 @@ describe('.files (the MFS API part)', function () {
   it('.add with only-hash=true', async () => {
     const content = String(Math.random() + Date.now())
 
-    const files = await ipfs.add(Buffer.from(content), { onlyHash: true })
+    const files = await all(ipfs.add(Buffer.from(content), { onlyHash: true }))
     expect(files).to.have.length(1)
 
     // 'ipfs.object.get(<hash>)' should timeout because content wasn't actually added
-    await expectTimeout(ipfs.object.get(files[0].hash), 4000)
+    return expect(ipfs.object.get(files[0].hash, { timeout: 2000 }))
+      .to.be.rejectedWith(TimeoutError)
   })
 
   it('.add with options', async () => {
-    const res = await ipfs.add(testfile, { pin: false })
+    const res = await all(ipfs.add(testfile, { pin: false }))
 
     expect(res).to.have.length(1)
     expect(res[0].hash).to.equal(expectedMultihash)
@@ -116,23 +116,23 @@ describe('.files (the MFS API part)', function () {
   it('.add pins by default', async () => {
     const newContent = Buffer.from(String(Math.random()))
 
-    const initialPins = await ipfs.pin.ls()
+    const initialPins = await all(ipfs.pin.ls())
 
-    await ipfs.add(newContent)
+    await all(ipfs.add(newContent))
 
-    const pinsAfterAdd = await ipfs.pin.ls()
+    const pinsAfterAdd = await all(ipfs.pin.ls())
 
     expect(pinsAfterAdd.length).to.eql(initialPins.length + 1)
   })
 
   it('.add with pin=false', async () => {
     const newContent = Buffer.from(String(Math.random()))
 
-    const initialPins = await ipfs.pin.ls()
+    const initialPins = await all(ipfs.pin.ls())
 
-    await ipfs.add(newContent, { pin: false })
+    await all(ipfs.add(newContent, { pin: false }))
 
-    const pinsAfterAdd = await ipfs.pin.ls()
+    const pinsAfterAdd = await all(ipfs.pin.ls())
 
     expect(pinsAfterAdd.length).to.eql(initialPins.length)
   })
@@ -146,7 +146,7 @@ describe('.files (the MFS API part)', function () {
       }
       const options = { hashAlg: name, rawLeaves: false }
 
-      const res = await ipfs.add([file], options)
+      const res = await all(ipfs.add([file], options))
 
       expect(res).to.have.length(1)
       const cid = new CID(res[0].hash)
@@ -163,7 +163,7 @@ describe('.files (the MFS API part)', function () {
       progress = p
     }
 
-    const res = await ipfs.add(testfile, { progress: progressHandler })
+    const res = await all(ipfs.add(testfile, { progress: progressHandler }))
 
     expect(res).to.have.length(1)
     expect(progress).to.be.equal(testfile.byteLength)
@@ -180,7 +180,7 @@ describe('.files (the MFS API part)', function () {
     }
 
     // TODO: needs to be using a big file
-    const res = await ipfs.add(testfile, { progress: progressHandler })
+    const res = await all(ipfs.add(testfile, { progress: progressHandler }))
 
     expect(res).to.have.length(1)
     expect(progress).to.be.equal(testfile.byteLength)
@@ -197,15 +197,15 @@ describe('.files (the MFS API part)', function () {
     }
 
     // TODO: needs to be using a directory
-    const res = await ipfs.add(testfile, { progress: progressHandler })
+    const res = await all(ipfs.add(testfile, { progress: progressHandler }))
 
     expect(res).to.have.length(1)
     expect(progress).to.be.equal(testfile.byteLength)
     expect(progressCount).to.be.equal(1)
   })
 
   it('.add without progress options', async () => {
-    const res = await ipfs.add(testfile)
+    const res = await all(ipfs.add(testfile))
 
     expect(res).to.have.length(1)
   })
@@ -219,44 +219,33 @@ describe('.files (the MFS API part)', function () {
       }
       const options = { hashAlg: name, rawLeaves: false }
 
-      const res = await ipfs.add([file], options)
+      const res = await all(ipfs.add([file], options))
 
       expect(res).to.have.length(1)
       const cid = new CID(res[0].hash)
       expect(mh.decode(cid.multihash).name).to.equal(name)
     })
   })
 
-  it('.addPullStream with object chunks and pull stream content', (done) => {
+  it('.add with object chunks and iterable content', async () => {
     const expectedCid = 'QmRf22bZar3WKmojipms22PkXH1MZGmvsqzQtuSvQE3uhm'
 
-    pull(
-      values([{ content: values([Buffer.from('test')]) }]),
-      ipfs.addPullStream(),
-      collect((err, res) => {
-        expect(err).to.not.exist()
-
-        expect(res).to.have.length(1)
-        expect(res[0]).to.deep.equal({ path: expectedCid, hash: expectedCid, size: 12 })
-        done()
-      })
+    const res = await pipe(
+      [{ content: [Buffer.from('test')] }],
+      ipfs.add,
+      all
     )
-  })
-
-  it('.add with pull stream', async () => {
-    const expectedCid = 'QmRf22bZar3WKmojipms22PkXH1MZGmvsqzQtuSvQE3uhm'
-    const res = await ipfs.add(values([Buffer.from('test')]))
 
     expect(res).to.have.length(1)
     expect(res[0]).to.deep.equal({ path: expectedCid, hash: expectedCid, size: 12 })
   })
 
-  it('.add with array of objects with pull stream content', async () => {
+  it('.add with iterable', async () => {
     const expectedCid = 'QmRf22bZar3WKmojipms22PkXH1MZGmvsqzQtuSvQE3uhm'
-    const res = await ipfs.add([{ content: values([Buffer.from('test')]) }])
+    const res = await all(ipfs.add([Buffer.from('test')]))
 
     expect(res).to.have.length(1)
-    expect(res[0]).to.eql({ path: expectedCid, hash: expectedCid, size: 12 })
+    expect(res[0]).to.deep.equal({ path: expectedCid, hash: expectedCid, size: 12 })
   })
 
   it('files.mkdir', async () => {
@@ -327,7 +316,7 @@ describe('.files (the MFS API part)', function () {
     await ipfs.files.write(file, Buffer.from('Hello, world'), {
       create: true
     })
-    const files = await ipfs.files.ls(folder)
+    const files = await all(ipfs.files.ls(folder))
 
     expect(files.length).to.equal(1)
   })
@@ -336,7 +325,7 @@ describe('.files (the MFS API part)', function () {
     const folder = `test-folder-${Math.random()}`
 
     await ipfs.files.mkdir(`/${folder}`)
-    const files = await ipfs.files.ls()
+    const files = await all(ipfs.files.ls())
 
     expect(files.find(file => file.name === folder)).to.be.ok()
   })
@@ -346,15 +335,15 @@ describe('.files (the MFS API part)', function () {
       create: true
     })
 
-    const buf = await ipfs.files.read('/test-folder/test-file-2.txt')
+    const buf = Buffer.concat(await all(ipfs.files.read('/test-folder/test-file-2.txt')))
 
     expect(buf.toString()).to.be.equal('hello world')
   })
 
   it('files.write without options', async () => {
     await ipfs.files.write('/test-folder/test-file-2.txt', Buffer.from('hello world'))
 
-    const buf = await ipfs.files.read('/test-folder/test-file-2.txt')
+    const buf = Buffer.concat(await all(ipfs.files.read('/test-folder/test-file-2.txt')))
 
     expect(buf.toString()).to.be.equal('hello world')
   })
@@ -395,7 +384,7 @@ describe('.files (the MFS API part)', function () {
     await ipfs.files.write(file, testfile, {
       create: true
     })
-    const buf = await ipfs.files.read(file)
+    const buf = Buffer.concat(await all(ipfs.files.read(file)))
 
     expect(Buffer.from(buf)).to.deep.equal(testfile)
   })