Async Crypto Endeavour (#82)

* more fixes for merkledag changes

* fix some more tests

* next aegir

* ready for the next level

* ready

* chore: update deps

Author: dignifiedquire

.gitignore

@@ -30,4 +30,3 @@ build/Release
 node_modules
 
 dist
-lib

.travis.yml

@@ -1,9 +1,16 @@
 sudo: false
 language: node_js
-node_js:
-  - 4
-  - 5
-  - stable
+
+matrix:
+  include:
+  - node_js: 4
+    env: CXX=g++-4.8
+  - node_js: 6
+    env:
+      - SAUCE=true
+      - CXX=g++-4.8
+  - node_js: stable
+    env: CXX=g++-4.8
 
 # Make sure we have new NPM.
 before_install:
@@ -14,12 +21,17 @@ script:
   - npm test
   - npm run coverage
 
-addons:
-  firefox: 'latest'
-
 before_script:
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
 
 after_success:
   - npm run coverage-publish
+
+addons:
+  firefox: 'latest'
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8

README.md

@@ -2,17 +2,19 @@
   "name": "ipfs-unixfs-engine",
   "version": "0.12.0",
   "description": "JavaScript implementation of the unixfs Engine used by IPFS",
-  "main": "lib/index.js",
-  "jsnext:main": "src/index.js",
+  "main": "src/index.js",
+  "browser": {
+    "fs": false
+  },
   "scripts": {
     "lint": "aegir-lint",
     "build": "aegir-build",
-    "test": "PHANTOM=off aegir-test",
+    "test": "aegir-test",
     "test:node": "aegir-test --env node",
-    "test:browser": "PHANTOM=off aegir-test --env browser",
-    "release": "PHANTOM=off aegir-release",
-    "release-minor": "PHANTOM=off aegir-release --type minor",
-    "release-major": "PHANTOM=off aegir-release --type major",
+    "test:browser": "aegir-test --env browser",
+    "release": "aegir-release",
+    "release-minor": "aegir-release --type minor",
+    "release-major": "aegir-release --type major",
     "coverage": "aegir-coverage",
     "coverage-publish": "aegir-coverage publish"
   },
@@ -32,36 +34,37 @@
   "bugs": {
     "url": "https://github.com/ipfs/js-ipfs-unixfs-engine/issues"
   },
-  "homepage": "https://github.com/ipfs/js-ipfs-unixfs-engineg#readme",
+  "engines": {
+    "node": ">=4.0.0"
+  },
+  "homepage": "https://github.com/ipfs/js-ipfs-unixfs-engine#readme",
   "devDependencies": {
-    "aegir": "^8.1.2",
+    "aegir": "^9.1.0",
     "buffer-loader": "0.0.1",
     "chai": "^3.5.0",
     "fs-pull-blob-store": "^0.4.1",
     "idb-pull-blob-store": "^0.5.1",
     "ipfs-block-service": "^0.6.0",
-    "ipfs-repo": "^0.10.0",
+    "ipfs-repo": "^0.11.0",
     "ncp": "^2.0.0",
     "pre-commit": "^1.1.3",
     "pull-zip": "^2.0.1",
-    "raw-loader": "^0.5.1",
-    "rimraf": "^2.5.4",
-    "run-series": "^1.1.4"
+    "rimraf": "^2.5.4"
   },
   "dependencies": {
+    "async": "^2.1.2",
     "cids": "^0.2.0",
-    "ipfs-unixfs": "^0.1.4",
-    "ipld-dag-pb": "^0.1.3",
-    "ipld-resolver": "^0.1.1",
+    "ipfs-unixfs": "^0.1.5",
+    "ipld-dag-pb": "^0.8.0",
+    "ipld-resolver": "^0.2.0",
     "is-ipfs": "^0.2.1",
     "multihashes": "^0.2.2",
     "pull-block": "^1.0.2",
     "pull-paramap": "^1.2.0",
     "pull-pushable": "^2.0.1",
-    "pull-stream": "^3.4.5",
+    "pull-stream": "^3.5.0",
     "pull-traverse": "^1.0.3",
-    "pull-write": "^1.1.1",
-    "run-parallel": "^1.1.6"
+    "pull-write": "^1.1.1"
   },
   "contributors": [
     "David Dias <daviddias.p@gmail.com>",
@@ -73,4 +76,4 @@
     "jbenet <juan@benet.ai>",
     "nginnever <ginneversource@gmail.com>"
   ]
-}
+}

package.json

@@ -3,6 +3,7 @@
 const traverse = require('pull-traverse')
 const pull = require('pull-stream')
 const CID = require('cids')
+const isIPFS = require('is-ipfs')
 
 const util = require('./../util')
 const switchType = util.switchType
@@ -12,6 +13,10 @@ const dirExporter = require('./dir')
 const fileExporter = require('./file')
 
 module.exports = (hash, ipldResolver, options) => {
+  if (!isIPFS.multihash(hash)) {
+    return pull.error(new Error('not valid multihash'))
+  }
+
   hash = cleanMultihash(hash)
   options = options || {}
 

src/exporter/index.js

@@ -5,7 +5,7 @@ const assert = require('assert')
 const pull = require('pull-stream')
 const pullPushable = require('pull-pushable')
 const pullWrite = require('pull-write')
-const parallel = require('run-parallel')
+const parallel = require('async/parallel')
 const dagPB = require('ipld-dag-pb')
 const CID = require('cids')
 

src/importer/index.js

@@ -3,23 +3,23 @@
 const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
 const mh = require('multihashes')
-const isIPFS = require('is-ipfs')
 
 exports.switchType = (node, dirHandler, fileHandler) => {
   const data = UnixFS.unmarshal(node.data)
   const type = data.type
 
-  if (type === 'directory') return dirHandler()
-  if (type === 'file') return fileHandler()
+  if (type === 'directory') {
+    return dirHandler()
+  }
+
+  if (type === 'file') {
+    return fileHandler()
+  }
 
   return pull.error(new Error('Unkown node type'))
 }
 
 exports.cleanMultihash = (multihash) => {
-  if (!isIPFS.multihash(multihash)) {
-    throw new Error('not valid multihash')
-  }
-
   if (Buffer.isBuffer(multihash)) {
     return mh.toB58String(multihash)
   }

src/util.js

@@ -7,7 +7,7 @@ const path = require('path')
 const IPFSRepo = require('ipfs-repo')
 const Store = require('fs-pull-blob-store')
 const mkdirp = require('mkdirp')
-const series = require('run-series')
+const series = require('async/series')
 
 describe('core', () => {
   const repoExample = path.join(process.cwd(), '/test/repo-example')

test/test-data/1.2MiB.txt renamed to test/fixtures/1.2MiB.txt

@@ -5,22 +5,21 @@ const expect = require('chai').expect
 const BlockService = require('ipfs-block-service')
 const IPLDResolver = require('ipld-resolver')
 const UnixFS = require('ipfs-unixfs')
-const fs = require('fs')
-const path = require('path')
 const bs58 = require('bs58')
 const pull = require('pull-stream')
 const zip = require('pull-zip')
 const CID = require('cids')
+const loadFixture = require('aegir/fixtures')
 
 const unixFSEngine = require('./../src')
 const exporter = unixFSEngine.exporter
 
+const bigFile = loadFixture(__dirname, 'fixtures/1.2MiB.txt')
+
 module.exports = (repo) => {
   describe('exporter', () => {
     let ipldResolver
 
-    const bigFile = fs.readFileSync(path.join(__dirname, '/test-data/1.2MiB.txt'))
-
     before(() => {
       const bs = new BlockService(repo)
       ipldResolver = new IPLDResolver(bs)

test/test-data/1.2MiB.txt.block renamed to test/fixtures/1.2MiB.txt.block

@@ -2,14 +2,11 @@
 'use strict'
 
 const chunker = require('./../src/chunker/fixed-size')
-const fs = require('fs')
 const expect = require('chai').expect
-const path = require('path')
 const pull = require('pull-stream')
+const loadFixture = require('aegir/fixtures')
 
-const rawFile = fs.readFileSync(
-  path.join(__dirname, '/test-data/1MiB.txt')
-)
+const rawFile = loadFixture(__dirname, 'fixtures/1MiB.txt')
 
 describe('chunker: fixed size', () => {
   it('chunks non flat buffers', (done) => {

test/test-data/1.2MiB.txt.link-block0 renamed to test/fixtures/1.2MiB.txt.link-block0

@@ -4,11 +4,10 @@
 const importer = require('./../src').importer
 const expect = require('chai').expect
 const BlockService = require('ipfs-block-service')
-const fs = require('fs')
-const path = require('path')
 const pull = require('pull-stream')
 const mh = require('multihashes')
 const IPLDResolver = require('ipld-resolver')
+const loadFixture = require('aegir/fixtures')
 
 function stringifyMh (files) {
   return files.map((file) => {
@@ -17,17 +16,13 @@ function stringifyMh (files) {
   })
 }
 
+const bigFile = loadFixture(__dirname, 'fixtures/1.2MiB.txt')
+const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
+
 module.exports = function (repo) {
   describe('importer', function () {
     let ipldResolver
 
-    const bigFile = fs.readFileSync(path.join(__dirname, '/test-data/1.2MiB.txt'))
-    const smallFile = fs.readFileSync(path.join(__dirname, '/test-data/200Bytes.txt'))
-
-    // const dirSmall = path.join(__dirname, '/test-data/dir-small')
-    // const dirBig = path.join(__dirname, '/test-data/dir-big')
-    // const dirNested = path.join(__dirname, '/test-data/dir-nested')
-
     before(() => {
       const bs = new BlockService(repo)
       ipldResolver = new IPLDResolver(bs)