Merge pull request #223 from dordille/rabin-chunker

feat(importer): add rabin fingerprinting chunk algorithm

Author: achingbrain

package.json

@@ -5,7 +5,8 @@
   "leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
   "main": "src/index.js",
   "browser": {
-    "fs": false
+    "fs": false,
+    "rabin": false
   },
   "scripts": {
     "test": "aegir test",
@@ -72,7 +73,9 @@
     "pull-through": "^1.0.18",
     "pull-traverse": "^1.0.3",
     "pull-write": "^1.1.4",
-    "sparse-array": "^1.3.1"
+    "rabin": "^1.6.0",
+    "sparse-array": "^1.3.1",
+    "stream-to-pull-stream": "^1.7.2"
   },
   "contributors": [
     "Alan Shaw <alan@tableflip.io>",

src/builder/builder.js

@@ -16,7 +16,8 @@ const DAGNode = dagPB.DAGNode
 
 const defaultOptions = {
   chunkerOptions: {
-    maxChunkSize: 262144
+    maxChunkSize: 262144,
+    avgChunkSize: 262144
   },
   rawLeaves: false,
   hashAlg: 'sha2-256',

src/chunker/index.js

@@ -0,0 +1,8 @@
+'use strict'
+
+const chunkers = {
+  fixed: require('../chunker/fixed-size'),
+  rabin: require('../chunker/rabin')
+}
+
+module.exports = chunkers

src/chunker/rabin.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const createRabin = require('rabin')
+const toPull = require('stream-to-pull-stream')
+
+module.exports = (options) => {
+  let min, max, avg
+  if (options.minChunkSize && options.maxChunkSize && options.avgChunkSize) {
+    avg = options.avgChunkSize
+    min = options.minChunkSize
+    max = options.maxChunkSize
+  } else {
+    avg = options.avgChunkSize
+    min = avg / 3
+    max = avg + (avg / 2)
+  }
+
+  const sizepow = Math.floor(Math.log2(avg))
+  const rabin = createRabin({
+    min: min,
+    max: max,
+    bits: sizepow,
+    window: options.window || 16,
+    polynomial: options.polynomial || '0x3DF305DFB2A805'
+  })
+
+  return toPull.duplex(rabin)
+}

src/importer/index.js

@@ -8,10 +8,7 @@ const assert = require('assert')
 const setImmediate = require('async/setImmediate')
 const DAGBuilder = require('../builder')
 const createTreeBuilder = require('./tree-builder')
-
-const chunkers = {
-  fixed: require('../chunker/fixed-size')
-}
+const chunkers = require('../chunker')
 
 const defaultOptions = {
   chunker: 'fixed',

test/chunker-rabin.js

@@ -0,0 +1,79 @@
+/* eslint-env mocha */
+'use strict'
+
+const chunker = require('./../src/chunker/rabin')
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+const expect = chai.expect
+const pull = require('pull-stream')
+const loadFixture = require('aegir/fixtures')
+
+const rawFile = loadFixture('test/fixtures/1MiB.txt')
+
+describe('chunker: rabin', function () {
+  this.timeout(30000)
+
+  it('chunks non flat buffers', (done) => {
+    const b1 = Buffer.alloc(2 * 256)
+    const b2 = Buffer.alloc(1 * 256)
+    const b3 = Buffer.alloc(5 * 256)
+
+    b1.fill('a')
+    b2.fill('b')
+    b3.fill('c')
+
+    pull(
+      pull.values([b1, b2, b3]),
+      chunker({minChunkSize: 48, avgChunkSize: 96, maxChunkSize: 192}),
+      pull.collect((err, chunks) => {
+        expect(err).to.not.exist()
+        chunks.forEach((chunk) => {
+          expect(chunk).to.have.length.gte(48)
+          expect(chunk).to.have.length.lte(192)
+        })
+        done()
+      })
+    )
+  })
+
+  it('uses default min and max chunk size when only avgChunkSize is specified', (done) => {
+    const b1 = Buffer.alloc(10 * 256)
+    b1.fill('a')
+    pull(
+      pull.values([b1]),
+      chunker({avgChunkSize: 256}),
+      pull.collect((err, chunks) => {
+        expect(err).to.not.exist()
+        chunks.forEach((chunk) => {
+          expect(chunk).to.have.length.gte(256 / 3)
+          expect(chunk).to.have.length.lte(256 * (256 / 2))
+        })
+        done()
+      })
+    )
+  })
+
+  it('256 KiB avg chunks of non scalar filesize', (done) => {
+    const KiB256 = 262144
+    let file = Buffer.concat([rawFile, Buffer.from('hello')])
+    const opts = {
+      minChunkSize: KiB256 / 3,
+      avgChunkSize: KiB256,
+      maxChunkSize: KiB256 + (KiB256 / 2)
+    }
+    pull(
+      pull.values([file]),
+      chunker(opts),
+      pull.collect((err, chunks) => {
+        expect(err).to.not.exist()
+
+        chunks.forEach((chunk) => {
+          expect(chunk).to.have.length.gte(opts.minChunkSize)
+          expect(chunk).to.have.length.lte(opts.maxChunkSize)
+        })
+
+        done()
+      })
+    )
+  })
+})

test/node.js

@@ -44,6 +44,7 @@ describe('IPFS UnixFS Engine', () => {
 
   // Chunkers
   require('./chunker-fixed-size')
+  require('./chunker-rabin')
 
   // Graph Builders
   require('./builder')(repo)