diff --git a/callback-test/blockstore-test.js b/callback-test/blockstore-test.js new file mode 100644 index 00000000..a504d925 --- /dev/null +++ b/callback-test/blockstore-test.js @@ -0,0 +1,250 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const Block = require('ipfs-block') +const CID = require('cids') +const parallel = require('async/parallel') +const waterfall = require('async/waterfall') +const each = require('async/each') +const map = require('async/map') +const _ = require('lodash') +const multihashing = require('multihashing-async') + +module.exports = (repo) => { + describe('blockstore', () => { + const blockData = _.range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) + const bData = Buffer.from('hello world') + let b + + before((done) => { + multihashing(bData, 'sha2-256', (err, h) => { + if (err) { + return done(err) + } + + b = new Block(bData, new CID(h)) + done() + }) + }) + + describe('.put', () => { + it('simple', (done) => { + repo.blocks.put(b, done) + }) + + it('multi write (locks)', (done) => { + parallel([ + (cb) => repo.blocks.put(b, cb), + (cb) => repo.blocks.put(b, cb) + ], done) + }) + + it('empty value', (done) => { + const d = Buffer.alloc(0) + multihashing(d, 'sha2-256', (err, multihash) => { + expect(err).to.not.exist() + const empty = new Block(d, new CID(multihash)) + repo.blocks.put(empty, done) + }) + }) + + it('massive multiwrite', function (done) { + this.timeout(15000) // add time for ci + waterfall([ + (cb) => map(_.range(100), (i, cb) => { + multihashing(blockData[i], 'sha2-256', cb) + }, cb), + (hashes, cb) => each(_.range(100), (i, cb) => { + const block = new Block(blockData[i], new CID(hashes[i])) + repo.blocks.put(block, cb) + }, cb) + ], done) + }) + + it('.putMany', function (done) { + this.timeout(15000) // add time for ci + waterfall([ + (cb) => map(_.range(50), (i, cb) => { + const d = Buffer.from('many' + Math.random()) + multihashing(d, 'sha2-256', (err, hash) => { + if (err) { + return cb(err) + } + cb(null, new Block(d, new CID(hash))) + }) + }, cb), + (blocks, cb) => { + repo.blocks.putMany(blocks, (err) => { + expect(err).to.not.exist() + map(blocks, (b, cb) => { + repo.blocks.get(b.cid, cb) + }, (err, res) => { + expect(err).to.not.exist() + expect(res).to.be.eql(blocks) + cb() + }) + }) + } + ], done) + }) + + it('returns an error on invalid block', (done) => { + repo.blocks.put('hello', (err) => { + expect(err).to.exist() + done() + }) + }) + }) + + describe('.get', () => { + it('simple', (done) => { + repo.blocks.get(b.cid, (err, block) => { + expect(err).to.not.exist() + expect(block).to.be.eql(b) + done() + }) + }) + + it('massive read', function (done) { + this.timeout(15000) // add time for ci + parallel(_.range(20 * 100).map((i) => (cb) => { + const j = i % blockData.length + waterfall([ + (cb) => multihashing(blockData[j], 'sha2-256', cb), + (h, cb) => { + const cid = new CID(h) + repo.blocks.get(cid, cb) + }, + (block, cb) => { + expect(block.data).to.be.eql(blockData[j]) + cb() + } + ], cb) + }), done) + }) + + it('returns an error on invalid block', (done) => { + repo.blocks.get('woot', (err, val) => { + expect(err).to.exist() + expect(val).to.not.exist() + done() + }) + }) + + it('should get block stored under v0 CID with a v1 CID', done => { + const data = Buffer.from(`TEST${Date.now()}`) + + multihashing(data, 'sha2-256', (err, hash) => { + if (err) return done(err) + + const cid = new CID(hash) + + repo.blocks.put(new Block(data, cid), err => { + if (err) return done(err) + + repo.blocks.get(cid.toV1(), (err, block) => { + expect(err).to.not.exist() + expect(block.data).to.eql(data) + done() + }) + }) + }) + }) + + it('should get block stored under v1 CID with a v0 CID', done => { + const data = Buffer.from(`TEST${Date.now()}`) + + multihashing(data, 'sha2-256', (err, hash) => { + if (err) return done(err) + + const cid = new CID(1, 'dag-pb', hash) + + repo.blocks.put(new Block(data, cid), err => { + if (err) return done(err) + + repo.blocks.get(cid.toV0(), (err, block) => { + expect(err).to.not.exist() + expect(block.data).to.eql(data) + done() + }) + }) + }) + }) + }) + + describe('.has', () => { + it('existing block', (done) => { + repo.blocks.has(b.cid, (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(true) + done() + }) + }) + + it('non existent block', (done) => { + repo.blocks.has(new CID('QmbcpFjzamCj5ZZdduW32ctWUPvbGMwQZk2ghWK6PrKswE'), (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(false) + done() + }) + }) + + it('should have block stored under v0 CID with a v1 CID', done => { + const data = Buffer.from(`TEST${Date.now()}`) + + multihashing(data, 'sha2-256', (err, hash) => { + if (err) return done(err) + + const cid = new CID(hash) + + repo.blocks.put(new Block(data, cid), err => { + if (err) return done(err) + + repo.blocks.has(cid.toV1(), (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(true) + done() + }) + }) + }) + }) + + it('should have block stored under v1 CID with a v0 CID', done => { + const data = Buffer.from(`TEST${Date.now()}`) + + multihashing(data, 'sha2-256', (err, hash) => { + if (err) return done(err) + + const cid = new CID(1, 'dag-pb', hash) + + repo.blocks.put(new Block(data, cid), err => { + if (err) return done(err) + + repo.blocks.has(cid.toV0(), (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(true) + done() + }) + }) + }) + }) + }) + + describe('.delete', () => { + it('simple', (done) => { + waterfall([ + (cb) => repo.blocks.delete(b.cid, cb), + (cb) => repo.blocks.has(b.cid, cb) + ], (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.equal(false) + done() + }) + }) + }) + }) +} diff --git a/callback-test/browser.js b/callback-test/browser.js new file mode 100644 index 00000000..f5490093 --- /dev/null +++ b/callback-test/browser.js @@ -0,0 +1,28 @@ +/* eslint-env mocha */ + +'use strict' + +const series = require('async/series') + +const IPFSRepo = require('../src') + +describe('IPFS Repo Tests on the Browser', () => { + require('./options-test') + const repo = new IPFSRepo('myrepo') + + before((done) => { + series([ + (cb) => repo.init({}, cb), + (cb) => repo.open(cb) + ], done) + }) + + after((done) => { + repo.close(done) + }) + + require('./repo-test')(repo) + require('./blockstore-test')(repo) + require('./datastore-test')(repo) + require('./keystore-test')(repo) +}) diff --git a/callback-test/datastore-test.js b/callback-test/datastore-test.js new file mode 100644 index 00000000..b7fb8c26 --- /dev/null +++ b/callback-test/datastore-test.js @@ -0,0 +1,93 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const parallel = require('async/parallel') +const waterfall = require('async/waterfall') +const each = require('async/each') +const _ = require('lodash') +const Key = require('interface-datastore').Key + +module.exports = (repo) => { + describe('datastore', () => { + const dataList = _.range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) + const data = Buffer.from('hello world') + const b = new Key('hello') + + describe('.put', () => { + it('simple', (done) => { + repo.datastore.put(b, data, done) + }) + + it('multi write (locks)', (done) => { + parallel([ + (cb) => repo.datastore.put(b, data, cb), + (cb) => repo.datastore.put(b, data, cb) + ], done) + }) + + it('massive multiwrite', function (done) { + this.timeout(15000) // add time for ci + each(_.range(100), (i, cb) => { + repo.datastore.put(new Key('hello' + i), dataList[i], cb) + }, done) + }) + }) + + describe('.get', () => { + it('simple', (done) => { + repo.datastore.get(b, (err, val) => { + expect(err).to.not.exist() + expect(val).to.be.eql(data) + done() + }) + }) + + it('massive read', function (done) { + this.timeout(15000) // add time for ci + parallel(_.range(20 * 100).map((i) => (cb) => { + const j = i % dataList.length + repo.datastore.get(new Key('hello' + j), (err, val) => { + expect(err).to.not.exist() + expect(val).to.be.eql(dataList[j]) + cb() + }) + }), done) + }).timeout(10 * 1000) + }) + + describe('.has', () => { + it('existing entry', (done) => { + repo.datastore.has(b, (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(true) + done() + }) + }) + + it('non existent block', (done) => { + repo.datastore.has(new Key('world'), (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.eql(false) + done() + }) + }) + }) + + describe('.delete', () => { + it('simple', (done) => { + waterfall([ + (cb) => repo.datastore.delete(b, cb), + (cb) => repo.datastore.has(b, cb) + ], (err, exists) => { + expect(err).to.not.exist() + expect(exists).to.equal(false) + done() + }) + }) + }) + }) +} diff --git a/callback-test/interop-test.js b/callback-test/interop-test.js new file mode 100644 index 00000000..05920c10 --- /dev/null +++ b/callback-test/interop-test.js @@ -0,0 +1,56 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const mh = require('multihashes') +const CID = require('cids') +const Key = require('interface-datastore').Key +const map = require('async/map') + +module.exports = (repo) => { + describe('interop', () => { + it('reads welcome-to-ipfs', (done) => { + const welcomeHash = mh.fromHexString( + '1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec' + ) + + repo.blocks.get(new CID(welcomeHash), (err, val) => { + expect(err).to.not.exist() + expect(val.data.toString()).to.match(/Hello and Welcome to IPFS/) + done() + }) + }) + + it('reads a bunch of blocks', (done) => { + const cids = [ + 'QmUxpzJGJYTK5AzH36jV9ucM2WdF5KhjANb4FAhqnREzuC', + 'QmQbb26h9dcU5iNPMNEzYZnZN9YLTXBtFwuHmmo6YU4Aig' + ].map((hash) => new CID(mh.fromB58String(hash))) + + map(cids, repo.blocks.get, (err, values) => { + expect(err).to.not.exist() + expect(values.length).to.equal(2) + expect(values.map((value) => value.data.length)).to.eql([2659, 12783]) + done() + }) + }) + + it('reads pin set from the datastore', (done) => { + repo.datastore.get(new Key('/local/pins'), (err, val) => { + expect(err).to.not.exist() + expect(mh.toB58String(val)).to.equal('QmYAuyf2LzMba65NnhxLtGJxixKNUev9qYSu4MYM88hdwK') + done() + }) + }) + + it('reads DHT records from the datastore', (done) => { + repo.datastore.get(new Key('/AHE5I5B7TY'), (err, val) => { + expect(err).to.not.exist() + expect(val.toString('hex')).to.eql('0a0601c9d4743f9e12097465737476616c75651a2212201d22e2a5e140e5cd20d88fc59cd560f4887c7d9acf938ddb24d7207eac40fd2f') + done() + }) + }) + }) +} diff --git a/callback-test/keystore-test.js b/callback-test/keystore-test.js new file mode 100644 index 00000000..0c3469c6 --- /dev/null +++ b/callback-test/keystore-test.js @@ -0,0 +1,20 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect + +module.exports = (repo) => { + describe('keystore', () => { + it('exists', () => { + expect(repo).to.have.property('keys') + }) + it('implements interface-datastore', () => { + const keys = repo.keys + expect(keys.batch).to.exist() + expect(keys.query).to.exist() + }) + }) +} diff --git a/callback-test/lock-test.js b/callback-test/lock-test.js new file mode 100644 index 00000000..342d58e8 --- /dev/null +++ b/callback-test/lock-test.js @@ -0,0 +1,63 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const series = require('async/series') +const IPFSRepo = require('../') + +module.exports = (repo) => { + describe('Repo lock tests', () => { + it('should handle locking for a repo lifecycle', (done) => { + expect(repo.lockfile).to.not.equal(null) + series([ + (cb) => { + repo.close(cb) + }, + (cb) => { + expect(repo.lockfile).to.equal(null) + cb() + }, + (cb) => { + repo.open(cb) + } + ], done) + }) + + it('should prevent multiple repos from using the same path', (done) => { + const repoClone = new IPFSRepo(repo.path, repo.options) + + // Levelup throws an uncaughtException when a lock already exists, catch it + const mochaExceptionHandler = process.listeners('uncaughtException').pop() + process.removeListener('uncaughtException', mochaExceptionHandler) + process.once('uncaughtException', function (err) { + expect(err.message).to.match(/already held|IO error|already being held/) + }) + + series([ + (cb) => { + try { + repoClone.init({}, cb) + } catch (err) { + cb(err) + } + }, + (cb) => { + repoClone.open(cb) + } + ], function (err) { + // There will be no listeners if the uncaughtException was triggered + if (process.listeners('uncaughtException').length > 0) { + expect(err.message).to.match(/already locked|already held|already being held|ELOCKED/) + } + + // Reset listeners to maintain test integrity + process.removeAllListeners('uncaughtException') + process.addListener('uncaughtException', mochaExceptionHandler) + + done() + }) + }) + }) +} diff --git a/callback-test/node.js b/callback-test/node.js new file mode 100644 index 00000000..73235b43 --- /dev/null +++ b/callback-test/node.js @@ -0,0 +1,112 @@ +/* eslint-env mocha */ +'use strict' + +const ncp = require('ncp').ncp +const rimraf = require('rimraf') +const fs = require('fs') +const path = require('path') +const series = require('async/series') +const chai = require('chai') +chai.use(require('dirty-chai')) + +const IPFSRepo = require('../src') + +describe('IPFS Repo Tests onNode.js', () => { + require('./options-test') + + const customLock = { + lockName: 'test.lock', + lock: (dir, callback) => { + debugger + customLock.locked(dir, (err, isLocked) => { + if (err || isLocked) { + return callback(new Error('already locked')) + } + + const lockPath = path.join(dir, customLock.lockName) + fs.writeFileSync(lockPath, '') + + callback(null, { + close: (cb) => { + rimraf(lockPath, cb) + } + }) + }) + }, + locked: (dir, callback) => { + fs.stat(path.join(dir, customLock.lockName), (err, stats) => { + if (err) { + callback(null, false) + } else { + callback(null, true) + } + }) + } + } + + const repos = [ + { + name: 'default inited', + opts: undefined, + init: true + }, + { + name: 'memory', + opts: { + fs: require('interface-datastore').MemoryDatastore, + level: require('memdown'), + lock: 'memory' + }, + init: true + }, + { + name: 'custom locker', + opts: { + lock: customLock + }, + init: true + }, + { + name: 'default existing', + opts: undefined, + init: false + } + ] + repos.forEach((r) => describe(r.name, () => { + const testRepoPath = path.join(__dirname, 'test-repo') + const date = Date.now().toString() + const repoPath = testRepoPath + '-for-' + date + + const repo = new IPFSRepo(repoPath, r.opts) + + before((done) => { + series([ + (cb) => { + if (r.init) { + repo.init({}, cb) + } else { + ncp(testRepoPath, repoPath, cb) + } + }, + (cb) => repo.open(cb) + ], done) + }) + + after((done) => { + series([ + (cb) => repo.close(cb), + (cb) => rimraf(repoPath, cb) + ], done) + }) + + require('./repo-test')(repo) + require('./blockstore-test')(repo) + require('./datastore-test')(repo) + require('./keystore-test')(repo) + require('./stat-test')(repo) + require('./lock-test')(repo) + if (!r.init) { + require('./interop-test')(repo) + } + })) +}) diff --git a/callback-test/options-test.js b/callback-test/options-test.js new file mode 100644 index 00000000..b7afb606 --- /dev/null +++ b/callback-test/options-test.js @@ -0,0 +1,94 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const path = require('path') +const rimraf = require('rimraf') +if (!rimraf.sync) { + // browser + rimraf.sync = noop +} +const Repo = require('../') + +describe('custom options tests', () => { + const repoPath = path.join(__dirname, 'slash', 'path') + after(() => { + rimraf.sync(repoPath) + }) + + it('missing repoPath', () => { + expect( + () => new Repo() + ).to.throw('missing repoPath') + }) + + it('default options', () => { + const repo = new Repo(repoPath) + expect(repo.options).to.deep.equal(expectedRepoOptions()) + }) + + it('allows for a custom lock', () => { + const lock = { + lock: (path, callback) => { }, + locked: (path, callback) => { } + } + + const repo = new Repo(repoPath, { + lock + }) + + expect(repo._getLocker()).to.deep.equal(lock) + }) + + it('ensures a custom lock has a .close method', (done) => { + const lock = { + lock: (path, callback) => { + callback(null, {}) + } + } + + const repo = new Repo(repoPath, { + lock + }) + + expect( + () => repo._openLock(repo.path) + ).to.throw('Locks must have a close method') + done() + }) +}) + +function noop () {} + +function expectedRepoOptions () { + const options = { + lock: process.browser ? 'memory' : 'fs', + storageBackends: { + // packages are exchanged to browser-compatible + // equivalents via package.browser + root: require('datastore-fs'), + blocks: require('datastore-fs'), + keys: require('datastore-fs'), + datastore: require('datastore-level') + }, + storageBackendOptions: { + root: { + extension: '' + }, + keys: {}, + blocks: { + sharding: true, + extension: '.data' + } + } + } + + if (process.browser) { + options.storageBackendOptions.keys.sharding = false + delete options.storageBackendOptions.blocks.extension + options.storageBackendOptions.blocks.sharding = false + } + return options +} diff --git a/callback-test/repo-test.js b/callback-test/repo-test.js new file mode 100644 index 00000000..30eea4f0 --- /dev/null +++ b/callback-test/repo-test.js @@ -0,0 +1,131 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const series = require('async/series') +const waterfall = require('async/waterfall') + +module.exports = (repo) => { + describe('IPFS Repo Tests', () => { + it('check if Repo exists', (done) => { + repo.exists((err, exists) => { + expect(err).to.not.exist() + expect(exists).to.equal(true) + done() + }) + }) + + it('exposes the path', () => { + expect(repo.path).to.be.a('string') + }) + + describe('config', () => { + it('get config', (done) => { + repo.config.get((err, config) => { + expect(err).to.not.exist() + expect(config).to.be.a('object') + done() + }) + }) + + it('set config', (done) => { + series([ + (cb) => repo.config.set({ a: 'b' }, cb), + (cb) => repo.config.get((err, config) => { + if (err) return cb(err) + expect(config).to.deep.equal({ a: 'b' }) + cb() + }) + ], done) + }) + + it('get config key', (done) => { + repo.config.get('a', (err, value) => { + expect(err).to.not.exist() + expect(value).to.equal('b') + done() + }) + }) + + it('get config key should fail with non string key', (done) => { + repo.config.get(1111, (err, value) => { + expect(err).to.exist() + console.log(value) + + done() + }) + }) + + it('set config key', (done) => { + series([ + (cb) => repo.config.set('c.x', 'd', cb), + (cb) => repo.config.get((err, config) => { + if (err) return cb(err) + expect(config).to.deep.equal({ a: 'b', c: { x: 'd' } }) + cb() + }) + ], done) + }) + }) + + describe('spec', () => { + it('get spec', (done) => { + repo.spec.get((err) => { + expect(err).to.not.exist() + done() + }) + }) + + it('set spec', (done) => { + series([ + (cb) => repo.spec.set({ a: 'b' }, cb), + (cb) => repo.spec.get((err, spec) => { + if (err) return cb(err) + expect(spec).to.deep.equal({ a: 'b' }) + cb() + }) + ], done) + }) + }) + + describe('version', () => { + it('get version', (done) => { + repo.version.get((err, version) => { + expect(err).to.not.exist() + expect(version).to.equal(7) + done() + }) + }) + + it('set version', (done) => { + waterfall([ + (cb) => repo.version.set(9000, cb), + (cb) => repo.version.get(cb), + (version, cb) => { + expect(version).to.equal(9000) + cb() + }, + (cb) => repo.version.set(7, cb) + ], done) + }) + }) + + describe('lifecycle', () => { + it('close and open', (done) => { + waterfall([ + (cb) => repo.close(cb), + (cb) => repo.open(cb), + (cb) => repo.close(cb), + (cb) => repo.open(cb), + (cb) => repo.version.get(cb), + (version, cb) => { + expect(version).to.exist() + cb() + } + ], done) + }) + }) + }) +} diff --git a/callback-test/stat-test.js b/callback-test/stat-test.js new file mode 100644 index 00000000..b263c368 --- /dev/null +++ b/callback-test/stat-test.js @@ -0,0 +1,46 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect + +module.exports = (repo) => { + describe('stat', () => { + it('get stats', (done) => { + repo.stat((err, stats) => { + expect(err).to.not.exist() + expect(stats).to.exist() + expect(stats).to.have.property('numObjects') + expect(stats).to.have.property('version') + expect(stats).to.have.property('repoPath') + expect(stats).to.have.property('repoSize') + expect(stats).to.have.property('storageMax') + + expect(stats.numObjects > '0').to.eql(true) + expect(stats.version > '0').to.eql(true) + expect(stats.repoSize > '0').to.eql(true) + expect(stats.storageMax > '0').to.eql(true) + done() + }) + }) + + it('get human stats', (done) => { + repo.stat({ human: true }, (err, stats) => { + expect(err).to.not.exist() + expect(stats).to.exist() + expect(stats).to.have.property('numObjects') + expect(stats).to.have.property('version') + expect(stats).to.have.property('repoPath') + expect(stats).to.have.property('repoSize') + expect(stats).to.have.property('storageMax') + + expect(stats.numObjects > '0').to.eql(true) + expect(stats.version > '0').to.eql(true) + expect(stats.repoSize > '0').to.eql(true) + expect(stats.storageMax > '0').to.eql(true) + done() + }) + }) + }) +} diff --git a/callback-test/test-repo/api b/callback-test/test-repo/api new file mode 100644 index 00000000..ec31f33a --- /dev/null +++ b/callback-test/test-repo/api @@ -0,0 +1 @@ +/ip4/127.0.0.1/tcp/5001 \ No newline at end of file diff --git a/callback-test/test-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data b/callback-test/test-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data new file mode 100644 index 00000000..9290f631 --- /dev/null +++ b/callback-test/test-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data @@ -0,0 +1,56 @@ + +� +� + IPFS -- Inter-Planetary File system + +IPFS is a global, versioned, peer-to-peer filesystem. It combines good ideas +from Git, BitTorrent, Kademlia, SFS, and the Web. It is like a single bit- +torrent swarm, exchanging git objects. IPFS provides an interface as simple +as the HTTP web, but with permanence built in. You can also mount the world +at /ipfs. + +IPFS is a protocol: +- defines a content-addressed file system +- coordinates content delivery +- combines Kademlia + BitTorrent + Git + +IPFS is a filesystem: +- has directories and files +- mountable filesystem (via FUSE) + +IPFS is a web: +- can be used to view documents like the web +- files accessible via HTTP at `http://ipfs.io/` +- browsers or extensions can learn to use `ipfs://` directly +- hash-addressed content guarantees authenticity + +IPFS is modular: +- connection layer over any network protocol +- routing layer +- uses a routing layer DHT (kademlia/coral) +- uses a path-based naming service +- uses bittorrent-inspired block exchange + +IPFS uses crypto: +- cryptographic-hash content addressing +- block-level deduplication +- file integrity + versioning +- filesystem-level encryption + signing support + +IPFS is p2p: +- worldwide peer-to-peer file transfers +- completely decentralized architecture +- **no** central point of failure + +IPFS is a cdn: +- add a file to the filesystem locally, and it's now available to the world +- caching-friendly (content-hash naming) +- bittorrent-based bandwidth distribution + +IPFS has a name service: +- IPNS, an SFS inspired name system +- global namespace based on PKI +- serves to build trust chains +- compatible with other NSes +- can map DNS, .onion, .bit, etc to IPNS +� diff --git a/callback-test/test-repo/blocks/EP/CIQOEA4URV75WXHSIBY7SYTVOFIEO5657YM3T6DTHFRQYSUZAGRLEPY.data b/callback-test/test-repo/blocks/EP/CIQOEA4URV75WXHSIBY7SYTVOFIEO5657YM3T6DTHFRQYSUZAGRLEPY.data new file mode 100644 index 00000000..02f8aad3 Binary files /dev/null and b/callback-test/test-repo/blocks/EP/CIQOEA4URV75WXHSIBY7SYTVOFIEO5657YM3T6DTHFRQYSUZAGRLEPY.data differ diff --git a/callback-test/test-repo/blocks/FJ/CIQMSSWEB56MYVN7BGLPIMICMCRBABQCR6FC5CFHLZV74LDT2PZIFJQ.data b/callback-test/test-repo/blocks/FJ/CIQMSSWEB56MYVN7BGLPIMICMCRBABQCR6FC5CFHLZV74LDT2PZIFJQ.data new file mode 100644 index 00000000..65f9d26d --- /dev/null +++ b/callback-test/test-repo/blocks/FJ/CIQMSSWEB56MYVN7BGLPIMICMCRBABQCR6FC5CFHLZV74LDT2PZIFJQ.data @@ -0,0 +1,3 @@ +/ +" ;)=֝ޤ\SjXaJ,dUU8+vdirectT2 +" ^5'T)aX+L'TZBAJ| recursiveU \ No newline at end of file diff --git a/callback-test/test-repo/blocks/G7/CIQCDCXJ6563LCORPOAY63YIR5EM5R4TXRVPZC2BWPOB5FNVRYEXG7I.data b/callback-test/test-repo/blocks/G7/CIQCDCXJ6563LCORPOAY63YIR5EM5R4TXRVPZC2BWPOB5FNVRYEXG7I.data new file mode 100644 index 00000000..2aa38838 --- /dev/null +++ b/callback-test/test-repo/blocks/G7/CIQCDCXJ6563LCORPOAY63YIR5EM5R4TXRVPZC2BWPOB5FNVRYEXG7I.data @@ -0,0 +1,308 @@ + +cc# IPFS Repo JavaScript Implementation + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![Build Status](https://travis-ci.org/ipfs/js-ipfs-repo.svg)](https://travis-ci.org/ipfs/js-ipfs-repo) [![Circle CI](https://circleci.com/gh/ipfs/js-ipfs-repo.svg?style=svg)](https://circleci.com/gh/ipfs/js-ipfs-repo) +[![Coverage Status](https://coveralls.io/repos/github/ipfs/js-ipfs-repo/badge.svg?branch=master)](https://coveralls.io/github/ipfs/js-ipfs-repo?branch=master) [![Dependency Status](https://david-dm.org/diasdavid/js-peer-id.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-repo) +[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/feross/standard) +![](https://img.shields.io/badge/npm-%3E%3D3.0.0-orange.svg?style=flat-square) +![](https://img.shields.io/badge/Node.js-%3E%3D4.0.0-orange.svg?style=flat-square) + +[![Sauce Test Status](https://saucelabs.com/browser-matrix/ipfs-js-repo.svg)](https://saucelabs.com/u/ipfs-js-repo) + +> Implementation of the IPFS repo spec (https://github.com/ipfs/specs/tree/master/repo) in JavaScript + +This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs/tree/master/repo) in JavaScript. + +## Table of Contents + +- [Background](#background) +- [Install](#install) + - [npm](#npm) + - [Use in Node.js](#use-in-nodejs) + - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) + - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) +- [Usage](#usage) +- [API](#api) +- [Contribute](#contribute) +- [License](#license) + +## Background + +Here is the architectural reasoning for this repo: + +```bash + ┌────────────────────────────────────────┐ + │ IPFSRepo │ + └────────────────────────────────────────┘ + ┌─────────────────┐ + │ / │ + ├─────────────────┤ + │ Datastore │ + └─────────────────┘ + │ + ┌───────────┴───────────┐ + │ │ + ┌─────────────────┐ ┌─────────────────┐ + │ /blocks │ │ /datastore │ + ├─────────────────┤ ├─────────────────┤ + │ Datastore │ │ LevelDatastore │ + └─────────────────┘ └─────────────────┘ + +┌────────────────────────────────────────┐ ┌────────────────────────────────────────┐ +│ IPFSRepo - Default Node.js │ │ IPFSRepo - Default Browser │ +└────────────────────────────────────────┘ └────────────────────────────────────────┘ + ┌─────────────────┐ ┌─────────────────┐ + │ / │ │ / │ + ├─────────────────┤ ├─────────────────┤ + │ FsDatastore │ │LevelJSDatastore │ + └─────────────────┘ └─────────────────┘ + │ │ + ┌───────────┴───────────┐ ┌───────────┴───────────┐ + │ │ │ │ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ /blocks │ │ /datastore │ │ /blocks │ │ /datastore │ +├─────────────────┤ ├─────────────────┤ ├─────────────────┤ ├─────────────────┤ +│ FlatfsDatastore │ │LevelDBDatastore │ │LevelJSDatastore │ │LevelJSDatastore │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +This provides a well defined interface for creating and interacting with an IPFS repo. + +## Install + +### npm + +```sh +> npm install ipfs-repo +``` + +### Use in Node.js + +```js +var IPFSRepo = require('ipfs-repo') +``` + +### Use in a browser with browserify, webpack or any other bundler + +```js +var IPFSRepo = require('ipfs-repo') +``` + +### Use in a browser Using a script tag + +Loading this module through a script tag will make the `IpfsRepo` obj available in the global namespace. + +```html + + + +``` + +## Usage + +Example: + +```js +const Repo = require('ipfs-repo') +const repo = new Repo('/tmp/ipfs-repo') + +repo.init({ cool: 'config' }, (err) => { + if (err) { + throw err + } + + repo.open((err) => { + if (err) { + throw err + } + + console.log('repo is ready') + }) +}) +``` + +This now has created the following structure, either on disk or as an in memory representation: + +``` +├── blocks +│   ├── SHARDING +│ └── _README +├── config +├── datastore +└── version +``` + +## API + +### Setup + +#### `new Repo(path[, options])` + +Creates an IPFS Repo. + +Arguments: + +* `path` (string, mandatory): the path for this repo +* `options` (object, optional): may contain the following values + * `lock` (string, defaults to `"fs"` in Node.js, `"memory"` in the browser): what type of lock to use. Lock has to be acquired when opening. + * `storageBackends` (object, optional): may contain the following values, which should each be a class implementing the [datastore interface](https://github.com/ipfs/interface-datastore#readme): + * `root` (defaults to [`datastore-fs`](https://github.com/ipfs/js-datastore-fs#readme) in Node.js and [`datastore-level`](https://github.com/ipfs/js-datastore-level#readme) in the browser). Defines the back-end type used for gets and puts of values at the root (`repo.set()`, `repo.get()`) + * `blocks` (defaults to [`datastore-fs`](https://github.com/ipfs/js-datastore-fs#readme) in Node.js and [`datastore-level`](https://github.com/ipfs/js-datastore-level#readme) in the browser). Defines the back-end type used for gets and puts of values at `repo.blocks`. + * `datastore` (defaults to [`datastore-level`](https://github.com/ipfs/js-datastore-level#readme)). Defines the back-end type used as the key-valye store used for gets and puts of values at `repo.datastore`. + +```js +const repo = new Repo('path/to/repo') +``` + +#### `repo.init (callback)` + +Creates the necessary folder structure inside the repo. + +#### `repo.open (callback)` + +Locks the repo. + +#### `repo.close (callback)` + +Unlocks the repo. + +#### `repo.exists (callback)` + +Tells whether this repo exists or not. Calls back with `(err, bool)`. + +### Repos + +Root repo: + +#### `repo.put (key, value:Buffer, callback)` + +Put a value at the root of the repo. + +* `key` can be a buffer, a string or a [Key](https://github.com/ipfs/interface-datastore#keys). + +#### `repo.get (key, callback)` + +Get a value at the root of the repo. + +* `key` can be a buffer, a string or a [Key](https://github.com/ipfs/interface-datastore#keys). +* `callback` is a callback function `function (err, result:Buffer)` + +[Blocks](https://github.com/ipfs/js-ipfs-block#readme): + +#### `repo.blocks.put (block:Block, callback)` + +* `block` should be of type [Block](https://github.com/ipfs/js-ipfs-block#readme). + +#### `repo.blocks.putMany (blocks, callback)` + +Put many blocks. + +* `block` should be an array of type [Block](https://github.com/ipfs/js-ipfs-block#readme). + +#### `repo.blocks.get (cid, callback)` + +Get block. + +* `cid` is the content id of [type CID](https://github.com/ipld/js-cid#readme). +* `callback` is a callback function `function (err, result:Buffer)` + +Datastore: + +#### `repo.datastore` + +This is contains a full implementation of [the `interface-datastore` API](https://github.com/ipfs/interface-datastore#api). + + +### Utils + +#### `repo.config` + +Instead of using `repo.set('config')` this exposes an API that allows you to set and get a decoded config object, as well as, in a safe manner, change any of the config values individually. + +##### `repo.config.set(key:string, value, callback)` + +Set a config value. `value` can be any object that is serializable to JSON. + +* `key` is a string specifying the object path. Example: + +```js +repo.config.set('a.b.c', 'c value', (err) => { + if (err) { throw err } + repo.config.get((err, config) => { + if (err) { throw err } + assert.equal(config.a.b.c, 'c value') + }) +}) +``` + +##### `repo.config.get(value, callback)` + +Set the whole config value. `value` can be any object that is serializable to JSON. + +##### `repo.config.get(key:string, callback)` + +Get a config value. `callback` is a function with the signature: `function (err, value)`, wehre the ` +value` is of the same type that was set before. + +* `key` is a string specifying the object path. Example: + +```js +repo.config.get('a.b.c', (err, value) => { + if (err) { throw err } + console.log('config.a.b.c = ', value) +}) +``` + +##### `repo.config.get(callback)` + +Get the entire config value. `callback` is a function with the signature: `function (err, configValue:Object)`. + +#### `repo.config.exists(callback)` + +Whether the config sub-repo exists. Calls back with `(err, bool)`. + +#### `repo.version` + +##### `repo.version.get (callback)` + +Gets the repo version. + +##### `repo.version.set (version:number, callback)` + +Sets the repo version + +#### `repo.apiAddr` + +#### `repo.apiAddr.get (callback)` + +Gets the API address. + +#### `repo.apiAddr.set (value, callback)` + +Sets the API address. + +* `value` should be a [Multiaddr](https://github.com/multiformats/js-multiaddr) or a String representing a valid one. + +## Notes + +- [Explanation of how repo is structured](https://github.com/ipfs/js-ipfs-repo/pull/111#issuecomment-279948247) + +## Contribute + +There are some ways you can make this module better: + +- Consult our [open issues](https://github.com/ipfs/js-ipfs-repo/issues) and take on one of them +- Help our tests reach 100% coverage! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +[MIT](LICENSE) +c \ No newline at end of file diff --git a/callback-test/test-repo/blocks/I7/CIQGE2VHIPKM4NFWKSH2DGUZIRE66VZHMPSOBVY2YU67EWMSHH4CI7Y.data b/callback-test/test-repo/blocks/I7/CIQGE2VHIPKM4NFWKSH2DGUZIRE66VZHMPSOBVY2YU67EWMSHH4CI7Y.data new file mode 100644 index 00000000..39b8f166 --- /dev/null +++ b/callback-test/test-repo/blocks/I7/CIQGE2VHIPKM4NFWKSH2DGUZIRE66VZHMPSOBVY2YU67EWMSHH4CI7Y.data @@ -0,0 +1,93 @@ + +{ + "name": "ipfs-repo", + "version": "0.14.0", + "description": "IPFS Repo implementation", + "main": "src/index.js", + "browser": { + "rimraf": false, + "datastore-fs": "datastore-level", + "leveldown": "level-js", + "./src/lock.js": "./src/lock-memory.js", + "./src/default-options.js": "./src/default-options-browser.js" + }, + "scripts": { + "test": "aegir-test", + "test:node": "aegir-test node", + "test:browser": "aegir-test browser", + "build": "aegir-build", + "lint": "aegir-lint", + "release": "aegir-release --docs", + "release-minor": "aegir-release --type minor --docs", + "release-major": "aegir-release --type major --docs", + "coverage": "aegir-coverage", + "coverage-publish": "aegir-coverage publish", + "docs": "aegir-docs" + }, + "repository": { + "type": "git", + "url": "https://github.com/ipfs/js-ipfs-repo.git" + }, + "keywords": [ + "IPFS", + "libp2p", + "datastore" + ], + "pre-commit": [ + "lint", + "test" + ], + "homepage": "https://github.com/ipfs/js-ipfs-repo", + "engines": { + "node": ">=4.0.0", + "npm": ">=3.0.0" + }, + "devDependencies": { + "aegir": "^11.0.2", + "chai": "^4.0.2", + "dirty-chai": "^2.0.0", + "lodash": "^4.17.4", + "memdown": "^1.2.4", + "multihashes": "~0.4.5", + "ncp": "^2.0.0", + "pre-commit": "^1.2.2", + "rimraf": "^2.6.1" + }, + "dependencies": { + "async": "^2.5.0", + "base32.js": "^0.1.0", + "cids": "^0.5.0", + "datastore-core": "^0.2.0", + "datastore-fs": "^0.2.0", + "datastore-level": "^0.4.2", + "debug": "^2.6.8", + "interface-datastore": "^0.2.2", + "ipfs-block": "~0.6.0", + "level-js": "timkuijsten/level.js#idbunwrapper", + "leveldown": "^1.7.2", + "lock-me": "^1.0.2", + "lodash.get": "^4.4.2", + "lodash.has": "^4.5.2", + "lodash.set": "^4.3.2", + "multiaddr": "^2.3.0", + "safe-buffer": "^5.1.1" + }, + "license": "MIT", + "contributors": [ + "Brian Hoffman ", + "David Dias ", + "Dmitriy Ryajov ", + "Francisco Baio Dias ", + "Friedel Ziegelmayer ", + "Greenkeeper ", + "Justin Chase ", + "Lars-Magnus Skog ", + "Pau Ramon Revilla ", + "Richard Littauer ", + "Stephen Whitmore ", + "greenkeeper[bot] ", + "nginnever ", + "npmcdn-to-unpkg-bot " + ] +} + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data b/callback-test/test-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data new file mode 100644 index 00000000..62d1c297 --- /dev/null +++ b/callback-test/test-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data @@ -0,0 +1,8 @@ + +Come hang out in our IRC chat room if you have any questions. + +Contact the ipfs dev team: +- Bugs: https://github.com/ipfs/go-ipfs/issues +- Help: irc.freenode.org/#ipfs +- Email: dev@ipfs.io + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/JA/CIQHS34HCZMO72ZWZBHY5SBVURQIA4LDBBVVTHZMZE5KPUQSVNVHJAQ.data b/callback-test/test-repo/blocks/JA/CIQHS34HCZMO72ZWZBHY5SBVURQIA4LDBBVVTHZMZE5KPUQSVNVHJAQ.data new file mode 100644 index 00000000..2469a1a1 --- /dev/null +++ b/callback-test/test-repo/blocks/JA/CIQHS34HCZMO72ZWZBHY5SBVURQIA4LDBBVVTHZMZE5KPUQSVNVHJAQ.data @@ -0,0 +1,3 @@ +5 +" bjC4TDIW'c=Y9$ package.json + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/JN/CIQPHMHGQLLZXC32FQQW2YVM4KGFORVFJAQYY55VK3WJGLZ2MS4RJNQ.data b/callback-test/test-repo/blocks/JN/CIQPHMHGQLLZXC32FQQW2YVM4KGFORVFJAQYY55VK3WJGLZ2MS4RJNQ.data new file mode 100644 index 00000000..27088364 --- /dev/null +++ b/callback-test/test-repo/blocks/JN/CIQPHMHGQLLZXC32FQQW2YVM4KGFORVFJAQYY55VK3WJGLZ2MS4RJNQ.data @@ -0,0 +1,3 @@ + + +ipfs \ No newline at end of file diff --git a/callback-test/test-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data b/callback-test/test-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data new file mode 100644 index 00000000..71be805f --- /dev/null +++ b/callback-test/test-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data @@ -0,0 +1,9 @@ + +Some helpful resources for finding your way around ipfs: + +- quick-start: a quick show of various ipfs features. +- ipfs commands: a list of all commands +- ipfs --help: every command describes itself +- https://github.com/ipfs/go-ipfs -- the src repository +- #ipfs on irc.freenode.org -- the community irc channel + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data b/callback-test/test-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data new file mode 100644 index 00000000..f2bf4f8b --- /dev/null +++ b/callback-test/test-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data @@ -0,0 +1,115 @@ + +  # 0.1 - Quick Start + +This is a set of short examples with minimal explanation. It is meant as +a "quick start". Soon, we'll write a longer tour :-) + + +Add a file to ipfs: + + echo "hello world" >hello + ipfs add hello + + +View it: + + ipfs cat + + +Try a directory: + + mkdir foo + mkdir foo/bar + echo "baz" > foo/baz + echo "baz" > foo/bar/baz + ipfs add -r foo + + +View things: + + ipfs ls + ipfs ls /bar + ipfs cat /baz + ipfs cat /bar/baz + ipfs cat /bar + ipfs ls /baz + + +References: + + ipfs refs + ipfs refs -r + ipfs refs --help + + +Get: + + ipfs get -o foo2 + diff foo foo2 + + +Objects: + + ipfs object get + ipfs object get /foo2 + ipfs object --help + + +Pin + GC: + + ipfs pin add + ipfs repo gc + ipfs ls + ipfs pin rm + ipfs repo gc + + +Daemon: + + ipfs daemon (in another terminal) + ipfs id + + +Network: + + (must be online) + ipfs swarm peers + ipfs id + ipfs cat + + +Mount: + + (warning: fuse is finicky!) + ipfs mount + cd /ipfs/ + ls + + +Tool: + + ipfs version + ipfs update + ipfs commands + ipfs config --help + open http://localhost:5001/webui + + +Browse: + + webui: + + http://localhost:5001/webui + + video: + + http://localhost:8080/ipfs/QmVc6zuAneKJzicnJpfrqCH9gSy6bz54JhcypfJYhGUFQu/play#/ipfs/QmTKZgRNwDNZwHtJSjCp6r5FYefzpULfy37JvMt9DwvXse + + images: + + http://localhost:8080/ipfs/QmZpc3HvfjEXvLWGQPWbHk3AjD5j8NEN4gmFN8Jmrd5g83/cs + + markdown renderer app: + + http://localhost:8080/ipfs/QmX7M9CiYXjVeFnkfVGf3y5ixTZ2ACeSGyL1vBJY1HvQPp/mdown + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/N2/CIQDWKPBHXLJ3XVELRJZA2SYY7OGCSX6FRSIZS2VQQPVKOA2Z4VXN2I.data b/callback-test/test-repo/blocks/N2/CIQDWKPBHXLJ3XVELRJZA2SYY7OGCSX6FRSIZS2VQQPVKOA2Z4VXN2I.data new file mode 100644 index 00000000..7e1e7f1e Binary files /dev/null and b/callback-test/test-repo/blocks/N2/CIQDWKPBHXLJ3XVELRJZA2SYY7OGCSX6FRSIZS2VQQPVKOA2Z4VXN2I.data differ diff --git a/callback-test/test-repo/blocks/OO/CIQBT4N7PS5IZ5IG2ZOUGKFK27IE33WKGJNDW2TY3LSBNQ34R6OVOOQ.data b/callback-test/test-repo/blocks/OO/CIQBT4N7PS5IZ5IG2ZOUGKFK27IE33WKGJNDW2TY3LSBNQ34R6OVOOQ.data new file mode 100644 index 00000000..77034827 --- /dev/null +++ b/callback-test/test-repo/blocks/OO/CIQBT4N7PS5IZ5IG2ZOUGKFK27IE33WKGJNDW2TY3LSBNQ34R6OVOOQ.data @@ -0,0 +1,27 @@ + +  IPFS Alpha Security Notes + +We try hard to ensure our system is safe and robust, but all software +has bugs, especially new software. This distribution is meant to be an +alpha preview, don't use it for anything mission critical. + +Please note the following: + +- This is alpha software and has not been audited. It is our goal + to conduct a proper security audit once we close in on a 1.0 release. + +- ipfs is a networked program, and may have serious undiscovered + vulnerabilities. It is written in Go, and we do not execute any + user provided data. But please point any problems out to us in a + github issue, or email security@ipfs.io privately. + +- security@ipfs.io GPG key: + - 4B9665FB 92636D17 7C7A86D3 50AAE8A9 59B13AF3 + - https://pgp.mit.edu/pks/lookup?op=get&search=0x50AAE8A959B13AF3 + +- ipfs uses encryption for all communication, but it's NOT PROVEN SECURE + YET! It may be totally broken. For now, the code is included to make + sure we benchmark our operations with encryption in mind. In the future, + there will be an "unsafe" mode for high performance intranet apps. + If this is a blocking feature for you, please contact us. + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data b/callback-test/test-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data new file mode 100644 index 00000000..e69de29b diff --git a/callback-test/test-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data b/callback-test/test-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data new file mode 100644 index 00000000..389e1117 --- /dev/null +++ b/callback-test/test-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data @@ -0,0 +1,28 @@ + +Hello and Welcome to IPFS! + +██╗██████╗ ███████╗███████╗ +██║██╔══██╗██╔════╝██╔════╝ +██║██████╔╝█████╗ ███████╗ +██║██╔═══╝ ██╔══╝ ╚════██║ +██║██║ ██║ ███████║ +╚═╝╚═╝ ╚═╝ ╚══════╝ + +If you're seeing this, you have successfully installed +IPFS and are now interfacing with the ipfs merkledag! + + ------------------------------------------------------- +| Warning: | +| This is alpha software. Use at your own discretion! | +| Much is missing or lacking polish. There are bugs. | +| Not yet secure. Read the security notes for more. | + ------------------------------------------------------- + +Check out some of the other files in this directory: + + ./about + ./help + ./quick-start <-- usage examples + ./readme <-- this file + ./security-notes + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/RQ/CIQGPZHNWKMUTP4X4VJYCKHWEL3A6UHYWGT2SVEOIQYOVOC2RMJBRQQ.data b/callback-test/test-repo/blocks/RQ/CIQGPZHNWKMUTP4X4VJYCKHWEL3A6UHYWGT2SVEOIQYOVOC2RMJBRQQ.data new file mode 100644 index 00000000..33e7bc4f Binary files /dev/null and b/callback-test/test-repo/blocks/RQ/CIQGPZHNWKMUTP4X4VJYCKHWEL3A6UHYWGT2SVEOIQYOVOC2RMJBRQQ.data differ diff --git a/callback-test/test-repo/blocks/RY/CIQKVNGIGQHPMLDVIATUPL3YPUYSLNHT5HJ2USSAZZO5GUTAVSYQRYY.data b/callback-test/test-repo/blocks/RY/CIQKVNGIGQHPMLDVIATUPL3YPUYSLNHT5HJ2USSAZZO5GUTAVSYQRYY.data new file mode 100644 index 00000000..5cebb55f Binary files /dev/null and b/callback-test/test-repo/blocks/RY/CIQKVNGIGQHPMLDVIATUPL3YPUYSLNHT5HJ2USSAZZO5GUTAVSYQRYY.data differ diff --git a/callback-test/test-repo/blocks/SB/CIQMNAV4TX2DAU5DPES65DFU3KLKFK6EJVAIKNVWIDKR3UCGX2LOSBI.data b/callback-test/test-repo/blocks/SB/CIQMNAV4TX2DAU5DPES65DFU3KLKFK6EJVAIKNVWIDKR3UCGX2LOSBI.data new file mode 100644 index 00000000..4c619f21 --- /dev/null +++ b/callback-test/test-repo/blocks/SB/CIQMNAV4TX2DAU5DPES65DFU3KLKFK6EJVAIKNVWIDKR3UCGX2LOSBI.data @@ -0,0 +1,3 @@ +/ +" ;)=֝ޤ\SjXaJ,dUU8+vdirectT2 +" 4,u@'Gx}1%ӪJ@]R` recursiveT \ No newline at end of file diff --git a/callback-test/test-repo/blocks/SHARDING b/callback-test/test-repo/blocks/SHARDING new file mode 100644 index 00000000..a153331d --- /dev/null +++ b/callback-test/test-repo/blocks/SHARDING @@ -0,0 +1 @@ +/repo/flatfs/shard/v1/next-to-last/2 diff --git a/callback-test/test-repo/blocks/U7/CIQF5KHIGUT3ZN2UFEDYQYKYYGMCX2ISX3WLLY4UJQTVIWSCB5AUU7A.data b/callback-test/test-repo/blocks/U7/CIQF5KHIGUT3ZN2UFEDYQYKYYGMCX2ISX3WLLY4UJQTVIWSCB5AUU7A.data new file mode 100644 index 00000000..44191cf9 Binary files /dev/null and b/callback-test/test-repo/blocks/U7/CIQF5KHIGUT3ZN2UFEDYQYKYYGMCX2ISX3WLLY4UJQTVIWSCB5AUU7A.data differ diff --git a/callback-test/test-repo/blocks/V2/CIQFIFKJWK4S7LN2N6IHF64AEZZDKWGGUVG7JNRNTYBQJ4ALX6JGV2Q.data b/callback-test/test-repo/blocks/V2/CIQFIFKJWK4S7LN2N6IHF64AEZZDKWGGUVG7JNRNTYBQJ4ALX6JGV2Q.data new file mode 100644 index 00000000..c2aa971d --- /dev/null +++ b/callback-test/test-repo/blocks/V2/CIQFIFKJWK4S7LN2N6IHF64AEZZDKWGGUVG7JNRNTYBQJ4ALX6JGV2Q.data @@ -0,0 +1,3 @@ +2 +" !}{oHǓjA s} README.mdc + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/V3/CIQJEFSTFKB33VFP24TN55TBQT5H7T37KCTQMVM6Z6IOJPWVPPYHV3Q.data b/callback-test/test-repo/blocks/V3/CIQJEFSTFKB33VFP24TN55TBQT5H7T37KCTQMVM6Z6IOJPWVPPYHV3Q.data new file mode 100644 index 00000000..9c9015c3 --- /dev/null +++ b/callback-test/test-repo/blocks/V3/CIQJEFSTFKB33VFP24TN55TBQT5H7T37KCTQMVM6Z6IOJPWVPPYHV3Q.data @@ -0,0 +1,3 @@ +/ +" ;)=֝ޤ\SjXaJ,dUU8+vdirectT2 +" \@qbuqPGws9c J? recursiveU \ No newline at end of file diff --git a/callback-test/test-repo/blocks/VP/CIQL7ROGT73FOVB23WYGP4F6LU5WSQKMNZHPN6IGKKQU62FWLSH7VPY.data b/callback-test/test-repo/blocks/VP/CIQL7ROGT73FOVB23WYGP4F6LU5WSQKMNZHPN6IGKKQU62FWLSH7VPY.data new file mode 100644 index 00000000..6affa52f Binary files /dev/null and b/callback-test/test-repo/blocks/VP/CIQL7ROGT73FOVB23WYGP4F6LU5WSQKMNZHPN6IGKKQU62FWLSH7VPY.data differ diff --git a/callback-test/test-repo/blocks/WS/CIQDIIAHGSIWR6TXBONXKL3LKJHZWQMXKJO7ZJOGM6MYLW3OGOLIWSA.data b/callback-test/test-repo/blocks/WS/CIQDIIAHGSIWR6TXBONXKL3LKJHZWQMXKJO7ZJOGM6MYLW3OGOLIWSA.data new file mode 100644 index 00000000..a28611f4 --- /dev/null +++ b/callback-test/test-repo/blocks/WS/CIQDIIAHGSIWR6TXBONXKL3LKJHZWQMXKJO7ZJOGM6MYLW3OGOLIWSA.data @@ -0,0 +1,3 @@ +/ +" ;)=֝ޤ\SjXaJ,dUU8+vdirectT2 +" ƟWT:ݰg];iALnNROh\ recursiveT \ No newline at end of file diff --git a/callback-test/test-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data b/callback-test/test-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data new file mode 100644 index 00000000..9553a942 --- /dev/null +++ b/callback-test/test-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/callback-test/test-repo/blocks/_README b/callback-test/test-repo/blocks/_README new file mode 100644 index 00000000..23cb0909 --- /dev/null +++ b/callback-test/test-repo/blocks/_README @@ -0,0 +1,30 @@ +This is a repository of IPLD objects. Each IPLD object is in a single file, +named .data. Where is the +"base32" encoding of the CID (as specified in +https://github.com/multiformats/multibase) without the 'B' prefix. +All the object files are placed in a tree of directories, based on a +function of the CID. This is a form of sharding similar to +the objects directory in git repositories. Previously, we used +prefixes, we now use the next-to-last two charters. + + func NextToLast(base32cid string) { + nextToLastLen := 2 + offset := len(base32cid) - nextToLastLen - 1 + return str[offset : offset+nextToLastLen] + } + +For example, an object with a base58 CIDv1 of + + zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f + +has a base32 CIDv1 of + + BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA + +and will be placed at + + SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data + +with 'SC' being the last-to-next two characters and the 'B' at the +beginning of the CIDv1 string is the multibase prefix that is not +stored in the filename. diff --git a/callback-test/test-repo/config b/callback-test/test-repo/config new file mode 100644 index 00000000..5a9765f5 --- /dev/null +++ b/callback-test/test-repo/config @@ -0,0 +1,99 @@ +{ + "Identity": { + "PeerID": "QmQJPPKEd1a1zLrsDzmzKMnbkkNNmCziUMsXvvkLbjPg1c", + "PrivKey": "CAASqQkwggSlAgEAAoIBAQC6rWI46zRrseaaQq94Y14LJczstp0boYRr7vzg3jugJOqGhqzD+t88vmelYuyI0bJkoMI8WkfSBT0OYbmwszgzNFvYvZNPvfgzt2qLODU1/3PP0ihYcO42yAoX3KWlExuX1xi/nNaUvMPGF4oMucp6PN2o/A2uAv0jsnAeb8hJM4MNs5aQoI5gby2yxtM9mc78UkdV1MZ00IvOdrU2IViNwTVre/PwjbaR4RPcF40+/E2zVJG/z8wOMnpkKAkIxCSjJcaRcxR1bAfP/cKLbpxrpnQlHgVEBVhIaqkd3i5i1+dG8VgsD5LZPh/hLjldXiSYrNMO70Ksne6sc7nL/3UhAgMBAAECggEBAKqKe5FETz+T5vhGs8doACJvBie7LQxxImj4jE1iuVY0Y41Cu9LO/eKgkE+ncDAOYstLkidQ0wwrfPwGi33CPTWKP95iUpInGvkkN1G4p+QM2+CgPfuOMBIb7hyelzWXnz24ZAOpZN+9E52FI7k8gp3pvRcELfsq/9f8zDECLhewRjZitiY5ewfBKsK5LFKQSRg8DIVIKq4iqi7QMRRwbFDtDLcUHJepXSTRhmhWr5b/23O8OxnHExtjXMFqtBzvaAuZPnw2whr8ujV3W+5PyY+btx6M0k/dDQe2dFSJWm8AKLF8AL+houl2vtnFQ47yeYisWd02BcR91DyF5u6hxYECgYEA8b1UlUgCJ1h4MLTt0HOk+JwccJtyLoofe/y8/i4sMtjUZ4kiyIlEUrmPaE3qcNsUOG5eldz4nfN4AYRG4UQwmMFehpGC5pxH/kW8xwC9iFxDbkrNwJ8T5Jc8EQmFZ9BTXviQ3d5ViO06gtOiAdf2L/ci/qazCR7eowdjvbUso0MCgYEAxbCFZjj0L7+Zuyl6Znv7A1raJUKTGR8J5OsP6lIcRq0EfC6OcoCS6bx7FIbM0gkgGj+F/1l1bonfCcCDv5x4tRZzVUCz2KrDEwo/RCv1Y47ipuusao8fTTlO5tgESl4jAvaD1mfZom9BslO4sk3CxXJyAuMJUCc/8To6HLPclcsCgYEAwcuRknd+2oIPIKT7fcAoAfJMzm2f1PWAFgrgjDkBz+WKKlKDIxcAQf1mnSzRhJXtGOxKQYQ7f4zeqQCdPhxHhT6IBAOFWFDzWkIX+8V5LGci27l2RzSAYyJm0hW68CXPoHRO1r9V/QaJgLYey7GROJS8Zj/HiclInJPg/wXOejcCgYBjiwcg+fy7LldSjwg7IqK6hbzilitLsv1gb5yL+NyUGfrwLOE9dtCDMY0oQNhRypaHoQTwFzOyfWn5lx7AFSISzUM14mas6w9fPwGsahYeE4y9UF55KagxUnIQeyVt7QjkLQ0loRVNXYhPKykNX2p70aznFztRSPJBnXg1i7u/EQKBgQC8iZQF/5vwgVN3FgEugru9DKUOWAXMXWcMENRgQ9kiUNU7cwqszQyOk/zmNXrrIbdzn+4/H4gBNRw+dy0t1D5nG7+AuMn1J9WmRS0NF6eF43ttRRmERxiYwRssBs0JLaQWaobmEqcMNCygm1BCqQrKfmY2oI3HDOukMwgMpfZsSQ==" + }, + "Datastore": { + "Type": "leveldb", + "Path": "/Users/pedroteixeira/.ipfs/datastore", + "StorageMax": "10GB", + "StorageGCWatermark": 90, + "GCPeriod": "1h", + "Params": null, + "NoSync": false, + "HashOnRead": false, + "BloomFilterSize": 0 + }, + "Addresses": { + "Swarm": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001" + ], + "API": "/ip4/127.0.0.1/tcp/5001", + "Gateway": "/ip4/127.0.0.1/tcp/8080" + }, + "Mounts": { + "IPFS": "/ipfs", + "IPNS": "/ipns", + "FuseAllowOther": false + }, + "Discovery": { + "MDNS": { + "Enabled": true, + "Interval": 10 + } + }, + "Ipns": { + "RepublishPeriod": "", + "RecordLifetime": "", + "ResolveCacheSize": 128 + }, + "Bootstrap": [ + "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.236.176.52/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z", + "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip4/162.243.248.213/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm", + "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + "/ip4/178.62.61.185/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3", + "/ip4/104.236.151.122/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx", + "/ip6/2604:a880:1:20::1f9:9001/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z", + "/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip6/2604:a880:0:1010::23:d001/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm", + "/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + "/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3", + "/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx" + ], + "Tour": { + "Last": "" + }, + "Gateway": { + "HTTPHeaders": { + "Access-Control-Allow-Headers": [ + "X-Requested-With", + "Range" + ], + "Access-Control-Allow-Methods": [ + "GET" + ], + "Access-Control-Allow-Origin": [ + "*" + ] + }, + "RootRedirect": "", + "Writable": false, + "PathPrefixes": [] + }, + "SupernodeRouting": { + "Servers": null + }, + "API": { + "HTTPHeaders": null + }, + "Swarm": { + "AddrFilters": null, + "DisableBandwidthMetrics": false, + "DisableNatPortMap": false + }, + "Reprovider": { + "Interval": "12h" + }, + "Experimental": { + "FilestoreEnabled": false, + "ShardingEnabled": false, + "Libp2pStreamMounting": false + } +} \ No newline at end of file diff --git a/callback-test/test-repo/datastore/000053.ldb b/callback-test/test-repo/datastore/000053.ldb new file mode 100644 index 00000000..ee8b8a50 Binary files /dev/null and b/callback-test/test-repo/datastore/000053.ldb differ diff --git a/callback-test/test-repo/datastore/CURRENT b/callback-test/test-repo/datastore/CURRENT new file mode 100644 index 00000000..f774e854 --- /dev/null +++ b/callback-test/test-repo/datastore/CURRENT @@ -0,0 +1 @@ +MANIFEST-000052 diff --git a/callback-test/test-repo/datastore/LOCK b/callback-test/test-repo/datastore/LOCK new file mode 100644 index 00000000..e69de29b diff --git a/callback-test/test-repo/datastore/LOG b/callback-test/test-repo/datastore/LOG new file mode 100644 index 00000000..d1562018 --- /dev/null +++ b/callback-test/test-repo/datastore/LOG @@ -0,0 +1,233 @@ +=============== Jul 4, 2017 (WEST) =============== +15:31:37.730802 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:31:37.732768 db@open opening +15:31:37.737378 version@stat F·[] S·0B[] Sc·[] +15:31:37.737913 db@janitor F·2 G·0 +15:31:37.737949 db@open done T·5.159368ms +15:31:37.757377 db@close closing +15:31:37.757461 db@close done T·81.924µs +=============== Jul 4, 2017 (WEST) =============== +15:31:37.757924 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:31:37.758028 version@stat F·[] S·0B[] Sc·[] +15:31:37.758036 db@open opening +15:31:37.758071 journal@recovery F·1 +15:31:37.758373 journal@recovery recovering @1 +15:31:37.759037 memdb@flush created L0@2 N·2 S·211B "/lo..oot,v2":"/lo..ins,v1" +15:31:37.762454 version@stat F·[1] S·211B[211B] Sc·[0.25] +15:31:37.763474 db@janitor F·3 G·0 +15:31:37.763495 db@open done T·5.453738ms +15:31:37.772607 db@close closing +15:31:37.772721 db@close done T·109.542µs +=============== Jul 4, 2017 (WEST) =============== +15:40:00.761297 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:40:00.762271 version@stat F·[1] S·211B[211B] Sc·[0.25] +15:40:00.762288 db@open opening +15:40:00.762340 journal@recovery F·1 +15:40:00.764790 journal@recovery recovering @3 +15:40:00.767080 memdb@flush created L0@5 N·4 S·1KiB "/F5..7JP,v5":"/lo..ins,v4" +15:40:00.767521 version@stat F·[2] S·1KiB[1KiB] Sc·[0.50] +15:40:00.768169 db@janitor F·4 G·0 +15:40:00.768186 db@open done T·5.891465ms +15:40:00.785309 db@close closing +15:40:00.785473 db@close done T·160.472µs +=============== Jul 4, 2017 (WEST) =============== +15:40:07.774348 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:40:07.774557 version@stat F·[2] S·1KiB[1KiB] Sc·[0.50] +15:40:07.774568 db@open opening +15:40:07.774611 journal@recovery F·1 +15:40:07.774966 journal@recovery recovering @6 +15:40:07.776842 memdb@flush created L0@8 N·2 S·211B "/lo..oot,v10":"/lo..ins,v9" +15:40:07.777345 version@stat F·[3] S·1KiB[1KiB] Sc·[0.75] +15:40:07.778166 db@janitor F·5 G·0 +15:40:07.778186 db@open done T·3.611268ms +15:40:07.783537 db@close closing +15:40:07.783656 db@close done T·116.559µs +=============== Jul 4, 2017 (WEST) =============== +15:41:22.547017 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:41:22.547314 version@stat F·[3] S·1KiB[1KiB] Sc·[0.75] +15:41:22.547334 db@open opening +15:41:22.547446 journal@recovery F·1 +15:41:22.547991 journal@recovery recovering @9 +15:41:22.548511 memdb@flush created L0@11 N·2 S·211B "/lo..oot,v13":"/lo..ins,v12" +15:41:22.548980 version@stat F·[4] S·1KiB[1KiB] Sc·[1.00] +15:41:22.549702 db@janitor F·6 G·0 +15:41:22.549719 db@open done T·2.377875ms +15:41:22.549788 table@compaction L0·4 -> L1·0 S·1KiB Q·14 +15:41:22.550450 table@build created L1@14 N·4 S·1KiB "/F5..7JP,v5":"/lo..ins,v12" +15:41:22.550488 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:41:22.550730 table@compaction committed F-3 S-633B Ke·0 D·6 T·918.111µs +15:41:22.550806 table@remove removed @11 +15:41:22.550862 table@remove removed @8 +15:41:22.550932 table@remove removed @5 +15:41:22.551030 table@remove removed @2 +15:41:22.551680 db@close closing +15:41:22.551796 db@close done T·112.63µs +=============== Jul 4, 2017 (WEST) =============== +15:41:39.052965 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:41:39.053171 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:41:39.053188 db@open opening +15:41:39.053255 journal@recovery F·1 +15:41:39.053684 journal@recovery recovering @12 +15:41:39.056117 memdb@flush created L0@15 N·2 S·211B "/lo..oot,v16":"/lo..ins,v15" +15:41:39.056892 version@stat F·[1 1] S·1KiB[211B 1KiB] Sc·[0.25 0.00] +15:41:39.058262 db@janitor F·4 G·0 +15:41:39.058279 db@open done T·5.082794ms +15:41:39.061267 db@close closing +15:41:39.061388 db@close done T·118.126µs +=============== Jul 4, 2017 (WEST) =============== +15:41:48.623766 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:41:48.623946 version@stat F·[1 1] S·1KiB[211B 1KiB] Sc·[0.25 0.00] +15:41:48.623957 db@open opening +15:41:48.623999 journal@recovery F·1 +15:41:48.624407 journal@recovery recovering @16 +15:41:48.626176 memdb@flush created L0@18 N·2 S·211B "/lo..oot,v19":"/lo..ins,v18" +15:41:48.627325 version@stat F·[2 1] S·1KiB[422B 1KiB] Sc·[0.50 0.00] +15:41:48.627922 db@janitor F·5 G·0 +15:41:48.627942 db@open done T·3.97763ms +15:41:48.629261 db@close closing +15:41:48.629399 db@close done T·136.301µs +=============== Jul 4, 2017 (WEST) =============== +15:53:08.378238 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:53:08.378408 version@stat F·[2 1] S·1KiB[422B 1KiB] Sc·[0.50 0.00] +15:53:08.378420 db@open opening +15:53:08.378468 journal@recovery F·1 +15:53:08.378990 journal@recovery recovering @19 +15:53:08.380800 memdb@flush created L0@21 N·2 S·211B "/lo..oot,v22":"/lo..ins,v21" +15:53:08.381531 version@stat F·[3 1] S·1KiB[633B 1KiB] Sc·[0.75 0.00] +15:53:08.382277 db@janitor F·6 G·0 +15:53:08.382295 db@open done T·3.868457ms +15:53:08.383570 db@close closing +15:53:08.383702 db@close done T·130.606µs +=============== Jul 4, 2017 (WEST) =============== +15:54:14.936403 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:54:14.936572 version@stat F·[3 1] S·1KiB[633B 1KiB] Sc·[0.75 0.00] +15:54:14.936584 db@open opening +15:54:14.936635 journal@recovery F·1 +15:54:14.936913 journal@recovery recovering @22 +15:54:14.938940 memdb@flush created L0@24 N·1 S·162B "/lo..oot,v24":"/lo..oot,v24" +15:54:14.939787 version@stat F·[4 1] S·1KiB[795B 1KiB] Sc·[1.00 0.00] +15:54:14.941728 db@janitor F·7 G·0 +15:54:14.941804 db@open done T·5.208439ms +15:54:14.941863 table@compaction L0·4 -> L1·1 S·1KiB Q·25 +15:54:14.942716 table@build created L1@27 N·4 S·1KiB "/F5..7JP,v5":"/lo..ins,v21" +15:54:14.942754 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:54:14.943032 table@compaction committed F-4 S-795B Ke·0 D·7 T·1.128534ms +15:54:14.943205 table@remove removed @24 +15:54:14.943295 table@remove removed @21 +15:54:14.943640 table@remove removed @18 +15:54:14.943818 table@remove removed @15 +15:54:14.943942 table@remove removed @14 +15:54:14.944068 db@close closing +15:54:14.944171 db@close done T·100.355µs +=============== Jul 4, 2017 (WEST) =============== +15:54:32.988001 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:54:32.988316 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:54:32.988336 db@open opening +15:54:32.988423 journal@recovery F·1 +15:54:32.988947 journal@recovery recovering @25 +15:54:32.991002 memdb@flush created L0@28 N·1 S·162B "/lo..oot,v26":"/lo..oot,v26" +15:54:32.991603 version@stat F·[1 1] S·1KiB[162B 1KiB] Sc·[0.25 0.00] +15:54:32.992495 db@janitor F·4 G·0 +15:54:32.992515 db@open done T·4.171213ms +15:54:32.993870 db@close closing +15:54:32.993989 db@close done T·114.28µs +=============== Jul 4, 2017 (WEST) =============== +15:54:50.017866 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:54:50.018029 version@stat F·[1 1] S·1KiB[162B 1KiB] Sc·[0.25 0.00] +15:54:50.018041 db@open opening +15:54:50.018085 journal@recovery F·1 +15:54:50.018498 journal@recovery recovering @29 +15:54:50.020254 memdb@flush created L0@31 N·1 S·162B "/lo..oot,v28":"/lo..oot,v28" +15:54:50.020663 version@stat F·[2 1] S·1KiB[324B 1KiB] Sc·[0.50 0.00] +15:54:50.021342 db@janitor F·5 G·0 +15:54:50.021365 db@open done T·3.314444ms +15:54:50.022280 db@close closing +15:54:50.022484 db@close done T·186.342µs +=============== Jul 4, 2017 (WEST) =============== +15:56:06.923937 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:56:06.924230 version@stat F·[2 1] S·1KiB[324B 1KiB] Sc·[0.50 0.00] +15:56:06.924251 db@open opening +15:56:06.924325 journal@recovery F·1 +15:56:06.924883 journal@recovery recovering @32 +15:56:06.926702 memdb@flush created L0@34 N·1 S·162B "/lo..oot,v30":"/lo..oot,v30" +15:56:06.927201 version@stat F·[3 1] S·1KiB[486B 1KiB] Sc·[0.75 0.00] +15:56:06.928181 db@janitor F·6 G·0 +15:56:06.928221 db@open done T·3.959635ms +15:56:06.929783 db@close closing +15:56:06.929949 db@close done T·163.655µs +=============== Jul 4, 2017 (WEST) =============== +15:56:23.797050 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:56:23.797250 version@stat F·[3 1] S·1KiB[486B 1KiB] Sc·[0.75 0.00] +15:56:23.797260 db@open opening +15:56:23.797307 journal@recovery F·1 +15:56:23.797703 journal@recovery recovering @35 +15:56:23.799545 memdb@flush created L0@37 N·1 S·162B "/lo..oot,v32":"/lo..oot,v32" +15:56:23.800045 version@stat F·[4 1] S·1KiB[648B 1KiB] Sc·[1.00 0.00] +15:56:23.800988 db@janitor F·7 G·0 +15:56:23.801011 db@open done T·3.743272ms +15:56:23.801104 table@compaction L0·4 -> L1·1 S·1KiB Q·33 +15:56:23.801542 table@build created L1@40 N·4 S·1KiB "/F5..7JP,v5":"/lo..ins,v21" +15:56:23.801571 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:56:23.801657 table@compaction committed F-4 S-648B Ke·0 D·4 T·523.359µs +15:56:23.801745 table@remove removed @37 +15:56:23.801794 table@remove removed @34 +15:56:23.801894 table@remove removed @31 +15:56:23.801997 table@remove removed @28 +15:56:23.802077 table@remove removed @27 +15:56:23.802676 db@close closing +15:56:23.802806 db@close done T·127.71µs +=============== Jul 4, 2017 (WEST) =============== +15:56:40.131595 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:56:40.131793 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +15:56:40.131804 db@open opening +15:56:40.131847 journal@recovery F·1 +15:56:40.132170 journal@recovery recovering @38 +15:56:40.134004 memdb@flush created L0@41 N·1 S·162B "/lo..oot,v34":"/lo..oot,v34" +15:56:40.134473 version@stat F·[1 1] S·1KiB[162B 1KiB] Sc·[0.25 0.00] +15:56:40.135186 db@janitor F·4 G·0 +15:56:40.135204 db@open done T·3.393383ms +15:56:40.142721 db@close closing +15:56:40.142910 db@close done T·182.273µs +=============== Jul 4, 2017 (WEST) =============== +15:56:55.053800 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:56:55.053959 version@stat F·[1 1] S·1KiB[162B 1KiB] Sc·[0.25 0.00] +15:56:55.053971 db@open opening +15:56:55.054018 journal@recovery F·1 +15:56:55.054340 journal@recovery recovering @42 +15:56:55.056275 memdb@flush created L0@44 N·1 S·162B "/lo..oot,v36":"/lo..oot,v36" +15:56:55.056865 version@stat F·[2 1] S·1KiB[324B 1KiB] Sc·[0.50 0.00] +15:56:55.057913 db@janitor F·5 G·0 +15:56:55.057935 db@open done T·3.957331ms +15:56:55.067574 db@close closing +15:56:55.067860 db@close done T·277.099µs +=============== Jul 4, 2017 (WEST) =============== +16:00:39.662902 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +16:00:39.664111 version@stat F·[2 1] S·1KiB[324B 1KiB] Sc·[0.50 0.00] +16:00:39.664131 db@open opening +16:00:39.664211 journal@recovery F·1 +16:00:39.664643 journal@recovery recovering @45 +16:00:39.665526 memdb@flush created L0@47 N·1 S·162B "/lo..oot,v38":"/lo..oot,v38" +16:00:39.666652 version@stat F·[3 1] S·1KiB[486B 1KiB] Sc·[0.75 0.00] +16:00:39.671269 db@janitor F·6 G·0 +16:00:39.671306 db@open done T·7.162312ms +16:00:39.672402 db@close closing +16:00:39.672694 db@close done T·288.456µs +=============== Jul 4, 2017 (WEST) =============== +16:01:03.403625 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +16:01:03.403841 version@stat F·[3 1] S·1KiB[486B 1KiB] Sc·[0.75 0.00] +16:01:03.403853 db@open opening +16:01:03.403895 journal@recovery F·1 +16:01:03.404257 journal@recovery recovering @48 +16:01:03.406212 memdb@flush created L0@50 N·1 S·162B "/lo..oot,v40":"/lo..oot,v40" +16:01:03.406706 version@stat F·[4 1] S·1KiB[648B 1KiB] Sc·[1.00 0.00] +16:01:03.407485 db@janitor F·7 G·0 +16:01:03.407505 db@open done T·3.645072ms +16:01:03.407599 table@compaction L0·4 -> L1·1 S·1KiB Q·41 +16:01:03.408141 table@build created L1@53 N·4 S·1KiB "/F5..7JP,v5":"/lo..ins,v21" +16:01:03.408173 version@stat F·[0 1] S·1KiB[0B 1KiB] Sc·[0.00 0.00] +16:01:03.408402 table@compaction committed F-4 S-648B Ke·0 D·4 T·777.962µs +16:01:03.408479 table@remove removed @50 +16:01:03.408534 table@remove removed @47 +16:01:03.410564 table@remove removed @44 +16:01:03.411741 table@remove removed @41 +16:01:03.412602 table@remove removed @40 diff --git a/callback-test/test-repo/datastore/MANIFEST-000052 b/callback-test/test-repo/datastore/MANIFEST-000052 new file mode 100644 index 00000000..aa43ddef Binary files /dev/null and b/callback-test/test-repo/datastore/MANIFEST-000052 differ diff --git a/callback-test/test-repo/datastore_spec b/callback-test/test-repo/datastore_spec new file mode 100644 index 00000000..7bf9626c --- /dev/null +++ b/callback-test/test-repo/datastore_spec @@ -0,0 +1 @@ +{"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"} \ No newline at end of file diff --git a/callback-test/test-repo/version b/callback-test/test-repo/version new file mode 100644 index 00000000..7f8f011e --- /dev/null +++ b/callback-test/test-repo/version @@ -0,0 +1 @@ +7 diff --git a/package.json b/package.json index c31b8f08..7c586cfa 100644 --- a/package.json +++ b/package.json @@ -49,13 +49,14 @@ "lodash": "^4.17.11", "memdown": "^4.0.0", "multihashes": "~0.4.14", - "multihashing-async": "~0.7.0", + "multihashing-async": "~0.6.0", "ncp": "^2.0.0", "rimraf": "^2.6.3" }, "dependencies": { "base32.js": "~0.1.0", "bignumber.js": "^9.0.0", + "callbackify": "^1.1.0", "cids": "~0.7.0", "datastore-core": "~0.7.0", "datastore-fs": "~0.9.0", diff --git a/src/api-addr.js b/src/api-addr.js index 48d005a2..263a845e 100644 --- a/src/api-addr.js +++ b/src/api-addr.js @@ -1,11 +1,12 @@ 'use strict' const Key = require('interface-datastore').Key +const callbackify = require('callbackify') const apiFile = new Key('api') module.exports = (store) => { - return { + const apiAddrStore = { /** * Get the current configuration from the repo. * @@ -33,4 +34,12 @@ module.exports = (store) => { return store.delete(apiFile) } } + + const callbackifiedApiAddrStore = { + get: callbackify(apiAddrStore.get), + set: callbackify(apiAddrStore.set), + delete: callbackify(apiAddrStore.delete) + } + + return callbackifiedApiAddrStore } diff --git a/src/blockstore.js b/src/blockstore.js index 27fc5415..44b60df0 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -5,11 +5,12 @@ const ShardingStore = core.ShardingDatastore const Block = require('ipfs-block') const CID = require('cids') const errcode = require('err-code') +const callbackify = require('callbackify') const { cidToKey } = require('./blockstore-utils') module.exports = async (filestore, options) => { const store = await maybeWithSharding(filestore, options) - return createBaseStore(store) + return CallbackifiedCreateBaseStore(store) } function maybeWithSharding (filestore, options) { @@ -148,6 +149,19 @@ function createBaseStore (store) { } } +function CallbackifiedCreateBaseStore (store) { + const baseStore = createBaseStore(store) + return { + query: callbackify(baseStore.query), + get: callbackify(baseStore.get), + put: callbackify(baseStore.put), + putMany: callbackify(baseStore.putMany), + has: callbackify(baseStore.has), + delete: callbackify(baseStore.delete), + close: callbackify(baseStore.close) + } +} + function cidToOtherVersion (cid) { try { return cid.version === 0 ? cid.toV1() : cid.toV0() diff --git a/src/config.js b/src/config.js index 2292ccb6..031bf585 100644 --- a/src/config.js +++ b/src/config.js @@ -6,6 +6,7 @@ const _get = require('just-safe-get') const _set = require('just-safe-set') const _has = require('lodash.has') const errcode = require('err-code') +const callbackify = require('callbackify') const errors = require('./errors') const configKey = new Key('config') @@ -70,7 +71,13 @@ module.exports = (store) => { } } - return configStore + const callbackifiedConfigStore = { + get: callbackify.variadic(configStore.get), + set: callbackify.variadic(configStore.set), + exists: callbackify(configStore.exists) + } + + return callbackifiedConfigStore async function _doSet (m) { const key = m.key diff --git a/src/index.js b/src/index.js index 5b203be2..de083ffd 100644 --- a/src/index.js +++ b/src/index.js @@ -6,6 +6,7 @@ const path = require('path') const debug = require('debug') const Big = require('bignumber.js') const errcode = require('err-code') +const callbackify = require('callbackify') const backends = require('./backends') const version = require('./version') @@ -87,6 +88,11 @@ class IpfsRepo { log('acquired repo.lock') log('creating datastore') this.datastore = backends.create('datastore', path.join(this.path, 'datastore'), this.options) +//console.log('vmx: this.datastore:', this.datastore) + debugger + callbackifyDatastore(this.datastore) + + log('creating blocks') const blocksBaseStore = backends.create('blocks', path.join(this.path, 'blocks'), this.options) this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks) @@ -297,7 +303,71 @@ async function getSize (queryFn) { return sum } -module.exports = IpfsRepo +function callbackifyDatastore (store) { + if (store.prototype) { + store.prototype.query = callbackify(store.query) + store.prototype.get = callbackify(store.get) + store.prototype.put = callbackify(store.put) + store.prototype.putMany = callbackify(store.putMany) + store.prototype.has = callbackify(store.has) + store.prototype.delete = callbackify(store.delete) + store.prototype.close = callbackify(store.close) + } else { + store.query = callbackify(store.query) + store.get = callbackify(store.get) + store.put = callbackify(store.put) + // NOTE vmx 2019-09-09: I have no idea why there isn't a putMany in this + // cast, but it makes the tests pass + //store.putMany = callbackify(store.putMany) + store.has = callbackify(store.has) + store.delete = callbackify(store.delete) + store.close = callbackify(store.close) + } +} + +// Based on https://github.com/junosuarez/callbackify +function callbackifyUndefinedSuccess(fn) { + var fnLength = fn.length + return function () { + var args = [].slice.call(arguments) + var ctx = this + if (args.length === fnLength + 1 && + typeof args[fnLength] === 'function') { + // callback mode + var cb = args.pop() + fn.apply(this, args) + .then( + function (val) { + if (val === undefined) { + //cb.call(ctx, undefined) + cb.call(ctx) + } else { + cb.call(ctx, null, val) + } + }, + function (err) { cb.call(ctx, err) } + ) + return + } + // promise mode + return fn.apply(ctx, arguments) + } +} + +class CallbackifiedIpfsRepo extends IpfsRepo { } + +CallbackifiedIpfsRepo.prototype.init = callbackify(IpfsRepo.prototype.init) +CallbackifiedIpfsRepo.prototype.open = callbackifyUndefinedSuccess(IpfsRepo.prototype.open) +//CallbackifiedIpfsRepo.prototype._openRoot = callbackify(IpfsRepo.prototype._openRoot) +//CallbackifiedIpfsRepo.prototype._openLock = callbackify(IpfsRepo.prototype._openLock) +//CallbackifiedIpfsRepo.prototype._checkInitialized = callbackify(IpfsRepo.prototype._checkInitialized) +CallbackifiedIpfsRepo.prototype.close = callbackifyUndefinedSuccess(IpfsRepo.prototype.close) +CallbackifiedIpfsRepo.prototype.exists = callbackifyUndefinedSuccess(IpfsRepo.prototype.exists) +CallbackifiedIpfsRepo.prototype.stat = callbackify.variadic(IpfsRepo.prototype.stat) +//CallbackifiedIpfsRepo.prototype._storageMaxStat = callbackify(IpfsRepo.prototype._storageMaxStat) +//CallbackifiedIpfsRepo.prototype._blockStat = callbackify(IpfsRepo.prototype._blockStat) + +module.exports = CallbackifiedIpfsRepo module.exports.utils = { blockstore: require('./blockstore-utils') } module.exports.repoVersion = repoVersion module.exports.errors = ERRORS diff --git a/src/spec.js b/src/spec.js index 881fc5a4..3e2f6c5d 100644 --- a/src/spec.js +++ b/src/spec.js @@ -2,11 +2,12 @@ const Key = require('interface-datastore').Key const sortKeys = require('sort-keys') +const callbackify = require('callbackify') const specKey = new Key('datastore_spec') module.exports = (store) => { - return { + const specStore = { /** * Check if a datastore spec file exists. * @@ -34,4 +35,11 @@ module.exports = (store) => { return store.put(specKey, Buffer.from(JSON.stringify(sortKeys(spec, { deep: true })))) } } + const callbackifiedSpecStore = { + get: callbackify(specStore.get), + set: callbackify(specStore.set), + exists: callbackify(specStore.exists) + } + + return callbackifiedSpecStore } diff --git a/src/version.js b/src/version.js index cd0db0ce..05b0e307 100644 --- a/src/version.js +++ b/src/version.js @@ -4,11 +4,12 @@ const Key = require('interface-datastore').Key const debug = require('debug') const log = debug('repo:version') const errcode = require('err-code') +const callbackify = require('callbackify') const versionKey = new Key('version') module.exports = (store) => { - return { + const versionStore = { /** * Check if a version file exists. * @@ -52,4 +53,13 @@ module.exports = (store) => { } } } + + const callbackifiedVersionStore = { + exists: callbackify(versionStore.exists), + get: callbackify(versionStore.get), + set: callbackify(versionStore.set), + check: callbackify(versionStore.check) + } + + return callbackifiedVersionStore }