From 0346abda142ce07d65f9eea889dc7d25c8cdd5b9 Mon Sep 17 00:00:00 2001 From: Zane Starr Date: Fri, 1 Mar 2019 10:46:17 -0800 Subject: [PATCH 1/2] refactor: update datastore core to use async/await instead of callbacks --- .travis.yml | 6 +- appveyor.yml | 3 +- package.json | 7 ++- src/keytransform.js | 44 +++++++------- src/mount.js | 100 ++++++++++++++----------------- src/namespace.js | 2 +- src/shard.js | 19 +----- src/sharding.js | 122 +++++++++++++++----------------------- src/tiered.js | 110 ++++++++++++---------------------- test/keytransform.spec.js | 60 ++++++------------- test/mount.spec.js | 118 +++++++++++++----------------------- test/namespace.spec.js | 64 +++++++------------- test/sharding.spec.js | 81 ++++++++----------------- test/tiered.spec.js | 81 +++++++------------------ test/util.js | 9 +++ 15 files changed, 300 insertions(+), 526 deletions(-) create mode 100644 test/util.js diff --git a/.travis.yml b/.travis.yml index 5102ee5..e04d5a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,8 @@ language: node_js matrix: include: - - node_js: 6 + - node_js: 10 env: CXX=g++-4.8 - - node_js: 8 - env: CXX=g++-4.8 - # - node_js: stable - # env: CXX=g++-4.8 script: - npm run lint diff --git a/appveyor.yml b/appveyor.yml index 046bf91..53522d6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,8 +3,7 @@ version: "{build}" environment: matrix: - - nodejs_version: "6" - - nodejs_version: "8" + - nodejs_version: "10" matrix: fast_finish: true diff --git a/package.json b/package.json index d05901d..0e9fb3a 100644 --- a/package.json +++ b/package.json @@ -35,14 +35,15 @@ }, "homepage": "https://github.com/ipfs/js-datastore-core#readme", "devDependencies": { - "aegir": "^15.3.1", + "aegir": "^v18.2.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "flow-bin": "~0.83.0" + "flow-bin": "~0.83.0", + "highlight.js": "https://github.com/highlightjs/highlight.js/tarball/9.15.4/highlight.js.tar.gz" }, "dependencies": { "async": "^2.6.1", - "interface-datastore": "~0.6.0", + "interface-datastore": "git://github.com/ipfs/interface-datastore.git#refactor/async-iterators", "left-pad": "^1.3.0", "pull-many": "^1.0.8", "pull-stream": "^3.6.9" diff --git a/src/keytransform.js b/src/keytransform.js index 9dfd919..ae184b0 100644 --- a/src/keytransform.js +++ b/src/keytransform.js @@ -1,7 +1,8 @@ /* @flow */ 'use strict' -const pull = require('pull-stream') +const utils = require('interface-datastore').utils +const map = utils.map /* :: import type {Key, Datastore, Batch, Query, QueryResult, Callback} from 'interface-datastore' @@ -38,24 +39,24 @@ class KeyTransformDatastore /* :: */ { this.transform = transform } - open (callback /* : Callback */) /* : void */ { - this.child.open(callback) + open () /* : Promise */ { + return this.child.open() } - put (key /* : Key */, val /* : Value */, callback /* : Callback */) /* : void */ { - this.child.put(this.transform.convert(key), val, callback) + async put (key /* : Key */, val /* : Value */) /* : Promise */ { + return this.child.put(this.transform.convert(key), val) } - get (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.get(this.transform.convert(key), callback) + async get (key /* : Key */) /* : Promise */ { + return this.child.get(this.transform.convert(key)) } - has (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.has(this.transform.convert(key), callback) + async has (key /* : Key */) /* : Promise */ { + return this.child.has(this.transform.convert(key)) } - delete (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.delete(this.transform.convert(key), callback) + async delete (key /* : Key */) /* : Promise */ { + return this.child.delete(this.transform.convert(key)) } batch () /* : Batch */ { @@ -67,24 +68,21 @@ class KeyTransformDatastore /* :: */ { delete: (key /* : Key */) /* : void */ => { b.delete(this.transform.convert(key)) }, - commit: (callback /* : Callback */) /* : void */ => { - b.commit(callback) + commit: async () /* : Promise */ => { + return b.commit() } } } - query (q /* : Query */) /* : QueryResult */ { - return pull( - this.child.query(q), - pull.map(e => { - e.key = this.transform.invert(e.key) - return e - }) - ) + query (q /* : Query */) /* : Iterator */ { + return map(this.child.query(q), e => { + e.key = this.transform.invert(e.key) + return e + }) } - close (callback /* : Callback */) /* : void */ { - this.child.close(callback) + async close () /* : Promise */ { + return this.child.close() } } diff --git a/src/mount.js b/src/mount.js index 7c461d2..15dc034 100644 --- a/src/mount.js +++ b/src/mount.js @@ -1,15 +1,12 @@ /* @flow */ 'use strict' -const each = require('async/each') -const many = require('pull-many') -const pull = require('pull-stream') - const Key = require('interface-datastore').Key const Errors = require('interface-datastore').Errors const utils = require('interface-datastore').utils -const asyncFilter = utils.asyncFilter -const asyncSort = utils.asyncSort +const filter = utils.filter +const take = utils.take +const sortAll = utils.sortAll const replaceStartWith = utils.replaceStartWith const Keytransform = require('./keytransform') @@ -34,10 +31,8 @@ class MountDatastore /* :: */ { this.mounts = mounts.slice() } - open (callback /* : Callback */) /* : void */ { - each(this.mounts, (m, cb) => { - m.datastore.open(cb) - }, callback) + async open () /* : Promise */ { + return Promise.all(this.mounts.map((m) => m.datastore.open())) } /** @@ -60,53 +55,44 @@ class MountDatastore /* :: */ { } } - put (key /* : Key */, value /* : Value */, callback /* : Callback */) /* : void */ { + async put (key /* : Key */, value /* : Value */) /* : Promise */ { const match = this._lookup(key) if (match == null) { - return callback( - Errors.dbWriteFailedError(new Error('No datastore mounted for this key')) - ) + throw Errors.dbWriteFailedError(new Error('No datastore mounted for this key')) } - match.datastore.put(match.rest, value, callback) + return match.datastore.put(match.rest, value) } - get (key /* : Key */, callback /* : Callback */) /* : void */ { + async get (key /* : Key */) /* : Promise */ { const match = this._lookup(key) if (match == null) { - return callback( - Errors.notFoundError(new Error('No datastore mounted for this key')) - ) + throw Errors.notFoundError(new Error('No datastore mounted for this key')) } - - match.datastore.get(match.rest, callback) + return match.datastore.get(match.rest) } - has (key /* : Key */, callback /* : Callback */) /* : void */ { + async has (key /* : Key */) /* : Promise */ { const match = this._lookup(key) if (match == null) { - callback(null, false) - return + return false } - - match.datastore.has(match.rest, callback) + return match.datastore.has(match.rest) } - delete (key /* : Key */, callback /* : Callback */) /* : void */ { + async delete (key /* : Key */) /* : Promise */ { const match = this._lookup(key) if (match == null) { - return callback( - Errors.dbDeleteFailedError(new Error('No datastore mounted for this key')) - ) + throw Errors.dbDeleteFailedError(new Error('No datastore mounted for this key')) } - match.datastore.delete(match.rest, callback) + return match.datastore.delete(match.rest) } - close (callback /* : Callback */) /* : void */ { - each(this.mounts, (m, cb) => { - m.datastore.close(cb) - }, callback) + async close () /* : Promise */ { + return Promise.all(this.mounts.map((m) => { + return m.datastore.close() + })) } batch () /* : Batch */ { @@ -137,10 +123,8 @@ class MountDatastore /* :: */ { const match = lookup(key) match.batch.delete(match.rest) }, - commit: (callback /* : Callback */) /* : void */ => { - each(Object.keys(batchMounts), (p, cb) => { - batchMounts[p].commit(cb) - }, callback) + commit: async () /* : Promise */ => { + return Promise.all(Object.keys(batchMounts).map(p => batchMounts[p].commit())) } } } @@ -168,27 +152,33 @@ class MountDatastore /* :: */ { }) }) - let tasks = [many(qs)] - - if (q.filters != null) { - tasks = tasks.concat(q.filters.map(f => asyncFilter(f))) - } - - if (q.orders != null) { - tasks = tasks.concat(q.orders.map(o => asyncSort(o))) - } - + let it = _many(qs) + if (q.filters) q.filters.forEach(f => { it = filter(it, f) }) + if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) }) if (q.offset != null) { let i = 0 - tasks.push(pull.filter(() => i++ >= q.offset)) + it = filter(it, () => i++ >= q.offset) } + if (q.limit != null) it = take(it, q.limit) - if (q.limit != null) { - tasks.push(pull.take(q.limit)) - } - - return pull.apply(null, tasks) + return it } } +function _many (iterable) { + return (async function * () { + let completed = iterable.map(() => false) + while (!completed.every(Boolean)) { + for (const [idx, itr] of iterable.entries()) { + const it = await itr.next() + if (it.done) { + completed[idx] = true + continue + } + yield it.value + } + } + })() +} + module.exports = MountDatastore diff --git a/src/namespace.js b/src/namespace.js index 13b0a3e..a3c83c9 100644 --- a/src/namespace.js +++ b/src/namespace.js @@ -41,7 +41,7 @@ class NamespaceDatastore/* :: */ extends KeytransformDatastore /* :: */)/* : QueryResult */ { + query (q /* : Query */)/* : Iterator */ { if (q.prefix && this.prefix.toString() !== '/') { return super.query(Object.assign({}, q, { prefix: this.prefix.child(new Key(q.prefix)).toString() diff --git a/src/shard.js b/src/shard.js index 8b45c1a..9cccaff 100644 --- a/src/shard.js +++ b/src/shard.js @@ -122,24 +122,11 @@ function parseShardFun (str /* : string */) /* : ShardV1 */ { } } -exports.readShardFun = (path /* : string */, store /* : Datastore */, callback /* : Callback */) /* : void */ => { +exports.readShardFun = async (path /* : string */, store /* : Datastore */) /* : Promise */ => { const key = new Key(path).child(new Key(SHARDING_FN)) const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) - - get(key, (err, res) => { - if (err) { - return callback(err) - } - - let shard - try { - shard = parseShardFun((res || '').toString().trim()) - } catch (err) { - return callback(err) - } - - callback(null, shard) - }) + const res = await get(key) + return parseShardFun((res || '').toString().trim()) } exports.readme = readme diff --git a/src/sharding.js b/src/sharding.js index a4df6b7..7537d6e 100644 --- a/src/sharding.js +++ b/src/sharding.js @@ -1,8 +1,6 @@ /* @flow */ 'use strict' -const waterfall = require('async/waterfall') -const parallel = require('async/parallel') const Key = require('interface-datastore').Key const sh = require('./shard') @@ -12,7 +10,7 @@ const shardKey = new Key(sh.SHARDING_FN) const shardReadmeKey = new Key(sh.README_FN) /* :: -import type {Datastore, Batch, Query, QueryResult, Callback} from 'interface-datastore' +import type {Datastore, Batch, Query, QueryResult} from 'interface-datastore' import type {ShardV1} from './shard' */ @@ -35,8 +33,8 @@ class ShardingDatastore { this.shard = shard } - open (callback /* : Callback */) /* : void */ { - this.child.open(callback) + async open () /* : Promise */ { + return this.child.open() } _convertKey (key/* : Key */)/* : Key */ { @@ -57,119 +55,95 @@ class ShardingDatastore { return Key.withNamespaces(key.list().slice(1)) } - static createOrOpen (store /* : Datastore */, shard /* : ShardV1 */, callback /* : Callback */) /* : void */ { - ShardingDatastore.create(store, shard, err => { - if (err && err.message !== 'datastore exists') { - return callback(err) - } - - ShardingDatastore.open(store, callback) - }) + static async createOrOpen (store /* : Datastore */, shard /* : ShardV1 */) /* : Promise */ { + try { + await ShardingDatastore.create(store, shard) + } catch (err) { + if (err && err.message !== 'datastore exists') throw err + } + return ShardingDatastore.open(store) } - static open (store /* : Datastore */, callback /* : Callback */) /* : void */ { - waterfall([ - (cb) => sh.readShardFun('/', store, cb), - (shard, cb) => { - cb(null, new ShardingDatastore(store, shard)) - } - ], callback) + static async open (store /* : Datastore */) /* : Promise */ { + const shard = await sh.readShardFun('/', store) + return new ShardingDatastore(store, shard) } - static create (store /* : Datastore */, shard /* : ShardV1 */, callback /* : Callback */) /* : void */ { - store.has(shardKey, (err, exists) => { - if (err) { - return callback(err) - } - - if (!exists) { - const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store) - return parallel([ - (cb) => put(shardKey, Buffer.from(shard.toString() + '\n'), cb), - (cb) => put(shardReadmeKey, Buffer.from(sh.readme), cb) - ], err => callback(err)) - } - - sh.readShardFun('/', store, (err, diskShard) => { - if (err) { - return callback(err) - } - - const a = (diskShard || '').toString() - const b = shard.toString() - if (a !== b) { - return callback(new Error(`specified fun ${b} does not match repo shard fun ${a}`)) - } - - callback(new Error('datastore exists')) - }) - }) + static async create (store /* : Datastore */, shard /* : ShardV1 */) /* : Promise */ { + const exists = await store.has(shardKey) + if (!exists) { + const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store) + return Promise.all([put(shardKey, Buffer.from(shard.toString() + '\n')), + put(shardReadmeKey, Buffer.from(sh.readme))]) + } + + const diskShard = await sh.readShardFun('/', store) + const a = (diskShard || '').toString() + const b = shard.toString() + if (a !== b) throw new Error(`specified fun ${b} does not match repo shard fun ${a}`) + throw new Error('datastore exists') } - put (key /* : Key */, val /* : Buffer */, callback /* : Callback */) /* : void */ { - this.child.put(key, val, callback) + async put (key /* : Key */, val /* : Buffer */) /* : Promise */ { + return this.child.put(key, val) } - get (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.get(key, callback) + async get (key /* : Key */) /* : Promise */ { + return this.child.get(key) } - has (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.has(key, callback) + async has (key /* : Key */) /* : Promise */ { + return this.child.has(key) } - delete (key /* : Key */, callback /* : Callback */) /* : void */ { - this.child.delete(key, callback) + async delete (key /* : Key */) /* : Promise */ { + return this.child.delete(key) } batch () /* : Batch */ { return this.child.batch() } - query (q /* : Query */) /* : QueryResult */ { + query (q /* : Query */) /* : Iterator */ { const tq/* : Query */ = { keysOnly: q.keysOnly, offset: q.offset, limit: q.limit, filters: [ - (e, cb) => cb(null, e.key.toString() !== shardKey.toString()), - (e, cb) => cb(null, e.key.toString() !== shardReadmeKey.toString()) + e => e.key.toString() !== shardKey.toString(), + e => e.key.toString() !== shardReadmeKey.toString() ] } if (q.prefix != null) { - tq.filters.push((e, cb) => { - cb(null, this._invertKey(e.key).toString().startsWith(q.prefix)) + tq.filters.push((e) => { + return this._invertKey(e.key).toString().startsWith(q.prefix) }) } if (q.filters != null) { - const filters = q.filters.map((f) => (e, cb) => { - f(Object.assign({}, e, { + const filters = q.filters.map((f) => (e) => { + return f(Object.assign({}, e, { key: this._invertKey(e.key) - }), cb) + })) }) tq.filters = tq.filters.concat(filters) } if (q.orders != null) { - tq.orders = q.orders.map((o) => (res, cb) => { + tq.orders = q.orders.map((o) => async (res) => { res.forEach((e) => { e.key = this._invertKey(e.key) }) - o(res, (err, ordered) => { - if (err) { - return cb(err) - } - ordered.forEach((e) => { e.key = this._convertKey(e.key) }) - cb(null, ordered) - }) + const ordered = await o(res) + ordered.forEach((e) => { e.key = this._convertKey(e.key) }) + return ordered }) } return this.child.query(tq) } - close (callback /* : Callback */) /* : void */ { - this.child.close(callback) + async close () /* : Promise */ { + return this.child.close() } } diff --git a/src/tiered.js b/src/tiered.js index ddf93be..ea22166 100644 --- a/src/tiered.js +++ b/src/tiered.js @@ -1,8 +1,6 @@ /* @flow */ 'use strict' -const each = require('async/each') -const whilst = require('async/whilst') const Errors = require('interface-datastore').Errors /* :: @@ -23,80 +21,50 @@ class TieredDatastore /* :: */ { this.stores = stores.slice() } - open (callback /* : Callback */) /* : void */ { - each(this.stores, (store, cb) => { - store.open(cb) - }, (err) => { - if (err) { - return callback(Errors.dbOpenFailedError()) - } - callback() - }) + async open () /* : void */ { + try { + await (this.stores.map((store) => store.open())) + } catch (err) { + throw Errors.dbOpenFailedError() + } } - put (key /* : Key */, value /* : Value */, callback /* : Callback */) /* : void */ { - each(this.stores, (store, cb) => { - store.put(key, value, cb) - }, (err) => { - if (err) { - return callback(Errors.dbWriteFailedError()) - } - callback() - }) + async put (key /* : Key */, value /* : Value */) /* : void */ { + try { + await Promise.all(this.stores.map(store => store.put(key, value))) + } catch (err) { + throw Errors.dbWriteFailedError() + } } - get (key /* : Key */, callback /* : Callback */) /* : void */ { - const storeLength = this.stores.length - let done = false - let i = 0 - whilst(() => !done && i < storeLength, cb => { - const store = this.stores[i++] - store.get(key, (err, res) => { - if (err == null) { - done = true - return cb(null, res) - } - cb() - }) - }, (err, res) => { - if (err || !res) { - return callback(Errors.notFoundError()) - } - callback(null, res) - }) + async get (key /* : Key */) /* : Promise */ { + for (const store of this.stores) { + try { + const res = await store.get(key) + if (res) return res + } catch (err) {} + } + throw Errors.notFoundError() } - has (key /* : Key */, callback /* : Callback */) /* : void */ { - const storeLength = this.stores.length - let done = false - let i = 0 - whilst(() => !done && i < storeLength, cb => { - const store = this.stores[i++] - store.has(key, (err, exists) => { - if (err == null) { - done = true - return cb(null, exists) - } - cb() - }) - }, callback) + async has (key /* : Key */) /* : Promise */ { + for (const store of this.stores) { + const exists = await store.has(key) + if (exists) return true + } + return false } - delete (key /* : Key */, callback /* : Callback */) /* : void */ { - each(this.stores, (store, cb) => { - store.delete(key, cb) - }, (err) => { - if (err) { - return callback(Errors.dbDeleteFailedError()) - } - callback() - }) + async delete (key /* : Key */) /* : Promise */ { + try { + await Promise.all(this.stores.map(store => store.delete(key))) + } catch (err) { + throw Errors.dbDeleteFailedError() + } } - close (callback /* : Callback */) /* : void */ { - each(this.stores, (store, cb) => { - store.close(cb) - }, callback) + async close () /* : Promise */ { + await Promise.all(this.stores.map(store => store.close())) } batch () /* : Batch */ { @@ -109,15 +77,15 @@ class TieredDatastore /* :: */ { delete: (key /* : Key */) /* : void */ => { batches.forEach(b => b.delete(key)) }, - commit: (callback /* : Callback */) /* : void */ => { - each(batches, (b, cb) => { - b.commit(cb) - }, callback) + commit: async () /* : Promise */ => { + for (const batch of batches) { + await batch.commit() + } } } } - query (q /* : Query */) /* : QueryResult */ { + query (q /* : Query */) /* : Iterator */ { return this.stores[this.stores.length - 1].query(q) } } diff --git a/test/keytransform.spec.js b/test/keytransform.spec.js index f9bc191..a936dcd 100644 --- a/test/keytransform.spec.js +++ b/test/keytransform.spec.js @@ -5,18 +5,14 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const series = require('async/series') -const each = require('async/each') -const map = require('async/map') -const parallel = require('async/parallel') -const pull = require('pull-stream') const Memory = require('interface-datastore').MemoryDatastore const Key = require('interface-datastore').Key +const collect = require('./util').collect const KeytransformStore = require('../src/').KeytransformDatastore describe('KeyTransformDatastore', () => { - it('basic', (done) => { + it('basic', async () => { const mStore = new Memory() const transform = { convert (key) { @@ -41,40 +37,22 @@ describe('KeyTransformDatastore', () => { 'foo/bar/bazb', 'foo/bar/baz/barb' ].map((s) => new Key(s)) - - series([ - (cb) => each(keys, (k, cb) => { - kStore.put(k, Buffer.from(k.toString()), cb) - }, cb), - (cb) => parallel([ - (cb) => map(keys, (k, cb) => { - kStore.get(k, cb) - }, cb), - (cb) => map(keys, (k, cb) => { - mStore.get(new Key('abc').child(k), cb) - }, cb) - ], (err, res) => { - expect(err).to.not.exist() - expect(res[0]).to.eql(res[1]) - cb() - }), - (cb) => parallel([ - (cb) => pull(mStore.query({}), pull.collect(cb)), - (cb) => pull(kStore.query({}), pull.collect(cb)) - ], (err, res) => { - expect(err).to.not.exist() - expect(res[0]).to.have.length(res[1].length) - - res[0].forEach((a, i) => { - const kA = a.key - const kB = res[1][i].key - expect(transform.invert(kA)).to.eql(kB) - expect(kA).to.eql(transform.convert(kB)) - }) - - cb() - }), - (cb) => kStore.close(cb) - ], done) + await Promise.all(keys.map((key) => kStore.put(key, Buffer.from(key.toString())))) + let kResults = Promise.all(keys.map((key) => kStore.get(key))) + let mResults = Promise.all(keys.map((key) => mStore.get(new Key('abc').child(key)))) + let results = await Promise.all([kResults, mResults]) + expect(results[0]).to.eql(results[1]) + + const mRes = await collect(mStore.query({})) + const kRes = await collect(kStore.query({})) + expect(kRes).to.have.length(mRes.length) + + mRes.forEach((a, i) => { + const kA = a.key + const kB = kRes[i].key + expect(transform.invert(kA)).to.eql(kB) + expect(kA).to.eql(transform.convert(kB)) + }) + await kStore.close() }) }) diff --git a/test/mount.spec.js b/test/mount.spec.js index 5e88da5..c693bc8 100644 --- a/test/mount.spec.js +++ b/test/mount.spec.js @@ -5,9 +5,9 @@ const chai = require('chai') chai.use(require('dirty-chai')) +const assert = chai.assert const expect = chai.expect -const pull = require('pull-stream') -const series = require('async/series') +const collect = require('./util').collect const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore @@ -15,28 +15,30 @@ const MemoryStore = require('interface-datastore').MemoryDatastore const MountStore = require('../src').MountDatastore describe('MountStore', () => { - it('put - no mount', (done) => { + it('put - no mount', async () => { const m = new MountStore([]) - - m.put(new Key('hello'), Buffer.from('foo'), (err) => { + try { + await m.put(new Key('hello'), Buffer.from('foo')) + assert(false, 'Failed to throw error on no mount') + } catch (err) { expect(err).to.be.an('Error') - done() - }) + } }) - it('put - wrong mount', (done) => { + it('put - wrong mount', async () => { const m = new MountStore([{ datastore: new MemoryStore(), prefix: new Key('cool') }]) - - m.put(new Key('/fail/hello'), Buffer.from('foo'), (err) => { + try { + await m.put(new Key('/fail/hello'), Buffer.from('foo')) + assert(false, 'Failed to throw error on wrong mount') + } catch (err) { expect(err).to.be.an('Error') - done() - }) + } }) - it('put', (done) => { + it('put', async () => { const mds = new MemoryStore() const m = new MountStore([{ datastore: mds, @@ -44,17 +46,12 @@ describe('MountStore', () => { }]) const val = Buffer.from('hello') - series([ - (cb) => m.put(new Key('/cool/hello'), val, cb), - (cb) => mds.get(new Key('/hello'), (err, res) => { - expect(err).to.not.exist() - expect(res).to.eql(val) - cb() - }) - ], done) + await m.put(new Key('/cool/hello'), val) + const res = await mds.get(new Key('/hello')) + expect(res).to.eql(val) }) - it('get', (done) => { + it('get', async () => { const mds = new MemoryStore() const m = new MountStore([{ datastore: mds, @@ -62,17 +59,12 @@ describe('MountStore', () => { }]) const val = Buffer.from('hello') - series([ - (cb) => mds.put(new Key('/hello'), val, cb), - (cb) => m.get(new Key('/cool/hello'), (err, res) => { - expect(err).to.not.exist() - expect(res).to.eql(val) - cb() - }) - ], done) + await mds.put(new Key('/hello'), val) + const res = await m.get(new Key('/cool/hello')) + expect(res).to.eql(val) }) - it('has', (done) => { + it('has', async () => { const mds = new MemoryStore() const m = new MountStore([{ datastore: mds, @@ -80,17 +72,12 @@ describe('MountStore', () => { }]) const val = Buffer.from('hello') - series([ - (cb) => mds.put(new Key('/hello'), val, cb), - (cb) => m.has(new Key('/cool/hello'), (err, exists) => { - expect(err).to.not.exist() - expect(exists).to.eql(true) - cb() - }) - ], done) + await mds.put(new Key('/hello'), val) + const exists = await m.has(new Key('/cool/hello')) + expect(exists).to.eql(true) }) - it('delete', (done) => { + it('delete', async () => { const mds = new MemoryStore() const m = new MountStore([{ datastore: mds, @@ -98,23 +85,15 @@ describe('MountStore', () => { }]) const val = Buffer.from('hello') - series([ - (cb) => m.put(new Key('/cool/hello'), val, cb), - (cb) => m.delete(new Key('/cool/hello'), cb), - (cb) => m.has(new Key('/cool/hello'), (err, exists) => { - expect(err).to.not.exist() - expect(exists).to.eql(false) - cb() - }), - (cb) => mds.has(new Key('/hello'), (err, exists) => { - expect(err).to.not.exist() - expect(exists).to.eql(false) - cb() - }) - ], done) + await m.put(new Key('/cool/hello'), val) + await m.delete(new Key('/cool/hello')) + let exists = await m.has(new Key('/cool/hello')) + expect(exists).to.eql(false) + exists = await mds.has(new Key('/hello')) + expect(exists).to.eql(false) }) - it('query simple', (done) => { + it('query simple', async () => { const mds = new MemoryStore() const m = new MountStore([{ datastore: mds, @@ -122,28 +101,15 @@ describe('MountStore', () => { }]) const val = Buffer.from('hello') - series([ - (cb) => m.put(new Key('/cool/hello'), val, cb), - (cb) => { - pull( - m.query({prefix: '/cool'}), - pull.collect((err, res) => { - expect(err).to.not.exist() - expect(res).to.eql([{ - key: new Key('/cool/hello'), - value: val - }]) - cb() - }) - ) - } - ], done) + await m.put(new Key('/cool/hello'), val) + const res = await collect(m.query({ prefix: '/cool' })) + expect(res).to.eql([{ key: new Key('/cool/hello'), value: val }]) }) describe('interface-datastore', () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new MountStore([{ + async setup () { + return new MountStore([{ prefix: new Key('/a'), datastore: new MemoryStore() }, { @@ -152,11 +118,9 @@ describe('MountStore', () => { }, { prefix: new Key('/q'), datastore: new MemoryStore() - }])) + }]) }, - teardown (callback) { - callback() - } + async teardown () { } }) }) }) diff --git a/test/namespace.spec.js b/test/namespace.spec.js index dff48f0..92b9284 100644 --- a/test/namespace.spec.js +++ b/test/namespace.spec.js @@ -5,23 +5,19 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const series = require('async/series') -const each = require('async/each') -const map = require('async/map') -const parallel = require('async/parallel') -const pull = require('pull-stream') const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore const NamespaceStore = require('../src/').NamespaceDatastore +const collect = require('./util').collect describe('KeyTransformDatastore', () => { const prefixes = [ 'abc', '' ] - prefixes.forEach((prefix) => it(`basic '${prefix}'`, (done) => { + prefixes.forEach((prefix) => it(`basic '${prefix}'`, async () => { const mStore = new MemoryStore() const store = new NamespaceStore(mStore, new Key(prefix)) @@ -34,51 +30,33 @@ describe('KeyTransformDatastore', () => { 'foo/bar/baz/barb' ].map((s) => new Key(s)) - series([ - (cb) => each(keys, (k, cb) => { - store.put(k, Buffer.from(k.toString()), cb) - }, cb), - (cb) => parallel([ - (cb) => map(keys, (k, cb) => { - store.get(k, cb) - }, cb), - (cb) => map(keys, (k, cb) => { - mStore.get(new Key(prefix).child(k), cb) - }, cb) - ], (err, res) => { - expect(err).to.not.exist() - expect(res[0]).to.eql(res[1]) - cb() - }), - (cb) => parallel([ - (cb) => pull(mStore.query({}), pull.collect(cb)), - (cb) => pull(store.query({}), pull.collect(cb)) - ], (err, res) => { - expect(err).to.not.exist() - expect(res[0]).to.have.length(res[1].length) + await Promise.all(keys.map(key => store.put(key, Buffer.from(key.toString())))) + let nResults = Promise.all(keys.map((key) => store.get(key))) + let mResults = Promise.all(keys.map((key) => mStore.get(new Key(prefix).child(key)))) + let results = await Promise.all([nResults, mResults]) + const mRes = await collect(mStore.query({})) + const nRes = await collect(store.query({})) - res[0].forEach((a, i) => { - const kA = a.key - const kB = res[1][i].key - expect(store.transform.invert(kA)).to.eql(kB) - expect(kA).to.eql(store.transform.convert(kB)) - }) + expect(nRes).to.have.length(mRes.length) - cb() - }), - (cb) => store.close(cb) - ], done) + mRes.forEach((a, i) => { + const kA = a.key + const kB = nRes[i].key + expect(store.transform.invert(kA)).to.eql(kB) + expect(kA).to.eql(store.transform.convert(kB)) + }) + await store.close() + + expect(results[0]).to.eql(results[1]) })) prefixes.forEach((prefix) => { describe(`interface-datastore: '${prefix}'`, () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new NamespaceStore(new MemoryStore(), new Key(prefix))) + async setup () { + return new NamespaceStore(new MemoryStore(), new Key(prefix)) }, - teardown (callback) { - callback() - } + async teardown () { } }) }) }) diff --git a/test/sharding.spec.js b/test/sharding.spec.js index 17c3e33..0c6dd2f 100644 --- a/test/sharding.spec.js +++ b/test/sharding.spec.js @@ -6,9 +6,7 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const series = require('async/series') -const parallel = require('async/parallel') -const waterfall = require('async/waterfall') +const assert = chai.expect const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore @@ -17,80 +15,49 @@ const ShardingStore = require('../src').ShardingDatastore const sh = require('../src').shard describe('ShardingStore', () => { - it('create', (done) => { + it('create', async () => { const ms = new MemoryStore() const shard = new sh.NextToLast(2) - - waterfall([ - (cb) => ShardingStore.create(ms, shard, cb), - (cb) => parallel([ - (cb) => ms.get(new Key(sh.SHARDING_FN), cb), - (cb) => ms.get(new Key(sh.README_FN), cb) - ], cb), - (res, cb) => { - expect( - res[0].toString() - ).to.eql( - shard.toString() + '\n' - ) - expect( - res[1].toString() - ).to.eql( - sh.readme - ) - cb() - } - ], done) + await ShardingStore.create(ms, shard) + const res = await Promise.all([ms.get(new Key(sh.SHARDING_FN)), ms.get(new Key(sh.README_FN))]) + expect(res[0].toString()).to.eql(shard.toString() + '\n') + expect(res[1].toString()).to.eql(sh.readme) }) - it('open - empty', (done) => { + it('open - empty', async () => { const ms = new MemoryStore() - - ShardingStore.open(ms, (err, ss) => { - expect(err).to.exist() - expect(ss).to.not.exist() - done() - }) + try { + await ShardingStore.open(ms) + assert(false, 'Failed to throw error on ShardStore.open') + } catch (err) {} }) - it('open - existing', (done) => { + it('open - existing', async () => { const ms = new MemoryStore() const shard = new sh.NextToLast(2) - waterfall([ - (cb) => ShardingStore.create(ms, shard, cb), - (cb) => ShardingStore.open(ms, cb) - ], done) + await ShardingStore.create(ms, shard) + await ShardingStore.open(ms) }) - it('basics', (done) => { + it('basics', async () => { const ms = new MemoryStore() const shard = new sh.NextToLast(2) - ShardingStore.createOrOpen(ms, shard, (err, ss) => { - expect(err).to.not.exist() - expect(ss).to.exist() - const store = ss - - series([ - (cb) => store.put(new Key('hello'), Buffer.from('test'), cb), - (cb) => ms.get(new Key('ll').child(new Key('hello')), (err, res) => { - expect(err).to.not.exist() - expect(res).to.eql(Buffer.from('test')) - cb() - }) - ], done) - }) + const store = await ShardingStore.createOrOpen(ms, shard) + expect(store).to.exist() + await ShardingStore.createOrOpen(ms, shard) + await store.put(new Key('hello'), Buffer.from('test')) + const res = await ms.get(new Key('ll').child(new Key('hello'))) + expect(res).to.eql(Buffer.from('test')) }) describe('interface-datastore', () => { require('interface-datastore/src/tests')({ - setup (callback) { + setup () { const shard = new sh.NextToLast(2) - ShardingStore.createOrOpen(new MemoryStore(), shard, callback) + return ShardingStore.createOrOpen(new MemoryStore(), shard) }, - teardown (callback) { - callback() - } + teardown () { } }) }) }) diff --git a/test/tiered.spec.js b/test/tiered.spec.js index acf9e9c..5ff8e3f 100644 --- a/test/tiered.spec.js +++ b/test/tiered.spec.js @@ -6,8 +6,6 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const series = require('async/series') -const parallel = require('async/parallel') const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore @@ -24,80 +22,47 @@ describe('Tiered', () => { store = new TieredStore(ms) }) - it('put', (done) => { + it('put', async () => { const k = new Key('hello') const v = Buffer.from('world') - series([ - (cb) => store.put(k, v, cb), - (cb) => parallel([ - (cb) => ms[0].get(k, cb), - (cb) => ms[1].get(k, cb) - ], (err, res) => { - expect(err).to.not.exist() - res.forEach((val) => { - expect(val).to.be.eql(v) - }) - cb() - }) - ], done) + await store.put(k, v) + const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) + res.forEach((val) => { + expect(val).to.be.eql(v) + }) }) - it('get and has, where available', (done) => { + it('get and has, where available', async () => { const k = new Key('hello') const v = Buffer.from('world') - - series([ - (cb) => ms[1].put(k, v, cb), - (cb) => store.get(k, (err, val) => { - expect(err).to.not.exist() - expect(val).to.be.eql(v) - cb() - }), - (cb) => store.has(k, (err, exists) => { - expect(err).to.not.exist() - expect(exists).to.be.eql(true) - cb() - }) - ], done) + await ms[1].put(k, v) + const val = await store.get(k) + expect(val).to.be.eql(v) + const exists = await store.has(k) + expect(exists).to.be.eql(true) }) - it('has and delete', (done) => { + it('has and delete', async () => { const k = new Key('hello') const v = Buffer.from('world') - series([ - (cb) => store.put(k, v, cb), - (cb) => parallel([ - (cb) => ms[0].has(k, cb), - (cb) => ms[1].has(k, cb) - ], (err, res) => { - expect(err).to.not.exist() - expect(res).to.be.eql([true, true]) - cb() - }), - (cb) => store.delete(k, cb), - (cb) => parallel([ - (cb) => ms[0].has(k, cb), - (cb) => ms[1].has(k, cb) - ], (err, res) => { - expect(err).to.not.exist() - expect(res).to.be.eql([false, false]) - cb() - }) - ], done) + await store.put(k, v) + let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) + expect(res).to.be.eql([true, true]) + await store.delete(k) + res = await Promise.all([ms[0].has(k), ms[1].has(k)]) + expect(res).to.be.eql([false, false]) }) }) describe('inteface-datastore-single', () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new TieredStore([ + setup () { + return new TieredStore([ new MemoryStore(), new MemoryStore() - ])) + ]) }, - teardown (callback) { - callback() - } + teardown () { } }) }) }) diff --git a/test/util.js b/test/util.js new file mode 100644 index 0000000..39c421f --- /dev/null +++ b/test/util.js @@ -0,0 +1,9 @@ +'use strict' + +const collect = async (iterable) => { + const results = [] + for await (const value of iterable) results.push(value) + return results +} + +exports.collect = collect From 99b4a783932f100062f13c218778b39276d9de39 Mon Sep 17 00:00:00 2001 From: Zane Starr Date: Thu, 14 Mar 2019 09:53:48 -0700 Subject: [PATCH 2/2] ci: update ci config --- .appveyor.yml | 23 ------------------ .travis.yml | 64 ++++++++++++++++++++++++++++++-------------------- appveyor.yml | 28 ---------------------- ci/Jenkinsfile | 2 -- circle.yml | 15 ------------ 5 files changed, 39 insertions(+), 93 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 appveyor.yml delete mode 100644 ci/Jenkinsfile delete mode 100644 circle.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index de3e378..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,23 +0,0 @@ -environment: - matrix: - - nodejs_version: "6" - - nodejs_version: "8" - -# cache: -# - node_modules - -platform: - - x64 - -install: - - ps: Install-Product node $env:nodejs_version $env:platform - - npm install - -test_script: - - node --version - - npm --version - - npm test - -build: off - -version: "{build}" diff --git a/.travis.yml b/.travis.yml index e04d5a1..d1036e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,42 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -sudo: false language: node_js +cache: npm +stages: + - check + - test + - cov -matrix: +node_js: + - '10' + +os: + - linux + - osx + +script: npx nyc -s npm run test:node -- --bail +after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov + +jobs: include: - - node_js: 10 - env: CXX=g++-4.8 - -script: - - npm run lint - - npm run test - - npm run coverage - -before_script: - - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start - -after_success: - - npm run coverage-publish - -addons: - firefox: 'latest' - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 + - os: windows + cache: false + + - stage: check + script: + - npx aegir commitlint --travis + - npx aegir dep-check + - npm run lint + + - stage: test + name: chrome + addons: + chrome: stable + script: npx aegir test -t browser -t webworker + + - stage: test + name: firefox + addons: + firefox: latest + script: npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless + +notifications: + email: false \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 53522d6..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -version: "{build}" - -environment: - matrix: - - nodejs_version: "10" - -matrix: - fast_finish: true - -install: - # Install Node.js - - ps: Install-Product node $env:nodejs_version - - # Upgrade npm - - npm install -g npm - - # Output our current versions for debugging - - node --version - - npm --version - - # Install our package dependencies - - npm install - -test_script: - - npm run test:node - -build: off diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile deleted file mode 100644 index a7da2e5..0000000 --- a/ci/Jenkinsfile +++ /dev/null @@ -1,2 +0,0 @@ -// Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -javascript() diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 0009693..0000000 --- a/circle.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -machine: - node: - version: stable - -dependencies: - pre: - - google-chrome --version - - curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - - sudo dpkg -i google-chrome.deb || true - - sudo apt-get update - - sudo apt-get install -f - - sudo apt-get install --only-upgrade lsb-base - - sudo dpkg -i google-chrome.deb - - google-chrome --version