Skip to content

Commit 311551a

Browse files
daviddiasdaviddias
committed
feat: no optional extension + simplify some of blockstore code
1 parent ca51a92 commit 311551a

File tree

3 files changed

+119
-131
lines changed

3 files changed

+119
-131
lines changed

‎src/stores/blockstore.js

Lines changed: 84 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,16 @@ const pull = require('pull-stream')
55
const Lock = require('lock')
66
const base32 = require('base32.js')
77
const path = require('path')
8-
const write = require('pull-write')
8+
const pullWrite = require('pull-write')
99
const parallel = require('run-parallel')
10-
const defer = require('pull-defer/source')
10+
const pullDefer = require('pull-defer/source')
1111

1212
const PREFIX_LENGTH = 5
1313

1414
exports = module.exports
1515

16-
function multihashToPath (multihash, extension) {
17-
extension = extension || 'data'
16+
function multihashToPath (multihash) {
17+
const extension = 'data'
1818
const encoder = new base32.Encoder()
1919
const hash = encoder.write(multihash).finalize()
2020
const filename = `${hash}.${extension}`
@@ -27,82 +27,105 @@ exports.setUp = (basePath, BlobStore, locks) => {
2727
const store = new BlobStore(basePath + '/blocks')
2828
const lock = new Lock()
2929

30-
function writeBlock (block, cb) {
30+
function writeBlock (block, callback) {
3131
if (!block || !block.data) {
32-
return cb(new Error('Invalid block'))
32+
return callback(new Error('Invalid block'))
3333
}
3434

35-
const key = multihashToPath(block.key, block.extension)
36-
37-
lock(key, (release) => pull(
38-
pull.values([block.data]),
39-
store.write(key, release((err) => {
40-
if (err) {
41-
return cb(err)
42-
}
43-
cb(null, {key})
44-
}))
45-
))
35+
const key = multihashToPath(block.key())
36+
37+
lock(key, (release) => {
38+
pull(
39+
pull.values([
40+
block.data
41+
]),
42+
store.write(key, release(released))
43+
)
44+
})
45+
46+
// called once the lock is released
47+
function released (err) {
48+
if (err) {
49+
return callback(err)
50+
}
51+
callback(null, { key: key })
52+
}
4653
}
4754

4855
return {
49-
getStream (key, extension) {
56+
// returns a pull-stream of one block being read
57+
getStream (key) {
5058
if (!key) {
5159
return pull.error(new Error('Invalid key'))
5260
}
5361

54-
const p = multihashToPath(key, extension)
55-
const deferred = defer()
62+
const blockPath = multihashToPath(key)
63+
const deferred = pullDefer()
5664

57-
lock(p, (release) => {
58-
const ext = extension === 'data' ? 'protobuf' : extension
65+
lock(blockPath, (release) => {
5966
pull(
60-
store.read(p),
61-
pull.collect(release((err, data) => {
62-
if (err) {
63-
return deferred.abort(err)
64-
}
65-
66-
deferred.resolve(pull.values([
67-
new Block(Buffer.concat(data), ext)
68-
]))
69-
}))
67+
store.read(blockPath),
68+
pull.collect(release(released))
7069
)
7170
})
7271

72+
function released (err, data) {
73+
if (err) {
74+
return deferred.abort(err)
75+
}
76+
77+
deferred.resolve(
78+
pull.values([
79+
new Block(Buffer.concat(data))
80+
])
81+
)
82+
}
83+
7384
return deferred
7485
},
7586

87+
// returns a pull-stream to write blocks into
88+
// TODO use a more explicit name, given that getStream is just for
89+
// one block, multiple blocks should have different naming
7690
putStream () {
7791
let ended = false
7892
let written = []
7993
let push = null
8094

81-
const sink = write((blocks, cb) => {
82-
parallel(blocks.map((block) => (cb) => {
83-
writeBlock(block, (err, meta) => {
84-
if (err) {
85-
return cb(err)
86-
}
87-
88-
if (push) {
89-
const read = push
90-
push = null
91-
read(null, meta)
92-
return cb()
93-
}
94-
95-
written.push(meta)
96-
cb()
97-
})
98-
}), cb)
95+
const sink = pullWrite((blocks, cb) => {
96+
const tasks = blocks.map((block) => {
97+
return (cb) => {
98+
writeBlock(block, (err, meta) => {
99+
if (err) {
100+
return cb(err)
101+
}
102+
103+
if (push) {
104+
const read = push
105+
push = null
106+
read(null, meta)
107+
return cb()
108+
}
109+
110+
written.push(meta)
111+
cb()
112+
})
113+
}
114+
})
115+
116+
parallel(tasks, cb)
99117
}, null, 100, (err) => {
100118
ended = err || true
101-
if (push) push(ended)
119+
if (push) {
120+
push(ended)
121+
}
102122
})
103123

124+
// TODO ??Why does a putStream need to be a source as well??
104125
const source = (end, cb) => {
105-
if (end) ended = end
126+
if (end) {
127+
ended = end
128+
}
106129
if (ended) {
107130
return cb(ended)
108131
}
@@ -114,35 +137,25 @@ exports.setUp = (basePath, BlobStore, locks) => {
114137
push = cb
115138
}
116139

117-
return {source, sink}
140+
return { source: source, sink: sink }
118141
},
119142

120-
has (key, extension, cb) {
121-
if (typeof extension === 'function') {
122-
cb = extension
123-
extension = undefined
124-
}
125-
143+
has (key, callback) {
126144
if (!key) {
127-
return cb(new Error('Invalid key'))
145+
return callback(new Error('Invalid key'))
128146
}
129147

130-
const p = multihashToPath(key, extension)
131-
store.exists(p, cb)
148+
const blockPath = multihashToPath(key)
149+
store.exists(blockPath, callback)
132150
},
133151

134-
delete (key, extension, cb) {
135-
if (typeof extension === 'function') {
136-
cb = extension
137-
extension = undefined
138-
}
139-
152+
delete (key, callback) {
140153
if (!key) {
141-
return cb(new Error('Invalid key'))
154+
return callback(new Error('Invalid key'))
142155
}
143156

144-
const p = multihashToPath(key, extension)
145-
store.remove(p, cb)
157+
const blockPath = multihashToPath(key)
158+
store.remove(blockPath, callback)
146159
}
147160
}
148161
}

‎src/stores/locks.js

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,17 @@ exports.setUp = (basePath, BlobStore) => {
1313
lock (callback) {
1414
function createLock () {
1515
pull(
16-
pull.values([new Buffer('LOCK')]),
16+
pull.values([
17+
new Buffer('LOCK')
18+
]),
1719
store.write(lockFile, callback)
1820
)
1921
}
2022

2123
function doesExist (err, exists) {
22-
if (err) return callback(err)
24+
if (err) {
25+
return callback(err)
26+
}
2327

2428
if (exists) {
2529
// default 100ms
@@ -37,16 +41,22 @@ exports.setUp = (basePath, BlobStore) => {
3741

3842
unlock (callback) {
3943
series([
40-
(cb) => store.remove(lockFile, cb),
41-
(cb) => store.exists(lockFile, (err, exists) => {
42-
if (err) return cb(err)
43-
44-
if (exists) {
45-
return cb(new Error('failed to remove lock'))
46-
}
47-
48-
cb()
49-
})
44+
(cb) => {
45+
store.remove(lockFile, cb)
46+
},
47+
(cb) => {
48+
store.exists(lockFile, (err, exists) => {
49+
if (err) {
50+
return cb(err)
51+
}
52+
53+
if (exists) {
54+
return cb(new Error('failed to remove lock'))
55+
}
56+
57+
cb()
58+
})
59+
}
5060
], callback)
5161
}
5262
}

‎test/blockstore-test.js

Lines changed: 13 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ const _ = require('lodash')
1111
module.exports = (repo) => {
1212
describe('blockstore', () => {
1313
const helloKey = 'CIQLS/CIQLSTJHXGJU2PQIUUXFFV62PWV7VREE57RXUU4A52IIR55M4LX432I.data'
14-
const helloIpldKey = 'CIQO2/CIQO2EUTF47PSTAHSL54KUTDS2AAN2DH4URM7H5KRATUGQFCM4OUIQI.ipld'
14+
15+
const helloIpldKey = 'CIQO2/CIQO2EUTF47PSTAHSL54KUTDS2AAN2DH4URM7H5KRATUGQFCM4OUIQI.data'
16+
1517
const blockCollection = _.range(100).map((i) => new Block(new Buffer(`hello-${i}-${Math.random()}`)))
1618

1719
describe('.putStream', () => {
@@ -68,7 +70,7 @@ module.exports = (repo) => {
6870
})
6971

7072
it('custom extension', function (done) {
71-
const b = new Block('hello world 2', 'ipld')
73+
const b = new Block('hello world 2')
7274
pull(
7375
pull.values([b]),
7476
repo.blockstore.putStream(),
@@ -97,10 +99,10 @@ module.exports = (repo) => {
9799
const b = new Block('hello world')
98100

99101
pull(
100-
repo.blockstore.getStream(b.key),
102+
repo.blockstore.getStream(b.key()),
101103
pull.collect((err, data) => {
102104
expect(err).to.not.exist
103-
expect(data[0]).to.be.eql(b)
105+
expect(data[0].key()).to.be.eql(b.key())
104106

105107
done()
106108
})
@@ -111,30 +113,17 @@ module.exports = (repo) => {
111113
parallel(_.range(20 * 100).map((i) => (cb) => {
112114
const j = i % blockCollection.length
113115
pull(
114-
repo.blockstore.getStream(blockCollection[j].key),
116+
repo.blockstore.getStream(blockCollection[j].key()),
115117
pull.collect((err, meta) => {
116118
expect(err).to.not.exist
117-
expect(meta).to.be.eql([blockCollection[j]])
119+
expect(meta[0].key())
120+
.to.be.eql(blockCollection[j].key())
118121
cb()
119122
})
120123
)
121124
}), done)
122125
})
123126

124-
it('custom extension', (done) => {
125-
const b = new Block('hello world 2', 'ipld')
126-
127-
pull(
128-
repo.blockstore.getStream(b.key, b.extension),
129-
pull.collect((err, data) => {
130-
expect(err).to.not.exist
131-
expect(data[0]).to.be.eql(b)
132-
133-
done()
134-
})
135-
)
136-
})
137-
138127
it('returns an error on invalid block', (done) => {
139128
pull(
140129
repo.blockstore.getStream(),
@@ -150,17 +139,7 @@ module.exports = (repo) => {
150139
it('existing block', (done) => {
151140
const b = new Block('hello world')
152141

153-
repo.blockstore.has(b.key, (err, exists) => {
154-
expect(err).to.not.exist
155-
expect(exists).to.equal(true)
156-
done()
157-
})
158-
})
159-
160-
it('with extension', (done) => {
161-
const b = new Block('hello world')
162-
163-
repo.blockstore.has(b.key, 'data', (err, exists) => {
142+
repo.blockstore.has(b.key(), (err, exists) => {
164143
expect(err).to.not.exist
165144
expect(exists).to.equal(true)
166145
done()
@@ -170,7 +149,7 @@ module.exports = (repo) => {
170149
it('non existent block', (done) => {
171150
const b = new Block('wooot')
172151

173-
repo.blockstore.has(b.key, (err, exists) => {
152+
repo.blockstore.has(b.key(), (err, exists) => {
174153
expect(err).to.not.exist
175154
expect(exists).to.equal(false)
176155
done()
@@ -182,24 +161,10 @@ module.exports = (repo) => {
182161
it('simple', (done) => {
183162
const b = new Block('hello world')
184163

185-
repo.blockstore.delete(b.key, (err) => {
186-
expect(err).to.not.exist
187-
188-
repo.blockstore.has(b.key, (err, exists) => {
189-
expect(err).to.not.exist
190-
expect(exists).to.equal(false)
191-
done()
192-
})
193-
})
194-
})
195-
196-
it('custom extension', (done) => {
197-
const b = new Block('hello world', 'ipld')
198-
199-
repo.blockstore.delete(b.key, b.extension, (err) => {
164+
repo.blockstore.delete(b.key(), (err) => {
200165
expect(err).to.not.exist
201166

202-
repo.blockstore.has(b.key, b.extension, (err, exists) => {
167+
repo.blockstore.has(b.key(), (err, exists) => {
203168
expect(err).to.not.exist
204169
expect(exists).to.equal(false)
205170
done()

0 commit comments

Comments
 (0)