Skip to content
This repository was archived by the owner on Aug 12, 2020. It is now read-only.

Commit 4aaae9a

Browse files
achingbrainachingbrain
committed
fix: updates ipld-dag-pb dep to version without .cid properties
Follows on from ipld/js-ipld-dag-pb#99 and updates this module to not rely on DAGNodes having knowledge of their CIDs. Bonus: also removes use of js-ipfs from this module breaking another circular dependency from the project.
1 parent c5c0720 commit 4aaae9a

File tree

16 files changed

+252
-318
lines changed

16 files changed

+252
-318
lines changed

‎package.json

Lines changed: 53 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,55 @@
11
{
2-
"name": "ipfs-unixfs-engine",
3-
"version": "0.33.0",
4-
"description": "JavaScript implementation of the unixfs Engine used by IPFS",
5-
"leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
6-
"main": "src/index.js",
7-
"browser": {
8-
"fs": false,
9-
"rabin": false
10-
},
11-
"scripts": {
12-
"test": "aegir test",
13-
"test:node": "aegir test -t node",
14-
"test:browser": "aegir test -t browser",
15-
"test:webworker": "aegir test -t webworker",
16-
"build": "aegir build",
17-
"lint": "aegir lint",
18-
"release": "aegir release",
19-
"release-minor": "aegir release --type minor",
20-
"release-major": "aegir release --type major",
21-
"coverage": "aegir coverage"
22-
},
23-
"repository": {
24-
"type": "git",
25-
"url": "git+https://github.com/ipfs/js-ipfs-unixfs-engine.git"
26-
},
27-
"keywords": [
28-
"IPFS"
29-
],
30-
"license": "MIT",
31-
"bugs": {
32-
"url": "https://github.com/ipfs/js-ipfs-unixfs-engine/issues"
33-
},
34-
"engines": {
35-
"node": ">=8.0.0",
36-
"npm": ">=3.0.0"
37-
},
38-
"homepage": "https://github.com/ipfs/js-ipfs-unixfs-engine#readme",
39-
"devDependencies": {
40-
"aegir": "^17.0.0",
41-
"chai": "^4.2.0",
42-
"dirty-chai": "^2.0.1",
43-
"ipfs": "~0.32.3",
44-
"ipfs-block-service": "~0.15.1",
45-
"ipfs-repo": "~0.25.0",
46-
"ipld": "~0.19.1",
47-
"mkdirp": "~0.5.1",
48-
"multihashes": "~0.4.14",
49-
"ncp": "^2.0.0",
50-
"pull-generate": "^2.2.0",
51-
"pull-stream-to-stream": "^1.3.4",
52-
"pull-zip": "^2.0.1",
53-
"rimraf": "^2.6.2",
54-
"sinon": "^7.1.0"
55-
},
56-
"dependencies": {
57-
"async": "^2.6.1",
58-
"cids": "~0.5.5",
59-
"deep-extend": "~0.6.0",
60-
"ipfs-unixfs": "~0.1.16",
61-
"ipld-dag-pb": "~0.14.11",
62-
"left-pad": "^1.3.0",
63-
"multihashing-async": "~0.5.1",
64-
"pull-batch": "^1.0.0",
65-
"pull-block": "^1.4.0",
66-
"pull-cat": "^1.1.11",
67-
"pull-pair": "^1.1.0",
68-
"pull-paramap": "^1.2.2",
69-
"pull-pause": "0.0.2",
70-
"pull-pushable": "^2.2.0",
71-
"pull-stream": "^3.6.9",
72-
"pull-through": "^1.0.18",
73-
"pull-traverse": "^1.0.3",
74-
"pull-write": "^1.1.4",
75-
"sparse-array": "^1.3.1",
76-
"stream-to-pull-stream": "^1.7.2"
77-
},
78-
"optionalDependencies": {
79-
"rabin": "^1.6.0"
80-
},
81-
"contributors": [
82-
"Alan Shaw <alan@tableflip.io>",
83-
"Arpit Agarwal <atvanguard@users.noreply.github.com>",
84-
"Bernard Mordan <bernard@tableflip.io>",
85-
"Dan Ordille <dordille@gmail.com>",
86-
"David Dias <daviddias.p@gmail.com>",
87-
"Diogo Silva <fsdiogo@gmail.com>",
88-
"Francisco Baio Dias <xicombd@gmail.com>",
89-
"Friedel Ziegelmayer <dignifiedquire@gmail.com>",
90-
"Greenkeeper <support@greenkeeper.io>",
91-
"Marcin Rataj <lidel@lidel.org>",
92-
"Pedro Teixeira <i@pgte.me>",
93-
"Richard Littauer <richard.littauer@gmail.com>",
94-
"Richard Schneider <makaretu@gmail.com>",
95-
"Stephen Whitmore <stephen.whitmore@gmail.com>",
96-
"Volker Mische <volker.mische@gmail.com>",
97-
"achingbrain <alex@achingbrain.net>",
98-
"greenkeeper[bot] <greenkeeper[bot]@users.noreply.github.com>",
99-
"jbenet <juan@benet.ai>",
100-
"nginnever <ginneversource@gmail.com>",
101-
"ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>"
102-
]
2+
"name": "ipfs-unixfs-engine",
3+
"version": "0.33.0",
4+
"description": "JavaScript implementation of the unixfs Engine used by IPFS",
5+
"leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
6+
"main": "src/index.js",
7+
"browser": {
8+
"fs": false,
9+
"rabin": false
10+
},
11+
"repository": {
12+
"type": "git",
13+
"url": "git+https://github.com/ipfs/js-ipfs-unixfs-engine.git"
14+
},
15+
"keywords": [
16+
"IPFS"
17+
],
18+
"license": "MIT",
19+
"bugs": {
20+
"url": "https://github.com/ipfs/js-ipfs-unixfs-engine/issues"
21+
},
22+
"engines": {
23+
"node": ">=8.0.0",
24+
"npm": ">=3.0.0"
25+
},
26+
"homepage": "https://github.com/ipfs/js-ipfs-unixfs-engine#readme",
27+
"devDependencies": {
28+
"ipld": "~0.20.0"
29+
},
30+
"dependencies": {
31+
"ipld-dag-pb": "~0.15.0"
32+
},
33+
"contributors": [
34+
"Alan Shaw <alan@tableflip.io>",
35+
"Arpit Agarwal <atvanguard@users.noreply.github.com>",
36+
"Bernard Mordan <bernard@tableflip.io>",
37+
"Dan Ordille <dordille@gmail.com>",
38+
"David Dias <daviddias.p@gmail.com>",
39+
"Diogo Silva <fsdiogo@gmail.com>",
40+
"Francisco Baio Dias <xicombd@gmail.com>",
41+
"Friedel Ziegelmayer <dignifiedquire@gmail.com>",
42+
"Greenkeeper <support@greenkeeper.io>",
43+
"Marcin Rataj <lidel@lidel.org>",
44+
"Pedro Teixeira <i@pgte.me>",
45+
"Richard Littauer <richard.littauer@gmail.com>",
46+
"Richard Schneider <makaretu@gmail.com>",
47+
"Stephen Whitmore <stephen.whitmore@gmail.com>",
48+
"Volker Mische <volker.mische@gmail.com>",
49+
"achingbrain <alex@achingbrain.net>",
50+
"greenkeeper[bot] <greenkeeper[bot]@users.noreply.github.com>",
51+
"jbenet <juan@benet.ai>",
52+
"nginnever <ginneversource@gmail.com>",
53+
"ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>"
54+
]
10355
}

‎src/builder/builder.js

Lines changed: 28 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,11 @@ const pull = require('pull-stream')
66
const through = require('pull-through')
77
const parallel = require('async/parallel')
88
const waterfall = require('async/waterfall')
9-
const dagPB = require('ipld-dag-pb')
10-
const CID = require('cids')
11-
const multihash = require('multihashing-async')
12-
9+
const persist = require('../utils/persist')
1310
const reduce = require('./reduce')
14-
15-
const DAGNode = dagPB.DAGNode
11+
const {
12+
DAGNode
13+
} = require('ipld-dag-pb')
1614

1715
const defaultOptions = {
1816
chunkerOptions: {
@@ -27,12 +25,6 @@ const defaultOptions = {
2725

2826
module.exports = function builder (createChunker, ipld, createReducer, _options) {
2927
const options = extend({}, defaultOptions, _options)
30-
options.cidVersion = options.cidVersion || options.cidVersion
31-
options.hashAlg = options.hashAlg || defaultOptions.hashAlg
32-
33-
if (options.hashAlg !== 'sha2-256') {
34-
options.cidVersion = 1
35-
}
3628

3729
return function (source) {
3830
return function (items, cb) {
@@ -71,33 +63,17 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
7163
const d = new UnixFS('directory')
7264

7365
waterfall([
74-
(cb) => DAGNode.create(d.marshal(), [], options.hashAlg, cb),
75-
(node, cb) => {
76-
if (options.onlyHash) {
77-
return cb(null, node)
78-
}
79-
80-
const cid = new CID(options.cidVersion, 'dag-pb', node.multihash)
81-
82-
node = new DAGNode(
83-
node.data,
84-
node.links,
85-
node.serialized,
86-
cid
87-
)
88-
89-
ipld.put(node, {
90-
cid
91-
}, (err) => cb(err, node))
92-
}
93-
], (err, node) => {
66+
(cb) => DAGNode.create(d.marshal(), [], cb),
67+
(node, cb) => persist(node, ipld, options, cb)
68+
], (err, result) => {
9469
if (err) {
9570
return callback(err)
9671
}
72+
9773
callback(null, {
9874
path: item.path,
99-
multihash: node.multihash,
100-
size: node.size
75+
multihash: result.cid.buffer,
76+
size: result.node.size
10177
})
10278
})
10379
}
@@ -134,55 +110,42 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
134110
}),
135111
pull.asyncMap((buffer, callback) => {
136112
if (options.rawLeaves) {
137-
return multihash(buffer, options.hashAlg, (error, hash) => {
138-
if (error) {
139-
return callback(error)
140-
}
141-
142-
return callback(null, {
143-
multihash: hash,
144-
size: buffer.length,
145-
leafSize: buffer.length,
146-
cid: new CID(1, 'raw', hash),
147-
data: buffer
148-
})
113+
return callback(null, {
114+
size: buffer.length,
115+
leafSize: buffer.length,
116+
data: buffer
149117
})
150118
}
151119

152120
const file = new UnixFS(options.leafType, buffer)
153121

154-
DAGNode.create(file.marshal(), [], options.hashAlg, (err, node) => {
122+
DAGNode.create(file.marshal(), [], (err, node) => {
155123
if (err) {
156124
return callback(err)
157125
}
158126

159127
callback(null, {
160-
multihash: node.multihash,
161128
size: node.size,
162129
leafSize: file.fileSize(),
163-
cid: new CID(options.cidVersion, 'dag-pb', node.multihash),
164130
data: node
165131
})
166132
})
167133
}),
168134
pull.asyncMap((leaf, callback) => {
169-
if (options.onlyHash) {
170-
return callback(null, leaf)
171-
}
135+
persist(leaf.data, ipld, options, (error, results) => {
136+
if (error) {
137+
return callback(error)
138+
}
172139

173-
ipld.put(leaf.data, {
174-
cid: leaf.cid
175-
}, (error) => callback(error, leaf))
176-
}),
177-
pull.map((leaf) => {
178-
return {
179-
path: file.path,
180-
multihash: leaf.cid.buffer,
181-
size: leaf.size,
182-
leafSize: leaf.leafSize,
183-
name: '',
184-
cid: leaf.cid
185-
}
140+
callback(null, {
141+
size: leaf.size,
142+
leafSize: leaf.leafSize,
143+
data: results.node,
144+
multihash: results.cid.buffer,
145+
path: leaf.path,
146+
name: ''
147+
})
148+
})
186149
}),
187150
through( // mark as single node if only one single node
188151
function onData (data) {

‎src/builder/reduce.js

Lines changed: 10 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
const waterfall = require('async/waterfall')
44
const dagPB = require('ipld-dag-pb')
55
const UnixFS = require('ipfs-unixfs')
6-
const CID = require('cids')
6+
const persist = require('../utils/persist')
77

88
const DAGLink = dagPB.DAGLink
99
const DAGNode = dagPB.DAGNode
@@ -14,10 +14,10 @@ module.exports = function reduce (file, ipld, options) {
1414
const leaf = leaves[0]
1515

1616
return callback(null, {
17-
path: file.path,
18-
multihash: leaf.multihash,
1917
size: leaf.size,
2018
leafSize: leaf.leafSize,
19+
multihash: leaf.multihash,
20+
path: file.path,
2121
name: leaf.name
2222
})
2323
}
@@ -28,44 +28,23 @@ module.exports = function reduce (file, ipld, options) {
2828
const links = leaves.map((leaf) => {
2929
f.addBlockSize(leaf.leafSize)
3030

31-
let cid = leaf.cid
32-
33-
if (!cid) {
34-
// we are an intermediate node
35-
cid = new CID(options.cidVersion, 'dag-pb', leaf.multihash)
36-
}
37-
38-
return new DAGLink(leaf.name, leaf.size, cid.buffer)
31+
return new DAGLink(leaf.name, leaf.size, leaf.multihash)
3932
})
4033

4134
waterfall([
42-
(cb) => DAGNode.create(f.marshal(), links, options.hashAlg, cb),
43-
(node, cb) => {
44-
const cid = new CID(options.cidVersion, 'dag-pb', node.multihash)
45-
46-
if (options.onlyHash) {
47-
return cb(null, {
48-
node, cid
49-
})
50-
}
51-
52-
ipld.put(node, {
53-
cid
54-
}, (error) => cb(error, {
55-
node, cid
56-
}))
57-
}
35+
(cb) => DAGNode.create(f.marshal(), links, cb),
36+
(node, cb) => persist(node, ipld, options, cb)
5837
], (error, result) => {
5938
if (error) {
6039
return callback(error)
6140
}
6241

6342
callback(null, {
64-
name: '',
65-
path: file.path,
66-
multihash: result.cid.buffer,
6743
size: result.node.size,
68-
leafSize: f.fileSize()
44+
leafSize: f.fileSize(),
45+
multihash: result.cid.buffer,
46+
path: file.path,
47+
name: ''
6948
})
7049
})
7150
}

‎src/exporter/dir-flat.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
1313
name: name,
1414
depth: depth,
1515
path: path,
16-
hash: cid,
16+
multihash: cid.buffer,
1717
size: node.size,
1818
type: 'dir'
1919
}
@@ -26,7 +26,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
2626
size: link.size,
2727
name: link.name,
2828
path: path + '/' + link.name,
29-
multihash: link.multihash,
29+
multihash: link.cid.buffer,
3030
linkName: link.name,
3131
pathRest: pathRest.slice(1),
3232
type: 'dir'

0 commit comments

Comments
 (0)