From a9045f76cbc4098532f5f1a006b598db2da68e2e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 3 May 2019 22:58:19 +0800 Subject: [PATCH 1/5] test: add test for Object.links with CBOR --- src/object/links.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/object/links.js b/src/object/links.js index 6cac58f96..4a849ddd5 100644 --- a/src/object/links.js +++ b/src/object/links.js @@ -209,5 +209,42 @@ module.exports = (createCommon, options) => { }) }) }) + + it('should get links from CBOR object', (done) => { + const hashes = [] + ipfs.add(Buffer.from('test data'), (err, res1) => { + expect(err).to.not.exist() + hashes.push(res1[0].hash) + ipfs.add(Buffer.from('more test data'), (err, res2) => { + hashes.push(res2[0].hash) + expect(err).to.not.exist() + const obj = { + some: 'data', + mylink: { '/': hashes[0] }, + myobj: { + anotherLink: { '/': hashes[1] } + } + } + ipfs.dag.put(obj, (err, cid) => { + expect(err).to.not.exist() + ipfs.object.links(cid, (err, links) => { + expect(err).to.not.exist() + expect(links.length).to.eql(2) + + // TODO: js-ipfs succeeds but go returns empty strings for link name + // const names = [links[0].name, links[1].name] + // expect(names).includes('mylink') + // expect(names).includes('myobj/anotherLink') + + const cids = [links[0].cid.toString(), links[1].cid.toString()] + expect(cids).includes(hashes[0]) + expect(cids).includes(hashes[1]) + + done() + }) + }) + }) + }) + }) }) } From 604a31e0662cef47b785ed2fafcbb04b70f84472 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 8 May 2019 17:25:01 +0800 Subject: [PATCH 2/5] feat: tests for GC --- src/repo/gc.js | 197 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/src/repo/gc.js b/src/repo/gc.js index fb108b2ba..46150024d 100644 --- a/src/repo/gc.js +++ b/src/repo/gc.js @@ -1,7 +1,21 @@ /* eslint-env mocha */ 'use strict' +const { promisify } = require('es6-promisify') const { getDescribe, getIt, expect } = require('../utils/mocha') +const { DAGNode, DAGLink } = require('ipld-dag-pb') + +const createDAGNode = promisify((data, links, cb) => { + DAGNode.create(data, links, cb) +}) + +const createDAGLink = promisify((name, size, cid, cb) => { + DAGLink.create(name, size, cid, cb) +}) + +const addDAGLink = promisify((node, link, cb) => { + DAGNode.addLink(node, link, cb) +}) module.exports = (createCommon, options) => { const describe = getDescribe(options) @@ -41,5 +55,188 @@ module.exports = (createCommon, options) => { expect(res).to.exist() }) }) + + it('should clean up unpinned data', async () => { + // Get initial list of local blocks + const refsBeforeAdd = await ipfs.refs.local() + + // Add some data. Note: this will implicitly pin the data, which causes + // some blocks to be added for the data itself and for the pinning + // information that refers to the blocks + const addRes = await ipfs.add(Buffer.from('apples')) + const hash = addRes[0].hash + + // Get the list of local blocks after the add, should be bigger than + // the initial list and contain hash + const refsAfterAdd = await ipfs.refs.local() + expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length) + expect(refsAfterAdd.map(r => r.ref)).includes(hash) + + // Run garbage collection + await ipfs.repo.gc() + + // Get the list of local blocks after GC, should still contain the hash, + // because the file is still pinned + const refsAfterGc = await ipfs.refs.local() + expect(refsAfterGc.map(r => r.ref)).includes(hash) + + // Unpin the data + await ipfs.pin.rm(hash) + + // Run garbage collection + await ipfs.repo.gc() + + // The list of local blocks should now be the same as at the start + const refsAfterUnpinAndGc = await ipfs.refs.local() + expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + }) + + it('should clean up removed MFS files', async () => { + // Get initial list of local blocks + const refsBeforeAdd = await ipfs.refs.local() + + // Add a file to MFS + await ipfs.files.write('/test', Buffer.from('oranges'), { create: true }) + const stats = await ipfs.files.stat('/test') + expect(stats.type).to.equal('file') + const hash = stats.hash + + // Get the list of local blocks after the add, should be bigger than + // the initial list and contain hash + const refsAfterAdd = await ipfs.refs.local() + expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length) + expect(refsAfterAdd.map(r => r.ref)).includes(hash) + + // Run garbage collection + await ipfs.repo.gc() + + // Get the list of local blocks after GC, should still contain the hash, + // because the file is in MFS + const refsAfterGc = await ipfs.refs.local() + expect(refsAfterGc.map(r => r.ref)).includes(hash) + + // Remove the file + await ipfs.files.rm('/test') + + // Run garbage collection + await ipfs.repo.gc() + + // The list of local blocks should now be the same as at the start + const refsAfterUnpinAndGc = await ipfs.refs.local() + expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + }) + + it('should clean up block only after unpinned and removed from MFS', async () => { + // Get initial list of local blocks + const refsBeforeAdd = await ipfs.refs.local() + + // Add a file to MFS + await ipfs.files.write('/test', Buffer.from('peaches'), { create: true }) + const stats = await ipfs.files.stat('/test') + expect(stats.type).to.equal('file') + const mfsFileHash = stats.hash + + // Get the CID of the data in the file + const block = await ipfs.block.get(mfsFileHash) + + // Add the data to IPFS (which implicitly pins the data) + const addRes = await ipfs.add(block.data) + const dataHash = addRes[0].hash + + // Get the list of local blocks after the add, should be bigger than + // the initial list and contain the data hash + const refsAfterAdd = await ipfs.refs.local() + expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length) + const hashesAfterAdd = refsAfterAdd.map(r => r.ref) + expect(hashesAfterAdd).includes(dataHash) + + // Run garbage collection + await ipfs.repo.gc() + + // Get the list of local blocks after GC, should still contain the hash, + // because the file is pinned and in MFS + const refsAfterGc = await ipfs.refs.local() + const hashesAfterGc = refsAfterGc.map(r => r.ref) + expect(hashesAfterGc).includes(dataHash) + + // Remove the file + await ipfs.files.rm('/test') + + // Run garbage collection + await ipfs.repo.gc() + + // Get the list of local blocks after GC, should still contain the hash, + // because the file is still pinned + const refsAfterRmAndGc = await ipfs.refs.local() + const hashesAfterRmAndGc = refsAfterRmAndGc.map(r => r.ref) + expect(hashesAfterRmAndGc).not.includes(mfsFileHash) + expect(hashesAfterRmAndGc).includes(dataHash) + + // Unpin the data + await ipfs.pin.rm(dataHash) + + // Run garbage collection + await ipfs.repo.gc() + + // The list of local blocks should now be the same as at the start + const refsAfterUnpinAndGc = await ipfs.refs.local() + expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + }) + + it('should clean up indirectly pinned data after recursive pin removal', async () => { + // Get initial list of local blocks + const refsBeforeAdd = await ipfs.refs.local() + + // Add some data + const addRes = await ipfs.add(Buffer.from('pears')) + const dataHash = addRes[0].hash + + // Unpin the data + await ipfs.pin.rm(dataHash) + + // Create a link to the data from an object + const link = await createDAGLink('p', addRes[0].size, dataHash) + const node = await createDAGNode(Buffer.from('fruit')) + const obj = await addDAGLink(node, link) + + // Put the object into IPFS + const objHash = (await ipfs.object.put(obj)).toString() + + // Putting an object doesn't pin it + expect((await ipfs.pin.ls()).map(p => p.hash)).not.includes(objHash) + + // Get the list of local blocks after the add, should be bigger than + // the initial list and contain data and object hash + const refsAfterAdd = await ipfs.refs.local() + expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length) + const hashesAfterAdd = refsAfterAdd.map(r => r.ref) + expect(hashesAfterAdd).includes(objHash) + expect(hashesAfterAdd).includes(dataHash) + + // Recursively pin the object + await ipfs.pin.add(objHash, { recursive: true }) + + // The data should now be indirectly pinned + const pins = await ipfs.pin.ls() + expect(pins.find(p => p.hash === dataHash).type).to.eql('indirect') + + // Run garbage collection + await ipfs.repo.gc() + + // Get the list of local blocks after GC, should still contain the data + // hash, because the data is still (indirectly) pinned + const refsAfterGc = await ipfs.refs.local() + expect(refsAfterGc.map(r => r.ref)).includes(dataHash) + + // Recursively unpin the object + await ipfs.pin.rm(objHash) + + // Run garbage collection + await ipfs.repo.gc() + + // The list of local blocks should now be the same as at the start + const refsAfterUnpinAndGc = await ipfs.refs.local() + expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + }) }) } From 15d1e7013d974cf0516bc71c4a309baf20079d31 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 22 May 2019 15:56:33 -0400 Subject: [PATCH 3/5] test: update gc tests with new ipld api --- src/repo/gc.js | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/repo/gc.js b/src/repo/gc.js index 46150024d..50b60c42a 100644 --- a/src/repo/gc.js +++ b/src/repo/gc.js @@ -1,21 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { promisify } = require('es6-promisify') const { getDescribe, getIt, expect } = require('../utils/mocha') -const { DAGNode, DAGLink } = require('ipld-dag-pb') - -const createDAGNode = promisify((data, links, cb) => { - DAGNode.create(data, links, cb) -}) - -const createDAGLink = promisify((name, size, cid, cb) => { - DAGLink.create(name, size, cid, cb) -}) - -const addDAGLink = promisify((node, link, cb) => { - DAGNode.addLink(node, link, cb) -}) +const { DAGNode } = require('ipld-dag-pb') module.exports = (createCommon, options) => { const describe = getDescribe(options) @@ -195,9 +182,11 @@ module.exports = (createCommon, options) => { await ipfs.pin.rm(dataHash) // Create a link to the data from an object - const link = await createDAGLink('p', addRes[0].size, dataHash) - const node = await createDAGNode(Buffer.from('fruit')) - const obj = await addDAGLink(node, link) + const obj = await DAGNode.create(Buffer.from('fruit'), [{ + Name: 'p', + Hash: dataHash, + TSize: addRes[0].size + }]) // Put the object into IPFS const objHash = (await ipfs.object.put(obj)).toString() From 630aa3cd10f9dc1f6444a99e204b655ab3adaafd Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 22 May 2019 15:58:31 -0400 Subject: [PATCH 4/5] chore: rebase onto master --- src/object/links.js | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/src/object/links.js b/src/object/links.js index 4a849ddd5..6cac58f96 100644 --- a/src/object/links.js +++ b/src/object/links.js @@ -209,42 +209,5 @@ module.exports = (createCommon, options) => { }) }) }) - - it('should get links from CBOR object', (done) => { - const hashes = [] - ipfs.add(Buffer.from('test data'), (err, res1) => { - expect(err).to.not.exist() - hashes.push(res1[0].hash) - ipfs.add(Buffer.from('more test data'), (err, res2) => { - hashes.push(res2[0].hash) - expect(err).to.not.exist() - const obj = { - some: 'data', - mylink: { '/': hashes[0] }, - myobj: { - anotherLink: { '/': hashes[1] } - } - } - ipfs.dag.put(obj, (err, cid) => { - expect(err).to.not.exist() - ipfs.object.links(cid, (err, links) => { - expect(err).to.not.exist() - expect(links.length).to.eql(2) - - // TODO: js-ipfs succeeds but go returns empty strings for link name - // const names = [links[0].name, links[1].name] - // expect(names).includes('mylink') - // expect(names).includes('myobj/anotherLink') - - const cids = [links[0].cid.toString(), links[1].cid.toString()] - expect(cids).includes(hashes[0]) - expect(cids).includes(hashes[1]) - - done() - }) - }) - }) - }) - }) }) } From 5818705b5bea2260e1bdc38c9a81633f52e38f86 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 22 May 2019 18:30:09 -0400 Subject: [PATCH 5/5] fix: make gc tests work in browser --- src/repo/gc.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/repo/gc.js b/src/repo/gc.js index 50b60c42a..2305e0b60 100644 --- a/src/repo/gc.js +++ b/src/repo/gc.js @@ -73,9 +73,9 @@ module.exports = (createCommon, options) => { // Run garbage collection await ipfs.repo.gc() - // The list of local blocks should now be the same as at the start + // The list of local blocks should no longer contain the hash const refsAfterUnpinAndGc = await ipfs.refs.local() - expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(hash) }) it('should clean up removed MFS files', async () => { @@ -108,9 +108,9 @@ module.exports = (createCommon, options) => { // Run garbage collection await ipfs.repo.gc() - // The list of local blocks should now be the same as at the start + // The list of local blocks should no longer contain the hash const refsAfterUnpinAndGc = await ipfs.refs.local() - expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(hash) }) it('should clean up block only after unpinned and removed from MFS', async () => { @@ -165,9 +165,11 @@ module.exports = (createCommon, options) => { // Run garbage collection await ipfs.repo.gc() - // The list of local blocks should now be the same as at the start + // The list of local blocks should no longer contain the hashes const refsAfterUnpinAndGc = await ipfs.refs.local() - expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref) + expect(hashesAfterUnpinAndGc).not.includes(mfsFileHash) + expect(hashesAfterUnpinAndGc).not.includes(dataHash) }) it('should clean up indirectly pinned data after recursive pin removal', async () => { @@ -223,9 +225,11 @@ module.exports = (createCommon, options) => { // Run garbage collection await ipfs.repo.gc() - // The list of local blocks should now be the same as at the start + // The list of local blocks should no longer contain the hashes const refsAfterUnpinAndGc = await ipfs.refs.local() - expect(refsAfterUnpinAndGc).to.eql(refsBeforeAdd) + const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref) + expect(hashesAfterUnpinAndGc).not.includes(objHash) + expect(hashesAfterUnpinAndGc).not.includes(dataHash) }) }) }