From b9e00d542a55630aea83accb74bd391f45c01e96 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Thu, 4 Apr 2019 16:38:49 -0700 Subject: [PATCH 01/12] Migration 8: Encoding key's names to base32 License: MIT Signed-off-by: Adam Uhlir --- migrations/migration-8/index.js | 77 +++++ .../node_modules/base32.js/.npmignore | 4 + .../node_modules/base32.js/.travis.yml | 4 + .../node_modules/base32.js/HISTORY.md | 4 + .../node_modules/base32.js/README.md | 71 ++++ .../node_modules/base32.js/base32.js | 312 ++++++++++++++++++ .../node_modules/base32.js/index.js | 16 + .../node_modules/base32.js/jsdoc.json | 33 ++ .../node_modules/base32.js/karma.conf.js | 33 ++ .../node_modules/base32.js/package.json | 73 ++++ .../base32.js/test/base32_test.js | 87 +++++ .../node_modules/base32.js/test/fixtures.js | 294 +++++++++++++++++ .../node_modules/base32.js/webpack.config.js | 17 + migrations/migration-8/package-lock.json | 13 + migrations/migration-8/package.json | 19 ++ 15 files changed, 1057 insertions(+) create mode 100644 migrations/migration-8/index.js create mode 100644 migrations/migration-8/node_modules/base32.js/.npmignore create mode 100644 migrations/migration-8/node_modules/base32.js/.travis.yml create mode 100644 migrations/migration-8/node_modules/base32.js/HISTORY.md create mode 100644 migrations/migration-8/node_modules/base32.js/README.md create mode 100644 migrations/migration-8/node_modules/base32.js/base32.js create mode 100644 migrations/migration-8/node_modules/base32.js/index.js create mode 100644 migrations/migration-8/node_modules/base32.js/jsdoc.json create mode 100644 migrations/migration-8/node_modules/base32.js/karma.conf.js create mode 100644 migrations/migration-8/node_modules/base32.js/package.json create mode 100644 migrations/migration-8/node_modules/base32.js/test/base32_test.js create mode 100644 migrations/migration-8/node_modules/base32.js/test/fixtures.js create mode 100644 migrations/migration-8/node_modules/base32.js/webpack.config.js create mode 100644 migrations/migration-8/package-lock.json create mode 100644 migrations/migration-8/package.json diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js new file mode 100644 index 0000000..c4eb6f7 --- /dev/null +++ b/migrations/migration-8/index.js @@ -0,0 +1,77 @@ +'use strict' + +const Datastore = require('datastore-fs') +const path = require('path') +const base32 = require('base32.js') +const Key = require('interface-datastore').Key +const log = require('debug')('jsipfs-repo-migrations:migration-8') + +const KEY_PREFIX = 'key_' + +function encode(name) { + name = Buffer.from(name) + const encoder = new base32.Encoder({type: "rfc4648"}); + return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() +} + +function decode(name) { + log(name) + if (!name.startsWith(KEY_PREFIX)) { + throw Error("Unknown format of key's name!") + } + + const decoder = new base32.Decoder({type: "rfc4648"}); + return decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) +} + +async function processFolder(store, prefix, fileNameProcessor) { + const query = { + prefix: `/${prefix}`, + } + + const files = await store.query(query) + for await (let file of files) { + const name = String(file.key._buf).replace(`/${prefix}/`, '') + const encodedFileName = fileNameProcessor(name) + const newKey = new Key(`${prefix}/${encodedFileName}`) + + await store.delete(file.key) + log(`Translating key's name '${file.key}' into '${newKey}'`) + await store.put(newKey, file.value) + } +} + +async function migrate(repoPath, isBrowser) { + const store = new Datastore(path.join(repoPath, 'keys'), {extension: '.data', createIfMissing: false}) + store.open() + try { + const info = processFolder(store, 'info', encode) + const data = processFolder(store, 'pkcs8', encode) + + await Promise.all([info, data]) + } finally { + await store.close() + } +} + +async function revert(repoPath, isBrowser) { + const store = new Datastore(path.join(repoPath, 'keys'), {extension: '.data', createIfMissing: false}) + store.open() + + try { + const info = processFolder(store, 'info', decode) + const data = processFolder(store, 'pkcs8', decode) + + await Promise.all([info, data]) + } finally { + await store.close() + } +} + +module.exports = { + version: 8, + description: 'Transforms key\'s names into base32 encoding.', + reversible: true, + migrate, + revert, +} diff --git a/migrations/migration-8/node_modules/base32.js/.npmignore b/migrations/migration-8/node_modules/base32.js/.npmignore new file mode 100644 index 0000000..0397291 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/.npmignore @@ -0,0 +1,4 @@ +bower_components +node_modules +public +webpack.stats.json diff --git a/migrations/migration-8/node_modules/base32.js/.travis.yml b/migrations/migration-8/node_modules/base32.js/.travis.yml new file mode 100644 index 0000000..b127718 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/.travis.yml @@ -0,0 +1,4 @@ +language: node_js +node_js: + - 0.12 + - iojs diff --git a/migrations/migration-8/node_modules/base32.js/HISTORY.md b/migrations/migration-8/node_modules/base32.js/HISTORY.md new file mode 100644 index 0000000..aca8598 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/HISTORY.md @@ -0,0 +1,4 @@ +0.0.1 / 2015-02-15 +================== + + * Initial release diff --git a/migrations/migration-8/node_modules/base32.js/README.md b/migrations/migration-8/node_modules/base32.js/README.md new file mode 100644 index 0000000..8db86b5 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/README.md @@ -0,0 +1,71 @@ +# Base 32 for JavaScript [![Build Status](https://travis-ci.org/mikepb/base32.js.svg)](http://travis-ci.org/mikepb/base32.js) + +[Wikipedia](https://en.wikipedia.org/wiki/Base32): + +> Base32 is a base 32 transfer encoding using the twenty-six letters A–Z and six digits 2–7. It is primarily used to encode binary data, but is able to encode binary text like ASCII. +> +> Base32 has number of advantages over Base64: +> +> 1. The resulting character set is all one case (usually represented as uppercase), which can often be beneficial when using a case-insensitive filesystem, spoken language, or human memory. +> +> 2. The result can be used as file name because it can not possibly contain '/' symbol which is usually acts as path separator in Unix-based operating systems. +> +> 3. The alphabet was selected to avoid similar-looking pairs of different symbols, so the strings can be accurately transcribed by hand. (For example, the symbol set omits the symbols for 1, 8 and zero, since they could be confused with the letters 'I', 'B', and 'O'.) +> +> 4. A result without padding can be included in a URL without encoding any characters. +> +> However, Base32 representation takes roughly 20% more space than Base64. + +## Documentation + +Full documentation at http://mikepb.github.io/base32.js/ + +## Installation + +```sh +$ npm install base32.js +``` + +## Usage + +Encoding an array of bytes using [Crockford][crock32]: + +```js +var base32 = require("base32.js"); + +var buf = [1, 2, 3, 4]; +var encoder = new base32.Encoder({ type: "crockford", lc: true }); +var str = encoder.write(buf).finalize(); +// str = "04106173" + +var decoder = new base32.Decoder({ type: "crockford" }); +var out = decoder.write(str).finalize(); +// out = [1, 2, 3, 4] +``` + +The default Base32 variant if no `type` is provided is `"rfc4648"` without +padding. + +## Browser support + +The browser versions of the library may be found under the `dist/` directory. +The browser files are updated on each versioned release, but not for +development. [Karma][karma] is used to run the [mocha][] tests in the browser. + +```sh +$ npm install -g karma-cli +$ npm run karma +``` + +## Related projects + +- [agnoster/base32-js][agnoster] + +## License + +MIT + +[agnoster]: https://github.com/agnoster/base32-js +[crock32]: http://www.crockford.com/wrmg/base32.html +[karma]: http://karma-runner.github.io +[mocha]: http://mochajs.org diff --git a/migrations/migration-8/node_modules/base32.js/base32.js b/migrations/migration-8/node_modules/base32.js/base32.js new file mode 100644 index 0000000..e64f896 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/base32.js @@ -0,0 +1,312 @@ +"use strict"; + +/** + * Generate a character map. + * @param {string} alphabet e.g. "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + * @param {object} mappings map overrides from key to value + * @method + */ + +var charmap = function (alphabet, mappings) { + mappings || (mappings = {}); + alphabet.split("").forEach(function (c, i) { + if (!(c in mappings)) mappings[c] = i; + }); + return mappings; +} + +/** + * The RFC 4648 base 32 alphabet and character map. + * @see {@link https://tools.ietf.org/html/rfc4648} + */ + +var rfc4648 = { + alphabet: "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", + charmap: { + 0: 14, + 1: 8 + } +}; + +rfc4648.charmap = charmap(rfc4648.alphabet, rfc4648.charmap); + +/** + * The Crockford base 32 alphabet and character map. + * @see {@link http://www.crockford.com/wrmg/base32.html} + */ + +var crockford = { + alphabet: "0123456789ABCDEFGHJKMNPQRSTVWXYZ", + charmap: { + O: 0, + I: 1, + L: 1 + } +}; + +crockford.charmap = charmap(crockford.alphabet, crockford.charmap); + +/** + * base32hex + * @see {@link https://en.wikipedia.org/wiki/Base32#base32hex} + */ + +var base32hex = { + alphabet: "0123456789ABCDEFGHIJKLMNOPQRSTUV", + charmap: {} +}; + +base32hex.charmap = charmap(base32hex.alphabet, base32hex.charmap); + +/** + * Create a new `Decoder` with the given options. + * + * @param {object} [options] + * @param {string} [type] Supported Base-32 variants are "rfc4648" and + * "crockford". + * @param {object} [charmap] Override the character map used in decoding. + * @constructor + */ + +function Decoder (options) { + this.buf = []; + this.shift = 8; + this.carry = 0; + + if (options) { + + switch (options.type) { + case "rfc4648": + this.charmap = exports.rfc4648.charmap; + break; + case "crockford": + this.charmap = exports.crockford.charmap; + break; + case "base32hex": + this.charmap = exports.base32hex.charmap; + break; + default: + throw new Error("invalid type"); + } + + if (options.charmap) this.charmap = options.charmap; + } +} + +/** + * The default character map coresponds to RFC4648. + */ + +Decoder.prototype.charmap = rfc4648.charmap; + +/** + * Decode a string, continuing from the previous state. + * + * @param {string} str + * @return {Decoder} this + */ + +Decoder.prototype.write = function (str) { + var charmap = this.charmap; + var buf = this.buf; + var shift = this.shift; + var carry = this.carry; + + // decode string + str.toUpperCase().split("").forEach(function (char) { + + // ignore padding + if (char == "=") return; + + // lookup symbol + var symbol = charmap[char] & 0xff; + + // 1: 00000 000 + // 2: 00 00000 0 + // 3: 0000 0000 + // 4: 0 00000 00 + // 5: 000 00000 + // 6: 00000 000 + // 7: 00 00000 0 + + shift -= 5; + if (shift > 0) { + carry |= symbol << shift; + } else if (shift < 0) { + buf.push(carry | (symbol >> -shift)); + shift += 8; + carry = (symbol << shift) & 0xff; + } else { + buf.push(carry | symbol); + shift = 8; + carry = 0; + } + }); + + // save state + this.shift = shift; + this.carry = carry; + + // for chaining + return this; +}; + +/** + * Finish decoding. + * + * @param {string} [str] The final string to decode. + * @return {Array} Decoded byte array. + */ + +Decoder.prototype.finalize = function (str) { + if (str) { + this.write(str); + } + if (this.shift !== 8 && this.carry !== 0) { + this.buf.push(this.carry); + this.shift = 8; + this.carry = 0; + } + return this.buf; +}; + +/** + * Create a new `Encoder` with the given options. + * + * @param {object} [options] + * @param {string} [type] Supported Base-32 variants are "rfc4648" and + * "crockford". + * @param {object} [alphabet] Override the alphabet used in encoding. + * @constructor + */ + +function Encoder (options) { + this.buf = ""; + this.shift = 3; + this.carry = 0; + + if (options) { + + switch (options.type) { + case "rfc4648": + this.alphabet = exports.rfc4648.alphabet; + break; + case "crockford": + this.alphabet = exports.crockford.alphabet; + break; + case "base32hex": + this.alphabet = exports.base32hex.alphabet; + break; + default: + throw new Error("invalid type"); + } + + if (options.alphabet) this.alphabet = options.alphabet; + else if (options.lc) this.alphabet = this.alphabet.toLowerCase(); + } +} + +/** + * The default alphabet coresponds to RFC4648. + */ + +Encoder.prototype.alphabet = rfc4648.alphabet; + +/** + * Encode a byte array, continuing from the previous state. + * + * @param {byte[]} buf The byte array to encode. + * @return {Encoder} this + */ + +Encoder.prototype.write = function (buf) { + var shift = this.shift; + var carry = this.carry; + var symbol; + var byte; + var i; + + // encode each byte in buf + for (i = 0; i < buf.length; i++) { + byte = buf[i]; + + // 1: 00000 000 + // 2: 00 00000 0 + // 3: 0000 0000 + // 4: 0 00000 00 + // 5: 000 00000 + // 6: 00000 000 + // 7: 00 00000 0 + + symbol = carry | (byte >> shift); + this.buf += this.alphabet[symbol & 0x1f]; + + if (shift > 5) { + shift -= 5; + symbol = byte >> shift; + this.buf += this.alphabet[symbol & 0x1f]; + } + + shift = 5 - shift; + carry = byte << shift; + shift = 8 - shift; + } + + // save state + this.shift = shift; + this.carry = carry; + + // for chaining + return this; +}; + +/** + * Finish encoding. + * + * @param {byte[]} [buf] The final byte array to encode. + * @return {string} The encoded byte array. + */ + +Encoder.prototype.finalize = function (buf) { + if (buf) { + this.write(buf); + } + if (this.shift !== 3) { + this.buf += this.alphabet[this.carry & 0x1f]; + this.shift = 3; + this.carry = 0; + } + return this.buf; +}; + +/** + * Convenience encoder. + * + * @param {byte[]} buf The byte array to encode. + * @param {object} [options] Options to pass to the encoder. + * @return {string} The encoded string. + */ + +exports.encode = function (buf, options) { + return new Encoder(options).finalize(buf); +}; + +/** + * Convenience decoder. + * + * @param {string} str The string to decode. + * @param {object} [options] Options to pass to the decoder. + * @return {byte[]} The decoded byte array. + */ + +exports.decode = function (str, options) { + return new Decoder(options).finalize(str); +}; + +// Exports. +exports.Decoder = Decoder; +exports.Encoder = Encoder; +exports.charmap = charmap; +exports.crockford = crockford; +exports.rfc4648 = rfc4648; +exports.base32hex = base32hex; diff --git a/migrations/migration-8/node_modules/base32.js/index.js b/migrations/migration-8/node_modules/base32.js/index.js new file mode 100644 index 0000000..e380c16 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/index.js @@ -0,0 +1,16 @@ +"use strict"; + +// Module dependencies. +var base32 = require("./base32"); + + +// Wrap decoder finalize to return a buffer; +var finalizeDecode = base32.Decoder.prototype.finalize; +base32.Decoder.prototype.finalize = function (buf) { + var bytes = finalizeDecode.call(this, buf); + return new Buffer(bytes); +}; + + +// Export Base32. +module.exports = base32; diff --git a/migrations/migration-8/node_modules/base32.js/jsdoc.json b/migrations/migration-8/node_modules/base32.js/jsdoc.json new file mode 100644 index 0000000..d576a70 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/jsdoc.json @@ -0,0 +1,33 @@ +{ + "source": { + "include": [ + "base32.js", + "package.json", + "README.md" + ] + }, + "plugins": ["plugins/markdown"], + "templates": { + "applicationName": "base32.js", + "meta": { + "title": "base32.js", + "description": "base32.js - Base 32 for JavaScript", + "keyword": [ + "base32", + "base32hex", + "crockford", + "rfc2938", + "rfc4648", + "encoding", + "decoding" + ] + }, + "default": { + "outputSourceFiles": true + }, + "linenums": true + }, + "opts": { + "destination": "docs" + } +} diff --git a/migrations/migration-8/node_modules/base32.js/karma.conf.js b/migrations/migration-8/node_modules/base32.js/karma.conf.js new file mode 100644 index 0000000..498312f --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/karma.conf.js @@ -0,0 +1,33 @@ +"use strict"; + +/** + * Karma configuration. + */ + +module.exports = function (config) { + config.set({ + + frameworks: ["mocha"], + + files: [ + "test/**_test.js" + ], + + preprocessors: { + "test/**_test.js": ["webpack"] + }, + + reporters: ["progress"], + + browsers: ["Chrome"], + + webpack: require("./webpack.config"), + + plugins: [ + "karma-chrome-launcher", + "karma-mocha", + "karma-webpack" + ] + + }); +}; diff --git a/migrations/migration-8/node_modules/base32.js/package.json b/migrations/migration-8/node_modules/base32.js/package.json new file mode 100644 index 0000000..682b055 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/package.json @@ -0,0 +1,73 @@ +{ + "_from": "base32.js@~0.1.0", + "_id": "base32.js@0.1.0", + "_inBundle": false, + "_integrity": "sha1-tYLexpPC8R6JPPBk7mrFthMaIgI=", + "_location": "/base32.js", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "base32.js@~0.1.0", + "name": "base32.js", + "escapedName": "base32.js", + "rawSpec": "~0.1.0", + "saveSpec": null, + "fetchSpec": "~0.1.0" + }, + "_requiredBy": [ + "/" + ], + "_resolved": "https://registry.npmjs.org/base32.js/-/base32.js-0.1.0.tgz", + "_shasum": "b582dec693c2f11e893cf064ee6ac5b6131a2202", + "_spec": "base32.js@~0.1.0", + "_where": "/Users/c5272397/dev/0_ipfs/js-ipfs-repo-migrations/migrations/migration-8", + "author": { + "name": "Michael Phan-Ba", + "email": "michael@mikepb.com" + }, + "browser": "base32.js", + "bugs": { + "url": "https://github.com/mikepb/base32.js/issues" + }, + "bundleDependencies": false, + "dependencies": {}, + "deprecated": false, + "description": "Base 32 encodings for JavaScript", + "devDependencies": { + "jsdoc": "*", + "karma": "*", + "karma-chrome-launcher": "*", + "karma-mocha": "*", + "karma-webpack": "*", + "mocha": "*", + "webpack": "*" + }, + "engines": { + "node": ">=0.12.0" + }, + "homepage": "https://github.com/mikepb/base32.js#readme", + "keywords": [ + "base32", + "base32hex", + "crockford", + "rfc2938", + "rfc4648", + "encoding", + "decoding" + ], + "license": "MIT", + "main": "index.js", + "name": "base32.js", + "repository": { + "type": "git", + "url": "git://github.com/mikepb/base32.js.git" + }, + "scripts": { + "dist": "webpack base32.js dist/base32.js && webpack --optimize-minimize base32.js dist/base32.min.js", + "doc": "jsdoc -c jsdoc.json", + "karma": "karma start --single-run", + "test": "mocha --reporter dot" + }, + "version": "0.1.0" +} diff --git a/migrations/migration-8/node_modules/base32.js/test/base32_test.js b/migrations/migration-8/node_modules/base32.js/test/base32_test.js new file mode 100644 index 0000000..0cfb33b --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/test/base32_test.js @@ -0,0 +1,87 @@ +"use strict"; + +var assert = require("assert"); +var base32 = require(".."); +var fixtures = require("./fixtures"); + +describe("Decoder", function () { + + fixtures.forEach(function (subject) { + var test = subject.buf; + + subject.rfc4648.forEach(function (str) { + it("should decode rfc4648 " + str, function () { + var decoder = new base32.Decoder({ type: "rfc4648" }); + var decoded = decoder.write(str).finalize(); + compare(decoded, test); + var s = new base32.Decoder().write(str).finalize(); + compare(s, test); + }); + }); + + subject.crock32.forEach(function (str) { + it("should decode crock32 " + str, function () { + var decoder = new base32.Decoder({ type: "crockford" }); + var decoded = decoder.write(str).finalize(); + compare(decoded, test); + }); + }); + + subject.base32hex.forEach(function (str) { + it("should decode base32hex " + str, function () { + var decoder = new base32.Decoder({ type: "base32hex" }); + var decoded = decoder.write(str).finalize(); + compare(decoded, test); + }); + }); + + }); + +}); + +describe("Encoder", function () { + + fixtures.forEach(function (subject) { + var buf = subject.buf; + + it("should encode rfc4648 " + buf, function () { + var test = subject.rfc4648[0]; + var encoder = new base32.Encoder({ type: "rfc4648" }); + var encode = encoder.write(buf).finalize(); + assert.equal(encode, test); + var s = new base32.Encoder().write(buf).finalize(); + assert.equal(s, test); + }); + + it("should encode crock32 " + buf, function () { + var test = subject.crock32[0]; + var encoder = new base32.Encoder({ type: "crockford" }); + var encoded = encoder.write(buf).finalize(); + assert.equal(encoded, test); + }); + + it("should encode crock32 " + buf + " with lower case", function () { + var test = subject.crock32[0]; + var encoder = new base32.Encoder({ type: "crockford", lc: true }); + var encoded = encoder.write(buf).finalize(); + assert.equal(encoded, test.toLowerCase()); + }); + + it("should encode base32hex " + buf + " with lower case", function () { + var test = subject.base32hex[0]; + var encoder = new base32.Encoder({ type: "base32hex", lc: true }); + var encoded = encoder.write(buf).finalize(); + assert.equal(encoded, test.toLowerCase()); + }); + + }); + +}); + +function compare (a, b) { + if (typeof Buffer != "undefined") { + b = new Buffer(b); + return assert.strictEqual(b.compare(a), 0); + } + assert.deepEqual(a, b); +} diff --git a/migrations/migration-8/node_modules/base32.js/test/fixtures.js b/migrations/migration-8/node_modules/base32.js/test/fixtures.js new file mode 100644 index 0000000..172630e --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/test/fixtures.js @@ -0,0 +1,294 @@ +"use strict"; + +module.exports = [ + { + buf: [0], + rfc4648: ["AA", "aa"], + crock32: ["00", "0O", "0o"], + crock32int: ["0", "O", "o"], + base32hex: ["00"] + }, + { + buf: [1], + rfc4648: ["AE"], + crock32: ["04"], + crock32int: ["1", "I", "i", "L", "l"], + base32hex: ["04"] + }, + { + buf: [2], + rfc4648: ["AI", "ai", "aI", "Ai"], + crock32: ["08"], + crock32int: ["2"], + base32hex: ["08"] + }, + { + buf: [3], + rfc4648: ["AM", "am", "aM", "Am"], + crock32: ["0C"], + crock32int: ["3"], + base32hex: ["0C"] + }, + { + buf: [4], + rfc4648: ["AQ", "aq", "aQ", "Aq"], + crock32: ["0G"], + crock32int: ["4"], + base32hex: ["0G"] + }, + { + buf: [5], + rfc4648: ["AU", "au", "aU", "Au"], + crock32: ["0M"], + crock32int: ["5"], + base32hex: ["0K"] + }, + { + buf: [6], + rfc4648: ["AY", "ay", "aY", "Ay"], + crock32: ["0R"], + crock32int: ["6"], + base32hex: ["0O"] + }, + { + buf: [7], + rfc4648: ["A4", "a4"], + crock32: ["0W"], + crock32int: ["7"], + base32hex: ["0S"] + }, + { + buf: [8], + rfc4648: ["BA", "ba", "bA", "Ba"], + crock32: ["10"], + crock32int: ["8"], + base32hex: ["10"] + }, + { + buf: [9], + rfc4648: ["BE", "be", "bE", "Be"], + crock32: ["14"], + crock32int: ["9"], + base32hex: ["14"] + }, + { + buf: [10], + rfc4648: ["BI", "bi", "bI", "Bi"], + crock32: ["18"], + crock32int: ["A", "a"], + base32hex: ["18"] + }, + { + buf: [11], + rfc4648: ["BM", "bm", "bM", "Bm"], + crock32: ["1C"], + crock32int: ["B", "b"], + base32hex: ["1C"] + }, + { + buf: [12], + rfc4648: ["BQ", "bq", "bQ", "Bq"], + crock32: ["1G"], + crock32int: ["C", "c"], + base32hex: ["1G"] + }, + { + buf: [13], + rfc4648: ["BU", "bu", "bU", "Bu"], + crock32: ["1M"], + crock32int: ["D", "d"], + base32hex: ["1K"] + }, + { + buf: [14], + rfc4648: ["BY", "by", "bY", "By"], + crock32: ["1R"], + crock32int: ["E", "e"], + base32hex: ["1O"] + }, + { + buf: [15], + rfc4648: ["B4", "b4"], + crock32: ["1W"], + crock32int: ["F", "f"], + base32hex: ["1S"] + }, + { + buf: [16], + rfc4648: ["CA", "ca", "cA", "Ca"], + crock32: ["20"], + crock32int: ["G", "g"], + base32hex: ["20"] + }, + { + buf: [17], + rfc4648: ["CE", "ce", "cE", "Ce"], + crock32: ["24"], + crock32int: ["H", "h"], + base32hex: ["24"] + }, + { + buf: [18], + rfc4648: ["CI", "ci", "cI", "Ci"], + crock32: ["28"], + crock32int: ["J", "j"], + base32hex: ["28"] + }, + { + buf: [19], + rfc4648: ["CM", "cm", "cM", "Cm"], + crock32: ["2C"], + crock32int: ["K", "k"], + base32hex: ["2C"] + }, + { + buf: [20], + rfc4648: ["CQ", "cq", "cQ", "Cq"], + crock32: ["2G"], + crock32int: ["M", "m"], + base32hex: ["2G"] + }, + { + buf: [21], + rfc4648: ["CU", "cu", "cU", "Cu"], + crock32: ["2M"], + crock32int: ["N", "n"], + base32hex: ["2K"] + }, + { + buf: [22], + rfc4648: ["CY", "cy", "cY", "Cy"], + crock32: ["2R"], + crock32int: ["P", "p"], + base32hex: ["2O"] + }, + { + buf: [23], + rfc4648: ["C4", "c4"], + crock32: ["2W"], + crock32int: ["Q", "q"], + base32hex: ["2S"] + }, + { + buf: [24], + rfc4648: ["DA", "da", "dA", "Da"], + crock32: ["30"], + crock32int: ["R", "r"], + base32hex: ["30"] + }, + { + buf: [25], + rfc4648: ["DE", "de", "dE", "De"], + crock32: ["34"], + crock32int: ["S", "s"], + base32hex: ["34"] + }, + { + buf: [26], + rfc4648: ["DI", "di", "dI", "Di"], + crock32: ["38"], + crock32int: ["T", "t"], + base32hex: ["38"] + }, + { + buf: [27], + rfc4648: ["DM", "dm", "dM", "Dm"], + crock32: ["3C"], + crock32int: ["V", "v"], + base32hex: ["3C"] + }, + { + buf: [28], + rfc4648: ["DQ", "dq", "dQ", "Dq"], + crock32: ["3G"], + crock32int: ["W", "w"], + base32hex: ["3G"] + }, + { + buf: [29], + rfc4648: ["DU", "du", "dU", "Du"], + crock32: ["3M"], + crock32int: ["X", "x"], + base32hex: ["3k"] + }, + { + buf: [30], + rfc4648: ["DY", "dy", "dY", "Dy"], + crock32: ["3R"], + crock32int: ["Y", "y"], + base32hex: ["3O"] + }, + { + buf: [31], + rfc4648: ["D4", "d4"], + crock32: ["3W"], + crock32int: ["Z", "z"], + base32hex: ["3S"] + }, + { + buf: [0, 0], + rfc4648: ["AAAA", "aaaa", "AaAa", "aAAa"], + crock32: ["0000", "oooo", "OOOO", "0oO0"], + base32hex: ["0000"] + }, + { + buf: [1, 0], + rfc4648: ["AEAA", "aeaa", "AeAa", "aEAa"], + crock32: ["0400", "o4oo", "O4OO", "04oO"], + base32hex: ["0400"] + }, + { + buf: [0, 1], + rfc4648: ["AAAQ", "aaaq", "AaAQ", "aAAq"], + crock32: ["000G", "ooog", "OOOG", "0oOg"], + base32hex: ["000G"] + }, + { + buf: [1, 1], + rfc4648: ["AEAQ", "aeaq", "AeAQ", "aEAq"], + crock32: ["040G", "o4og", "O4og", "04Og"], + base32hex: ["040G"] + }, + { + buf: [136, 64], + rfc4648: ["RBAA", "rbaa", "RbAA", "rBAa"], + crock32: ["H100", "hio0", "HLOo"], + base32hex: ["H100"] + }, + { + buf: [139, 188], + rfc4648: ["RO6A", "r06a", "Ro6A", "r06A"], + crock32: ["HEY0", "heyo", "HeYO"], + base32hex: ["HEU0"] + }, + { + buf: [54, 31, 127], + rfc4648: ["GYPX6", "gypx6"], + crock32: ["6RFQY", "6rfqy"], + base32hex: ["6OFNU"] + }, + { + buf: [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33], + rfc4648: ["JBSWY3DPEBLW64TMMQQQ", "jbswy3dpeblw64tmmqqq"], + crock32: ["91JPRV3F41BPYWKCCGGG", "91jprv3f41bpywkccggg", "9Ljprv3f4ibpywkccggg"], + base32hex: ["91IMOR3F41BMUSJCCGGG"] + }, + { + buf: [139, 130, 16, 112, 24, 11, 64], + rfc4648: ["ROBBA4AYBNAA", "robba4aybnaa", "R0BBA4aybnaa"], + crock32: ["HE110W0R1D00", "helloworld00", "heiiOw0RidoO"], + base32hex: ["HE110S0O1D00"] + }, + { + buf: [139, 130, 16, 112, 24, 11], + rfc4648: ["ROBBA4AYBM", "robba4aybm", "R0BBA4aybm"], + crock32: ["HE110W0R1C", "helloworlc", "heiiOw0RiC"], + base32hex: ["HE110S0O1C"] + }, + { + buf: [139, 130, 16, 112, 24, 11, 0], + rfc4648: ["ROBBA4AYBMAA", "robba4aybmaa", "R0BBA4aybmaa"], + crock32: ["HE110W0R1C00", "helloworlc00", "heiiOw0RiC00"], + base32hex: ["HE110S0O1C00"] + } +]; diff --git a/migrations/migration-8/node_modules/base32.js/webpack.config.js b/migrations/migration-8/node_modules/base32.js/webpack.config.js new file mode 100644 index 0000000..c7bc926 --- /dev/null +++ b/migrations/migration-8/node_modules/base32.js/webpack.config.js @@ -0,0 +1,17 @@ +"use strict"; + +/** + * Webpack configuration. + */ + +exports = module.exports = { + output: { + library: "base32", + libraryTarget: "this", + sourcePrefix: "" + }, + devtool: "source-map", + node: { + Buffer: false + } +}; diff --git a/migrations/migration-8/package-lock.json b/migrations/migration-8/package-lock.json new file mode 100644 index 0000000..c37f9be --- /dev/null +++ b/migrations/migration-8/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "ipfs-repo-migration-8", + "version": "0.1.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "base32.js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base32.js/-/base32.js-0.1.0.tgz", + "integrity": "sha1-tYLexpPC8R6JPPBk7mrFthMaIgI=" + } + } +} diff --git a/migrations/migration-8/package.json b/migrations/migration-8/package.json new file mode 100644 index 0000000..f283e31 --- /dev/null +++ b/migrations/migration-8/package.json @@ -0,0 +1,19 @@ +{ + "name": "ipfs-repo-migration-8", + "version": "0.1.0", + "description": "Migration", + "author": "Adam Uhlir ", + "main": "index.js", + "files": [ + "index.js", + "node_modules" + ], + "engines": { + "node": ">=10.0.0", + "npm": ">=3.0.0" + }, + "dependencies": { + "base32.js": "~0.1.0" + }, + "license": "MIT" +} From e73704c58001c6f2f08e2d153e88a08dfeb283da Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Sat, 27 Apr 2019 10:01:26 -0700 Subject: [PATCH 02/12] Migration 8 tests License: MIT Signed-off-by: Adam Uhlir --- migrations/index.js | 1 + migrations/migration-8/index.js | 2 +- test/migrations/migration-8-test.js | 82 +++++++++++++++++++++++++++++ test/node.js | 4 ++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 test/migrations/migration-8-test.js diff --git a/migrations/index.js b/migrations/index.js index 17f5b82..d2b3b0e 100644 --- a/migrations/index.js +++ b/migrations/index.js @@ -11,4 +11,5 @@ const emptyMigration = { } module.exports = [ + require('./migration-8') ] diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index c4eb6f7..f154684 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -4,7 +4,7 @@ const Datastore = require('datastore-fs') const path = require('path') const base32 = require('base32.js') const Key = require('interface-datastore').Key -const log = require('debug')('jsipfs-repo-migrations:migration-8') +const log = require('debug')('ipfs-repo-migrations:migration-8') const KEY_PREFIX = 'key_' diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js new file mode 100644 index 0000000..1421878 --- /dev/null +++ b/test/migrations/migration-8-test.js @@ -0,0 +1,82 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const chaiAsPromised = require("chai-as-promised") +chai.use(chaiAsPromised) +const expect = chai.expect + +const path = require('path') +const migration = require('../../migrations/migration-8') +const Key = require('interface-datastore').Key +const Datastore = require('datastore-fs') + +const log = require('debug')('js-ipfs-repo-migrations:migration-8') + +const fixtures = [ + ['aAa', 'key_mfawc'], + ['bbb', 'key_mjrge'], + ['self', 'key_onswyzq'] +] + +async function bootstrapKeys (dir, encoded) { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) + await store.open() + + let name + for (let keyNames of fixtures) { + name = encoded ? keyNames[1] : keyNames[0] + await store.put(new Key(`/pkcs8/${name}`), '') + await store.put(new Key(`/info/${name}`), '') + } + + await store.close() +} + +async function validateKeys (dir, shouldBeEncoded) { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: false }) + await store.open() + + let name + for (let keyNames of fixtures) { + name = shouldBeEncoded ? keyNames[1] : keyNames[0] + expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true() + expect(await store.has(new Key(`/info/${name}`))).to.be.true() + } + + await store.close() +} + +module.exports = (setup, cleanup) => { + let dir + + beforeEach(async () => { + dir = await setup() + }) + afterEach(() => cleanup(dir)) + + it('should migrate forward', async () => { + await bootstrapKeys(dir, false) + await migration.migrate(dir) + await validateKeys(dir, true) + }) + + it('should migrate backward', async () => { + await bootstrapKeys(dir, true) + await migration.revert(dir) + await validateKeys(dir, false) + }) + + it('should fail to migrate backward with invalid key name', async () => { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) + await store.open() + + await store.put(new Key('/pkcs8/mfawc'), '') + await store.put(new Key('/info/mfawc'), '') + + await store.close() + + expect(migration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + }) +} diff --git a/test/node.js b/test/node.js index ca0ffad..194569e 100644 --- a/test/node.js +++ b/test/node.js @@ -43,6 +43,10 @@ describe('Node specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(repoSetup, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) }) From 01293a3cdabcae8b702a43480d408d1b3bebd09e Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Tue, 30 Apr 2019 18:04:56 -0700 Subject: [PATCH 03/12] Adding correct option recognition License: MIT Signed-off-by: Adam Uhlir --- migrations/migration-8/index.js | 43 +++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index f154684..c7ab9d1 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -8,25 +8,25 @@ const log = require('debug')('ipfs-repo-migrations:migration-8') const KEY_PREFIX = 'key_' -function encode(name) { +function encode (name) { name = Buffer.from(name) - const encoder = new base32.Encoder({type: "rfc4648"}); + const encoder = new base32.Encoder({ type: "rfc4648" }) return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() } -function decode(name) { +function decode (name) { log(name) if (!name.startsWith(KEY_PREFIX)) { throw Error("Unknown format of key's name!") } - const decoder = new base32.Decoder({type: "rfc4648"}); + const decoder = new base32.Decoder({ type: "rfc4648" }) return decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) } -async function processFolder(store, prefix, fileNameProcessor) { +async function processFolder (store, prefix, fileNameProcessor) { const query = { - prefix: `/${prefix}`, + prefix: `/${prefix}` } const files = await store.query(query) @@ -41,9 +41,18 @@ async function processFolder(store, prefix, fileNameProcessor) { } } -async function migrate(repoPath, isBrowser) { - const store = new Datastore(path.join(repoPath, 'keys'), {extension: '.data', createIfMissing: false}) - store.open() +async function migrate (repoPath, options, isBrowser) { + let storageBackend + if (options !== undefined + && options['storageBackends'] !== undefined + && options['storageBackends']['keys'] !== undefined + ) { + storageBackend = options['storageBackends']['keys'] + } else { + storageBackend = Datastore + } + + const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) try { const info = processFolder(store, 'info', encode) const data = processFolder(store, 'pkcs8', encode) @@ -54,10 +63,18 @@ async function migrate(repoPath, isBrowser) { } } -async function revert(repoPath, isBrowser) { - const store = new Datastore(path.join(repoPath, 'keys'), {extension: '.data', createIfMissing: false}) - store.open() +async function revert (repoPath, options, isBrowser) { +let storageBackend + if (options !== undefined + && options['storageBackends'] !== undefined + && options['storageBackends']['keys'] !== undefined + ) { + storageBackend = options['storageBackends']['keys'] + } else { + storageBackend = Datastore + } + const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) try { const info = processFolder(store, 'info', decode) const data = processFolder(store, 'pkcs8', decode) @@ -73,5 +90,5 @@ module.exports = { description: 'Transforms key\'s names into base32 encoding.', reversible: true, migrate, - revert, + revert } From 8f1e7c14f9b77b4f306a704604e5e472c38297bc Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Mon, 1 Jul 2019 14:30:17 +0200 Subject: [PATCH 04/12] Add usage of storageBackendOptions for correct datastore instantiation --- migrations/migration-8/index.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index c7ab9d1..cd7ec49 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -29,7 +29,7 @@ async function processFolder (store, prefix, fileNameProcessor) { prefix: `/${prefix}` } - const files = await store.query(query) + const files = store.query(query) for await (let file of files) { const name = String(file.key._buf).replace(`/${prefix}/`, '') const encodedFileName = fileNameProcessor(name) @@ -42,7 +42,7 @@ async function processFolder (store, prefix, fileNameProcessor) { } async function migrate (repoPath, options, isBrowser) { - let storageBackend + let storageBackend, storageBackendOptions if (options !== undefined && options['storageBackends'] !== undefined && options['storageBackends']['keys'] !== undefined @@ -52,7 +52,16 @@ async function migrate (repoPath, options, isBrowser) { storageBackend = Datastore } - const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) + if (options !== undefined + && options['storageBackendOptions'] !== undefined + && options['storageBackendOptions']['keys'] !== undefined + ) { + storageBackendOptions = options['storageBackendOptions']['keys'] + } else { + storageBackendOptions = {} + } + + const store = new storageBackend(path.join(repoPath, 'keys'), storageBackendOptions) try { const info = processFolder(store, 'info', encode) const data = processFolder(store, 'pkcs8', encode) @@ -71,7 +80,7 @@ let storageBackend ) { storageBackend = options['storageBackends']['keys'] } else { - storageBackend = Datastore + storageBackend = FSDatastore } const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) From b01e7efddff332a9b01c909c5a837db788ae49ba Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Mon, 1 Jul 2019 15:39:39 +0200 Subject: [PATCH 05/12] Dropping migration's package.json and specify dependency in root --- .../node_modules/base32.js/.npmignore | 4 - .../node_modules/base32.js/.travis.yml | 4 - .../node_modules/base32.js/HISTORY.md | 4 - .../node_modules/base32.js/README.md | 71 ---- .../node_modules/base32.js/base32.js | 312 ------------------ .../node_modules/base32.js/index.js | 16 - .../node_modules/base32.js/jsdoc.json | 33 -- .../node_modules/base32.js/karma.conf.js | 33 -- .../node_modules/base32.js/package.json | 73 ---- .../base32.js/test/base32_test.js | 87 ----- .../node_modules/base32.js/test/fixtures.js | 294 ----------------- .../node_modules/base32.js/webpack.config.js | 17 - migrations/migration-8/package-lock.json | 13 - migrations/migration-8/package.json | 19 -- package.json | 3 +- 15 files changed, 2 insertions(+), 981 deletions(-) delete mode 100644 migrations/migration-8/node_modules/base32.js/.npmignore delete mode 100644 migrations/migration-8/node_modules/base32.js/.travis.yml delete mode 100644 migrations/migration-8/node_modules/base32.js/HISTORY.md delete mode 100644 migrations/migration-8/node_modules/base32.js/README.md delete mode 100644 migrations/migration-8/node_modules/base32.js/base32.js delete mode 100644 migrations/migration-8/node_modules/base32.js/index.js delete mode 100644 migrations/migration-8/node_modules/base32.js/jsdoc.json delete mode 100644 migrations/migration-8/node_modules/base32.js/karma.conf.js delete mode 100644 migrations/migration-8/node_modules/base32.js/package.json delete mode 100644 migrations/migration-8/node_modules/base32.js/test/base32_test.js delete mode 100644 migrations/migration-8/node_modules/base32.js/test/fixtures.js delete mode 100644 migrations/migration-8/node_modules/base32.js/webpack.config.js delete mode 100644 migrations/migration-8/package-lock.json delete mode 100644 migrations/migration-8/package.json diff --git a/migrations/migration-8/node_modules/base32.js/.npmignore b/migrations/migration-8/node_modules/base32.js/.npmignore deleted file mode 100644 index 0397291..0000000 --- a/migrations/migration-8/node_modules/base32.js/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -bower_components -node_modules -public -webpack.stats.json diff --git a/migrations/migration-8/node_modules/base32.js/.travis.yml b/migrations/migration-8/node_modules/base32.js/.travis.yml deleted file mode 100644 index b127718..0000000 --- a/migrations/migration-8/node_modules/base32.js/.travis.yml +++ /dev/null @@ -1,4 +0,0 @@ -language: node_js -node_js: - - 0.12 - - iojs diff --git a/migrations/migration-8/node_modules/base32.js/HISTORY.md b/migrations/migration-8/node_modules/base32.js/HISTORY.md deleted file mode 100644 index aca8598..0000000 --- a/migrations/migration-8/node_modules/base32.js/HISTORY.md +++ /dev/null @@ -1,4 +0,0 @@ -0.0.1 / 2015-02-15 -================== - - * Initial release diff --git a/migrations/migration-8/node_modules/base32.js/README.md b/migrations/migration-8/node_modules/base32.js/README.md deleted file mode 100644 index 8db86b5..0000000 --- a/migrations/migration-8/node_modules/base32.js/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Base 32 for JavaScript [![Build Status](https://travis-ci.org/mikepb/base32.js.svg)](http://travis-ci.org/mikepb/base32.js) - -[Wikipedia](https://en.wikipedia.org/wiki/Base32): - -> Base32 is a base 32 transfer encoding using the twenty-six letters A–Z and six digits 2–7. It is primarily used to encode binary data, but is able to encode binary text like ASCII. -> -> Base32 has number of advantages over Base64: -> -> 1. The resulting character set is all one case (usually represented as uppercase), which can often be beneficial when using a case-insensitive filesystem, spoken language, or human memory. -> -> 2. The result can be used as file name because it can not possibly contain '/' symbol which is usually acts as path separator in Unix-based operating systems. -> -> 3. The alphabet was selected to avoid similar-looking pairs of different symbols, so the strings can be accurately transcribed by hand. (For example, the symbol set omits the symbols for 1, 8 and zero, since they could be confused with the letters 'I', 'B', and 'O'.) -> -> 4. A result without padding can be included in a URL without encoding any characters. -> -> However, Base32 representation takes roughly 20% more space than Base64. - -## Documentation - -Full documentation at http://mikepb.github.io/base32.js/ - -## Installation - -```sh -$ npm install base32.js -``` - -## Usage - -Encoding an array of bytes using [Crockford][crock32]: - -```js -var base32 = require("base32.js"); - -var buf = [1, 2, 3, 4]; -var encoder = new base32.Encoder({ type: "crockford", lc: true }); -var str = encoder.write(buf).finalize(); -// str = "04106173" - -var decoder = new base32.Decoder({ type: "crockford" }); -var out = decoder.write(str).finalize(); -// out = [1, 2, 3, 4] -``` - -The default Base32 variant if no `type` is provided is `"rfc4648"` without -padding. - -## Browser support - -The browser versions of the library may be found under the `dist/` directory. -The browser files are updated on each versioned release, but not for -development. [Karma][karma] is used to run the [mocha][] tests in the browser. - -```sh -$ npm install -g karma-cli -$ npm run karma -``` - -## Related projects - -- [agnoster/base32-js][agnoster] - -## License - -MIT - -[agnoster]: https://github.com/agnoster/base32-js -[crock32]: http://www.crockford.com/wrmg/base32.html -[karma]: http://karma-runner.github.io -[mocha]: http://mochajs.org diff --git a/migrations/migration-8/node_modules/base32.js/base32.js b/migrations/migration-8/node_modules/base32.js/base32.js deleted file mode 100644 index e64f896..0000000 --- a/migrations/migration-8/node_modules/base32.js/base32.js +++ /dev/null @@ -1,312 +0,0 @@ -"use strict"; - -/** - * Generate a character map. - * @param {string} alphabet e.g. "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" - * @param {object} mappings map overrides from key to value - * @method - */ - -var charmap = function (alphabet, mappings) { - mappings || (mappings = {}); - alphabet.split("").forEach(function (c, i) { - if (!(c in mappings)) mappings[c] = i; - }); - return mappings; -} - -/** - * The RFC 4648 base 32 alphabet and character map. - * @see {@link https://tools.ietf.org/html/rfc4648} - */ - -var rfc4648 = { - alphabet: "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", - charmap: { - 0: 14, - 1: 8 - } -}; - -rfc4648.charmap = charmap(rfc4648.alphabet, rfc4648.charmap); - -/** - * The Crockford base 32 alphabet and character map. - * @see {@link http://www.crockford.com/wrmg/base32.html} - */ - -var crockford = { - alphabet: "0123456789ABCDEFGHJKMNPQRSTVWXYZ", - charmap: { - O: 0, - I: 1, - L: 1 - } -}; - -crockford.charmap = charmap(crockford.alphabet, crockford.charmap); - -/** - * base32hex - * @see {@link https://en.wikipedia.org/wiki/Base32#base32hex} - */ - -var base32hex = { - alphabet: "0123456789ABCDEFGHIJKLMNOPQRSTUV", - charmap: {} -}; - -base32hex.charmap = charmap(base32hex.alphabet, base32hex.charmap); - -/** - * Create a new `Decoder` with the given options. - * - * @param {object} [options] - * @param {string} [type] Supported Base-32 variants are "rfc4648" and - * "crockford". - * @param {object} [charmap] Override the character map used in decoding. - * @constructor - */ - -function Decoder (options) { - this.buf = []; - this.shift = 8; - this.carry = 0; - - if (options) { - - switch (options.type) { - case "rfc4648": - this.charmap = exports.rfc4648.charmap; - break; - case "crockford": - this.charmap = exports.crockford.charmap; - break; - case "base32hex": - this.charmap = exports.base32hex.charmap; - break; - default: - throw new Error("invalid type"); - } - - if (options.charmap) this.charmap = options.charmap; - } -} - -/** - * The default character map coresponds to RFC4648. - */ - -Decoder.prototype.charmap = rfc4648.charmap; - -/** - * Decode a string, continuing from the previous state. - * - * @param {string} str - * @return {Decoder} this - */ - -Decoder.prototype.write = function (str) { - var charmap = this.charmap; - var buf = this.buf; - var shift = this.shift; - var carry = this.carry; - - // decode string - str.toUpperCase().split("").forEach(function (char) { - - // ignore padding - if (char == "=") return; - - // lookup symbol - var symbol = charmap[char] & 0xff; - - // 1: 00000 000 - // 2: 00 00000 0 - // 3: 0000 0000 - // 4: 0 00000 00 - // 5: 000 00000 - // 6: 00000 000 - // 7: 00 00000 0 - - shift -= 5; - if (shift > 0) { - carry |= symbol << shift; - } else if (shift < 0) { - buf.push(carry | (symbol >> -shift)); - shift += 8; - carry = (symbol << shift) & 0xff; - } else { - buf.push(carry | symbol); - shift = 8; - carry = 0; - } - }); - - // save state - this.shift = shift; - this.carry = carry; - - // for chaining - return this; -}; - -/** - * Finish decoding. - * - * @param {string} [str] The final string to decode. - * @return {Array} Decoded byte array. - */ - -Decoder.prototype.finalize = function (str) { - if (str) { - this.write(str); - } - if (this.shift !== 8 && this.carry !== 0) { - this.buf.push(this.carry); - this.shift = 8; - this.carry = 0; - } - return this.buf; -}; - -/** - * Create a new `Encoder` with the given options. - * - * @param {object} [options] - * @param {string} [type] Supported Base-32 variants are "rfc4648" and - * "crockford". - * @param {object} [alphabet] Override the alphabet used in encoding. - * @constructor - */ - -function Encoder (options) { - this.buf = ""; - this.shift = 3; - this.carry = 0; - - if (options) { - - switch (options.type) { - case "rfc4648": - this.alphabet = exports.rfc4648.alphabet; - break; - case "crockford": - this.alphabet = exports.crockford.alphabet; - break; - case "base32hex": - this.alphabet = exports.base32hex.alphabet; - break; - default: - throw new Error("invalid type"); - } - - if (options.alphabet) this.alphabet = options.alphabet; - else if (options.lc) this.alphabet = this.alphabet.toLowerCase(); - } -} - -/** - * The default alphabet coresponds to RFC4648. - */ - -Encoder.prototype.alphabet = rfc4648.alphabet; - -/** - * Encode a byte array, continuing from the previous state. - * - * @param {byte[]} buf The byte array to encode. - * @return {Encoder} this - */ - -Encoder.prototype.write = function (buf) { - var shift = this.shift; - var carry = this.carry; - var symbol; - var byte; - var i; - - // encode each byte in buf - for (i = 0; i < buf.length; i++) { - byte = buf[i]; - - // 1: 00000 000 - // 2: 00 00000 0 - // 3: 0000 0000 - // 4: 0 00000 00 - // 5: 000 00000 - // 6: 00000 000 - // 7: 00 00000 0 - - symbol = carry | (byte >> shift); - this.buf += this.alphabet[symbol & 0x1f]; - - if (shift > 5) { - shift -= 5; - symbol = byte >> shift; - this.buf += this.alphabet[symbol & 0x1f]; - } - - shift = 5 - shift; - carry = byte << shift; - shift = 8 - shift; - } - - // save state - this.shift = shift; - this.carry = carry; - - // for chaining - return this; -}; - -/** - * Finish encoding. - * - * @param {byte[]} [buf] The final byte array to encode. - * @return {string} The encoded byte array. - */ - -Encoder.prototype.finalize = function (buf) { - if (buf) { - this.write(buf); - } - if (this.shift !== 3) { - this.buf += this.alphabet[this.carry & 0x1f]; - this.shift = 3; - this.carry = 0; - } - return this.buf; -}; - -/** - * Convenience encoder. - * - * @param {byte[]} buf The byte array to encode. - * @param {object} [options] Options to pass to the encoder. - * @return {string} The encoded string. - */ - -exports.encode = function (buf, options) { - return new Encoder(options).finalize(buf); -}; - -/** - * Convenience decoder. - * - * @param {string} str The string to decode. - * @param {object} [options] Options to pass to the decoder. - * @return {byte[]} The decoded byte array. - */ - -exports.decode = function (str, options) { - return new Decoder(options).finalize(str); -}; - -// Exports. -exports.Decoder = Decoder; -exports.Encoder = Encoder; -exports.charmap = charmap; -exports.crockford = crockford; -exports.rfc4648 = rfc4648; -exports.base32hex = base32hex; diff --git a/migrations/migration-8/node_modules/base32.js/index.js b/migrations/migration-8/node_modules/base32.js/index.js deleted file mode 100644 index e380c16..0000000 --- a/migrations/migration-8/node_modules/base32.js/index.js +++ /dev/null @@ -1,16 +0,0 @@ -"use strict"; - -// Module dependencies. -var base32 = require("./base32"); - - -// Wrap decoder finalize to return a buffer; -var finalizeDecode = base32.Decoder.prototype.finalize; -base32.Decoder.prototype.finalize = function (buf) { - var bytes = finalizeDecode.call(this, buf); - return new Buffer(bytes); -}; - - -// Export Base32. -module.exports = base32; diff --git a/migrations/migration-8/node_modules/base32.js/jsdoc.json b/migrations/migration-8/node_modules/base32.js/jsdoc.json deleted file mode 100644 index d576a70..0000000 --- a/migrations/migration-8/node_modules/base32.js/jsdoc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "source": { - "include": [ - "base32.js", - "package.json", - "README.md" - ] - }, - "plugins": ["plugins/markdown"], - "templates": { - "applicationName": "base32.js", - "meta": { - "title": "base32.js", - "description": "base32.js - Base 32 for JavaScript", - "keyword": [ - "base32", - "base32hex", - "crockford", - "rfc2938", - "rfc4648", - "encoding", - "decoding" - ] - }, - "default": { - "outputSourceFiles": true - }, - "linenums": true - }, - "opts": { - "destination": "docs" - } -} diff --git a/migrations/migration-8/node_modules/base32.js/karma.conf.js b/migrations/migration-8/node_modules/base32.js/karma.conf.js deleted file mode 100644 index 498312f..0000000 --- a/migrations/migration-8/node_modules/base32.js/karma.conf.js +++ /dev/null @@ -1,33 +0,0 @@ -"use strict"; - -/** - * Karma configuration. - */ - -module.exports = function (config) { - config.set({ - - frameworks: ["mocha"], - - files: [ - "test/**_test.js" - ], - - preprocessors: { - "test/**_test.js": ["webpack"] - }, - - reporters: ["progress"], - - browsers: ["Chrome"], - - webpack: require("./webpack.config"), - - plugins: [ - "karma-chrome-launcher", - "karma-mocha", - "karma-webpack" - ] - - }); -}; diff --git a/migrations/migration-8/node_modules/base32.js/package.json b/migrations/migration-8/node_modules/base32.js/package.json deleted file mode 100644 index 682b055..0000000 --- a/migrations/migration-8/node_modules/base32.js/package.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "_from": "base32.js@~0.1.0", - "_id": "base32.js@0.1.0", - "_inBundle": false, - "_integrity": "sha1-tYLexpPC8R6JPPBk7mrFthMaIgI=", - "_location": "/base32.js", - "_phantomChildren": {}, - "_requested": { - "type": "range", - "registry": true, - "raw": "base32.js@~0.1.0", - "name": "base32.js", - "escapedName": "base32.js", - "rawSpec": "~0.1.0", - "saveSpec": null, - "fetchSpec": "~0.1.0" - }, - "_requiredBy": [ - "/" - ], - "_resolved": "https://registry.npmjs.org/base32.js/-/base32.js-0.1.0.tgz", - "_shasum": "b582dec693c2f11e893cf064ee6ac5b6131a2202", - "_spec": "base32.js@~0.1.0", - "_where": "/Users/c5272397/dev/0_ipfs/js-ipfs-repo-migrations/migrations/migration-8", - "author": { - "name": "Michael Phan-Ba", - "email": "michael@mikepb.com" - }, - "browser": "base32.js", - "bugs": { - "url": "https://github.com/mikepb/base32.js/issues" - }, - "bundleDependencies": false, - "dependencies": {}, - "deprecated": false, - "description": "Base 32 encodings for JavaScript", - "devDependencies": { - "jsdoc": "*", - "karma": "*", - "karma-chrome-launcher": "*", - "karma-mocha": "*", - "karma-webpack": "*", - "mocha": "*", - "webpack": "*" - }, - "engines": { - "node": ">=0.12.0" - }, - "homepage": "https://github.com/mikepb/base32.js#readme", - "keywords": [ - "base32", - "base32hex", - "crockford", - "rfc2938", - "rfc4648", - "encoding", - "decoding" - ], - "license": "MIT", - "main": "index.js", - "name": "base32.js", - "repository": { - "type": "git", - "url": "git://github.com/mikepb/base32.js.git" - }, - "scripts": { - "dist": "webpack base32.js dist/base32.js && webpack --optimize-minimize base32.js dist/base32.min.js", - "doc": "jsdoc -c jsdoc.json", - "karma": "karma start --single-run", - "test": "mocha --reporter dot" - }, - "version": "0.1.0" -} diff --git a/migrations/migration-8/node_modules/base32.js/test/base32_test.js b/migrations/migration-8/node_modules/base32.js/test/base32_test.js deleted file mode 100644 index 0cfb33b..0000000 --- a/migrations/migration-8/node_modules/base32.js/test/base32_test.js +++ /dev/null @@ -1,87 +0,0 @@ -"use strict"; - -var assert = require("assert"); -var base32 = require(".."); -var fixtures = require("./fixtures"); - -describe("Decoder", function () { - - fixtures.forEach(function (subject) { - var test = subject.buf; - - subject.rfc4648.forEach(function (str) { - it("should decode rfc4648 " + str, function () { - var decoder = new base32.Decoder({ type: "rfc4648" }); - var decoded = decoder.write(str).finalize(); - compare(decoded, test); - var s = new base32.Decoder().write(str).finalize(); - compare(s, test); - }); - }); - - subject.crock32.forEach(function (str) { - it("should decode crock32 " + str, function () { - var decoder = new base32.Decoder({ type: "crockford" }); - var decoded = decoder.write(str).finalize(); - compare(decoded, test); - }); - }); - - subject.base32hex.forEach(function (str) { - it("should decode base32hex " + str, function () { - var decoder = new base32.Decoder({ type: "base32hex" }); - var decoded = decoder.write(str).finalize(); - compare(decoded, test); - }); - }); - - }); - -}); - -describe("Encoder", function () { - - fixtures.forEach(function (subject) { - var buf = subject.buf; - - it("should encode rfc4648 " + buf, function () { - var test = subject.rfc4648[0]; - var encoder = new base32.Encoder({ type: "rfc4648" }); - var encode = encoder.write(buf).finalize(); - assert.equal(encode, test); - var s = new base32.Encoder().write(buf).finalize(); - assert.equal(s, test); - }); - - it("should encode crock32 " + buf, function () { - var test = subject.crock32[0]; - var encoder = new base32.Encoder({ type: "crockford" }); - var encoded = encoder.write(buf).finalize(); - assert.equal(encoded, test); - }); - - it("should encode crock32 " + buf + " with lower case", function () { - var test = subject.crock32[0]; - var encoder = new base32.Encoder({ type: "crockford", lc: true }); - var encoded = encoder.write(buf).finalize(); - assert.equal(encoded, test.toLowerCase()); - }); - - it("should encode base32hex " + buf + " with lower case", function () { - var test = subject.base32hex[0]; - var encoder = new base32.Encoder({ type: "base32hex", lc: true }); - var encoded = encoder.write(buf).finalize(); - assert.equal(encoded, test.toLowerCase()); - }); - - }); - -}); - -function compare (a, b) { - if (typeof Buffer != "undefined") { - b = new Buffer(b); - return assert.strictEqual(b.compare(a), 0); - } - assert.deepEqual(a, b); -} diff --git a/migrations/migration-8/node_modules/base32.js/test/fixtures.js b/migrations/migration-8/node_modules/base32.js/test/fixtures.js deleted file mode 100644 index 172630e..0000000 --- a/migrations/migration-8/node_modules/base32.js/test/fixtures.js +++ /dev/null @@ -1,294 +0,0 @@ -"use strict"; - -module.exports = [ - { - buf: [0], - rfc4648: ["AA", "aa"], - crock32: ["00", "0O", "0o"], - crock32int: ["0", "O", "o"], - base32hex: ["00"] - }, - { - buf: [1], - rfc4648: ["AE"], - crock32: ["04"], - crock32int: ["1", "I", "i", "L", "l"], - base32hex: ["04"] - }, - { - buf: [2], - rfc4648: ["AI", "ai", "aI", "Ai"], - crock32: ["08"], - crock32int: ["2"], - base32hex: ["08"] - }, - { - buf: [3], - rfc4648: ["AM", "am", "aM", "Am"], - crock32: ["0C"], - crock32int: ["3"], - base32hex: ["0C"] - }, - { - buf: [4], - rfc4648: ["AQ", "aq", "aQ", "Aq"], - crock32: ["0G"], - crock32int: ["4"], - base32hex: ["0G"] - }, - { - buf: [5], - rfc4648: ["AU", "au", "aU", "Au"], - crock32: ["0M"], - crock32int: ["5"], - base32hex: ["0K"] - }, - { - buf: [6], - rfc4648: ["AY", "ay", "aY", "Ay"], - crock32: ["0R"], - crock32int: ["6"], - base32hex: ["0O"] - }, - { - buf: [7], - rfc4648: ["A4", "a4"], - crock32: ["0W"], - crock32int: ["7"], - base32hex: ["0S"] - }, - { - buf: [8], - rfc4648: ["BA", "ba", "bA", "Ba"], - crock32: ["10"], - crock32int: ["8"], - base32hex: ["10"] - }, - { - buf: [9], - rfc4648: ["BE", "be", "bE", "Be"], - crock32: ["14"], - crock32int: ["9"], - base32hex: ["14"] - }, - { - buf: [10], - rfc4648: ["BI", "bi", "bI", "Bi"], - crock32: ["18"], - crock32int: ["A", "a"], - base32hex: ["18"] - }, - { - buf: [11], - rfc4648: ["BM", "bm", "bM", "Bm"], - crock32: ["1C"], - crock32int: ["B", "b"], - base32hex: ["1C"] - }, - { - buf: [12], - rfc4648: ["BQ", "bq", "bQ", "Bq"], - crock32: ["1G"], - crock32int: ["C", "c"], - base32hex: ["1G"] - }, - { - buf: [13], - rfc4648: ["BU", "bu", "bU", "Bu"], - crock32: ["1M"], - crock32int: ["D", "d"], - base32hex: ["1K"] - }, - { - buf: [14], - rfc4648: ["BY", "by", "bY", "By"], - crock32: ["1R"], - crock32int: ["E", "e"], - base32hex: ["1O"] - }, - { - buf: [15], - rfc4648: ["B4", "b4"], - crock32: ["1W"], - crock32int: ["F", "f"], - base32hex: ["1S"] - }, - { - buf: [16], - rfc4648: ["CA", "ca", "cA", "Ca"], - crock32: ["20"], - crock32int: ["G", "g"], - base32hex: ["20"] - }, - { - buf: [17], - rfc4648: ["CE", "ce", "cE", "Ce"], - crock32: ["24"], - crock32int: ["H", "h"], - base32hex: ["24"] - }, - { - buf: [18], - rfc4648: ["CI", "ci", "cI", "Ci"], - crock32: ["28"], - crock32int: ["J", "j"], - base32hex: ["28"] - }, - { - buf: [19], - rfc4648: ["CM", "cm", "cM", "Cm"], - crock32: ["2C"], - crock32int: ["K", "k"], - base32hex: ["2C"] - }, - { - buf: [20], - rfc4648: ["CQ", "cq", "cQ", "Cq"], - crock32: ["2G"], - crock32int: ["M", "m"], - base32hex: ["2G"] - }, - { - buf: [21], - rfc4648: ["CU", "cu", "cU", "Cu"], - crock32: ["2M"], - crock32int: ["N", "n"], - base32hex: ["2K"] - }, - { - buf: [22], - rfc4648: ["CY", "cy", "cY", "Cy"], - crock32: ["2R"], - crock32int: ["P", "p"], - base32hex: ["2O"] - }, - { - buf: [23], - rfc4648: ["C4", "c4"], - crock32: ["2W"], - crock32int: ["Q", "q"], - base32hex: ["2S"] - }, - { - buf: [24], - rfc4648: ["DA", "da", "dA", "Da"], - crock32: ["30"], - crock32int: ["R", "r"], - base32hex: ["30"] - }, - { - buf: [25], - rfc4648: ["DE", "de", "dE", "De"], - crock32: ["34"], - crock32int: ["S", "s"], - base32hex: ["34"] - }, - { - buf: [26], - rfc4648: ["DI", "di", "dI", "Di"], - crock32: ["38"], - crock32int: ["T", "t"], - base32hex: ["38"] - }, - { - buf: [27], - rfc4648: ["DM", "dm", "dM", "Dm"], - crock32: ["3C"], - crock32int: ["V", "v"], - base32hex: ["3C"] - }, - { - buf: [28], - rfc4648: ["DQ", "dq", "dQ", "Dq"], - crock32: ["3G"], - crock32int: ["W", "w"], - base32hex: ["3G"] - }, - { - buf: [29], - rfc4648: ["DU", "du", "dU", "Du"], - crock32: ["3M"], - crock32int: ["X", "x"], - base32hex: ["3k"] - }, - { - buf: [30], - rfc4648: ["DY", "dy", "dY", "Dy"], - crock32: ["3R"], - crock32int: ["Y", "y"], - base32hex: ["3O"] - }, - { - buf: [31], - rfc4648: ["D4", "d4"], - crock32: ["3W"], - crock32int: ["Z", "z"], - base32hex: ["3S"] - }, - { - buf: [0, 0], - rfc4648: ["AAAA", "aaaa", "AaAa", "aAAa"], - crock32: ["0000", "oooo", "OOOO", "0oO0"], - base32hex: ["0000"] - }, - { - buf: [1, 0], - rfc4648: ["AEAA", "aeaa", "AeAa", "aEAa"], - crock32: ["0400", "o4oo", "O4OO", "04oO"], - base32hex: ["0400"] - }, - { - buf: [0, 1], - rfc4648: ["AAAQ", "aaaq", "AaAQ", "aAAq"], - crock32: ["000G", "ooog", "OOOG", "0oOg"], - base32hex: ["000G"] - }, - { - buf: [1, 1], - rfc4648: ["AEAQ", "aeaq", "AeAQ", "aEAq"], - crock32: ["040G", "o4og", "O4og", "04Og"], - base32hex: ["040G"] - }, - { - buf: [136, 64], - rfc4648: ["RBAA", "rbaa", "RbAA", "rBAa"], - crock32: ["H100", "hio0", "HLOo"], - base32hex: ["H100"] - }, - { - buf: [139, 188], - rfc4648: ["RO6A", "r06a", "Ro6A", "r06A"], - crock32: ["HEY0", "heyo", "HeYO"], - base32hex: ["HEU0"] - }, - { - buf: [54, 31, 127], - rfc4648: ["GYPX6", "gypx6"], - crock32: ["6RFQY", "6rfqy"], - base32hex: ["6OFNU"] - }, - { - buf: [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33], - rfc4648: ["JBSWY3DPEBLW64TMMQQQ", "jbswy3dpeblw64tmmqqq"], - crock32: ["91JPRV3F41BPYWKCCGGG", "91jprv3f41bpywkccggg", "9Ljprv3f4ibpywkccggg"], - base32hex: ["91IMOR3F41BMUSJCCGGG"] - }, - { - buf: [139, 130, 16, 112, 24, 11, 64], - rfc4648: ["ROBBA4AYBNAA", "robba4aybnaa", "R0BBA4aybnaa"], - crock32: ["HE110W0R1D00", "helloworld00", "heiiOw0RidoO"], - base32hex: ["HE110S0O1D00"] - }, - { - buf: [139, 130, 16, 112, 24, 11], - rfc4648: ["ROBBA4AYBM", "robba4aybm", "R0BBA4aybm"], - crock32: ["HE110W0R1C", "helloworlc", "heiiOw0RiC"], - base32hex: ["HE110S0O1C"] - }, - { - buf: [139, 130, 16, 112, 24, 11, 0], - rfc4648: ["ROBBA4AYBMAA", "robba4aybmaa", "R0BBA4aybmaa"], - crock32: ["HE110W0R1C00", "helloworlc00", "heiiOw0RiC00"], - base32hex: ["HE110S0O1C00"] - } -]; diff --git a/migrations/migration-8/node_modules/base32.js/webpack.config.js b/migrations/migration-8/node_modules/base32.js/webpack.config.js deleted file mode 100644 index c7bc926..0000000 --- a/migrations/migration-8/node_modules/base32.js/webpack.config.js +++ /dev/null @@ -1,17 +0,0 @@ -"use strict"; - -/** - * Webpack configuration. - */ - -exports = module.exports = { - output: { - library: "base32", - libraryTarget: "this", - sourcePrefix: "" - }, - devtool: "source-map", - node: { - Buffer: false - } -}; diff --git a/migrations/migration-8/package-lock.json b/migrations/migration-8/package-lock.json deleted file mode 100644 index c37f9be..0000000 --- a/migrations/migration-8/package-lock.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "ipfs-repo-migration-8", - "version": "0.1.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "base32.js": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base32.js/-/base32.js-0.1.0.tgz", - "integrity": "sha1-tYLexpPC8R6JPPBk7mrFthMaIgI=" - } - } -} diff --git a/migrations/migration-8/package.json b/migrations/migration-8/package.json deleted file mode 100644 index f283e31..0000000 --- a/migrations/migration-8/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "ipfs-repo-migration-8", - "version": "0.1.0", - "description": "Migration", - "author": "Adam Uhlir ", - "main": "index.js", - "files": [ - "index.js", - "node_modules" - ], - "engines": { - "node": ">=10.0.0", - "npm": ">=3.0.0" - }, - "dependencies": { - "base32.js": "~0.1.0" - }, - "license": "MIT" -} diff --git a/package.json b/package.json index 46a8f9f..915acb3 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "interface-datastore": "~0.7.0", "proper-lockfile": "^3.2.0", "yargs": "^12.0.5", - "yargs-promise": "^1.1.0" + "yargs-promise": "^1.1.0", + "base32.js": "~0.1.0" }, "devDependencies": { "aegir": "^18.1.0", From 76b9fd50ba1f8b1205b42b23d1149d171b6037c6 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Wed, 10 Jul 2019 11:32:18 +0200 Subject: [PATCH 06/12] Fix wrong Datastore typo --- migrations/migration-8/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index cd7ec49..ba8d3e7 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -80,7 +80,7 @@ let storageBackend ) { storageBackend = options['storageBackends']['keys'] } else { - storageBackend = FSDatastore + storageBackend = Datastore } const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) From ce842796351a977a63b582262ea1cd60bfbe9a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 2 Sep 2019 16:14:38 +0200 Subject: [PATCH 07/12] fix: reversion in browsers --- migrations/migration-8/index.js | 4 +-- test/browser.js | 4 +++ test/migrations/migration-8-test.js | 56 +++++++++++++++-------------- test/node.js | 2 +- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index ba8d3e7..2e4b259 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -15,13 +15,13 @@ function encode (name) { } function decode (name) { - log(name) if (!name.startsWith(KEY_PREFIX)) { throw Error("Unknown format of key's name!") } const decoder = new base32.Decoder({ type: "rfc4648" }) - return decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) + const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) + return Buffer.from(decodedNameBuff).toString() } async function processFolder (store, prefix, fileNameProcessor) { diff --git a/test/browser.js b/test/browser.js index b341bf9..f0646cf 100644 --- a/test/browser.js +++ b/test/browser.js @@ -43,6 +43,10 @@ describe('Browser specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(createRepo, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) }) diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index 1421878..82925ab 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -3,7 +3,7 @@ const chai = require('chai') chai.use(require('dirty-chai')) -const chaiAsPromised = require("chai-as-promised") +const chaiAsPromised = require('chai-as-promised') chai.use(chaiAsPromised) const expect = chai.expect @@ -27,8 +27,8 @@ async function bootstrapKeys (dir, encoded) { let name for (let keyNames of fixtures) { name = encoded ? keyNames[1] : keyNames[0] - await store.put(new Key(`/pkcs8/${name}`), '') - await store.put(new Key(`/info/${name}`), '') + await store.put(new Key(`/pkcs8/${ name }`), '') + await store.put(new Key(`/info/${ name }`), '') } await store.close() @@ -41,42 +41,44 @@ async function validateKeys (dir, shouldBeEncoded) { let name for (let keyNames of fixtures) { name = shouldBeEncoded ? keyNames[1] : keyNames[0] - expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true() - expect(await store.has(new Key(`/info/${name}`))).to.be.true() + expect(await store.has(new Key(`/pkcs8/${ name }`))).to.be.true(name) + expect(await store.has(new Key(`/info/${ name }`))).to.be.true(name) } await store.close() } module.exports = (setup, cleanup) => { - let dir + describe('migration 8', () => { + let dir - beforeEach(async () => { - dir = await setup() - }) - afterEach(() => cleanup(dir)) + beforeEach(async () => { + dir = await setup() + }) + afterEach(() => cleanup(dir)) - it('should migrate forward', async () => { - await bootstrapKeys(dir, false) - await migration.migrate(dir) - await validateKeys(dir, true) - }) + it('should migrate forward', async () => { + await bootstrapKeys(dir, false) + await migration.migrate(dir) + await validateKeys(dir, true) + }) - it('should migrate backward', async () => { - await bootstrapKeys(dir, true) - await migration.revert(dir) - await validateKeys(dir, false) - }) + it('should migrate backward', async () => { + await bootstrapKeys(dir, true) + await migration.revert(dir) + await validateKeys(dir, false) + }) - it('should fail to migrate backward with invalid key name', async () => { - const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) - await store.open() + it('should fail to migrate backward with invalid key name', async () => { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) + await store.open() - await store.put(new Key('/pkcs8/mfawc'), '') - await store.put(new Key('/info/mfawc'), '') + await store.put(new Key('/pkcs8/mfawc'), '') + await store.put(new Key('/info/mfawc'), '') - await store.close() + await store.close() - expect(migration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + expect(migration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + }) }) } diff --git a/test/node.js b/test/node.js index 194569e..1d2244d 100644 --- a/test/node.js +++ b/test/node.js @@ -44,7 +44,7 @@ describe('Node specific tests', () => { }) describe('migrations tests', () => { - require('./migrations/migration-8-test')(repoSetup, repoCleanup) + require('./migrations/migration-8-test')(createRepo, repoCleanup) }) describe('init tests', () => { From 8edd2bb577142e6c2de733c74a95f0aeb294777e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 16 Sep 2019 14:46:55 +0200 Subject: [PATCH 08/12] feat: adding to migration blocks to multihash --- migrations/migration-8/blocks-to-multihash.js | 75 +++++++++++++ migrations/migration-8/index.js | 100 ++++-------------- migrations/migration-8/keys-encoding.js | 62 +++++++++++ test/migrations/migration-8-test.js | 75 ++++++++++--- 4 files changed, 218 insertions(+), 94 deletions(-) create mode 100644 migrations/migration-8/blocks-to-multihash.js create mode 100644 migrations/migration-8/keys-encoding.js diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js new file mode 100644 index 0000000..32c2f8f --- /dev/null +++ b/migrations/migration-8/blocks-to-multihash.js @@ -0,0 +1,75 @@ +const path = require('path') +const CID = require('cids') +const Key = require('interface-datastore').Key +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore +const base32 = require('base32.js') +const utils = require('../../src/utils') +const log = require('debug')('ipfs-repo-migrations:migration-8') + +// This function in js-ipfs-repo defaults to not using sharding +// but the default value of the options.sharding is True hence this +// function defaults to use sharding. +async function maybeWithSharding (filestore, options) { + if (options.sharding === false) { + return filestore + } + + const shard = new core.shard.NextToLast(2) + return await ShardingStore.createOrOpen(filestore, shard) +} + +function keyToMultihash(key){ + // Key to CID + const decoder = new base32.Decoder() + const buff = decoder.write(key.toString().slice(1)).finalize() + const cid = new CID(Buffer.from(buff)) + + // CID to multihash + const enc = new base32.Encoder() + return new Key('/' + enc.finalize(cid.multihash), false) +} + +function keyToCid(key){ + // Key to CID + const decoder = new base32.Decoder() + const buff = decoder.write(key.toString().slice(1)).finalize() + const cid = new CID(1, 'raw', Buffer.from(buff)) + + // CID to Key + const enc = new base32.Encoder() + return new Key('/' + enc.finalize(cid.buffer), false) +} + +async function process(repoPath, options, keyFunction){ + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') + + const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) + const store = await maybeWithSharding(baseStore, storageOptions) + + try { + const batch = store.batch() + let counter = 0 + for await (const block of store.query({})) { + batch.delete(block.key) + + counter += 1 + const newKey = keyFunction(block.key) + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + batch.put(newKey, block.value) + } + + log(`Changing ${ counter } blocks`) + await batch.commit() + } finally { + await store.close() + } +} + +exports.migrate = async function blocksMigrate (repoPath, options) { + return process(repoPath, options, keyToMultihash) +} + +exports.revert = async function blocksRevert (repoPath, options) { + return process(repoPath, options, keyToCid) +} diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 2e4b259..6821d77 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -1,103 +1,39 @@ 'use strict' -const Datastore = require('datastore-fs') -const path = require('path') -const base32 = require('base32.js') -const Key = require('interface-datastore').Key +const keysEncoding = require('./keys-encoding') +const blocksToMultihash = require('./blocks-to-multihash') const log = require('debug')('ipfs-repo-migrations:migration-8') -const KEY_PREFIX = 'key_' - -function encode (name) { - name = Buffer.from(name) - const encoder = new base32.Encoder({ type: "rfc4648" }) - return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() -} - -function decode (name) { - if (!name.startsWith(KEY_PREFIX)) { - throw Error("Unknown format of key's name!") - } - - const decoder = new base32.Decoder({ type: "rfc4648" }) - const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) - return Buffer.from(decodedNameBuff).toString() -} - -async function processFolder (store, prefix, fileNameProcessor) { - const query = { - prefix: `/${prefix}` - } - - const files = store.query(query) - for await (let file of files) { - const name = String(file.key._buf).replace(`/${prefix}/`, '') - const encodedFileName = fileNameProcessor(name) - const newKey = new Key(`${prefix}/${encodedFileName}`) - - await store.delete(file.key) - log(`Translating key's name '${file.key}' into '${newKey}'`) - await store.put(newKey, file.value) - } -} - async function migrate (repoPath, options, isBrowser) { - let storageBackend, storageBackendOptions - if (options !== undefined - && options['storageBackends'] !== undefined - && options['storageBackends']['keys'] !== undefined - ) { - storageBackend = options['storageBackends']['keys'] - } else { - storageBackend = Datastore - } - - if (options !== undefined - && options['storageBackendOptions'] !== undefined - && options['storageBackendOptions']['keys'] !== undefined - ) { - storageBackendOptions = options['storageBackendOptions']['keys'] - } else { - storageBackendOptions = {} - } + await keysEncoding.migrate(repoPath, options) - const store = new storageBackend(path.join(repoPath, 'keys'), storageBackendOptions) - try { - const info = processFolder(store, 'info', encode) - const data = processFolder(store, 'pkcs8', encode) + try{ + await blocksToMultihash.migrate(repoPath, options) + }catch (e) { + log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!') + await keysEncoding.revert(repoPath, options) - await Promise.all([info, data]) - } finally { - await store.close() + throw e } } async function revert (repoPath, options, isBrowser) { -let storageBackend - if (options !== undefined - && options['storageBackends'] !== undefined - && options['storageBackends']['keys'] !== undefined - ) { - storageBackend = options['storageBackends']['keys'] - } else { - storageBackend = Datastore - } + await keysEncoding.revert(repoPath, options) - const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' }) - try { - const info = processFolder(store, 'info', decode) - const data = processFolder(store, 'pkcs8', decode) + try{ + await blocksToMultihash.revert(repoPath, options) + }catch (e) { + log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!') + await keysEncoding.migrate(repoPath, options) - await Promise.all([info, data]) - } finally { - await store.close() + throw e } } module.exports = { version: 8, - description: 'Transforms key\'s names into base32 encoding.', - reversible: true, + description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes', + reversible: false, // TODO: Currently Block's CIDs are lost migrate, revert } diff --git a/migrations/migration-8/keys-encoding.js b/migrations/migration-8/keys-encoding.js new file mode 100644 index 0000000..029e17d --- /dev/null +++ b/migrations/migration-8/keys-encoding.js @@ -0,0 +1,62 @@ +const utils = require('../../src/utils') +const path = require('path') +const base32 = require('base32.js') +const Key = require('interface-datastore').Key +const log = require('debug')('ipfs-repo-migrations:migration-8') + +const KEY_PREFIX = 'key_' + +function encode (name) { + name = Buffer.from(name) + const encoder = new base32.Encoder({ type: 'rfc4648' }) + return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() +} + +function decode (name) { + if (!name.startsWith(KEY_PREFIX)) { + throw Error('Unknown format of key\'s name!') + } + + const decoder = new base32.Decoder({ type: 'rfc4648' }) + const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) + return Buffer.from(decodedNameBuff).toString() +} + +async function processFolder (store, prefix, fileNameProcessor) { + const query = { + prefix: `/${ prefix }` + } + + const files = store.query(query) + for await (let file of files) { + const name = String(file.key._buf).replace(`/${ prefix }/`, '') + const encodedFileName = fileNameProcessor(name) + const newKey = new Key(`${ prefix }/${ encodedFileName }`) + + await store.delete(file.key) + log(`Translating key's name '${ file.key }' into '${ newKey }'`) + await store.put(newKey, file.value) + } +} + +async function process (repoPath, options, processor) { + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys') + + const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions) + try { + const info = processFolder(store, 'info', processor) + const data = processFolder(store, 'pkcs8', processor) + + return await Promise.all([info, data]) + } finally { + await store.close() + } +} + +exports.migrate = async function keyEncode (repoPath, options) { + return process(repoPath, options, encode) +} + +exports.revert = async function keyDecode (repoPath, options) { + return process(repoPath, options, decode) +} diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index 82925ab..d85014b 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -8,24 +8,32 @@ chai.use(chaiAsPromised) const expect = chai.expect const path = require('path') -const migration = require('../../migrations/migration-8') +const keysMigration = require('../../migrations/migration-8/keys-encoding') +const blocksMigration = require('../../migrations/migration-8/blocks-to-multihash') const Key = require('interface-datastore').Key const Datastore = require('datastore-fs') +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore -const log = require('debug')('js-ipfs-repo-migrations:migration-8') - -const fixtures = [ +const keysFixtures = [ ['aAa', 'key_mfawc'], ['bbb', 'key_mjrge'], ['self', 'key_onswyzq'] ] +const blocksFixtures = [ + ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y', ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y']], + ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA', ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA']], + ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ', ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ']], + ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y', ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y']], +] + async function bootstrapKeys (dir, encoded) { const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) await store.open() let name - for (let keyNames of fixtures) { + for (let keyNames of keysFixtures) { name = encoded ? keyNames[1] : keyNames[0] await store.put(new Key(`/pkcs8/${ name }`), '') await store.put(new Key(`/info/${ name }`), '') @@ -39,7 +47,7 @@ async function validateKeys (dir, shouldBeEncoded) { await store.open() let name - for (let keyNames of fixtures) { + for (let keyNames of keysFixtures) { name = shouldBeEncoded ? keyNames[1] : keyNames[0] expect(await store.has(new Key(`/pkcs8/${ name }`))).to.be.true(name) expect(await store.has(new Key(`/info/${ name }`))).to.be.true(name) @@ -48,6 +56,36 @@ async function validateKeys (dir, shouldBeEncoded) { await store.close() } +async function bootstrapBlocks (dir, encoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true }) + const shard = new core.shard.NextToLast(2) + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let name + for (let blocksNames of blocksFixtures) { + name = encoded ? blocksNames[1] : blocksNames[0] + await store.put(new Key(name), '') + } + + await store.close() +} + +async function validateBlocks (dir, shouldBeEncoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false }) + const shard = new core.shard.NextToLast(2) + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let newName, oldName + for (let blockNames of blocksFixtures) { + newName = shouldBeEncoded ? blockNames[1] : blockNames[0] + oldName = shouldBeEncoded ? blockNames[0] : blockNames[1] + expect(await store.has(new Key(oldName))).to.be.false(oldName) + expect(await store.has(new Key(newName))).to.be.true(newName) + } + + await store.close() +} + module.exports = (setup, cleanup) => { describe('migration 8', () => { let dir @@ -57,19 +95,19 @@ module.exports = (setup, cleanup) => { }) afterEach(() => cleanup(dir)) - it('should migrate forward', async () => { + it('should migrate keys forward', async () => { await bootstrapKeys(dir, false) - await migration.migrate(dir) + await keysMigration.migrate(dir) await validateKeys(dir, true) }) - it('should migrate backward', async () => { + it('should migrate keys backward', async () => { await bootstrapKeys(dir, true) - await migration.revert(dir) + await keysMigration.revert(dir) await validateKeys(dir, false) }) - it('should fail to migrate backward with invalid key name', async () => { + it('should fail to migrate keys backward with invalid key name', async () => { const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) await store.open() @@ -78,7 +116,20 @@ module.exports = (setup, cleanup) => { await store.close() - expect(migration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + expect(keysMigration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + }) + + it('should migrate blocks forward', async () => { + await bootstrapBlocks(dir, false) + await blocksMigration.migrate(dir) + await validateBlocks(dir, true) }) + // + // it('should migrate blocks backward', async () => { + // await bootstrapKeys(dir, true) + // await blocksMigration.revert(dir) + // await validateKeys(dir, false) + // }) + }) } From ca620b239cb51dcab0ccedd1c768ed2ccd9a1d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 16 Sep 2019 15:04:40 +0200 Subject: [PATCH 09/12] style: lint package.json --- package.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 915acb3..8948766 100644 --- a/package.json +++ b/package.json @@ -45,15 +45,18 @@ "docs": "aegir docs" }, "dependencies": { + "base32.js": "~0.1.0", "chalk": "^2.4.2", - "datastore-fs": "~0.9.0", - "datastore-level": "~0.12.0", + "cids": "~0.7.0", + "datastore-core": "~0.7.0", + "datastore-fs": "^0.9.0", + "datastore-level": "^0.12.0", "debug": "^4.1.0", - "interface-datastore": "~0.7.0", + "fsevents": "^1.2.9", + "interface-datastore": "^0.7.0", "proper-lockfile": "^3.2.0", "yargs": "^12.0.5", - "yargs-promise": "^1.1.0", - "base32.js": "~0.1.0" + "yargs-promise": "^1.1.0" }, "devDependencies": { "aegir": "^18.1.0", From f79d59e7ea74544b516c078a4784432e8963cac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 16 Sep 2019 15:12:35 +0200 Subject: [PATCH 10/12] fix: remove reversibility flag from migration --- migrations/migration-8/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 6821d77..f1215fd 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -33,7 +33,6 @@ async function revert (repoPath, options, isBrowser) { module.exports = { version: 8, description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes', - reversible: false, // TODO: Currently Block's CIDs are lost migrate, revert } From 7c6277733c7f7c2d581af0aff4965540043e2106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 16 Sep 2019 16:59:30 +0200 Subject: [PATCH 11/12] style: lint --- package.json | 6 +++--- test/migrations/migration-8-test.js | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 8948766..38c472c 100644 --- a/package.json +++ b/package.json @@ -49,11 +49,11 @@ "chalk": "^2.4.2", "cids": "~0.7.0", "datastore-core": "~0.7.0", - "datastore-fs": "^0.9.0", - "datastore-level": "^0.12.0", + "datastore-fs": "~0.9.0", + "datastore-level": "~0.12.0", "debug": "^4.1.0", "fsevents": "^1.2.9", - "interface-datastore": "^0.7.0", + "interface-datastore": "~0.7.0", "proper-lockfile": "^3.2.0", "yargs": "^12.0.5", "yargs-promise": "^1.1.0" diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index d85014b..62cda68 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -25,7 +25,7 @@ const blocksFixtures = [ ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y', ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y']], ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA', ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA']], ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ', ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ']], - ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y', ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y']], + ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y', ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y']] ] async function bootstrapKeys (dir, encoded) { @@ -35,8 +35,8 @@ async function bootstrapKeys (dir, encoded) { let name for (let keyNames of keysFixtures) { name = encoded ? keyNames[1] : keyNames[0] - await store.put(new Key(`/pkcs8/${ name }`), '') - await store.put(new Key(`/info/${ name }`), '') + await store.put(new Key(`/pkcs8/${name}`), '') + await store.put(new Key(`/info/${name}`), '') } await store.close() @@ -49,8 +49,8 @@ async function validateKeys (dir, shouldBeEncoded) { let name for (let keyNames of keysFixtures) { name = shouldBeEncoded ? keyNames[1] : keyNames[0] - expect(await store.has(new Key(`/pkcs8/${ name }`))).to.be.true(name) - expect(await store.has(new Key(`/info/${ name }`))).to.be.true(name) + expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true(name) + expect(await store.has(new Key(`/info/${name}`))).to.be.true(name) } await store.close() @@ -130,6 +130,5 @@ module.exports = (setup, cleanup) => { // await blocksMigration.revert(dir) // await validateKeys(dir, false) // }) - }) } From 2077e4d5dec4b97daab6c89c7c58077e3b80cb7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Mon, 16 Sep 2019 18:21:37 +0200 Subject: [PATCH 12/12] fix: migrate blocks of CIDv1 --- migrations/migration-8/blocks-to-multihash.js | 20 +++++++++++-------- test/migrations/migration-8-test.js | 5 +---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js index 32c2f8f..d6897cd 100644 --- a/migrations/migration-8/blocks-to-multihash.js +++ b/migrations/migration-8/blocks-to-multihash.js @@ -22,7 +22,7 @@ async function maybeWithSharding (filestore, options) { function keyToMultihash(key){ // Key to CID const decoder = new base32.Decoder() - const buff = decoder.write(key.toString().slice(1)).finalize() + const buff = decoder.finalize(key.toString().slice(1)) const cid = new CID(Buffer.from(buff)) // CID to multihash @@ -51,12 +51,16 @@ async function process(repoPath, options, keyFunction){ const batch = store.batch() let counter = 0 for await (const block of store.query({})) { - batch.delete(block.key) - - counter += 1 const newKey = keyFunction(block.key) - log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) - batch.put(newKey, block.value) + + // If the Key is CIDv0 then it is raw multihash and nothing is changing + if(newKey.toString() !== block.key.toString()){ + counter += 1 + + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + batch.delete(block.key) + batch.put(newKey, block.value) + } } log(`Changing ${ counter } blocks`) @@ -66,10 +70,10 @@ async function process(repoPath, options, keyFunction){ } } -exports.migrate = async function blocksMigrate (repoPath, options) { +exports.migrate = function blocksMigrate (repoPath, options) { return process(repoPath, options, keyToMultihash) } -exports.revert = async function blocksRevert (repoPath, options) { +exports.revert = function blocksRevert (repoPath, options) { return process(repoPath, options, keyToCid) } diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index 62cda68..6f7f0e2 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -22,10 +22,7 @@ const keysFixtures = [ ] const blocksFixtures = [ - ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y', ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y']], - ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA', ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA']], - ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ', ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ']], - ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y', ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y']] + ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] ] async function bootstrapKeys (dir, encoded) {