feat: upgrade to the new multiformats (#3556)

- Replaces the old [interface-ipld-format](https://github.com/ipld/interface-ipld-format) stack with the new [multiformats](https://github.com/multiformats/js-multiformats) stack.
- The Block API takes/returns `Uint8Array`s instead of [ipld-block](https://github.com/ipld/js-ipld-block) objects

BREAKING CHANGE: ipld-formats no longer supported, use multiformat BlockCodecs instead

Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: achingbrain <alex@achingbrain.net>

Author: 3 people

browser-add-readable-stream/index.js

@@ -14,13 +14,14 @@ const main = async () => {
 
   const directoryHash = await streamFiles(ipfs, directoryName, inputFiles)
 
-  const fileList = await ipfs.ls(directoryHash)
-
   log(`\n--\n\nDirectory contents:\n\n${directoryName}/ ${directoryHash}`)
 
-  fileList.forEach((file, index) => {
-    log(` ${index < fileList.length - 1 ? '\u251C' : '\u2514'}\u2500 ${file.name} ${file.path} ${file.hash}`)
-  })
+  let index = 0
+
+  for await (const file of ipfs.ls(directoryHash)) {
+    log(` ${index < inputFiles.length - 1 ? '\u251C' : '\u2514'}\u2500 ${file.name} ${file.path} ${file.cid}`)
+    index++
+  }
 }
 
 const createFiles = (directory) => {
@@ -54,7 +55,7 @@ const streamFiles = async (ipfs, directory, files) => {
 
   const data = await ipfs.add(stream)
 
-  log(`Added ${data.path} hash: ${data.hash}`)
+  log(`Added ${data.path} hash: ${data.cid}`)
 
   // The last data event will contain the directory hash
   if (data.path === directory) {

browser-exchange-files/package.json

@@ -15,12 +15,12 @@
     "execa": "^5.0.0",
     "http-server": "^0.12.3",
     "ipfs-http-client": "^50.1.2",
-    "uint8arrays": "^2.1.3"
+    "uint8arrays": "^2.1.6"
   },
   "dependencies": {
     "ipfs": "^0.55.4",
     "it-all": "^1.0.4",
-    "libp2p-websockets": "^0.15.6",
+    "libp2p-websockets": "^0.16.1",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^3.0.0"
   }

browser-ipns-publish/index.js

@@ -8,6 +8,7 @@ const last = require("it-last");
 const cryptoKeys = require("human-crypto-keys"); // { getKeyPairFromSeed }
 const uint8ArrayToString = require('uint8arrays/to-string')
 const uint8ArrayFromString = require('uint8arrays/from-string')
+const { sha256 } = require('multiformats/hashes/sha2')
 
 const { sleep, Logger, onEnterPress, catchAndLog } = require("./util");
 
@@ -142,11 +143,10 @@ async function main() {
     return new Promise(async (resolve, reject) => {
       try {
         // quick and dirty key gen, don't do this in real life
-        const key = await IPFS.multihashing.digest(
+        const key = await sha256.digest(
           uint8ArrayFromString(keyName + Math.random().toString(36).substring(2)),
-          "sha2-256"
         );
-        const keyPair = await cryptoKeys.getKeyPairFromSeed(key, "rsa");
+        const keyPair = await cryptoKeys.getKeyPairFromSeed(key.bytes, "rsa");
 
         // put it on the browser IPNS keychain and name it
         await ipfsBrowser.key.import(keyName, keyPair.privateKey);

browser-ipns-publish/package.json

@@ -16,19 +16,19 @@
     "human-crypto-keys": "^0.1.4",
     "ipfs": "^0.55.4",
     "ipfs-http-client": "^50.1.2",
-    "ipfs-utils": "^8.1.2",
-    "ipns": "^0.12.0",
+    "ipfs-utils": "^8.1.4",
+    "ipns": "^0.13.2",
     "it-last": "^1.0.4",
     "p-retry": "^4.2.0",
-    "uint8arrays": "^2.1.3"
+    "uint8arrays": "^2.1.6"
   },
   "browserslist": [
     "last 2 versions and not dead and > 2%"
   ],
   "devDependencies": {
     "delay": "^5.0.0",
     "execa": "^5.0.0",
-    "ipfsd-ctl": "^8.0.1",
+    "ipfsd-ctl": "^9.0.0",
     "go-ipfs": "0.8.0",
     "parcel": "2.0.0-beta.2",
     "path": "^0.12.7",

circuit-relaying/package.json

@@ -17,8 +17,8 @@
     "delay": "^5.0.0",
     "ipfs": "^0.55.4",
     "ipfs-pubsub-room": "^2.0.1",
-    "libp2p-websockets": "^0.15.6",
-    "uint8arrays": "^2.1.3"
+    "libp2p-websockets": "^0.16.1",
+    "uint8arrays": "^2.1.6"
   },
   "devDependencies": {
     "execa": "^5.0.0",

custom-ipfs-repo/index.js

@@ -1,73 +1,87 @@
 'use strict'
 
 const IPFS = require('ipfs')
-const Repo = require('ipfs-repo')
-const fsLock = require('ipfs-repo/src/lock')
+const {
+  createRepo,
+  locks: {
+    fs: fsLock
+  }
+} = require('ipfs-repo')
 const all = require('it-all')
 const uint8ArrayFromString = require('uint8arrays/from-string')
 const uint8ArrayConcat = require('uint8arrays/concat')
+const DatastoreFS = require('datastore-fs')
+const BlockstoreDatastoreAdapter = require('blockstore-datastore-adapter')
 
-// Create our custom options
-const customRepositoryOptions = {
+// multiformat codecs to support
+const codecs = [
+  require('@ipld/dag-pb'),
+  require('@ipld/dag-cbor'),
+  require('multiformats/codecs/raw')
+].reduce((acc, curr) => {
+  acc[curr.name] = curr
+  acc[curr.code] = curr
 
-  /**
-   * IPFS nodes store different information in separate storageBackends, or datastores.
-   * Each storage backend can use the same type of datastore or a different one — you
-   * could store your keys in a levelDB database while everything else is in files,
-   * for example. (See https://github.com/ipfs/interface-datastore for more about datastores.)
-   */
-  storageBackends: {
-    root: require('datastore-fs'), // version and config data will be saved here
-    blocks: require('datastore-fs'),
-    keys: require('datastore-fs'),
-    datastore: require('datastore-fs')
-  },
+  return acc
+}, {})
 
-  /**
-   * Storage Backend Options will get passed into the instantiation of their counterpart
-   * in `storageBackends`. If you create a custom datastore, this is where you can pass in
-   * custom constructor arguments. You can see an S3 datastore example at:
-   * https://github.com/ipfs/js-datastore-s3/tree/master/examples/full-s3-repo
-   *
-   * NOTE: The following options are being overriden for demonstration purposes only.
-   * In most instances you can simply use the default options, by not passing in any
-   * overrides, which is recommended if you have no need to override.
-   */
-  storageBackendOptions: {
-    root: {
-      extension: '.ipfsroot', // Defaults to ''. Used by datastore-fs; Appended to all files
-      errorIfExists: false, // Used by datastore-fs; If the datastore exists, don't throw an error
-      createIfMissing: true // Used by datastore-fs; If the datastore doesn't exist yet, create it
-    },
-    blocks: {
-      sharding: false, // Used by IPFSRepo Blockstore to determine sharding; Ignored by datastore-fs
-      extension: '.ipfsblock', // Defaults to '.data'.
-      errorIfExists: false,
-      createIfMissing: true
-    },
-    keys: {
-      extension: '.ipfskey', // No extension by default
-      errorIfExists: false,
-      createIfMissing: true
-    },
-    datastore: {
-      extension: '.ipfsds', // No extension by default
-      errorIfExists: false,
-      createIfMissing: true
+async function main () {
+  const path = '/tmp/custom-repo/.ipfs'
+
+  // Support dag-pb and dag-cbor at a minimum
+  const loadCodec = (nameOrCode) => {
+    if (codecs[nameOrCode]) {
+      return codecs[nameOrCode]
     }
-  },
 
-  /**
-   * A custom lock can be added here. Or the build in Repo `fs` or `memory` locks can be used.
-   * See https://github.com/ipfs/js-ipfs-repo for more details on setting the lock.
-   */
-  lock: fsLock
-}
+    throw new Error(`Could not load codec for ${nameOrCode}`)
+  }
 
-async function main () {
   // Initialize our IPFS node with the custom repo options
   const node = await IPFS.create({
-    repo: new Repo('/tmp/custom-repo/.ipfs', customRepositoryOptions),
+    repo: createRepo(path, loadCodec, {
+      /**
+       * IPFS repos store different types of information in separate datastores.
+       * Each storage backend can use the same type of datastore or a different one — for example
+       * you could store your keys in a levelDB database while everything else is in files.
+       * See https://www.npmjs.com/package/interface-datastore for more about datastores.
+       */
+      root: new DatastoreFS(path, {
+        extension: '.ipfsroot', // Defaults to '', appended to all files
+        errorIfExists: false, // If the datastore exists, don't throw an error
+        createIfMissing: true // If the datastore doesn't exist yet, create it
+      }),
+      // blocks is a blockstore, all other backends are datastores - but we can wrap a datastore
+      // in an adapter to turn it into a blockstore
+      blocks: new BlockstoreDatastoreAdapter(
+        new DatastoreFS(`${path}/blocks`, {
+          extension: '.ipfsblock',
+          errorIfExists: false,
+          createIfMissing: true
+        })
+      ),
+      keys: new DatastoreFS(`${path}/keys`, {
+        extension: '.ipfskey',
+        errorIfExists: false,
+        createIfMissing: true
+      }),
+      datastore: new DatastoreFS(`${path}/datastore`, {
+        extension: '.ipfsds',
+        errorIfExists: false,
+        createIfMissing: true
+      }),
+      pins: new DatastoreFS(`${path}/pins`, {
+        extension: '.ipfspin',
+        errorIfExists: false,
+        createIfMissing: true
+      })
+    }, {
+      /**
+       * A custom lock can be added here. Or the build in Repo `fs` or `memory` locks can be used.
+       * See https://github.com/ipfs/js-ipfs-repo for more details on setting the lock.
+       */
+      lock: fsLock
+    }),
 
     // This just means we dont try to connect to the network which isn't necessary
     // to demonstrate custom repos

custom-ipfs-repo/package.json

@@ -10,10 +10,14 @@
   },
   "license": "MIT",
   "dependencies": {
-    "datastore-fs": "4.0.0",
+    "@ipld/dag-cbor": "^6.0.5",
+    "@ipld/dag-pb": "^2.1.3",
+    "blockstore-datastore-adapter": "^1.0.0",
+    "datastore-fs": "^5.0.1",
     "ipfs": "^0.55.4",
-    "ipfs-repo": "^9.1.6",
-    "it-all": "^1.0.4"
+    "ipfs-repo": "^11.0.0",
+    "it-all": "^1.0.4",
+    "multiformats": "^9.4.1"
   },
   "devDependencies": {
     "execa": "^5.0.0",

custom-ipld-formats/daemon-node.js

@@ -1,70 +1,33 @@
-// ordinarily we'd open a PR against the multicodec module to get our
-// codec number added but since we're just testing we shim our new
-// codec into the base-table.json file - this has to be done
-// before requiring other modules as the int table will become read-only
-const codecName = 'dag-test'
-const codecNumber = 392091
-
-const table = require('multicodec/src/base-table')
-// @ts-ignore
-table.baseTable = {
-  ...table.baseTable,
-  [codecName]: codecNumber
-}
-
-// now require modules as usual
 const IPFSDaemon = require('ipfs-daemon')
-const multihashing = require('multihashing-async')
-const multihash = multihashing.multihash
-const multicodec = require('multicodec')
-const CID = require('cids')
 const ipfsHttpClient = require('ipfs-http-client')
 const uint8ArrayToString = require('uint8arrays/to-string')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 async function main () {
-  // see https://github.com/ipld/interface-ipld-format for the interface definition
-  const format = {
-    codec: codecNumber,
-    defaultHashAlg: multicodec.SHA2_256,
-    util: {
-      serialize (data) {
-        return Buffer.from(JSON.stringify(data))
-      },
-      deserialize (buf) {
-        return JSON.parse(uint8ArrayToString(buf))
-      },
-      async cid (buf) {
-        const multihash = await multihashing(buf, format.defaultHashAlg)
-
-        return new CID(1, format.codec, multihash)
-      }
-    },
-    resolver: {
-      resolve: (buf, path) => {
-        return {
-          value: format.util.deserialize(buf),
-          remainderPath: path
-        }
-      }
-    }
+  // see https://github.com/multiformats/js-multiformats#multicodec-encoders--decoders--codecs for the interface definition
+  const codec = {
+    name: 'dag-test',
+    code: 392091,
+    encode: (data) => uint8ArrayFromString(JSON.stringify(data)),
+    decode: (buf) => JSON.parse(uint8ArrayToString(buf))
   }
 
   // start an IPFS Daemon
   const daemon = new IPFSDaemon({
     ipld: {
-      formats: [
-        format
+      codecs: [
+        codec
       ]
     }
   })
   await daemon.start()
 
   // in another process:
-  const client = ipfsHttpClient({
+  const client = ipfsHttpClient.create({
     url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`,
     ipld: {
-      formats: [
-        format
+      codecs: [
+        codec
       ]
     }
   })
@@ -74,8 +37,8 @@ async function main () {
   }
 
   const cid = await client.dag.put(data, {
-    format: codecName,
-    hashAlg: multihash.codes[format.defaultHashAlg]
+    format: 'dag-test',
+    hashAlg: 'sha2-256'
   })
 
   console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)