feat: add support for File DOM API to files-regular

Author: hugomrdias

examples/upload-file-via-browser/.gitignore

@@ -2,3 +2,4 @@ node_modules
 npm-debug.log
 .DS_Store
 dist
+yarn.lock

examples/upload-file-via-browser/package.json

@@ -16,8 +16,6 @@
     "@babel/preset-react": "^7.0.0",
     "eslint": "^5.16.0",
     "eslint-plugin-react": "^7.11.1",
-    "ipfs-http-client": "../../",
-    "pull-file-reader": "~1.0.2",
     "react": "~16.6.3",
     "react-dom": "~16.6.3",
     "webpack": "~4.30.0",

examples/upload-file-via-browser/src/App.js

@@ -2,16 +2,13 @@
 const React = require('react')
 const ipfsClient = require('../../../src')
 
-// create a stream from a file, which enables uploads of big files without allocating memory twice
-const fileReaderPullStream = require('pull-file-reader')
-
 class App extends React.Component {
   constructor () {
     super()
     this.state = {
       added_file_hash: null
     }
-    this.ipfs = ipfsClient('localhost', '50895')
+    this.ipfs = ipfsClient('localhost', '58041')
 
     // bind methods
     this.captureFile = this.captureFile.bind(this)
@@ -33,7 +30,7 @@ class App extends React.Component {
   // Add file to IPFS and return a CID
   saveToIpfs (files) {
     let ipfsId
-    this.ipfs.add([...files][0], { progress: (prog) => console.log(`received: ${prog}`) })
+    this.ipfs.add([...files], { progress: (prog) => console.log(`received: ${prog}`) })
       .then((response) => {
         console.log(response)
         ipfsId = response[0].hash
@@ -46,12 +43,12 @@ class App extends React.Component {
 
   // Example #2
   // Add file to IPFS and wrap it in a directory to keep the original filename
-  saveToIpfsWithFilename (file) {
+  saveToIpfsWithFilename (files) {
+    const file = [...files][0]
     let ipfsId
-    const fileStream = fileReaderPullStream(file)
     const fileDetails = {
       path: file.name,
-      content: fileStream
+      content: file
     }
     const options = {
       wrapWithDirectory: true,

package.json

@@ -35,9 +35,11 @@
     "err-code": "^1.1.2",
     "flatmap": "0.0.3",
     "glob": "^7.1.3",
+    "ipfs": "^0.35.0",
     "ipfs-block": "~0.8.0",
     "ipld-dag-cbor": "~0.13.1",
     "ipld-dag-pb": "~0.15.3",
+    "ipfs-utils": "github:hugomrdias/js-ipfs-utils#master",
     "is-ipfs": "~0.6.0",
     "is-pull-stream": "0.0.0",
     "is-stream": "^1.1.0",
@@ -80,7 +82,7 @@
     "chai": "^4.2.0",
     "cross-env": "^5.2.0",
     "dirty-chai": "^2.0.1",
-    "go-ipfs-dep": "~0.4.19",
+    "go-ipfs-dep": "0.4.19",
     "interface-ipfs-core": "~0.99.0",
     "ipfsd-ctl": "~0.42.0",
     "nock": "^10.0.2",

src/files-regular/add.js

@@ -2,12 +2,11 @@
 
 const promisify = require('promisify-es6')
 const ConcatStream = require('concat-stream')
-const kindOf = require('kind-of')
 const once = require('once')
-const isStream = require('is-stream')
-const isSource = require('is-pull-stream').isSource
+const { isSource } = require('is-pull-stream')
 const FileResultStreamConverter = require('../utils/file-result-stream-converter')
 const SendFilesStream = require('../utils/send-files-stream')
+const validateAddInput = require('ipfs-utils/src/files/add-input-validation')
 
 module.exports = (send) => {
   const createAddStream = SendFilesStream(send, 'add')
@@ -24,23 +23,10 @@ module.exports = (send) => {
     }
     options.converter = FileResultStreamConverter
 
-    // Buffer, pull stream or Node.js stream
-    const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj)
-    // An object like { content?, path? }, where content isBufferOrStream and path isString
-    const isContentObject = obj => {
-      if (typeof obj !== 'object') return false
-      // path is optional if content is present
-      if (obj.content) return isBufferOrStream(obj.content)
-      // path must be a non-empty string if no content
-      return Boolean(obj.path) && typeof obj.path === 'string'
-    }
-    // An input atom: a buffer, stream or content object
-    const isInput = obj => isBufferOrStream(obj) || isContentObject(obj) || kindOf(_files) === 'file'
-    // All is ok if data isInput or data is an array of isInput
-    const ok = isInput(_files) || (Array.isArray(_files) && _files.every(isInput))
-
-    if (!ok) {
-      return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects'))
+    try {
+      validateAddInput(_files)
+    } catch (err) {
+      return callback(err)
     }
 
     const files = [].concat(_files)

src/utils/multipart.js

@@ -77,12 +77,7 @@ class Multipart extends Transform {
       return callback() // early
     }
 
-    if (isSource(content)) {
-      content = toStream.readable(content)
-    }
-
     // From now on we assume content is a stream
-
     content.once('error', this.emit.bind(this, 'error'))
 
     content.once('end', () => {

src/utils/prepare-file.js

@@ -2,11 +2,14 @@
 
 const isNode = require('detect-node')
 const flatmap = require('flatmap')
-const { Readable } = require('stream')
+const { Readable } = require('readable-stream')
 const kindOf = require('kind-of')
+const { isSource } = require('is-pull-stream')
+const isStream = require('is-stream')
+const pullToStream = require('pull-to-stream')
+const { supportsFileReader } = require('ipfs-utils/src/supports')
+const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader')
 
-// eslint-disable-next-line no-undef
-const supportsFileReader = FileReader in self
 function loadPaths (opts, file) {
   const path = require('path')
   const fs = require('fs')
@@ -77,38 +80,36 @@ function loadPaths (opts, file) {
   }
 }
 
-function streamFromFileReader (file) {
-  class FileStream extends Readable {
-    constructor (file, options = {}) {
-      super(options)
-      this.offset = 0
-      this.chunkSize = 1024 * 1024
-      this.fileReader = new self.FileReader(file)
-      this.fileReader.onloadend = (event) => {
-        const data = event.target.result
-        if (data.byteLength === 0) {
-          this.push(null)
-        }
-        this.push(new Uint8Array(data))
+function contentToStream (content) {
+  if (supportsFileReader && kindOf(content) === 'file') {
+    return streamFromFileReader(content)
+  }
+
+  if (kindOf(content) === 'buffer') {
+    return new Readable({
+      read () {
+        this.push(content)
+        this.push(null)
       }
-      this.fileReader.onerror = (err) => this.emit('error', err)
-    }
+    })
+  }
 
-    _read (size) {
-      const end = this.offset + this.chunkSize
-      const slice = file.slice(this.offset, end)
-      this.fileReader.readAsArrayBuffer(slice)
-      this.offset = end
-    }
+  if (isSource(content)) {
+    return pullToStream.readable(content)
   }
 
-  return new FileStream(file)
+  if (isStream.readable(content)) {
+    return content
+  }
+
+  throw new Error(`Input not supported. Expected Buffer|ReadableStream|PullStream|File got ${kindOf(content)}. Check the documentation for more info https://github.com/ipfs/interface-js-ipfs-core/blob/master/SPEC/FILES.md#add`)
 }
 
 function prepareFile (file, opts) {
   let files = [].concat(file)
 
   return flatmap(files, (file) => {
+    // add from fs with file path
     if (typeof file === 'string') {
       if (!isNode) {
         throw new Error('Can only add file paths in node')
@@ -117,29 +118,34 @@ function prepareFile (file, opts) {
       return loadPaths(opts, file)
     }
 
-    if (file.path && !file.content) {
-      file.dir = true
-      return file
-    }
+    // add with object syntax { path : <string> , content: <Buffer|ReadableStream|PullStream|File }
+    if (kindOf(file) === 'object') {
+      // treat as an empty directory when path is a string and content undefined
+      if (file.path && kindOf(file.path) === 'string' && !file.content) {
+        file.dir = true
+        return file
+      }
 
-    if (file.content || file.dir) {
-      return file
-    }
+      // just return when directory
+      if (file.dir) {
+        return file
+      }
 
-    if (supportsFileReader && kindOf(file) === 'file') {
-      return {
-        path: '',
-        symlink: false,
-        dir: false,
-        content: streamFromFileReader(file, opts)
+      if (file.content) {
+        return {
+          path: file.path || '',
+          symlink: false,
+          dir: false,
+          content: contentToStream(file.content)
+        }
       }
     }
 
     return {
       path: '',
       symlink: false,
       dir: false,
-      content: file
+      content: contentToStream(file)
     }
   })
 }