refactor: remove addFromURL addFromFs and export globSource and urlSource

Author: Alan Shaw

README.md

@@ -53,6 +53,8 @@
     - [Additional Options](#additional-options)
     - [Instance Utils](#instance-utils)
     - [Static Types and Utils](#static-types-and-utils)
+      - [Glob source](#glob-source)
+      - [URL source](#url-source)
 - [Development](#development)
   - [Testing](#testing)
 - [Contribute](#contribute)
@@ -362,6 +364,8 @@ Aside from the default export, `ipfs-http-client` exports various types and util
 - [`multicodec`](https://www.npmjs.com/package/multicodec)
 - [`multihash`](https://www.npmjs.com/package/multihashes)
 - [`CID`](https://www.npmjs.com/package/cids)
+- [`globSource`](https://github.com/ipfs/js-ipfs-utils/blob/master/src/files/glob-source.js) (not available in the browser)
+- [`urlSource`](https://github.com/ipfs/js-ipfs-utils/blob/master/src/files/url-source.js)
 
 These can be accessed like this, for example:
 
@@ -371,6 +375,74 @@ const { CID } = require('ipfs-http-client')
 import { CID } from 'ipfs-http-client'
 ```
 
+##### Glob source
+
+A utility to allow files on the file system to be easily added to IPFS.
+
+###### `globSource(path, [options])`
+
+- `path`: A path to a single file or directory to glob from
+- `options`: Optional options
+- `options.recursive`: If `path` is a directory, use option `{ recursive: true }` to add the directory and all its sub-directories.
+- `options.ignore`: To exclude file globs from the directory, use option `{ ignore: ['ignore/this/folder/**', 'and/this/file'] }`.
+- `options.hidden`: Hidden/dot files (files or folders starting with a `.`, for example, `.git/`) are not included by default. To add them, use the option `{ hidden: true }`.
+
+Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
+
+###### Example
+
+```js
+const IpfsHttpClient = require('ipfs-http-client')
+const { globSource } = IpfsHttpClient
+const ipfs = IpfsHttpClient()
+
+for await (const file of ipfs.add(globSource('./docs', { recursive: true }))) {
+  console.log(file)
+}
+/*
+{
+  path: 'docs/assets/anchor.js',
+  hash: 'QmVHxRocoWgUChLEvfEyDuuD6qJ4PhdDL2dTLcpUy3dSC2',
+  size: 15347
+}
+{
+  path: 'docs/assets/bass-addons.css',
+  hash: 'QmPiLWKd6yseMWDTgHegb8T7wVS7zWGYgyvfj7dGNt2viQ',
+  size: 232
+}
+...
+*/
+```
+
+##### URL source
+
+A utility to allow content from the internet to be easily added to IPFS.
+
+###### `urlSource(url)`
+
+- `url`: A string URL or [`URL`](https://developer.mozilla.org/en-US/docs/Web/API/URL) instance to send HTTP GET request to
+
+Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
+
+###### Example
+
+```js
+const IpfsHttpClient = require('ipfs-http-client')
+const { urlSource } = IpfsHttpClient
+const ipfs = IpfsHttpClient()
+
+for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg'))) {
+  console.log(file)
+}
+/*
+{
+  path: 'ipfs-logo.svg',
+  hash: 'QmTqZhR6f7jzdhLgPArDPnsbZpvvgxzCZycXK7ywkLxSyU',
+  size: 3243
+}
+*/
+```
+
 ## Development
 
 ### Testing

package.json

@@ -15,13 +15,10 @@
   ],
   "main": "src/index.js",
   "browser": {
-    "glob": false,
-    "fs": false,
-    "stream": "readable-stream",
     "ky-universal": "ky/umd",
     "./src/add/form-data.js": "./src/add/form-data.browser.js",
-    "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js",
-    "./src/lib/buffer-to-form-data.js": "./src/lib/buffer-to-form-data.browser.js"
+    "./src/lib/buffer-to-form-data.js": "./src/lib/buffer-to-form-data.browser.js",
+    "ipfs-utils/src/files/glob-source": false
   },
   "repository": "github:ipfs/js-ipfs-http-client",
   "scripts": {

src/add-from-fs/index.browser.js

@@ -6,12 +6,12 @@ const multiaddr = require('multiaddr')
 const multibase = require('multibase')
 const multicodec = require('multicodec')
 const multihash = require('multihashes')
+const globSource = require('ipfs-utils/src/files/glob-source')
+const urlSource = require('./lib/url-source') // TODO: move to ipfs-utils
 
 function ipfsClient (config) {
   return {
     add: require('./add')(config),
-    addFromFs: require('./add-from-fs')(config),
-    addFromURL: require('./add-from-url')(config),
     bitswap: require('./bitswap')(config),
     block: require('./block')(config),
     bootstrap: require('./bootstrap')(config),
@@ -46,6 +46,6 @@ function ipfsClient (config) {
   }
 }
 
-Object.assign(ipfsClient, { Buffer, CID, multiaddr, multibase, multicodec, multihash })
+Object.assign(ipfsClient, { Buffer, CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource })
 
 module.exports = ipfsClient

src/add-from-fs/index.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { default: ky } = require('ky-universal')
+const toIterable = require('./stream-to-iterable')
+
+module.exports = async function * urlSource (url, options) {
+  options = options || {}
+
+  const { body } = await ky.get(url)
+
+  yield {
+    path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''),
+    content: toIterable(body)
+  }
+}