refactor: use it-glob

License: MIT
Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>

Author: Alan Shaw

package.json

@@ -51,6 +51,7 @@
     "err-code": "^2.0.0",
     "explain-error": "^1.0.4",
     "flatmap": "0.0.3",
+    "fs-extra": "^8.1.0",
     "glob": "^7.1.3",
     "ipfs-block": "~0.8.1",
     "ipfs-utils": "~0.0.3",
@@ -62,7 +63,7 @@
     "is-stream": "^2.0.0",
     "iso-stream-http": "~0.1.2",
     "iso-url": "~0.4.6",
-    "it-pushable": "^1.2.1",
+    "it-glob": "0.0.1",
     "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "just-kebab-case": "^1.1.0",

src/add-from-fs/glob-source.js

@@ -1,9 +1,8 @@
 'use strict'
 
-const Fs = require('fs')
+const fs = require('fs-extra')
+const glob = require('it-glob')
 const Path = require('path')
-const glob = require('glob')
-const pushable = require('it-pushable')
 const errCode = require('err-code')
 
 /**
@@ -16,9 +15,9 @@ const errCode = require('err-code')
 * @param {Boolean} [options.hidden] Include .dot files in matched paths
 * @param {Array<String>} [options.ignore] Glob paths to ignore
 * @param {Boolean} [options.followSymlinks] follow symlinks
-* @returns {AsyncIterable}
+* @returns {Iterable} source iterable
 */
-module.exports = (...args) => (async function * () {
+module.exports = async function * globSource (...args) {
   const options = typeof args[args.length - 1] === 'string' ? {} : args.pop()
   const paths = args
 
@@ -32,63 +31,49 @@ module.exports = (...args) => (async function * () {
   }
 
   // Check the input paths comply with options.recursive and convert to glob sources
-  const results = await Promise.all(paths.map(pathAndType))
-  const globSources = results.map(r => toGlobSource(r, globSourceOptions))
-
-  for (const globSource of globSources) {
-    for await (const { path, contentPath } of globSource) {
-      yield { path, content: Fs.createReadStream(contentPath) }
-    }
+  for (const path of paths) {
+    const stat = await fs.stat(path)
+    const prefix = Path.dirname(path)
+    yield * toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions))
   }
-})()
-
-function toGlobSource ({ path, type }, options) {
-  return (async function * () {
-    options = options || {}
+}
 
-    const baseName = Path.basename(path)
+async function * toGlobSource ({ path, type, prefix }, options) {
+  options = options || {}
 
-    if (type === 'file') {
-      yield { path: baseName, contentPath: path }
-      return
-    }
+  const baseName = Path.basename(path)
 
-    if (type === 'dir' && !options.recursive) {
-      throw errCode(
-        new Error(`'${path}' is a directory and recursive option not set`),
-        'ERR_DIR_NON_RECURSIVE',
-        { path }
+  if (type === 'file') {
+    yield {
+      path: baseName.replace(prefix, ''),
+      content: fs.createReadStream(
+        Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)
       )
     }
+    return
+  }
 
-    const globOptions = Object.assign({}, options.glob, {
-      cwd: path,
-      nodir: true,
-      realpath: false,
-      absolute: false
-    })
-
-    // TODO: want to use pull-glob but it doesn't have the features...
-    const pusher = pushable()
+  if (type === 'dir' && !options.recursive) {
+    throw errCode(
+      new Error(`'${path}' is a directory and recursive option not set`),
+      'ERR_DIR_NON_RECURSIVE',
+      { path }
+    )
+  }
 
-    glob('**/*', globOptions)
-      .on('match', m => pusher.push(m))
-      .on('end', () => pusher.end())
-      .on('abort', () => pusher.end())
-      .on('error', err => pusher.end(err))
+  const globOptions = Object.assign({}, options.glob, {
+    cwd: path,
+    nodir: true,
+    realpath: false,
+    absolute: false
+  })
 
-    for await (const p of pusher) {
-      yield {
-        path: `${baseName}/${toPosix(p)}`,
-        contentPath: Path.join(path, p)
-      }
+  for await (const p of glob(path, '**/*', globOptions)) {
+    yield {
+      path: toPosix(p.replace(prefix, '')),
+      content: fs.createReadStream(p)
     }
-  })()
-}
-
-async function pathAndType (path) {
-  const stat = await Fs.promises.stat(path)
-  return { path, type: stat.isDirectory() ? 'dir' : 'file' }
+  }
 }
 
 const toPosix = path => path.replace(/\\/g, '/')

src/add-from-url.js

@@ -17,8 +17,6 @@ module.exports = configure(({ ky }) => {
       content: toIterable(body)
     }
 
-    for await (const file of add(input, options)) {
-      yield file
-    }
+    yield * add(input, options)
   })()
 })