Repository: webpack-contrib/copy-webpack-plugin
Pull request #466 (2020-05-12 13:55:44)

feat: concurrency
<!--
  HOLY CRAP a Pull Request. We ❤️ those!

  If you remove or skip this template, you'll make the 🐼 sad and the mighty god
  of Github will appear and pile-drive the close button from a great height
  while making animal noises.

  Please place an x (no spaces!) in all [ ] that apply
-->

This PR contains a:

- [ ] **bugfix**
- [x] new **feature**
- [ ] **code refactor**
- [x] **test update** <!-- if bug or feature is checked, this should be too -->
- [ ] **typo fix**
- [x] **metadata update**

### Motivation / Use-Case

Limits the number of simultaneous requests to fs

### Breaking Changes

No

### Additional Info

No


codecov[bot]: # [Codecov](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=h1) Report
> Merging [#466](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=desc) into [master](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/commit/0be64701ef458116b4d228e6ee8053f4e7e4eaa9&el=desc) will **increase** coverage by `0.02%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466/graphs/tree.svg?width=650&height=150&src=pr&token=fEoU9iBFGL)](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #466      +/-   ##
==========================================
+ Coverage   94.54%   94.57%   +0.02%     
==========================================
  Files           8        8              
  Lines         220      221       +1     
  Branches       56       56              
==========================================
+ Hits          208      209       +1     
  Misses         11       11              
  Partials        1        1              
```


| [Impacted Files](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [src/index.js](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466/diff?src=pr&el=tree#diff-c3JjL2luZGV4Lmpz) | `100.00% <100.00%> (ø)` | |
| [src/processPattern.js](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466/diff?src=pr&el=tree#diff-c3JjL3Byb2Nlc3NQYXR0ZXJuLmpz) | `96.77% <100.00%> (-0.11%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=footer). Last update [0be6470...2a9d307](https://codecov.io/gh/webpack-contrib/copy-webpack-plugin/pull/466?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/README.md b/README.md
index 410ee9a..970d250 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ module.exports = {
         { from: 'other', to: 'public' },
       ],
       options: {
-        ignore: ['*.bin'],
+        concurrency: 100,
       },
     }),
   ],
@@ -562,9 +562,15 @@ module.exports = {
 };
 ```
 
-#### `ignore`
+### Options
 
-Array of globs to ignore (applied to `from`).
+|             Name              |    Type    | Default | Description                                      |
+| :---------------------------: | :--------: | :-----: | :----------------------------------------------- |
+| [`concurrency`](#concurrency) | `{Number}` |  `100`  | Limits the number of simultaneous requests to fs |
+
+#### `concurrency`
+
+limits the number of simultaneous requests to fs
 
 **webpack.config.js**
 
@@ -573,7 +579,7 @@ module.exports = {
   plugins: [
     new CopyPlugin({
       patterns: [...patterns],
-      options: { ignore: ['*.js', '*.css'] },
+      options: { concurrency: 50 },
     }),
   ],
 };
diff --git a/src/index.js b/src/index.js
index 1dadcf2..c13c577 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,4 +1,5 @@
 import validateOptions from 'schema-utils';
+import pLimit from 'p-limit';
 
 import schema from './options.json';
 import preProcessPattern from './preProcessPattern';
@@ -18,6 +19,7 @@ class CopyPlugin {
 
   apply(compiler) {
     const plugin = { name: 'CopyPlugin' };
+    const limit = pLimit(this.options.concurrency || 100);
 
     compiler.hooks.compilation.tap(plugin, (compilation) => {
       const logger = compilation.getLogger('copy-webpack-plugin');
@@ -33,38 +35,39 @@ class CopyPlugin {
             compilation,
             inputFileSystem: compiler.inputFileSystem,
             output: compiler.options.output.path,
-            concurrency: this.options.concurrency,
           };
 
           try {
             await Promise.all(
-              this.patterns.map(async (pattern) => {
-                const patternAfterPreProcess = await preProcessPattern(
-                  globalRef,
-                  pattern
-                );
+              this.patterns.map((pattern) =>
+                limit(async () => {
+                  const patternAfterPreProcess = await preProcessPattern(
+                    globalRef,
+                    pattern
+                  );
 
-                const files = await processPattern(
-                  globalRef,
-                  patternAfterPreProcess
-                );
+                  const files = await processPattern(
+                    globalRef,
+                    patternAfterPreProcess
+                  );
 
-                if (!files) {
-                  return Promise.resolve();
-                }
+                  if (!files) {
+                    return Promise.resolve();
+                  }
 
-                return Promise.all(
-                  files
-                    .filter(Boolean)
-                    .map((file) =>
-                      postProcessPattern(
-                        globalRef,
-                        patternAfterPreProcess,
-                        file
-                      )
+                  return Promise.all(
+                    files.filter(Boolean).map((file) =>
+                      limit(() => {
+                        return postProcessPattern(
+                          globalRef,
+                          patternAfterPreProcess,
+                          file
+                        );
+                      })
                     )
-                );
-              })
+                  );
+                })
+              )
             );
 
             logger.debug('end to adding additionalAssets');
diff --git a/src/options.json b/src/options.json
index 8eb8d99..3409991 100644
--- a/src/options.json
+++ b/src/options.json
@@ -76,7 +76,11 @@
     "options": {
       "type": "object",
       "additionalProperties": false,
-      "properties": {}
+      "properties": {
+        "concurrency": {
+          "type": "number"
+        }
+      }
     }
   }
 }
diff --git a/src/postProcessPattern.js b/src/postProcessPattern.js
index dbf0944..8e093fb 100644
--- a/src/postProcessPattern.js
+++ b/src/postProcessPattern.js
@@ -14,7 +14,7 @@ import { stat, readFile } from './utils/promisify';
 
 /* eslint-disable no-param-reassign */
 
-export default function postProcessPattern(globalRef, pattern, file) {
+export default async function postProcessPattern(globalRef, pattern, file) {
   const { logger, compilation, inputFileSystem } = globalRef;
 
   logger.debug(`getting stats for '${file.absoluteFrom}' to write to assets`);
@@ -23,145 +23,136 @@ export default function postProcessPattern(globalRef, pattern, file) {
     ? Promise.resolve().then(() => pattern.stats)
     : stat(inputFileSystem, file.absoluteFrom);
 
-  return getStats.then((stats) => {
-    // We don't write empty directories
-    if (stats.isDirectory()) {
-      logger.debug(
-        `skipping '${file.absoluteFrom}' because it is empty directory`
-      );
+  let stats;
+
+  try {
+    stats = await getStats;
+  } catch (error) {
+    compilation.errors.push(error);
+    return;
+  }
+
+  if (stats.isDirectory()) {
+    logger.debug(
+      `skipping '${file.absoluteFrom}' because it is empty directory`
+    );
+  }
+
+  // If this came from a glob, add it to the file watchlist
+  if (pattern.fromType === 'glob') {
+    logger.debug(`add ${file.absoluteFrom} as fileDependencies`);
+    compilation.fileDependencies.add(file.absoluteFrom);
+  }
+
+  logger.debug(`reading '${file.absoluteFrom}' to write to assets`);
+
+  let content;
+
+  try {
+    content = await readFile(inputFileSystem, file.absoluteFrom);
+  } catch (error) {
+    compilation.errors.push(error);
+    return;
+  }
+
+  if (pattern.transform) {
+    logger.log(`transforming content for '${file.absoluteFrom}'`);
+
+    // eslint-disable-next-line no-shadow
+    const transform = (content, absoluteFrom) =>
+      pattern.transform(content, absoluteFrom);
+
+    if (pattern.cacheTransform) {
+      if (!globalRef.cacheDir) {
+        globalRef.cacheDir =
+          findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
+      }
+
+      const cacheKey = pattern.cacheTransform.key
+        ? pattern.cacheTransform.key
+        : serialize({
+            name,
+            version,
+            pattern,
+            hash: crypto.createHash('md4').update(content).digest('hex'),
+          });
+
+      try {
+        const result = await cacache.get(globalRef.cacheDir, cacheKey);
+
+        logger.debug(
+          `getting cached transformation for '${file.absoluteFrom}'`
+        );
 
-      return Promise.resolve();
-    }
+        content = result.data;
+      } catch (e) {
+        content = await transform(content, file.absoluteFrom);
+
+        logger.debug(`caching transformation for '${file.absoluteFrom}'`);
 
-    // If this came from a glob, add it to the file watchlist
-    if (pattern.fromType === 'glob') {
-      logger.debug(`add ${file.absoluteFrom} as fileDependencies`);
-      compilation.fileDependencies.add(file.absoluteFrom);
+        content = await cacache
+          .put(globalRef.cacheDir, cacheKey, content)
+          .then(() => content);
+      }
+    } else {
+      content = await transform(content, file.absoluteFrom);
     }
+  }
 
-    logger.debug(`reading '${file.absoluteFrom}' to write to assets`);
-
-    return readFile(inputFileSystem, file.absoluteFrom)
-      .then((content) => {
-        if (pattern.transform) {
-          logger.log(`transforming content for '${file.absoluteFrom}'`);
-
-          // eslint-disable-next-line no-shadow
-          const transform = (content, absoluteFrom) =>
-            pattern.transform(content, absoluteFrom);
-
-          if (pattern.cacheTransform) {
-            if (!globalRef.cacheDir) {
-              globalRef.cacheDir =
-                findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
-            }
-
-            const cacheKey = pattern.cacheTransform.key
-              ? pattern.cacheTransform.key
-              : serialize({
-                  name,
-                  version,
-                  pattern,
-                  hash: crypto.createHash('md4').update(content).digest('hex'),
-                });
-
-            return cacache.get(globalRef.cacheDir, cacheKey).then(
-              (result) => {
-                logger.debug(
-                  `getting cached transformation for '${file.absoluteFrom}'`
-                );
-
-                return result.data;
-              },
-              () =>
-                Promise.resolve()
-                  .then(() => transform(content, file.absoluteFrom))
-                  // eslint-disable-next-line no-shadow
-                  .then((content) => {
-                    logger.debug(
-                      `caching transformation for '${file.absoluteFrom}'`
-                    );
-
-                    return cacache
-                      .put(globalRef.cacheDir, cacheKey, content)
-                      .then(() => content);
-                  })
-            );
-          }
-
-          content = transform(content, file.absoluteFrom);
-        }
-
-        return content;
-      })
-      .then((content) => {
-        if (pattern.toType === 'template') {
-          logger.log(
-            `interpolating template '${file.webpackTo}' for '${file.relativeFrom}'`
-          );
-
-          // If it doesn't have an extension, remove it from the pattern
-          // ie. [name].[ext] or [name][ext] both become [name]
-          if (!path.extname(file.relativeFrom)) {
-            file.webpackTo = file.webpackTo.replace(/\.?\[ext\]/g, '');
-          }
-
-          file.webpackTo = loaderUtils.interpolateName(
-            { resourcePath: file.absoluteFrom },
-            file.webpackTo,
-            {
-              content,
-              regExp: file.webpackToRegExp,
-              context: pattern.context,
-            }
-          );
-
-          // Bug in `loader-utils`, package convert `\\` to `/`, need fix in loader-utils
-          file.webpackTo = path.normalize(file.webpackTo);
-        }
-
-        return content;
-      })
-      .then((content) => {
-        if (pattern.transformPath) {
-          logger.log(
-            `transforming path '${file.webpackTo}' for '${file.absoluteFrom}'`
-          );
-
-          return Promise.resolve()
-            .then(() =>
-              pattern.transformPath(file.webpackTo, file.absoluteFrom)
-            )
-            .then((newPath) => {
-              file.webpackTo = newPath;
-
-              return content;
-            });
-        }
-
-        return content;
-      })
-      .then((content) => {
-        const targetPath = normalizePath(file.webpackTo);
-
-        if (compilation.assets[targetPath] && !file.force) {
-          logger.log(`skipping '${file.webpackTo}', because it already exists`);
-
-          return;
-        }
-
-        logger.log(
-          `writing '${file.webpackTo}' to compilation assets from '${file.absoluteFrom}'`
-        );
+  if (pattern.toType === 'template') {
+    logger.log(
+      `interpolating template '${file.webpackTo}' for '${file.relativeFrom}'`
+    );
+
+    // If it doesn't have an extension, remove it from the pattern
+    // ie. [name].[ext] or [name][ext] both become [name]
+    if (!path.extname(file.relativeFrom)) {
+      file.webpackTo = file.webpackTo.replace(/\.?\[ext\]/g, '');
+    }
 
-        compilation.assets[targetPath] = {
-          size() {
-            return stats.size;
-          },
-          source() {
-            return content;
-          },
-        };
-      });
-  });
+    file.webpackTo = loaderUtils.interpolateName(
+      { resourcePath: file.absoluteFrom },
+      file.webpackTo,
+      {
+        content,
+        regExp: file.webpackToRegExp,
+        context: pattern.context,
+      }
+    );
+
+    // Bug in `loader-utils`, package convert `\\` to `/`, need fix in loader-utils
+    file.webpackTo = path.normalize(file.webpackTo);
+  }
+
+  if (pattern.transformPath) {
+    logger.log(
+      `transforming path '${file.webpackTo}' for '${file.absoluteFrom}'`
+    );
+
+    file.webpackTo = await pattern.transformPath(
+      file.webpackTo,
+      file.absoluteFrom
+    );
+  }
+
+  const targetPath = normalizePath(file.webpackTo);
+
+  if (compilation.assets[targetPath] && !file.force) {
+    logger.log(`skipping '${file.webpackTo}', because it already exists`);
+
+    return;
+  }
+
+  logger.log(
+    `writing '${file.webpackTo}' to compilation assets from '${file.absoluteFrom}'`
+  );
+
+  compilation.assets[targetPath] = {
+    size() {
+      return stats.size;
+    },
+    source() {
+      return content;
+    },
+  };
 }
diff --git a/src/processPattern.js b/src/processPattern.js
index 3359bc1..631e5a0 100644
--- a/src/processPattern.js
+++ b/src/processPattern.js
@@ -1,18 +1,15 @@
 import path from 'path';
 
 import globby from 'globby';
-import pLimit from 'p-limit';
 
 import createPatternGlob from './utils/createPatternGlob';
 
 /* eslint-disable no-param-reassign */
 
 export default async function processPattern(globalRef, pattern) {
-  const { logger, output, concurrency, compilation } = globalRef;
+  const { logger, output, compilation } = globalRef;
   createPatternGlob(pattern, globalRef);
 
-  const limit = pLimit(concurrency || 100);
-
   logger.log(
     `begin globbing '${pattern.glob}' with a context of '${pattern.context}'`
   );
@@ -38,42 +35,36 @@ export default async function processPattern(globalRef, pattern) {
     return Promise.resolve();
   }
 
-  return Promise.all(
-    paths.map((from) =>
-      limit(() => {
-        const file = {
-          force: pattern.force,
-          absoluteFrom: path.resolve(pattern.context, from),
-        };
-
-        file.relativeFrom = path.relative(pattern.context, file.absoluteFrom);
-
-        if (pattern.flatten) {
-          file.relativeFrom = path.basename(file.relativeFrom);
-        }
-
-        logger.debug(`found ${from}`);
-
-        // Change the to path to be relative for webpack
-        if (pattern.toType === 'dir') {
-          file.webpackTo = path.join(pattern.to, file.relativeFrom);
-        } else if (pattern.toType === 'file') {
-          file.webpackTo = pattern.to || file.relativeFrom;
-        } else if (pattern.toType === 'template') {
-          file.webpackTo = pattern.to;
-          file.webpackToRegExp = pattern.test;
-        }
-
-        if (path.isAbsolute(file.webpackTo)) {
-          file.webpackTo = path.relative(output, file.webpackTo);
-        }
-
-        logger.log(
-          `determined that '${from}' should write to '${file.webpackTo}'`
-        );
-
-        return file;
-      })
-    )
-  );
+  return paths.map((from) => {
+    const file = {
+      force: pattern.force,
+      absoluteFrom: path.resolve(pattern.context, from),
+    };
+
+    file.relativeFrom = path.relative(pattern.context, file.absoluteFrom);
+
+    if (pattern.flatten) {
+      file.relativeFrom = path.basename(file.relativeFrom);
+    }
+
+    logger.debug(`found ${from}`);
+
+    // Change the to path to be relative for webpack
+    if (pattern.toType === 'dir') {
+      file.webpackTo = path.join(pattern.to, file.relativeFrom);
+    } else if (pattern.toType === 'file') {
+      file.webpackTo = pattern.to || file.relativeFrom;
+    } else if (pattern.toType === 'template') {
+      file.webpackTo = pattern.to;
+      file.webpackToRegExp = pattern.test;
+    }
+
+    if (path.isAbsolute(file.webpackTo)) {
+      file.webpackTo = path.relative(output, file.webpackTo);
+    }
+
+    logger.log(`determined that '${from}' should write to '${file.webpackTo}'`);
+
+    return file;
+  });
 }


Test patch:
diff --git a/test/__snapshots__/validate-options.test.js.snap b/test/__snapshots__/validate-options.test.js.snap
index 4bfa80b..4c895eb 100644
--- a/test/__snapshots__/validate-options.test.js.snap
+++ b/test/__snapshots__/validate-options.test.js.snap
@@ -1,9 +1,14 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`validate options should throw an error on the "options" option with "{"concurrency":true}" value 1`] = `
+"Invalid options object. Copy Plugin has been initialized using an options object that does not match the API schema.
+ - options.options.concurrency should be a number."
+`;
+
 exports[`validate options should throw an error on the "options" option with "{"unknown":true}" value 1`] = `
 "Invalid options object. Copy Plugin has been initialized using an options object that does not match the API schema.
  - options.options has an unknown property 'unknown'. These properties are valid:
-   object {}"
+   object { concurrency? }"
 `;
 
 exports[`validate options should throw an error on the "patterns" option with "" value 1`] = `
diff --git a/test/validate-options.test.js b/test/validate-options.test.js
index bd87b48..69dd911 100644
--- a/test/validate-options.test.js
+++ b/test/validate-options.test.js
@@ -203,8 +203,8 @@ describe('validate options', () => {
       ],
     },
     options: {
-      success: [{}],
-      failure: [{ unknown: true }],
+      success: [{ concurrency: 50 }],
+      failure: [{ unknown: true }, { concurrency: true }],
     },
     unknown: {
       success: [],