Repository: ember-cli/ember-cli
Pull request #7349 (2017-10-03 19:15:49)

Limit allowed concurrency in CI environments.
Ideally we could determine the actually allowed concurrency count, but I am not aware of a mechanism that works consistently here. If we could detect this, we should update [this line](https://github.com/stefanpenner/broccoli-persistent-filter/blob/57d8f901e955b5d08de72417263fd14fb1d53803/index.js#L92) to use the "right" value by default.

Happy to revert this if docker/kubernetes/xen/kvm experts can point us to a better solution than `require('os').cpus().length - 1` for default concurrency...

Fixes https://github.com/emberjs/ember.js/issues/15641

Turbo87: @rwjblue shouldn't we fix this in `broccoli-babel-transpiler` itself?
rwjblue: After a *full 5 minutes* of googling @hjdivad pointed me to https://github.com/moby/moby/issues/20688#issuecomment-188923858, which we might be able to leverage in `broccoli-babel-transpiler`.

rwjblue: @Turbo87 - Ya, probably, but we can only choose a default in `broccoli-persistent-filter` and I'm not sure that we can actually detect this perfectly yet.  The changes in this PR are safe and work right now, I'm happy to tweak as needed but I feel that this PR is roughly the safest path at the moment...
Turbo87: > I feel that this PR is roughly the safest path at the moment...

can we set `JOBS=1` for the whole environment instead of patching it into each `ember test` call? is `ember-try` correctly passing that env var to `ember test`?
hjdivad: > After a full 5 minutes of googling @hjdivad pointed me to moby/moby#20688 (comment), which we might be able to leverage in broccoli-persistent-filter.

The tl;dr here seems to be to parse `/sys/fs/cgroup/cpuset/cpuset.cpus` (see `man cpuset`)
Turbo87: @hjdivad if I understand correctly the issue here is not incorrect CPU readings, but the container running out of memory. imagine a container that has 64 CPUs but only 512MB of memory, it would still fail even though we have lots of CPUs, so reading the correct amount of CPUs might not help that much either 🤷‍♂️ 
hjdivad: > @hjdivad if I understand correctly the issue here is not incorrect CPU readings, but the container running out of memory. imagine a container that has 64 CPUs but only 512MB of memory, it would still fail even though we have lots of CPUs, so reading the correct amount of CPUs might not help that much either 🤷‍♂️

Yes although I expect these values to be related: 64 CPU available to a container but only 512MB memory is a pretty weird configuration.
rwjblue: > can we set JOBS=1 for the whole environment instead of patching it into each ember test call? is ember-try correctly passing that env var to ember test?

Good point.  Updated.
Turbo87: I'm wondering if we should add a comment to this to explain why this is necessary on TravisCI 🤔 
rwjblue: How about this:

```
    # the normal mechanism for detecting maximum number of concurrent jobs does not work
    # properly on CI (where the concurrency is limited to 2 but available CPU's is a much
    # larger number). this prevents the build from exhausting resources
```
rwjblue: Updated with that comment inline.
rwjblue: Updated instead to add a `docs/build-concurrency.md` file and link to that from the `.travis.yml`.  This allows us to tweak the messaging over time as needed, and avoids a lengthy comment in the configuration file...

Patch:
diff --git a/blueprints/addon/files/.travis.yml b/blueprints/addon/files/.travis.yml
index 06233357c..7208dea5a 100644
--- a/blueprints/addon/files/.travis.yml
+++ b/blueprints/addon/files/.travis.yml
@@ -19,6 +19,9 @@ cache:
     - $HOME/.npm
 <% } %>
 env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
   # we recommend new addons test the current and previous LTS
   # as well as latest stable release (bonus points to beta/canary)
   - EMBER_TRY_SCENARIO=ember-lts-2.8
diff --git a/blueprints/addon/index.js b/blueprints/addon/index.js
index fab616769..2bba7f39c 100644
--- a/blueprints/addon/index.js
+++ b/blueprints/addon/index.js
@@ -11,7 +11,6 @@ const sortPackageJson = require('sort-package-json');
 let date = new Date();
 
 const normalizeEntityName = require('ember-cli-normalize-entity-name');
-const stringifyAndNormalize = require('../../lib/utilities/stringify-and-normalize');
 const FileInfo = require('../../lib/models/file-info');
 
 const replacers = {
@@ -68,7 +67,7 @@ module.exports = {
     contents['ember-addon'] = contents['ember-addon'] || {};
     contents['ember-addon'].configPath = 'tests/dummy/config';
 
-    return stringifyAndNormalize(sortPackageJson(contents));
+    return JSON.stringify(sortPackageJson(contents), null, 2);
   },
 
   buildFileInfo(intoDir, templateVariables, file) {
diff --git a/blueprints/app/files/.travis.yml b/blueprints/app/files/.travis.yml
index dff8bed40..75eef5779 100644
--- a/blueprints/app/files/.travis.yml
+++ b/blueprints/app/files/.travis.yml
@@ -16,6 +16,11 @@ cache:
   directories:
     - $HOME/.npm
 <% } %>
+env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
+
 before_install:<% if (yarn) { %>
   - curl -o- -L https://yarnpkg.com/install.sh | bash
   - export PATH=$HOME/.yarn/bin:$PATH
diff --git a/blueprints/app/files/package.json b/blueprints/app/files/package.json
index 55c1ed7c0..48dd56d5a 100644
--- a/blueprints/app/files/package.json
+++ b/blueprints/app/files/package.json
@@ -29,8 +29,8 @@
     "ember-cli-qunit": "^4.0.0",
     "ember-cli-shims": "^1.1.0",
     "ember-cli-sri": "^2.1.0",
-    "ember-cli-uglify": "^2.0.0",
-    "ember-data": "~2.16.2",
+    "ember-cli-uglify": "^1.2.0",
+    "ember-data": "~2.15.0",
     "ember-export-application-global": "^2.0.0",
     "ember-load-initializers": "^1.0.0",
     "ember-resolver": "^4.0.0",
diff --git a/docs/build-concurrency.md b/docs/build-concurrency.md
new file mode 100644
index 000000000..3f91c60eb
--- /dev/null
+++ b/docs/build-concurrency.md
@@ -0,0 +1,15 @@
+# Build Concurrency
+
+In order to speed up your project's build time, Ember CLI has added a bit of
+concurrency throughout the build system. The exact number of parallel
+transpilation jobs that will be used can be customized via the `JOBS` process
+environment variable.
+
+The default value for `process.env.JOBS` is (max concurrency) - 1 (via
+`require('os').cpus().length - 1`), however there may be times when you need to
+customize this value to avoid issues.
+
+The most common case for this is in CI systems like TravisCI and CircleCI where
+the total number of CPU's available on the system is very large (> 32) but the
+individual CI jobs are limited to only 1.5 or 2 concurrent processes.
+


Test patch:
diff --git a/blueprints/app/files/testem.js b/blueprints/app/files/testem.js
index e79c22e85..a40a53053 100644
--- a/blueprints/app/files/testem.js
+++ b/blueprints/app/files/testem.js
@@ -9,14 +9,11 @@ module.exports = {
     'Chrome'
   ],
   browser_args: {
-    Chrome: {
-      mode: 'ci',
-      args: [
-        '--disable-gpu',
-        '--headless',
-        '--remote-debugging-port=9222',
-        '--window-size=1440,900'
-      ]
-    },
+    Chrome: [
+      '--disable-gpu',
+      '--headless',
+      '--remote-debugging-port=9222',
+      '--window-size=1440,900'
+    ]
   }
 };
diff --git a/tests/fixtures/addon/npm/.travis.yml b/tests/fixtures/addon/npm/.travis.yml
index d04ff8a6a..3987f3124 100644
--- a/tests/fixtures/addon/npm/.travis.yml
+++ b/tests/fixtures/addon/npm/.travis.yml
@@ -16,6 +16,9 @@ cache:
     - $HOME/.npm
 
 env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
   # we recommend new addons test the current and previous LTS
   # as well as latest stable release (bonus points to beta/canary)
   - EMBER_TRY_SCENARIO=ember-lts-2.8
diff --git a/tests/fixtures/addon/yarn/.travis.yml b/tests/fixtures/addon/yarn/.travis.yml
index 7a742ed5d..29a280707 100644
--- a/tests/fixtures/addon/yarn/.travis.yml
+++ b/tests/fixtures/addon/yarn/.travis.yml
@@ -15,6 +15,9 @@ cache:
   yarn: true
 
 env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
   # we recommend new addons test the current and previous LTS
   # as well as latest stable release (bonus points to beta/canary)
   - EMBER_TRY_SCENARIO=ember-lts-2.8
diff --git a/tests/fixtures/app-import/package.json b/tests/fixtures/app-import/package.json
index 75c44576d..08706ccd6 100644
--- a/tests/fixtures/app-import/package.json
+++ b/tests/fixtures/app-import/package.json
@@ -28,7 +28,7 @@
     "ember-cli-qunit": "^4.0.0",
     "ember-cli-shims": "^1.1.0",
     "ember-cli-sri": "^2.1.0",
-    "ember-cli-uglify": "^2.0.0",
+    "ember-cli-uglify": "^1.2.0",
     "ember-data": "~2.13.0",
     "ember-export-application-global": "^2.0.0",
     "ember-load-initializers": "^1.0.0",
diff --git a/tests/fixtures/app/npm/.travis.yml b/tests/fixtures/app/npm/.travis.yml
index 35340c417..7a6dc7128 100644
--- a/tests/fixtures/app/npm/.travis.yml
+++ b/tests/fixtures/app/npm/.travis.yml
@@ -13,5 +13,10 @@ cache:
   directories:
     - $HOME/.npm
 
+env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
+
 before_install:
   - npm config set spin false
diff --git a/tests/fixtures/app/yarn/.travis.yml b/tests/fixtures/app/yarn/.travis.yml
index 47a97a568..224eafc8b 100644
--- a/tests/fixtures/app/yarn/.travis.yml
+++ b/tests/fixtures/app/yarn/.travis.yml
@@ -12,6 +12,11 @@ addons:
 cache:
   yarn: true
 
+env:
+  global:
+    # See https://git.io/vdao3 for details.
+    - JOBS=1
+
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
   - export PATH=$HOME/.yarn/bin:$PATH
diff --git a/tests/unit/blueprints/addon-test.js b/tests/unit/blueprints/addon-test.js
index 51f4c6b5b..7c4e4f969 100644
--- a/tests/unit/blueprints/addon-test.js
+++ b/tests/unit/blueprints/addon-test.js
@@ -79,7 +79,7 @@ describe('blueprint - addon', function() {
   "ember-addon": {\n\
     "configPath": "tests/dummy/config"\n\
   }\n\
-}\n');
+}');
       });
 
       it('removes the `private` property', function() {