Repository: codecov/uploader
Pull request #729 (2022-04-21 20:51:06)

fix: increase maxBuffer of spawnSync
nodeJS limits the size of the stdout of child processes to 1MB: https://nodejs.org/api/child_process.html#child_processspawnsynccommand-args-options

The output of `git ls-files` exceeds 1MB in my case, falling back to using `glob`. However the glob call [hardcodes](https://github.com/codecov/uploader/blob/2ad72d4c48aed30338c00e848d856c481beee7ba/src/helpers/files.ts#L249) a list of files that doesn't respect .gitignore and causes weird behaviour.

I'm setting 100MB as an arbitrary large value. There is some discussion about the default value of this argument here: https://github.com/nodejs/node/issues/9829

Motivated by https://github.com/codecov/uploader/issues/704

## Tests

I tried to find tests for this behaviour but it looks like there are none. I manually ran the uploader on a large repository with more than 1MB output:

```
$ git ls-files > ls_files.txt
$ du -sh ls_files.txt
6.8M	ls_files.txt
```

**Before**:
```
./uploader ...
...
<--- Last few GCs --->

[22463:0x7fe065602300]   168398 ms: Mark-sweep 4052.5 (4135.6) -> 4042.5 (4141.9) MB, 2926.0 / 0.0 ms  (average mu = 0.584, current mu = 0.255) allocation failure scavenge might not succeed
[22463:0x7fe065602300]   172823 ms: Mark-sweep 4060.3 (4143.6) -> 4049.4 (4148.6) MB, 4104.4 / 0.0 ms  (average mu = 0.367, current mu = 0.072) allocation failure scavenge might not succeed


<--- JS stacktrace --->

FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
```

**After**:
```
./uploader ...
...
[2022-04-21T20:58:15.604Z] ['info'] Uploading...
[2022-04-21T20:58:17.644Z] ['info'] {"status":"success","resultURL":""}
```



Patch:
diff --git a/src/helpers/constansts.ts b/src/helpers/constansts.ts
index 841860c..c29d6ff 100644
--- a/src/helpers/constansts.ts
+++ b/src/helpers/constansts.ts
@@ -1,1 +1,3 @@
 export const DEFAULT_UPLOAD_HOST = 'https://codecov.io'
+
+export const SPAWNPROCESSBUFFERSIZE = 1_048_576 * 100 // 100 MiB
diff --git a/src/helpers/files.ts b/src/helpers/files.ts
index b2ac57e..b3084f3 100644
--- a/src/helpers/files.ts
+++ b/src/helpers/files.ts
@@ -7,6 +7,7 @@ import { UploaderArgs } from '../types'
 import { logError, UploadLogger } from './logger'
 import { runExternalProgram } from './util'
 import micromatch from "../vendor/micromatch/index.js";
+import { SPAWNPROCESSBUFFERSIZE } from './constansts'
 
 export const MARKER_NETWORK_END = '\n<<<<<< network\n'
 export const MARKER_FILE_END = '<<<<<< EOF\n'
@@ -235,11 +236,11 @@ export function getAllFiles(
 ): string[] {
   UploadLogger.verbose(`Searching for files in ${dirPath}`)
 
-  const {
-    stdout,
-    status,
-    error,
-  } = spawnSync('git', ['-C', dirPath, 'ls-files'], { encoding: 'utf8' })
+  const { stdout, status, error } = spawnSync(
+    'git',
+    ['-C', dirPath, 'ls-files'],
+    { encoding: 'utf8', maxBuffer: SPAWNPROCESSBUFFERSIZE },
+  )
 
   let files = []
   if (error instanceof Error || status !== 0) {
diff --git a/src/helpers/util.ts b/src/helpers/util.ts
index d5d8cdf..3d861a2 100644
--- a/src/helpers/util.ts
+++ b/src/helpers/util.ts
@@ -1,6 +1,8 @@
 import childprocess from 'child_process'
+import { SPAWNPROCESSBUFFERSIZE } from './constansts'
+export { SPAWNPROCESSBUFFERSIZE } from './constansts'
+
 
-export const SPAWNPROCESSBUFFERSIZE = 1_048_576 * 10 // 10 MiB
 
 export function isProgramInstalled(programName: string): boolean {
   return !childprocess.spawnSync(programName).error


Test patch:
diff --git a/test/helpers/util.test.ts b/test/helpers/util.test.ts
index 60b1b52..c4c407a 100644
--- a/test/helpers/util.test.ts
+++ b/test/helpers/util.test.ts
@@ -1,61 +1,60 @@
 import childProcess from 'child_process'
 import td from 'testdouble'
-import {
-    isProgramInstalled,
-    runExternalProgram,
-    SPAWNPROCESSBUFFERSIZE,
-} from '../../src/helpers/util'
+import { isProgramInstalled, runExternalProgram } from '../../src/helpers/util'
+import { SPAWNPROCESSBUFFERSIZE } from '../../src/helpers/constansts'
 
 describe('isProgramInstalled()', () => {
-    afterEach(() => {
-        td.reset()
-    })
-
-    it('should throw an error when program is not found', () => {
-        const spawnSync = td.replace(childProcess, 'spawnSync')
-        td.when(spawnSync('fooProg')).thenReturn({
-          error: 'Huh?',
-        })
-        expect(isProgramInstalled('fooProg')).toBeFalsy()
-
+  afterEach(() => {
+    td.reset()
+  })
+
+  it('should throw an error when program is not found', () => {
+    const spawnSync = td.replace(childProcess, 'spawnSync')
+    td.when(spawnSync('fooProg')).thenReturn({
+      error: 'Huh?',
     })
+    expect(isProgramInstalled('fooProg')).toBeFalsy()
+  })
 })
 
 describe('runExternalProgram()', () => {
-    afterEach(() => {
-        td.reset()
+  afterEach(() => {
+    td.reset()
+  })
+
+  it('should throw an error when program is not found', () => {
+    const spawnSync = td.replace(childProcess, 'spawnSync')
+    td.when(
+      spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE }),
+    ).thenReturn({
+      error: 'Huh?',
     })
-
-    it('should throw an error when program is not found', () => {
-        const spawnSync = td.replace(childProcess, 'spawnSync')
-        td.when(spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE })).thenReturn({
-          error: 'Huh?',
-        })
-        expect(() => runExternalProgram('ls', ['-geewhiz'])).toThrowError(/Huh\?/)
-
+    expect(() => runExternalProgram('ls', ['-geewhiz'])).toThrowError(/Huh\?/)
+  })
+
+  it('should return stdout on success', () => {
+    const spawnSync = td.replace(childProcess, 'spawnSync')
+    td.when(
+      spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE }),
+    ).thenReturn({
+      stdout: 'I am output',
     })
+    expect(runExternalProgram('ls', ['-geewhiz'])).toEqual('I am output')
+  })
 
-    it('should return stdout on success', () => {
-        const spawnSync = td.replace(childProcess, 'spawnSync')
-        td.when(spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE })).thenReturn({
-          stdout: 'I am output',
-        })
-        expect(runExternalProgram('ls', ['-geewhiz'])).toEqual("I am output")
-
-    })
-
-    it('should return a trimmed string with no newlines', () => {
-        const spawnSync = td.replace(childProcess, 'spawnSync')
-        td.when(spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE })).thenReturn({
-          stdout: `
+  it('should return a trimmed string with no newlines', () => {
+    const spawnSync = td.replace(childProcess, 'spawnSync')
+    td.when(
+      spawnSync('ls', ['-geewhiz'], { maxBuffer: SPAWNPROCESSBUFFERSIZE }),
+    ).thenReturn({
+      stdout: `
 
 
           I am output
 
 
           `,
-        })
-        expect(runExternalProgram('ls', ['-geewhiz'])).toEqual("I am output")
-
     })
+    expect(runExternalProgram('ls', ['-geewhiz'])).toEqual('I am output')
+  })
 })