Repository: okonet/lint-staged
Pull request #778 (2020-01-25 19:23:31)

fix: make sure deleted files aren't restored due to git bugs
Fixes https://github.com/okonet/lint-staged/issues/561

Currently, version 10 only fixes some of the issues with deleted files getting restored when running lint-staged. This PR adds yet more checks to fix the remaining issues. The issue itself seems to be with the way git tracks files, and you can read more [here in Stack Overflow](https://stackoverflow.com/questions/48446845/git-stash-restoring-index-state-of-deleted-and-renamed-files).

I also updated some eslint rules to take into account the Node.js version bump, and after that refactored the file api to use `fs.promises` to simplify the code.

codecov[bot]: # [Codecov](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=h1) Report
> Merging [#778](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=desc) into [master](https://codecov.io/gh/okonet/lint-staged/commit/0f2a1c01ca7bc599ea69744259c0de04b4e223b3?src=pr&el=desc) will **not change** coverage.
> The diff coverage is `100%`.

[![Impacted file tree graph](https://codecov.io/gh/okonet/lint-staged/pull/778/graphs/tree.svg?width=650&token=LSoEgsjk4O&height=150&src=pr)](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=tree)

```diff
@@          Coverage Diff          @@
##           master   #778   +/-   ##
=====================================
  Coverage     100%   100%           
=====================================
  Files          13     13           
  Lines         449    451    +2     
  Branches      104    101    -3     
=====================================
+ Hits          449    451    +2
```


| [Impacted Files](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [lib/index.js](https://codecov.io/gh/okonet/lint-staged/pull/778/diff?src=pr&el=tree#diff-bGliL2luZGV4Lmpz) | `100% <ø> (ø)` | :arrow_up: |
| [lib/getStagedFiles.js](https://codecov.io/gh/okonet/lint-staged/pull/778/diff?src=pr&el=tree#diff-bGliL2dldFN0YWdlZEZpbGVzLmpz) | `100% <ø> (ø)` | :arrow_up: |
| [lib/resolveGitRepo.js](https://codecov.io/gh/okonet/lint-staged/pull/778/diff?src=pr&el=tree#diff-bGliL3Jlc29sdmVHaXRSZXBvLmpz) | `100% <100%> (ø)` | :arrow_up: |
| [lib/gitWorkflow.js](https://codecov.io/gh/okonet/lint-staged/pull/778/diff?src=pr&el=tree#diff-bGliL2dpdFdvcmtmbG93Lmpz) | `100% <100%> (ø)` | :arrow_up: |
| [lib/file.js](https://codecov.io/gh/okonet/lint-staged/pull/778/diff?src=pr&el=tree#diff-bGliL2ZpbGUuanM=) | `100% <100%> (ø)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=footer). Last update [0f2a1c0...712e55b](https://codecov.io/gh/okonet/lint-staged/pull/778?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

iiroj: The test coverage decrease is due to current git versions not hitting the special case. I'll have to extract it back into a separate function so that it can be tested via a mock.
iiroj: @tremby If you don't mind, could you test this again after the rebase? There were some other related changes put into master.
iiroj: What about this, @okonet?

Patch:
diff --git a/lib/file.js b/lib/file.js
index 462b2c2..68339b3 100644
--- a/lib/file.js
+++ b/lib/file.js
@@ -1,68 +1,75 @@
 'use strict'
 
 const debug = require('debug')('lint-staged:file')
-const fs = require('fs')
-
-const fsPromises = fs.promises
+const fs = require('fs').promises
 
 /**
- * Check if a file exists. Returns the filepath if exists.
- * @param {string} filepath
+ * Check if a file exists. Returns the filename if exists.
+ * @param {String} filename
+ * @returns {String|Boolean}
  */
-const exists = async filepath => {
+const exists = async filename => {
   try {
-    await fsPromises.access(filepath)
-    return filepath
+    await fs.access(filename)
+    return filename
   } catch {
     return false
   }
 }
 
 /**
+ * Read contents of a file to buffer
  * @param {String} filename
- * @returns {Promise<Buffer|Null>}
+ * @param {Boolean} [ignoreENOENT=true] — Whether to throw if the file doesn't exist
+ * @returns {Promise<Buffer>}
  */
-const readBufferFromFile = (filename, rejectENOENT = false) =>
-  new Promise(resolve => {
-    debug('Reading buffer from file `%s`', filename)
-    fs.readFile(filename, (error, buffer) => {
-      if (!rejectENOENT && error && error.code === 'ENOENT') {
-        debug("File `%s` doesn't exist, ignoring...", filename)
-        return resolve(null) // no-op file doesn't exist
-      }
-      debug('Done reading buffer from file `%s`!', filename)
-      resolve(buffer)
-    })
-  })
+const readFile = async (filename, ignoreENOENT = true) => {
+  debug('Reading file `%s`', filename)
+  try {
+    return await fs.readFile(filename)
+  } catch (error) {
+    if (ignoreENOENT && error.code === 'ENOENT') {
+      debug("File `%s` doesn't exist, ignoring...", filename)
+      return null // no-op file doesn't exist
+    } else {
+      throw error
+    }
+  }
+}
 
 /**
  * Unlink a file if it exists
- * @param {*} filepath
+ * @param {String} filename
+ * @param {Boolean} [ignoreENOENT=true] — Whether to throw if the file doesn't exist
  */
-const unlink = async filepath => {
-  if (filepath) {
-    await fsPromises.access(filepath)
-    await fsPromises.unlink(filepath)
+const unlink = async (filename, ignoreENOENT = true) => {
+  if (filename) {
+    debug('Unlinking file `%s`', filename)
+    try {
+      await fs.unlink(filename)
+    } catch (error) {
+      if (ignoreENOENT && error.code === 'ENOENT') {
+        debug("File `%s` doesn't exist, ignoring...", filename)
+      } else {
+        throw error
+      }
+    }
   }
 }
 
 /**
+ * Write buffer to file
  * @param {String} filename
  * @param {Buffer} buffer
- * @returns {Promise<Void>}
  */
-const writeBufferToFile = (filename, buffer) =>
-  new Promise(resolve => {
-    debug('Writing buffer to file `%s`', filename)
-    fs.writeFile(filename, buffer, () => {
-      debug('Done writing buffer to file `%s`!', filename)
-      resolve()
-    })
-  })
+const writeFile = async (filename, buffer) => {
+  debug('Writing file `%s`', filename)
+  await fs.writeFile(filename, buffer)
+}
 
 module.exports = {
   exists,
-  readBufferFromFile,
+  readFile,
   unlink,
-  writeBufferToFile
+  writeFile
 }
diff --git a/lib/getStagedFiles.js b/lib/getStagedFiles.js
index 1a7b81e..61f2dd6 100644
--- a/lib/getStagedFiles.js
+++ b/lib/getStagedFiles.js
@@ -6,7 +6,7 @@ module.exports = async function getStagedFiles(options) {
   try {
     const lines = await execGit(['diff', '--staged', '--diff-filter=ACMR', '--name-only'], options)
     return lines ? lines.split('\n') : []
-  } catch (error) {
+  } catch {
     return null
   }
 }
diff --git a/lib/gitWorkflow.js b/lib/gitWorkflow.js
index e650ca2..c8570f2 100644
--- a/lib/gitWorkflow.js
+++ b/lib/gitWorkflow.js
@@ -4,7 +4,7 @@ const debug = require('debug')('lint-staged:git')
 const path = require('path')
 
 const execGit = require('./execGit')
-const { exists, readBufferFromFile, unlink, writeBufferToFile } = require('./file')
+const { exists, readFile, unlink, writeFile } = require('./file')
 
 const MERGE_HEAD = 'MERGE_HEAD'
 const MERGE_MODE = 'MERGE_MODE'
@@ -18,23 +18,6 @@ const PATCH_UNTRACKED = 'lint-staged_untracked.patch'
 const GIT_APPLY_ARGS = ['apply', '-v', '--whitespace=nowarn', '--recount', '--unidiff-zero']
 const GIT_DIFF_ARGS = ['--binary', '--unified=0', '--no-color', '--no-ext-diff', '--patch']
 
-/**
- * Delete untracked files using `git clean`
- * @param {Function} execGit function for executing git commands using execa
- * @returns {Promise<void>}
- */
-const cleanUntrackedFiles = async execGit => {
-  const untrackedFiles = await execGit(['ls-files', '--others', '--exclude-standard'])
-  if (untrackedFiles) {
-    debug('Detected unstaged, untracked files: ', untrackedFiles)
-    debug(
-      'This is probably due to a bug in git =< 2.13.0 where `git stash --keep-index` resurrects deleted files.'
-    )
-    debug('Deleting the files using `git clean`...')
-    await execGit(['clean', '--force', ...untrackedFiles.split('\n')])
-  }
-}
-
 const handleError = (error, ctx) => {
   ctx.gitError = true
   throw error
@@ -44,6 +27,7 @@ class GitWorkflow {
   constructor({ allowEmpty, gitConfigDir, gitDir, stagedFileChunks }) {
     this.execGit = (args, options = {}) => execGit(args, { ...options, cwd: gitDir })
     this.gitConfigDir = gitConfigDir
+    this.gitDir = gitDir
     this.unstagedDiff = null
     this.allowEmpty = allowEmpty
     this.stagedFileChunks = stagedFileChunks
@@ -73,7 +57,7 @@ class GitWorkflow {
     const resolved = this.getHiddenFilepath(filename)
     const pathIfExists = await exists(resolved)
     if (!pathIfExists) return false
-    const buffer = await readBufferFromFile(pathIfExists)
+    const buffer = await readFile(pathIfExists)
     const patch = buffer.toString().trim()
     return patch.length ? filename : false
   }
@@ -97,9 +81,9 @@ class GitWorkflow {
   async backupMergeStatus() {
     debug('Backing up merge state...')
     await Promise.all([
-      readBufferFromFile(this.mergeHeadFilename).then(buffer => (this.mergeHeadBuffer = buffer)),
-      readBufferFromFile(this.mergeModeFilename).then(buffer => (this.mergeModeBuffer = buffer)),
-      readBufferFromFile(this.mergeMsgFilename).then(buffer => (this.mergeMsgBuffer = buffer))
+      readFile(this.mergeHeadFilename).then(buffer => (this.mergeHeadBuffer = buffer)),
+      readFile(this.mergeModeFilename).then(buffer => (this.mergeModeBuffer = buffer)),
+      readFile(this.mergeMsgFilename).then(buffer => (this.mergeMsgBuffer = buffer))
     ])
     debug('Done backing up merge state!')
   }
@@ -111,9 +95,9 @@ class GitWorkflow {
     debug('Restoring merge state...')
     try {
       await Promise.all([
-        this.mergeHeadBuffer && writeBufferToFile(this.mergeHeadFilename, this.mergeHeadBuffer),
-        this.mergeModeBuffer && writeBufferToFile(this.mergeModeFilename, this.mergeModeBuffer),
-        this.mergeMsgBuffer && writeBufferToFile(this.mergeMsgFilename, this.mergeMsgBuffer)
+        this.mergeHeadBuffer && writeFile(this.mergeHeadFilename, this.mergeHeadBuffer),
+        this.mergeModeBuffer && writeFile(this.mergeModeFilename, this.mergeModeBuffer),
+        this.mergeMsgBuffer && writeFile(this.mergeMsgFilename, this.mergeMsgBuffer)
       ])
       debug('Done restoring merge state!')
     } catch (error) {
@@ -123,6 +107,18 @@ class GitWorkflow {
     }
   }
 
+  /**
+   * List and delete untracked files
+   */
+  async cleanUntrackedFiles() {
+    const lsFiles = await this.execGit(['ls-files', '--others', '--exclude-standard'])
+    const untrackedFiles = lsFiles
+      .split('\n')
+      .filter(Boolean)
+      .map(file => path.resolve(this.gitDir, file))
+    await Promise.all(untrackedFiles.map(file => unlink(file)))
+  }
+
   /**
    * Create backup stashes, one of everything and one of only staged changes
    * Staged files are left in the index for running tasks
@@ -135,6 +131,14 @@ class GitWorkflow {
       // Manually check and backup if necessary
       await this.backupMergeStatus()
 
+      // Get a list of unstaged deleted files, because certain bugs might cause them to reappear:
+      // - in git versions =< 2.13.0 the `--keep-index` flag resurrects deleted files
+      // - git stash can't infer RD or MD states correctly, and will lose the deletion
+      this.deletedFiles = (await this.execGit(['ls-files', '--deleted']))
+        .split('\n')
+        .filter(Boolean)
+        .map(file => path.resolve(this.gitDir, file))
+
       // Save stash of entire original state, including unstaged and untracked changes.
       // `--keep-index leaves only staged files on disk, for tasks.`
       await this.execGit(['stash', 'save', '--include-untracked', '--keep-index', STASH])
@@ -144,7 +148,7 @@ class GitWorkflow {
 
       // There is a bug in git =< 2.13.0 where `--keep-index` resurrects deleted files.
       // These files should be listed and deleted before proceeding.
-      await cleanUntrackedFiles(this.execGit)
+      await this.cleanUntrackedFiles()
 
       // Get a diff of unstaged changes by diffing the saved stash against what's left on disk.
       await this.execGit([
@@ -238,6 +242,9 @@ class GitWorkflow {
       ctx.gitRestoreUntrackedError = true
       handleError(error, ctx)
     }
+
+    // If stashing resurrected deleted files, clean them out
+    await Promise.all(this.deletedFiles.map(file => unlink(file)))
   }
 
   /**
@@ -251,6 +258,9 @@ class GitWorkflow {
       await this.execGit(['stash', 'apply', '--quiet', '--index', backupStash])
       debug('Done restoring original state!')
 
+      // If stashing resurrected deleted files, clean them out
+      await Promise.all(this.deletedFiles.map(file => unlink(file)))
+
       // Restore meta information about ongoing git merge
       await this.restoreMergeStatus()
     } catch (error) {
@@ -278,4 +288,3 @@ class GitWorkflow {
 }
 
 module.exports = GitWorkflow
-module.exports.cleanUntrackedFiles = cleanUntrackedFiles
diff --git a/lib/index.js b/lib/index.js
index b33b1c7..e5ca47b 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -14,7 +14,7 @@ const errConfigNotFound = new Error('Config could not be found')
 function resolveConfig(configPath) {
   try {
     return require.resolve(configPath)
-  } catch (ignore) {
+  } catch {
     return configPath
   }
 }
diff --git a/lib/resolveGitRepo.js b/lib/resolveGitRepo.js
index fa06498..b65013e 100644
--- a/lib/resolveGitRepo.js
+++ b/lib/resolveGitRepo.js
@@ -7,7 +7,7 @@ const path = require('path')
 const debugLog = require('debug')('lint-staged:resolveGitRepo')
 
 const execGit = require('./execGit')
-const { readBufferFromFile } = require('./file')
+const { readFile } = require('./file')
 
 /**
  * Resolve path to the .git directory, with special handling for
@@ -19,7 +19,7 @@ const resolveGitConfigDir = async gitDir => {
   // If .git is a directory, use it
   if (stats.isDirectory()) return defaultDir
   // Otherwise .git is a file containing path to real location
-  const file = (await readBufferFromFile(defaultDir)).toString()
+  const file = (await readFile(defaultDir)).toString()
   return path.resolve(gitDir, file.replace(/^gitdir: /, '')).trim()
 }
 


Test patch:
diff --git a/test/file.spec.js b/test/file.spec.js
new file mode 100644
index 0000000..93f3430
--- /dev/null
+++ b/test/file.spec.js
@@ -0,0 +1,13 @@
+import { unlink, readFile } from '../lib/file'
+
+describe('unlink', () => {
+  it('should throw when second argument is false and file is not found', async () => {
+    await expect(unlink('example', false)).rejects.toThrow('ENOENT')
+  })
+})
+
+describe('readFile', () => {
+  it('should throw when second argument is false and file is not found', async () => {
+    await expect(readFile('example', false)).rejects.toThrow('ENOENT')
+  })
+})
diff --git a/test/gitWorkflow.spec.js b/test/gitWorkflow.spec.js
index c49e2ae..97d7cb5 100644
--- a/test/gitWorkflow.spec.js
+++ b/test/gitWorkflow.spec.js
@@ -5,7 +5,7 @@ import path from 'path'
 import nanoid from 'nanoid'
 
 import execGitBase from '../lib/execGit'
-import GitWorkflow, { cleanUntrackedFiles } from '../lib/gitWorkflow'
+import GitWorkflow from '../lib/gitWorkflow'
 
 jest.unmock('execa')
 
@@ -65,16 +65,6 @@ describe('gitWorkflow', () => {
     }
   })
 
-  describe('cleanUntrackedFiles', () => {
-    it('should delete untracked, unstaged files', async () => {
-      const testFile = path.resolve(cwd, 'test.js')
-      await appendFile(testFile, 'test')
-      expect(await fs.exists(testFile)).toEqual(true)
-      await cleanUntrackedFiles(execGit)
-      expect(await fs.exists(testFile)).toEqual(false)
-    })
-  })
-
   describe('hasPatch', () => {
     it('should return false when patch file not found', async () => {
       const gitWorkflow = new GitWorkflow({
@@ -101,4 +91,19 @@ describe('gitWorkflow', () => {
       })
     })
   })
+
+  describe('cleanUntrackedFiles', () => {
+    it('should remove untracked files', async () => {
+      const tempFile = path.resolve(cwd, 'tempFile')
+      await fs.writeFile(tempFile, 'Hello')
+
+      const gitWorkflow = new GitWorkflow({
+        gitDir: cwd,
+        gitConfigDir: path.resolve(cwd, './.git')
+      })
+
+      await gitWorkflow.cleanUntrackedFiles()
+      await expect(fs.access(tempFile)).rejects.toThrow('ENOENT')
+    })
+  })
 })
diff --git a/test/runAll.unmocked.2.spec.js b/test/runAll.unmocked.2.spec.js
index 202449e..86aea65 100644
--- a/test/runAll.unmocked.2.spec.js
+++ b/test/runAll.unmocked.2.spec.js
@@ -8,7 +8,7 @@ import nanoid from 'nanoid'
 jest.mock('../lib/file')
 
 import execGitBase from '../lib/execGit'
-import { readBufferFromFile, writeBufferToFile } from '../lib/file'
+import { readFile, writeFile } from '../lib/file'
 import runAll from '../lib/runAll'
 
 jest.unmock('execa')
@@ -89,8 +89,8 @@ describe('runAll', () => {
   })
 
   it.only('Should throw when restoring untracked files fails', async () => {
-    readBufferFromFile.mockImplementation(async () => Buffer.from('test'))
-    writeBufferToFile.mockImplementation(async () => Promise.reject('test'))
+    readFile.mockImplementation(async () => Buffer.from('test'))
+    writeFile.mockImplementation(async () => Promise.reject('test'))
 
     // Stage pretty file
     await appendFile('test.js', testJsFilePretty)
diff --git a/test/runAll.unmocked.spec.js b/test/runAll.unmocked.spec.js
index fd96f65..afff847 100644
--- a/test/runAll.unmocked.spec.js
+++ b/test/runAll.unmocked.spec.js
@@ -604,11 +604,53 @@ describe('runAll', () => {
     expect(status).toMatch('no changes added to commit')
   })
 
-  it('should not resurrect removed files due to git bug', async () => {
+  it('should not resurrect removed files due to git bug when tasks pass', async () => {
     const readmeFile = path.resolve(cwd, 'README.md')
     await fs.remove(readmeFile) // Remove file from previous commit
-    await execGit(['add', '.'])
-    await gitCommit({ config: { '*.{js,md}': 'prettier --list-different' } })
+    await appendFile('test.js', testJsFilePretty)
+    await execGit(['add', 'test.js'])
+    await runAll({ cwd, config: { '*.{js,md}': 'prettier --list-different' } })
+    const exists = await fs.exists(readmeFile)
+    expect(exists).toEqual(false)
+  })
+
+  it('should not resurrect removed files in complex case', async () => {
+    // Add file to index, and remove it from disk
+    await appendFile('test.js', testJsFilePretty)
+    await execGit(['add', 'test.js'])
+    const testFile = path.resolve(cwd, 'test.js')
+    await fs.remove(testFile)
+
+    // Rename file in index, and remove it from disk
+    const readmeFile = path.resolve(cwd, 'README.md')
+    const readme = await readFile(readmeFile)
+    await fs.remove(readmeFile)
+    await execGit(['add', readmeFile])
+    const newReadmeFile = path.resolve(cwd, 'README_NEW.md')
+    await appendFile(newReadmeFile, readme)
+    await execGit(['add', newReadmeFile])
+    await fs.remove(newReadmeFile)
+
+    const status = await execGit(['status', '--porcelain'])
+    expect(status).toMatchInlineSnapshot(`
+      "RD README.md -> README_NEW.md
+      AD test.js"
+    `)
+
+    await runAll({ cwd, config: { '*.{js,md}': 'prettier --list-different' } })
+    expect(await fs.exists(testFile)).toEqual(false)
+    expect(await fs.exists(newReadmeFile)).toEqual(false)
+    expect(await execGit(['status', '--porcelain'])).toEqual(status)
+  })
+
+  it('should not resurrect removed files due to git bug when tasks fail', async () => {
+    const readmeFile = path.resolve(cwd, 'README.md')
+    await fs.remove(readmeFile) // Remove file from previous commit
+    await appendFile('test.js', testJsFileUgly)
+    await execGit(['add', 'test.js'])
+    await expect(
+      runAll({ allowEmpty: true, cwd, config: { '*.{js,md}': 'prettier --list-different' } })
+    ).rejects.toThrowError()
     const exists = await fs.exists(readmeFile)
     expect(exists).toEqual(false)
   })