Repository: steveukx/git-js
Pull request #576 (2021-02-16 09:34:43)

feat: Progress Handler
The `SimpleGitOptions` can now be used to supply a `progress` handler, called whenever progress data is received from one of the tasks. Setting the `progress` option will automatically add the `--progress` command to any `checkout` / `clone` or `pull` task when not already supplied in the `TaskOptions`.

For any task that ran with `--progress` as a command (ie: even `git.raw` so long as the `--progress` command was present), monitors the `stdErr` for progress data.


Patch:
diff --git a/readme.md b/readme.md
index a29f3ae..61bc236 100644
--- a/readme.md
+++ b/readme.md
@@ -77,6 +77,36 @@ const git: SimpleGit = simpleGit('/some/path', { config: ['http.proxy=someproxy'
 await git.pull();
 ```
 
+## Progress Events
+
+To receive progress updates, pass a `progress` configuration option to the `simpleGit` instance:
+
+```typescript
+import simpleGit, { SimpleGit, SimpleGitProgressEvent } from 'simple-git';
+
+const progress = ({method, stage, progress}: SimpleGitProgressEvent) => {
+   console.log(`git.${method} ${stage} stage ${progress}% complete`);
+}
+const git: SimpleGit = simpleGit({baseDir: '/some/path', progress});
+
+// pull automatically triggers progress events when the progress plugin is configured
+await git.pull();
+
+// supply the `--progress` option to any other command that supports it to receive
+// progress events into your handler
+await git.raw('pull', '--progress');
+```
+
+The `checkout`, `clone`, `pull`, `push` methods will automatically enable progress events when
+a progress handler has been set. For any other method that _can_ support progress events, set
+`--progress` in the task's `TaskOptions` for example to receive progress events when running
+submodule tasks:
+
+```typescript
+await git.submoduleUpdate('submodule-name', { '--progress': null });
+await git.submoduleUpdate('submodule-name', ['--progress']);
+```
+
 ## Using task callbacks
 
 Each of the methods in the API listing below can be called in a chain and will be called in series after each other.
diff --git a/src/git-factory.js b/src/git-factory.js
index b4ef6c5..e5577be 100644
--- a/src/git-factory.js
+++ b/src/git-factory.js
@@ -1,7 +1,9 @@
 const Git = require('./git');
+
 const {GitConstructError} = require('./lib/api');
 const {PluginStore} = require("./lib/plugins/plugin-store");
 const {commandConfigPrefixingPlugin} = require('./lib/plugins/command-config-prefixing-plugin');
+const {progressMonitorPlugin} = require('./lib/plugins/progress-monitor-plugin');
 const {createInstanceConfig, folderExists} = require('./lib/utils');
 
 const api = Object.create(null);
@@ -45,9 +47,11 @@ module.exports.gitInstanceFactory = function gitInstanceFactory (baseDir, option
       throw new GitConstructError(config, `Cannot use simple-git on a directory that does not exist`);
    }
 
-   if (config.config) {
+   if (Array.isArray(config.config)) {
       plugins.add(commandConfigPrefixingPlugin(config.config));
    }
 
+   config.progress && plugins.add(progressMonitorPlugin(config.progress));
+
    return new Git(config, plugins);
 };
diff --git a/src/lib/plugins/plugin-store.ts b/src/lib/plugins/plugin-store.ts
index 6ea53d1..590719c 100644
--- a/src/lib/plugins/plugin-store.ts
+++ b/src/lib/plugins/plugin-store.ts
@@ -1,13 +1,16 @@
 import { SimpleGitPlugin, SimpleGitPluginType, SimpleGitPluginTypes } from './simple-git-plugin';
+import { asArray } from '../utils';
 
 export class PluginStore {
 
    private plugins: Set<SimpleGitPlugin<SimpleGitPluginType>> = new Set();
 
-   public add<T extends SimpleGitPluginType>(plugin: SimpleGitPlugin<T>) {
-      this.plugins.add(plugin);
+   public add<T extends SimpleGitPluginType>(plugin: SimpleGitPlugin<T> | SimpleGitPlugin<T>[]) {
+      const plugins = asArray(plugin);
+      plugins.forEach(plugin => this.plugins.add(plugin));
+
       return () => {
-         this.plugins.delete(plugin);
+         plugins.forEach(plugin => this.plugins.delete(plugin));
       };
    }
 
diff --git a/src/lib/plugins/progress-monitor-plugin.ts b/src/lib/plugins/progress-monitor-plugin.ts
new file mode 100644
index 0000000..e60cf38
--- /dev/null
+++ b/src/lib/plugins/progress-monitor-plugin.ts
@@ -0,0 +1,50 @@
+import { SimpleGitOptions } from '../types';
+import { asNumber, including } from '../utils';
+
+import { SimpleGitPlugin } from './simple-git-plugin';
+
+export function progressMonitorPlugin(progress: Exclude<SimpleGitOptions['progress'], void>) {
+   const progressCommand = '--progress';
+   const progressMethods = ['checkout', 'clone', 'pull', 'push'];
+
+   const onProgress: SimpleGitPlugin<'spawn.after'> = {
+      type: 'spawn.after',
+      action(_data, context) {
+         if (!context.commands.includes(progressCommand)) {
+            return;
+         }
+
+         context.spawned.stderr?.on('data', (chunk: Buffer) => {
+            const message = /^([a-zA-Z ]+):\s*(\d+)% \((\d+)\/(\d+)\)/.exec(chunk.toString('utf8'));
+            if (!message) {
+               return;
+            }
+
+            progress({
+               method: context.method,
+               stage: progressEventStage(message[1]),
+               progress: asNumber(message[2]),
+               processed: asNumber(message[3]),
+               total: asNumber(message[4]),
+            });
+         });
+      }
+   };
+
+   const onArgs: SimpleGitPlugin<'spawn.args'> = {
+      type: 'spawn.args',
+      action(args, context) {
+         if (!progressMethods.includes(context.method)) {
+            return args;
+         }
+
+         return including(args, progressCommand);
+      }
+   }
+
+   return [onArgs, onProgress];
+}
+
+function progressEventStage (input: string) {
+   return String(input.toLowerCase().split(' ', 1)) || 'unknown';
+}
diff --git a/src/lib/plugins/simple-git-plugin.ts b/src/lib/plugins/simple-git-plugin.ts
index d40e888..6ea8c54 100644
--- a/src/lib/plugins/simple-git-plugin.ts
+++ b/src/lib/plugins/simple-git-plugin.ts
@@ -1,8 +1,21 @@
+import { ChildProcess } from 'child_process';
+
+type SimpleGitTaskPluginContext = {
+   readonly method: string;
+   readonly commands: string[];
+}
+
 export interface SimpleGitPluginTypes {
    'spawn.args': {
       data: string[];
-      context: {};
+      context: SimpleGitTaskPluginContext & {};
    };
+   'spawn.after': {
+      data: void;
+      context: SimpleGitTaskPluginContext & {
+         spawned: ChildProcess,
+      };
+   }
 }
 
 export type SimpleGitPluginType = keyof SimpleGitPluginTypes;
diff --git a/src/lib/runners/git-executor-chain.ts b/src/lib/runners/git-executor-chain.ts
index 0efd841..2a62eba 100644
--- a/src/lib/runners/git-executor-chain.ts
+++ b/src/lib/runners/git-executor-chain.ts
@@ -1,9 +1,8 @@
 import { spawn, SpawnOptions } from 'child_process';
 import { GitError } from '../api';
 import { OutputLogger } from '../git-logger';
+import { PluginStore } from '../plugins';
 import { EmptyTask, isBufferTask, isEmptyTask, } from '../tasks/task';
-import { Scheduler } from './scheduler';
-import { TasksPendingQueue } from './tasks-pending-queue';
 import {
    GitExecutorResult,
    Maybe,
@@ -13,8 +12,9 @@ import {
    SimpleGitTask,
    TaskResponseFormat
 } from '../types';
-import { callTaskParser, GitOutputStreams, objectToString } from '../utils';
-import { PluginStore } from '../plugins/plugin-store';
+import { callTaskParser, first, GitOutputStreams, objectToString } from '../utils';
+import { Scheduler } from './scheduler';
+import { TasksPendingQueue } from './tasks-pending-queue';
 
 export class GitExecutorChain implements SimpleGitExecutor {
 
@@ -82,9 +82,10 @@ export class GitExecutorChain implements SimpleGitExecutor {
    }
 
    private async attemptRemoteTask<R>(task: RunnableTask<R>, logger: OutputLogger) {
-      const args = this._plugins.exec('spawn.args', task.commands, {});
+      const args = this._plugins.exec('spawn.args', [...task.commands], pluginContext(task, task.commands));
 
       const raw = await this.gitResponse(
+         task,
          this.binary, args, this.outputHandler, logger.step('SPAWN'),
       );
       const outputStreams = await this.handleTaskData(task, raw, logger.step('HANDLE'));
@@ -148,7 +149,7 @@ export class GitExecutorChain implements SimpleGitExecutor {
       });
    }
 
-   private async gitResponse(command: string, args: string[], outputHandler: Maybe<outputHandler>, logger: OutputLogger): Promise<GitExecutorResult> {
+   private async gitResponse<R>(task: SimpleGitTask<R>, command: string, args: string[], outputHandler: Maybe<outputHandler>, logger: OutputLogger): Promise<GitExecutorResult> {
       const outputLogger = logger.sibling('output');
       const spawnOptions: SpawnOptions = {
          cwd: this.cwd,
@@ -202,11 +203,23 @@ export class GitExecutorChain implements SimpleGitExecutor {
             outputHandler(command, spawned.stdout!, spawned.stderr!, [...args]);
          }
 
+         this._plugins.exec('spawn.after', undefined, {
+            ...pluginContext(task, args),
+            spawned,
+         });
+
       });
    }
 
 }
 
+function pluginContext<R>(task: SimpleGitTask<R>, commands: string[]) {
+   return {
+      method: first(task.commands) || '',
+      commands,
+   }
+}
+
 function onErrorReceived(target: Buffer[], logger: OutputLogger) {
    return (err: Error) => {
       logger(`[ERROR] child process exception %o`, err);
diff --git a/src/lib/types/handlers.ts b/src/lib/types/handlers.ts
index 7295092..2fc10f5 100644
--- a/src/lib/types/handlers.ts
+++ b/src/lib/types/handlers.ts
@@ -9,3 +9,19 @@ export interface SimpleGitTaskCallback<T = string, E extends GitError = GitError
 
    (err: E): void;
 }
+
+/**
+ * The event data emitted to the progress handler whenever progress detail is received.
+ */
+export interface SimpleGitProgressEvent {
+   /** The underlying method called - push, pull etc */
+   method: string;
+   /** The type of progress being reported, note that any one task may emit many stages - for example `git clone` emits both `receiving` and `resolving` */
+   stage: 'compressing' | 'counting' | 'receiving' | 'resolving' | 'unknown' | 'writing' | string;
+   /** The percent progressed as a number 0 - 100 */
+   progress: number;
+   /** The number of items processed so far */
+   processed: number;
+   /** The total number of items to be processed */
+   total: number;
+}
diff --git a/src/lib/types/index.ts b/src/lib/types/index.ts
index 268728d..b434323 100644
--- a/src/lib/types/index.ts
+++ b/src/lib/types/index.ts
@@ -1,4 +1,5 @@
 import { SimpleGitTask } from './tasks';
+import { SimpleGitProgressEvent } from './handlers';
 
 export * from './handlers';
 export * from './tasks';
@@ -37,9 +38,6 @@ export type outputHandler = (
 export type GitExecutorEnv = NodeJS.ProcessEnv | undefined;
 
 
-
-
-
 /**
  * Public interface of the Executor
  */
@@ -50,6 +48,7 @@ export interface SimpleGitExecutor {
    cwd: string;
 
    chain(): SimpleGitExecutor;
+
    push<R>(task: SimpleGitTask<R>): Promise<R>;
 }
 
@@ -71,6 +70,8 @@ export interface SimpleGitOptions {
    binary: string;
    maxConcurrentProcesses: number;
    config: string[];
+
+   progress?(data: SimpleGitProgressEvent): void;
 }
 
 export type Maybe<T> = T | undefined;
diff --git a/src/lib/utils/util.ts b/src/lib/utils/util.ts
index d8d0357..14bf470 100644
--- a/src/lib/utils/util.ts
+++ b/src/lib/utils/util.ts
@@ -75,7 +75,7 @@ export function folderExists(path: string): boolean {
 }
 
 /**
- * Adds `item` into the `target` `Array` or `Set` when it is not already present.
+ * Adds `item` into the `target` `Array` or `Set` when it is not already present and returns the `item`.
  */
 export function append<T>(target: T[] | Set<T>, item: T): typeof item {
    if (Array.isArray(target)) {
@@ -88,6 +88,17 @@ export function append<T>(target: T[] | Set<T>, item: T): typeof item {
    return item;
 }
 
+/**
+ * Adds `item` into the `target` `Array` when it is not already present and returns the `target`.
+ */
+export function including<T>(target: T[], item: T): typeof target {
+   if (Array.isArray(target) && !target.includes(item)) {
+      target.push(item);
+   }
+
+   return target;
+}
+
 export function remove<T>(target: Set<T> | T[], item: T): T {
    if (Array.isArray(target)) {
       const index = target.indexOf(item);


Test patch:
diff --git a/test/integration/progress-plugin.spec.ts b/test/integration/progress-plugin.spec.ts
new file mode 100644
index 0000000..5d918f7
--- /dev/null
+++ b/test/integration/progress-plugin.spec.ts
@@ -0,0 +1,43 @@
+import { createTestContext, newSimpleGit, SimpleGitTestContext } from '../__fixtures__';
+import { SimpleGitOptions } from '../../src/lib/types';
+
+describe('progress-monitor', () => {
+
+   const upstream = 'https://github.com/steveukx/git-js.git';
+
+   let context: SimpleGitTestContext;
+
+   beforeEach(async () => context = await createTestContext());
+
+   it('emits progress events', async () => {
+      const progress = jest.fn();
+      const opt: Partial<SimpleGitOptions> = {
+         baseDir: context.root,
+         progress,
+      };
+
+      await newSimpleGit(opt).clone(upstream);
+
+      const receivingUpdates = progressEventsAtStage(progress, 'receiving');
+
+      expect(receivingUpdates.length).toBeGreaterThan(0);
+
+      receivingUpdates.reduce((previous, update) => {
+         expect(update).toEqual({
+            method: 'clone',
+            stage: 'receiving',
+            progress: expect.any(Number),
+            processed: expect.any(Number),
+            total: expect.any(Number),
+         });
+
+         expect(update.progress).toBeGreaterThanOrEqual(previous);
+         return update.progress;
+      }, 0);
+   });
+
+});
+
+function progressEventsAtStage (mock: jest.Mock, stage: string) {
+   return mock.mock.calls.filter(c => c[0].stage === stage).map(c => c[0]);
+}
diff --git a/test/unit/__fixtures__/child-processes.ts b/test/unit/__fixtures__/child-processes.ts
index aed4c9e..e337caf 100644
--- a/test/unit/__fixtures__/child-processes.ts
+++ b/test/unit/__fixtures__/child-processes.ts
@@ -4,6 +4,21 @@ import { wait } from '../../__fixtures__';
 const EXIT_CODE_SUCCESS = 0;
 const EXIT_CODE_ERROR = 1;
 
+export async function writeToStdErr (data = '') {
+   await wait();
+   const proc = mockChildProcessModule.$mostRecent();
+
+   if (!proc) {
+      throw new Error(`writeToStdErr unable to find matching child process`);
+   }
+
+   if (proc.$emitted('exit')) {
+      throw new Error('writeToStdErr: attempting to write to an already closed process');
+   }
+
+   proc.stderr.$emit('data', Buffer.from(data));
+}
+
 export async function closeWithError (stack = 'CLOSING WITH ERROR', code = EXIT_CODE_ERROR) {
    await wait();
    const match = mockChildProcessModule.$mostRecent();
diff --git a/test/unit/__fixtures__/expectations.ts b/test/unit/__fixtures__/expectations.ts
index 11a79bf..088aee1 100644
--- a/test/unit/__fixtures__/expectations.ts
+++ b/test/unit/__fixtures__/expectations.ts
@@ -27,6 +27,10 @@ export function assertExecutedCommandsContains(command: string) {
    expect(mockChildProcessModule.$mostRecent().$args.indexOf(command)).not.toBe(-1);
 }
 
+export function assertExecutedCommandsContainsOnce(command: string) {
+   expect(mockChildProcessModule.$mostRecent().$args.filter(c => c === command)).toHaveLength(1);
+}
+
 export function assertChildProcessEnvironmentVariables(env: any) {
    expect(mockChildProcessModule.$mostRecent()).toHaveProperty('$env', env);
 }
diff --git a/test/unit/plugins.spec.ts b/test/unit/plugins.spec.ts
index d3590b7..f9f2604 100644
--- a/test/unit/plugins.spec.ts
+++ b/test/unit/plugins.spec.ts
@@ -1,16 +1,114 @@
 import { SimpleGit } from '../../typings';
-import { assertExecutedCommands, closeWithSuccess, newSimpleGit } from './__fixtures__';
+import {
+   assertExecutedCommands,
+   assertExecutedCommandsContainsOnce,
+   closeWithSuccess,
+   newSimpleGit,
+   writeToStdErr
+} from './__fixtures__';
 
 describe('plugins', () => {
 
    let git: SimpleGit;
+   let fn: jest.Mock;
+
+   beforeEach(() => fn = jest.fn());
 
    it('allows configuration prefixing', async () => {
-      git = newSimpleGit({ config: ['a', 'bcd'] });
+      git = newSimpleGit({config: ['a', 'bcd']});
       git.raw('foo');
 
       await closeWithSuccess();
       assertExecutedCommands('-c', 'a', '-c', 'bcd', 'foo');
-   })
+   });
+
+   describe('progress', () => {
+
+      it('emits progress events when counting objects', async () => {
+         newSimpleGit({progress: fn}).raw('something', '--progress');
+
+         await writeToStdErr(`Counting objects: 90% (180/200)`);
+
+         expect(fn).toHaveBeenCalledWith({
+            method: 'something',
+            progress: 90,
+            processed: 180,
+            stage: 'counting',
+            total: 200,
+         });
+      });
+
+      it('emits progress events when writing objects', async () => {
+         newSimpleGit({progress: fn}).push();
+
+         await writeToStdErr(`Writing objects: 90% (180/200)`);
+
+         expect(fn).toHaveBeenCalledWith({
+            method: 'push',
+            progress: 90,
+            processed: 180,
+            stage: 'writing',
+            total: 200,
+         });
+      });
+
+      it('emits progress events when receiving objects', async () => {
+         newSimpleGit({progress: fn}).raw('something', '--progress');
+
+         await writeToStdErr(`Receiving objects: 5% (1/20)`);
+
+         expect(fn).toHaveBeenCalledWith({
+            method: 'something',
+            progress: 5,
+            processed: 1,
+            stage: 'receiving',
+            total: 20,
+         });
+      });
+
+      it('no progress events emitted if --progress flag is not used', async () => {
+         newSimpleGit({progress: fn}).raw('other');
+
+         await writeToStdErr(`Receiving objects: 5% (1/20)`);
+
+         expect(fn).not.toHaveBeenCalled();
+      });
+
+      it('handles progress with custom config', async () => {
+         git = newSimpleGit({
+            progress: fn,
+            config: ['foo', '--progress', 'bar'],
+         });
+         git.raw('baz');
+
+         await writeToStdErr(`Receiving objects: 10% (100/1000)`);
+         await closeWithSuccess();
+
+         expect(fn).toHaveBeenCalledWith({
+            method: 'baz',
+            progress: 10,
+            processed: 100,
+            stage: 'receiving',
+            total: 1000,
+         });
+      });
+
+      it.each<[string, (git: SimpleGit) => unknown]>([
+         ['checkout', (git) => git.checkout('main')],
+         ['pull', (git) => git.pull()],
+         ['push', (git) => git.push()],
+         ['clone', (git) => git.clone('some-remote.git')],
+         ['checkout - progress set', (git) => git.checkout('main', ['--progress', 'blah'])],
+         ['pull - progress set', (git) => git.pull(['--progress', 'blah'])],
+         ['push - progress set', (git) => git.push(['--progress', 'blah'])],
+         ['clone - progress set', (git) => git.clone('some-remote.git', ['--progress', 'blah'])],
+         ['raw - progress set', (git) => git.raw('foo','--progress', 'blah')],
+      ])(`auto-adds to %s`, async (_name, use) => {
+         use(newSimpleGit({progress: fn}));
+
+         await closeWithSuccess();
+         assertExecutedCommandsContainsOnce('--progress');
+      });
+   });
 
-})
+});
diff --git a/test/unit/utils.spec.ts b/test/unit/utils.spec.ts
index d659212..694d3af 100644
--- a/test/unit/utils.spec.ts
+++ b/test/unit/utils.spec.ts
@@ -8,6 +8,7 @@ import {
    filterPrimitives,
    first,
    forEachLineWithContent,
+   including,
    last,
    NOOP,
    toLinesWithContent
@@ -73,7 +74,7 @@ describe('utils', () => {
 
    describe('arrays', () => {
 
-      function test<T> (target: T[] | Set<T>, itemA: T, itemB: T) {
+      function test<T>(target: T[] | Set<T>, itemA: T, itemB: T) {
          expect(append(target, itemA)).toBe(itemA);
          expect(Array.from(target)).toEqual([itemA]);
 
@@ -87,17 +88,40 @@ describe('utils', () => {
       it('appends objects into an array', () => {
          test([], {a: 1}, {b: 1});
       });
+
       it('appends primitives into an array', () => {
          test<string>([], 'A', 'B');
       });
+
       it('appends objects into a set', () => {
          test(new Set(), {a: 1}, {b: 1});
       });
+
       it('appends primitives into a set', () => {
          test(new Set(), 'A', 'B');
       });
    });
 
+   describe('including', () => {
+
+      it('does nothing when the item already exists', () => {
+         const input = ['abc', 'foo', 'bar'];
+         const output = including(input, 'foo');
+
+         expect(input).toBe(output);
+         expect(output).toEqual(['abc', 'foo', 'bar']);
+      });
+
+      it('appends when the item does not exist', () => {
+         const input = ['abc', 'bar'];
+         const output = including(input, 'foo');
+
+         expect(input).toBe(output);
+         expect(output).toEqual(['abc', 'bar', 'foo']);
+      });
+
+   });
+
    describe('argument filtering', () => {
 
       it('recognises arrays', () => {