Repository: Azure/azure-functions-durable-js (ts)

Issue:
Exception in activity function fails orchestrator with non-deterministic error
**Describe the bug**
An exception in an activity results in a non-deterministic error.

**Investigative information**

- Durable Functions extension version: 2.0.0
- durable-functions npm module version: 1.3.1
- Language (JavaScript/TypeScript) and version: JS
- Node.js version: 10.16.3

**To Reproduce**
Steps to reproduce the behavior:

1. Clone this repo: https://github.com/anthonychu/test-durable-js
1. ```bash
    npm i
    func extensions install
    cp local.settings.SAMPLE.json local.settings.json
    # Add storage connection string to local.settings.json
    func start
    ```
1. Call `http://localhost:7071/api/orchestrators/OrchestratorWithBadActivity` to start the orchestrator that calls an activity that throws an exception.
1. Get the `statusQueryGetUri` in the response and use it to check the status. I think it should show the actual error, but instead it shows a non-deterministic error:
    ```json
    {
      "name": "OrchestratorWithBadActivity",
      "instanceId": "4db326c0a2d34fdbad47cf134e6ddf1c",
      "runtimeStatus": "Failed",
      "input": null,
      "customStatus": null,
      "output": "Non-Deterministic workflow detected: TaskScheduledEvent: 0 TaskScheduled BadActivity",
      "createdTime": "2019-11-20T22:27:37Z",
      "lastUpdatedTime": "2019-11-20T22:27:38Z"
    }
    ```

@ConnorMcMahon @cgillum 

Pull request:
Fix nondeterminism on Activity/Suborchestration failures
When we throw errors in the JS layer, we need some way to embed the orchestration replay state in the errors so that the C# extension can replay behavior. This is necessary so that the Durable Task does not throw an non-determinism exception.

This also requires a fix on the C# extension to be able to read this embedded data from the RPC exception. Here is the PR for the fix in Durable 1.x: https://github.com/Azure/azure-functions-durable-extension/pull/1171. 

Resolve #125 

Patch:
diff --git a/src/durableerror.ts b/src/durableerror.ts
new file mode 100644
index 0000000..de36d1e
--- /dev/null
+++ b/src/durableerror.ts
@@ -0,0 +1,10 @@
+/**
+ * A specfic error thrown when a scheduled activity or suborchestrator has failed.
+ * This error can be checked for via `instanceof` guards to catch only exceptions thrown
+ * by the DurableJS library.
+ */
+export class DurableError extends Error {
+    constructor(message: string | undefined) {
+        super(message);
+    }
+}
diff --git a/src/orchestrationfailureerror.ts b/src/orchestrationfailureerror.ts
new file mode 100644
index 0000000..1c35db0
--- /dev/null
+++ b/src/orchestrationfailureerror.ts
@@ -0,0 +1,29 @@
+import { OrchestratorState } from "./classes";
+
+const outOfProcDataLabel = "\n\n$OutOfProcData$:";
+
+/**
+ * A wrapper for all errors thrown within an orchestrator function. This exception will embed
+ * the orchestrator state in a way that the C# extension knows how to read so that it can replay the
+ * actions scheduled before throwing an exception. This prevents non-determinism errors in Durable Task.
+ * 
+ * Note that making any changes to the following schema to OrchestrationFailureError.message could be considered a breaking change:
+ * 
+ * "<error message as a string>\n\n$OutOfProcData$<json representation of state>"
+ */
+export class OrchestrationFailureError extends Error {
+    constructor(error: any, state: OrchestratorState) {
+        let errorMessage : String;
+        if (error instanceof Error) {
+            errorMessage= error.message;
+        } else if (error instanceof String) {
+            errorMessage = error;
+        } else {
+            errorMessage = JSON.stringify(error);
+        }
+
+        const message = `${errorMessage}${outOfProcDataLabel}${JSON.stringify(state)}`;
+        super(message);
+        this.stack = error.stack;
+    }
+}
diff --git a/src/orchestrator.ts b/src/orchestrator.ts
index 35c4196..a34550a 100644
--- a/src/orchestrator.ts
+++ b/src/orchestrator.ts
@@ -9,6 +9,8 @@ import { CallActivityAction, CallActivityWithRetryAction, CallEntityAction, Call
     TaskScheduledEvent, TaskSet, TimerCreatedEvent, TimerFiredEvent, TimerTask,
     Utils, WaitForExternalEventAction,
 } from "./classes";
+import { DurableError } from "./durableerror";
+import { OrchestrationFailureError } from "./orchestrationfailureerror";
 import { FailedSingleTask } from "./tasks/taskinterfaces";
 import { TokenSource } from "./tokensource";
 
@@ -98,7 +100,7 @@ export class Orchestrator {
                         log("Iterator is done");
                         // The customer returned an absolute type.
                         context.done(
-                            null,
+                            undefined,
                             new OrchestratorState({
                                 isDone: true,
                                 output: g.value,
@@ -127,7 +129,7 @@ export class Orchestrator {
                 // Return continue as new events as completed, as the execution itself is now completed.
                 if (TaskFilter.isSingleTask(partialResult) && partialResult.action instanceof ContinueAsNewAction) {
                     context.done(
-                        null,
+                        undefined,
                         new OrchestratorState({
                             isDone: true,
                             output: undefined,
@@ -140,7 +142,7 @@ export class Orchestrator {
 
                 if (!TaskFilter.isCompletedTask(partialResult)) {
                     context.done(
-                        null,
+                        undefined,
                         new OrchestratorState({
                             isDone: false,
                             output: undefined,
@@ -164,7 +166,7 @@ export class Orchestrator {
                 // would yield the task before returning out of the generator function.
                 if (g.done) {
                     log("Iterator is done");
-                    context.done(null,
+                    context.done(undefined,
                         new OrchestratorState({
                             isDone: true,
                             actions,
@@ -185,17 +187,16 @@ export class Orchestrator {
                 g = gen.next(partialResult.result);
             }
         } catch (error) {
-            log(`Error: ${error}`);
-            context.done(
-                error,
-                new OrchestratorState({
-                    isDone: false,
-                    output: undefined,
-                    actions,
-                    error: error.stack,
-                    customStatus: this.customStatus,
-                }),
-            );
+            // Wrap orchestration state in OutOfProcErrorWrapper to ensure data
+            // gets embedded in error message received by C# extension.
+            const errorState = new OrchestratorState({
+                isDone: false,
+                output: undefined,
+                actions,
+                error: error.message,
+                customStatus: this.customStatus,
+            });
+            context.done(new OrchestrationFailureError(error, errorState), undefined);
             return;
         }
     }
@@ -225,7 +226,7 @@ export class Orchestrator {
                 taskFailed.Timestamp,
                 taskFailed.TaskScheduledId,
                 state.indexOf(taskFailed),
-                new Error(taskFailed.Reason),
+                new DurableError(taskFailed.Reason),
             );
         } else {
             return TaskFactory.UncompletedTask(
@@ -256,14 +257,16 @@ export class Orchestrator {
                 const result = this.parseHistoryEvent(taskCompleted);
 
                 return TaskFactory.SuccessfulTask(newAction, result, taskCompleted.Timestamp, taskCompleted.TaskScheduledId, state.indexOf(taskCompleted));
-            } else if (taskFailed && taskRetryTimer && attempt >= retryOptions.maxNumberOfAttempts) {
+            } else if (taskFailed
+                && taskRetryTimer
+                && attempt >= retryOptions.maxNumberOfAttempts) {
                 return TaskFactory.FailedTask(
                     newAction,
                     taskFailed.Reason,
                     taskFailed.Timestamp,
                     taskFailed.TaskScheduledId,
                     state.indexOf(taskFailed),
-                    new Error(taskFailed.Reason),
+                    new DurableError(taskFailed.Reason),
                 );
             }
         }
@@ -278,7 +281,7 @@ export class Orchestrator {
         const eventSent = this.findEventSent(state, schedulerId, "op");
         let eventRaised;
         if (eventSent) {
-            const eventSentInput = eventSent && eventSent.Input !== undefined ? JSON.parse(eventSent.Input) as RequestMessage : undefined;
+            const eventSentInput = eventSent && eventSent.Input ? JSON.parse(eventSent.Input) as RequestMessage : undefined;
             eventRaised = eventSentInput ? this.findEventRaised(state, eventSentInput.id) : undefined;
         }
         this.setProcessed([ eventSent, eventRaised ]);
@@ -325,7 +328,7 @@ export class Orchestrator {
                 subOrchestratorFailed.Timestamp,
                 subOrchestratorFailed.TaskScheduledId,
                 state.indexOf(subOrchestratorFailed),
-                new Error(subOrchestratorFailed.Reason),
+                new DurableError(subOrchestratorFailed.Reason),
             );
         } else {
             return TaskFactory.UncompletedTask(
@@ -373,14 +376,16 @@ export class Orchestrator {
                     subOrchestratorCompleted.TaskScheduledId,
                     state.indexOf(subOrchestratorCompleted),
                 );
-            } else if (subOrchestratorFailed && retryTimer && attempt >= retryOptions.maxNumberOfAttempts) {
+            } else if (subOrchestratorFailed
+                && retryTimer
+                && attempt >= retryOptions.maxNumberOfAttempts) {
                 return TaskFactory.FailedTask(
                     newAction,
                     subOrchestratorFailed.Reason,
                     subOrchestratorFailed.Timestamp,
                     subOrchestratorFailed.TaskScheduledId,
                     state.indexOf(subOrchestratorFailed),
-                    new Error(subOrchestratorFailed.Reason),
+                    new DurableError(subOrchestratorFailed.Reason),
                 );
             }
         }
@@ -427,7 +432,7 @@ export class Orchestrator {
                 httpFailed.Timestamp,
                 httpFailed.TaskScheduledId,
                 state.indexOf(httpFailed),
-                new Error(httpFailed.Reason),
+                new DurableError(httpFailed.Reason),
             );
         } else {
             return TaskFactory.UncompletedTask(
@@ -464,7 +469,7 @@ export class Orchestrator {
 
     private isLocked(contextLocks: EntityId[]): LockState {
         return new LockState(
-            contextLocks !== undefined && contextLocks !== null,
+            contextLocks && contextLocks !== null,
             contextLocks,
         );
     }
@@ -568,8 +573,8 @@ export class Orchestrator {
         const completedTasks = tasks
             .filter(TaskFilter.isSuccessfulSingleTask)
             .sort((a, b) => {
-                if (a.completedHistoryEventIndex > b.completedHistoryEventIndex) { return 1; }
-                if (a.completedHistoryEventIndex < b.completedHistoryEventIndex) { return -1; }
+                if (a.completionIndex > b.completionIndex) { return 1; }
+                if (a.completionIndex < b.completionIndex) { return -1; }
                 return 0;
             });
 
@@ -587,7 +592,8 @@ export class Orchestrator {
         switch (directiveResult.EventType) {
             case (HistoryEventType.EventRaised):
                 const eventRaised = directiveResult as EventRaisedEvent;
-                parsedDirectiveResult = (eventRaised && eventRaised.Input !== undefined) ? JSON.parse(eventRaised.Input) : undefined;
+                parsedDirectiveResult = (eventRaised && eventRaised.Input) 
+                    ? JSON.parse(eventRaised.Input) : undefined;
                 break;
             case (HistoryEventType.SubOrchestrationInstanceCompleted):
                 parsedDirectiveResult = JSON.parse((directiveResult as SubOrchestrationInstanceCompletedEvent).Result);
@@ -708,37 +714,45 @@ export class Orchestrator {
     }
 
     /* Returns undefined if not found. */
-    private findTaskScheduled(state: HistoryEvent[], name: string): TaskScheduledEvent {
+    private findTaskScheduled(state: HistoryEvent[], name: string): TaskScheduledEvent | undefined {
         const returnValue = name
             ? state.filter((val: HistoryEvent) => {
                 return val.EventType === HistoryEventType.TaskScheduled
                     && (val as TaskScheduledEvent).Name === name
                     && !val.IsProcessed;
-            })[0]
+            })[0] as TaskScheduledEvent
             : undefined;
-        return returnValue as TaskScheduledEvent;
+        return returnValue;
     }
 
     /* Returns undefined if not found. */
-    private findTaskCompleted(state: HistoryEvent[], scheduledTask: TaskScheduledEvent): TaskCompletedEvent {
+    private findTaskCompleted(state: HistoryEvent[], scheduledTask: TaskScheduledEvent | undefined): TaskCompletedEvent | undefined {
+        if (scheduledTask === undefined) {
+            return undefined;
+        }
+
         const returnValue = scheduledTask
             ? state.filter((val: HistoryEvent) => {
                 return val.EventType === HistoryEventType.TaskCompleted
                     && (val as TaskCompletedEvent).TaskScheduledId === scheduledTask.EventId;
-            })[0]
+            })[0] as TaskCompletedEvent
             : undefined;
-        return returnValue as TaskCompletedEvent;
+        return returnValue;
     }
 
     /* Returns undefined if not found. */
-    private findTaskFailed(state: HistoryEvent[], scheduledTask: HistoryEvent): TaskFailedEvent {
+    private findTaskFailed(state: HistoryEvent[], scheduledTask: TaskScheduledEvent | undefined): TaskFailedEvent | undefined {
+        if (scheduledTask === undefined) {
+            return undefined;
+        }
+
         const returnValue = scheduledTask
         ? state.filter((val: HistoryEvent) => {
                 return val.EventType === HistoryEventType.TaskFailed
                     && (val as TaskFailedEvent).TaskScheduledId === scheduledTask.EventId;
-            })[0]
+            })[0] as TaskFailedEvent
             : undefined;
-        return returnValue as TaskFailedEvent;
+        return returnValue;
     }
 
     /* Returns undefined if not found. */
@@ -758,9 +772,9 @@ export class Orchestrator {
             ? state.filter((val: HistoryEvent) => {
                 return val.EventType === HistoryEventType.TimerFired
                     && (val as TimerFiredEvent).TimerId === createdTimer.EventId;
-            })[0]
+            })[0] as TimerFiredEvent
             : undefined;
-        return returnValue as TimerFiredEvent;
+        return returnValue;
     }
 
     private setProcessed(events: Array<HistoryEvent | undefined>): void {
diff --git a/src/tasks/taskfactory.ts b/src/tasks/taskfactory.ts
index b78fcc1..4b35395 100644
--- a/src/tasks/taskfactory.ts
+++ b/src/tasks/taskfactory.ts
@@ -52,7 +52,7 @@ export class TaskFactory {
         );
     }
 
-    public static FailedTaskSet(tasks: Task[], exception: unknown): TaskSet {
+    public static FailedTaskSet(tasks: Task[], exception: Error): TaskSet {
         return new TaskSet(
             true,
             true,
diff --git a/src/tasks/taskfilter.ts b/src/tasks/taskfilter.ts
index 306cec3..e2c00bb 100644
--- a/src/tasks/taskfilter.ts
+++ b/src/tasks/taskfilter.ts
@@ -33,7 +33,7 @@ export class TaskFilter {
     }
 
     public static isSuccessfulSingleTask(task: Task): task is SuccessfulSingleTask  {
-        return TaskFilter.isSingleTask(task) && task.isCompleted === true && task.isFaulted === false && task.result !== undefined;
+        return TaskFilter.isSingleTask(task) && task.isCompleted === true && task.isFaulted === false && task.completionIndex !== undefined;
     }
 
     public static isFailedTask(task: Task | TaskSet): task is FailedTask {
diff --git a/src/tasks/taskinterfaces.ts b/src/tasks/taskinterfaces.ts
index 2760013..d24c565 100644
--- a/src/tasks/taskinterfaces.ts
+++ b/src/tasks/taskinterfaces.ts
@@ -17,7 +17,7 @@ export interface UncompletedSingleTask extends TaskBase {
 }
 
 export interface CompletedSingleTask extends SingleTask {
-    completedHistoryEventIndex: number;
+    completionIndex: number;
     readonly timestamp: Date;
     readonly id: number;
     readonly isCompleted: true;
@@ -31,7 +31,7 @@ export interface SuccessfulSingleTask extends CompletedSingleTask {
 
 export interface FailedSingleTask extends CompletedSingleTask {
     readonly isFaulted: true;
-    readonly exception: unknown;
+    readonly exception: Error;
     readonly result: undefined;
 }
 
@@ -51,7 +51,7 @@ export interface UncompletedTaskSet extends TaskCollection {
 
 export interface FailedTaskSet extends CompletedTaskSet {
     readonly isFaulted: true;
-    readonly exception: unknown;
+    readonly exception: Error;
     readonly result: undefined;
 }
 
diff --git a/src/tasks/taskset.ts b/src/tasks/taskset.ts
index daed0a1..c240b8a 100644
--- a/src/tasks/taskset.ts
+++ b/src/tasks/taskset.ts
@@ -7,6 +7,6 @@ export class TaskSet {
         public readonly isFaulted: boolean,
         public readonly tasks: SingleTask[],
         public result?: unknown,
-        public exception?: unknown,
+        public exception?: Error,
     ) { }
 }


Test patch:
diff --git a/test/integration/orchestrator-spec.ts b/test/integration/orchestrator-spec.ts
index 297f38a..12765a7 100644
--- a/test/integration/orchestrator-spec.ts
+++ b/test/integration/orchestrator-spec.ts
@@ -5,14 +5,16 @@ import * as uuidv1 from "uuid/v1";
 import { isUUID } from "validator";
 import { ManagedIdentityTokenSource } from "../../src";
 import {
-    CallActivityAction, CallActivityWithRetryAction, CallEntityAction, CallHttpAction, CallSubOrchestratorAction,
+    ActionType, CallActivityAction, CallActivityWithRetryAction, CallEntityAction, CallHttpAction, CallSubOrchestratorAction,
     CallSubOrchestratorWithRetryAction, ContinueAsNewAction, CreateTimerAction, DurableHttpRequest,
     DurableHttpResponse, DurableOrchestrationBindingInfo, DurableOrchestrationContext, EntityId,
     ExternalEventType, IOrchestratorState, LockState, OrchestratorState,
     RetryOptions, WaitForExternalEventAction,
 } from "../../src/classes";
+import { OrchestrationFailureError } from "../../src/orchestrationfailureerror";
 import { TestHistories } from "../testobjects/testhistories";
 import { TestOrchestrations } from "../testobjects/TestOrchestrations";
+import { TestUtils } from "../testobjects/testutils";
 
 describe("Orchestrator", () => {
     const falsyValues = [ false, 0, "", null, undefined, NaN ];
@@ -90,7 +92,7 @@ describe("Orchestrator", () => {
             } else {
                 expect(mockContext.doneValue!.output).to.equal(falsyValue);
             }
-            expect(mockContext.err).to.equal(null);
+            expect(mockContext.err).to.equal(undefined);
         });
     });
 
@@ -195,7 +197,7 @@ describe("Orchestrator", () => {
 
             expect(mockContext.doneValue!.error).to.equal(undefined);
 
-            expect(mockContext.err).to.equal(null);
+            expect(mockContext.err).to.equal(undefined);
         });
     });
 
@@ -214,13 +216,16 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
                 actions: [],
             });
-            expect(mockContext.doneValue!.error).to.include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("reports an unhandled exception from activity passed through orchestrator", async () => {
@@ -236,16 +241,24 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
                 actions:
                 [
-                    [ new CallActivityAction("ThrowsErrorActivity") ],
+                    [
+                        {
+                            actionType: ActionType.CallActivity,
+                            functionName: "ThrowsErrorActivity",
+                        },
+                    ],
                 ],
             });
-            expect(mockContext.doneValue!.error).to.include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("schedules an activity function after orchestrator catches an exception", async () => {
@@ -512,17 +525,18 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include(
-                new OrchestratorState({
-                    isDone: false,
-                    output: undefined,
-                    actions: [],
-                }),
-            );
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
 
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState).to.be.an("object").that.deep.include({
+                isDone: false,
+                actions: [],
+            });
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("schedules an activity function", async () => {
@@ -637,22 +651,16 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
-                actions:
-                [
-                    [
-                        new CallActivityWithRetryAction(
-                            "Hello",
-                            retryOptions,
-                            name),
-                    ],
-                ],
+                actions: [ [ new CallActivityWithRetryAction("Hello", retryOptions, name) ]],
             });
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("handles a completed activity function", async () => {
@@ -985,10 +993,12 @@ describe("Orchestrator", () => {
 
             const expectedErr = "The sub-orchestration call (n = 1) should be executed with a function name of SayHelloInline instead of the provided function name of SayHelloWithActivity. Check your code for non-deterministic behavior.";
 
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
 
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("replay does not match history (mismatched suborchestration instance id) and throws error.", async () => {
@@ -1014,10 +1024,12 @@ describe("Orchestrator", () => {
 
             const expectedErr = `The sub-orchestration call (n = 1) should be executed with an instance id of ${subId} instead of the provided instance id of ${id}:0. Check your code for non-deterministic behavior.`;
 
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
 
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("handles a completed suborchestrator function", async () => {
@@ -1075,18 +1087,19 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
                 actions:
                 [
                     [ new CallSubOrchestratorAction("SayHelloWithActivity", childId, name) ],
                 ],
             });
-
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
     });
 
@@ -1107,14 +1120,16 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
                 actions: [],
             });
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("schedules a suborchestrator function", async () => {
@@ -1252,7 +1267,12 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.an("object").that.deep.include({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.an("object").that.deep.include({
                 isDone: false,
                 actions:
                 [
@@ -1266,10 +1286,7 @@ describe("Orchestrator", () => {
                     ],
                 ],
             });
-            expect(mockContext.doneValue!.error).to
-                .include(expectedErr);
-
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("handles a completed suborchestrator function", async () => {
@@ -1717,19 +1734,22 @@ describe("Orchestrator", () => {
 
             orchestrator(mockContext);
 
-            expect(mockContext.doneValue).to.be.deep.include(
-                new OrchestratorState({
+            expect(mockContext.err).to.be.an.instanceOf(OrchestrationFailureError);
+
+            const orchestrationState =
+                TestUtils.extractStateFromError(mockContext.err as OrchestrationFailureError);
+
+            expect(orchestrationState).to.be.deep.include(
+                {
                     isDone: false,
-                    output: undefined,
                     actions:
                     [
                         [ new CallActivityAction("GetFileList", "C:\\Dev")],
                         filePaths.map((file) => new CallActivityAction("GetFileSize", file)),
                     ],
-                }),
+                },
             );
-            expect(mockContext.doneValue!.error).to.include(expectedErr);
-            expect(mockContext.err!.toString()).to.include(expectedErr);
+            expect(orchestrationState.error).to.include(expectedErr);
         });
 
         it("Task.any proceeds if a scheduled parallel task completes in order", async () => {
@@ -1851,7 +1871,7 @@ describe("Orchestrator", () => {
                         currentTime.toDate(),
                         1,
                     ),
-                    null,
+                    undefined,
                 ),
             });
 
@@ -1875,7 +1895,7 @@ describe("Orchestrator", () => {
                         currentTime.toDate(),
                         2,
                     ),
-                    null,
+                    undefined,
                 ),
             });
 
@@ -1899,7 +1919,7 @@ describe("Orchestrator", () => {
                         currentTime.toDate(),
                         3,
                     ),
-                    null,
+                    undefined,
                 ),
             });
 
@@ -1923,7 +1943,7 @@ describe("Orchestrator", () => {
                         currentTime.toDate(),
                         4,
                     ),
-                    null,
+                    undefined,
                 ),
             });
 
@@ -1940,6 +1960,59 @@ describe("Orchestrator", () => {
                 }),
             );
         });
+
+        it("Timer in combination with Task.any() timer wins", async () => {
+            const orchestrator = TestOrchestrations.TimerActivityRace;
+            const currentTime = moment.utc();
+
+            // first iteration
+            let mockContext = new MockContext({
+                context: new DurableOrchestrationBindingInfo(
+                    TestHistories.GetTimerActivityRaceTimerWinsHistory(
+                        currentTime.toDate(),
+                        1,
+                    ),
+                    null,
+                ),
+            });
+
+            orchestrator(mockContext);
+
+            expect(mockContext.doneValue).to.be.deep.equal(
+                new OrchestratorState({
+                    isDone: false,
+                    actions:
+                    [
+                        [ new CreateTimerAction(currentTime.add(1, "s").toDate()), new CallActivityAction("TaskA") ],
+                    ],
+                    output: undefined,
+                }),
+            );
+
+            // second iteration
+            mockContext = new MockContext({
+                context: new DurableOrchestrationBindingInfo(
+                    TestHistories.GetTimerActivityRaceTimerWinsHistory(
+                        currentTime.toDate(),
+                        2,
+                    ),
+                    null,
+                ),
+            });
+
+            orchestrator(mockContext);
+
+            expect(mockContext.doneValue).to.be.deep.equal(
+                new OrchestratorState({
+                    isDone: true,
+                    actions:
+                    [
+                        [ new CreateTimerAction(currentTime.add(1, "s").toDate()), new CallActivityAction("TaskA") ],
+                    ],
+                    output: "Timer finished",
+                }),
+            );
+        });
     });
 
     // rewind
diff --git a/test/testobjects/testhistories.ts b/test/testobjects/testhistories.ts
index b78da39..90efefc 100644
--- a/test/testobjects/testhistories.ts
+++ b/test/testobjects/testhistories.ts
@@ -180,6 +180,61 @@ export class TestHistories {
         return history;
     }
 
+    public static GetTimerActivityRaceTimerWinsHistory(firstTimestamp: Date, iteration: number): HistoryEvent[] {
+        const firstIteration = moment(firstTimestamp);
+        const fireAt = firstIteration.add(1, "s").toDate();
+        const secondIteration = firstIteration.add(1100, "ms").toDate();
+
+        const history = [];
+        if (iteration >= 1) {
+            history.push(new OrchestratorStartedEvent({
+                eventId: -1,
+                timestamp: firstTimestamp,
+                isPlayed: false,
+            }));
+            history.push(new ExecutionStartedEvent({
+                eventId: -1,
+                timestamp: firstTimestamp,
+                isPlayed: true,
+                name: "TimerActivityRace",
+            }));
+            history.push(new TimerCreatedEvent({
+                eventId: 0,
+                timestamp: firstTimestamp,
+                isPlayed: false,
+                fireAt,
+            }));
+            history.push(new TaskScheduledEvent({
+                eventId: 1,
+                timestamp: firstTimestamp,
+                isPlayed: iteration > 1,
+                name: "TaskA",
+            }));
+            history.push(new OrchestratorCompletedEvent({
+                eventId: -1,
+                timestamp: firstTimestamp,
+                isPlayed: iteration > 1,
+            }));
+        }
+
+        if (iteration >= 2) {
+            history.push(new OrchestratorStartedEvent({
+                eventId: -1,
+                timestamp: secondIteration,
+                isPlayed: iteration > 2,
+            }));
+            history.push(new TimerFiredEvent({
+                eventId: -1,
+                timestamp: firstTimestamp,
+                fireAt,
+                isPlayed: iteration > 2,
+                timerId: 0,
+            }));
+        }
+
+        return history;
+    }
+
     public static GetCallEntitySet(firstTimestamp: Date, entityId: EntityId) {
         const firstMoment = moment(firstTimestamp);
         const orchestratorId = uuidv1();
diff --git a/test/testobjects/testutils.ts b/test/testobjects/testutils.ts
index 71386c7..f5b175d 100644
--- a/test/testobjects/testutils.ts
+++ b/test/testobjects/testutils.ts
@@ -1,5 +1,6 @@
-import { HttpCreationPayload, HttpManagementPayload,
+import { HttpCreationPayload, HttpManagementPayload, IOrchestratorState,
     OrchestrationClientInputData } from "../../src/classes";
+import { OrchestrationFailureError } from "../../src/orchestrationfailureerror";
 import { TestConstants } from "./testconstants";
 
 export class TestUtils {
@@ -12,7 +13,8 @@ export class TestUtils {
             taskHub,
             TestUtils.createHttpCreationPayload(host, taskHub, connection),
             TestUtils.createHttpManagementPayload(id, host, taskHub, connection),
-            `${host}${TestConstants.webhookPath.replace(/\/$/, "")}`, // Returns baseURL with remaining whitespace trimmed.
+            // Returns baseURL with remaining whitespace trimmed.
+            `${host}${TestConstants.webhookPath.replace(/\/$/, "")}`,
             TestConstants.testCode,
         );
     }
@@ -72,4 +74,12 @@ export class TestUtils {
                 .replace(TestConstants.connectionPlaceholder, connection),
         );
     }
+
+    public static extractStateFromError(err: OrchestrationFailureError): IOrchestratorState {
+        const label = "\n\n$OutOfProcData$:";
+        const message = err.message;
+        const dataStart = message.indexOf(label) + label.length;
+        const dataJson = message.substr(dataStart);
+        return JSON.parse(dataJson) as IOrchestratorState;
+    }
 }