Repository: serverless/serverless-azure-functions
Pull request #215 (2019-07-25 17:22:45)

fix: Add retry support when listing deployed functions
When listing the deployed functions on 1st time deployment it may take some additional time to initialize the function app and extract the functions.  This adds a retry policy that will retry the function listing multiple times until the function app is fully ready.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/24796821/badge)](https://coveralls.io/builds/24796821)

Coverage decreased (-0.3%) to 84.146% when pulling **9164a913a58f3bb4b3854349eaf2c8ae6c23d01d on wabrez/async-retry** into **844ffb5c93224f34b986460937c9cd2015f5b5ee on dev**.

pjlittle: One small comment, otherwise LGTM! Hopefully this helps w/ some of the flakiness we've seen :shipit: 

Patch:
diff --git a/src/services/apimService.ts b/src/services/apimService.ts
index 9257b13..f66a722 100644
--- a/src/services/apimService.ts
+++ b/src/services/apimService.ts
@@ -216,7 +216,9 @@ export class ApimService extends BaseService {
         responses: options.operation.responses || [],
       };
 
-      const operationUrl = `${service.gatewayUrl}/${api.path}${operationConfig.urlTemplate}`;
+      // Ensure a single path seperator in the operation path
+      const operationPath = `/${api.path}/${operationConfig.urlTemplate}`.replace(/\/+/g, "/");
+      const operationUrl = `${service.gatewayUrl}${operationPath}`;
       this.log(`--> Deploying API operation ${options.function}: ${operationConfig.method.toUpperCase()} ${operationUrl}`);
 
       const operation = await client.apiOperation.createOrUpdate(
diff --git a/src/services/functionAppService.ts b/src/services/functionAppService.ts
index b0eb5cd..a94f575 100644
--- a/src/services/functionAppService.ts
+++ b/src/services/functionAppService.ts
@@ -10,8 +10,11 @@ import { Guard } from "../shared/guard";
 import { ArmService } from "./armService";
 import { AzureBlobStorageService } from "./azureBlobStorageService";
 import { BaseService } from "./baseService";
+import { Utils } from "../shared/utils";
 
 export class FunctionAppService extends BaseService {
+  private static readonly retryCount: number = 10;
+  private static readonly retryInterval: number = 5000;
   private webClient: WebSiteManagementClient;
   private blobService: AzureBlobStorageService;
 
@@ -90,27 +93,53 @@ export class FunctionAppService extends BaseService {
     Guard.null(functionApp);
 
     const getTokenUrl = `${this.baseUrl}${functionApp.id}/functions?api-version=2016-08-01`;
-    const response = await this.sendApiRequest("GET", getTokenUrl);
+    try {
+      const response = await Utils.runWithRetry(async () => {
+        const listFunctionsResponse = await this.sendApiRequest("GET", getTokenUrl);
 
-    if (response.status !== 200) {
-      return [];
-    }
+        if (listFunctionsResponse.status !== 200 || listFunctionsResponse.data.value.length === 0) {
+          this.log("-> Function App not ready. Retrying...");
+          throw new Error(listFunctionsResponse.data);
+        }
+
+        return listFunctionsResponse;
+      }, FunctionAppService.retryCount, FunctionAppService.retryInterval);
 
-    return response.data.value.map((functionConfig) => functionConfig.properties);
+      return response.data.value.map((functionConfig) => functionConfig.properties);
+    }
+    catch (e) {
+      this.log("-> Unable to retrieve function app list");
+      throw e;
+    }
   }
 
+  /**
+   * Gets the configuration of the specified function within the function app
+   * @param functionApp The parent function app
+   * @param functionName The name of hte function
+   */
   public async getFunction(functionApp: Site, functionName: string): Promise<FunctionEnvelope> {
     Guard.null(functionApp);
     Guard.empty(functionName);
 
     const getFunctionUrl = `${this.baseUrl}${functionApp.id}/functions/${functionName}?api-version=2016-08-01`;
-    const response = await this.sendApiRequest("GET", getFunctionUrl);
 
-    if (response.status !== 200) {
+    try {
+      const response = await Utils.runWithRetry(async () => {
+        const getFunctionResponse = await this.sendApiRequest("GET", getFunctionUrl);
+
+        if (getFunctionResponse.status !== 200) {
+          this.log("-> Function app not ready. Retrying...")
+          throw new Error(response.data);
+        }
+
+        return getFunctionResponse;
+      }, FunctionAppService.retryCount, FunctionAppService.retryInterval);
+
+      return response.data.properties;
+    } catch (e) {
       return null;
     }
-
-    return response.data.properties;
   }
 
   public async uploadFunctions(functionApp: Site): Promise<any> {
@@ -122,6 +151,23 @@ export class FunctionAppService extends BaseService {
     const uploadFunctionApp = this.uploadZippedArfifactToFunctionApp(functionApp, functionZipFile);
     const uploadBlobStorage = this.uploadZippedArtifactToBlobStorage(functionZipFile);
     await Promise.all([uploadFunctionApp, uploadBlobStorage]);
+
+
+    this.log("Deployed serverless functions:")
+    const serverlessFunctions = this.serverless.service.getAllFunctions();
+    const deployedFunctions = await this.listFunctions(functionApp);
+
+    // List functions that are part of the serverless yaml config
+    deployedFunctions.forEach((functionConfig) => {
+      if (serverlessFunctions.includes(functionConfig.name)) {
+        const httpConfig = this.getFunctionHttpTriggerConfig(functionApp, functionConfig);
+
+        if (httpConfig) {
+          const method = httpConfig.methods[0].toUpperCase();
+          this.log(`-> ${functionConfig.name}: ${method} ${httpConfig.url}`);
+        }
+      }
+    });
   }
 
   /**
@@ -166,23 +212,7 @@ export class FunctionAppService extends BaseService {
     };
 
     await this.sendFile(requestOptions, functionZipFile);
-
     this.log("-> Function package uploaded successfully");
-    const serverlessFunctions = this.serverless.service.getAllFunctions();
-    const deployedFunctions = await this.listFunctions(functionApp);
-
-    this.log("Deployed serverless functions:")
-    deployedFunctions.forEach((functionConfig) => {
-      // List functions that are part of the serverless yaml config
-      if (serverlessFunctions.includes(functionConfig.name)) {
-        const httpConfig = this.getFunctionHttpTriggerConfig(functionApp, functionConfig);
-
-        if (httpConfig) {
-          const method = httpConfig.methods[0].toUpperCase();
-          this.log(`-> ${functionConfig.name}: ${method} ${httpConfig.url}`);
-        }
-      }
-    });
   }
 
   /**
diff --git a/src/shared/utils.ts b/src/shared/utils.ts
index bf3717f..3d091fc 100644
--- a/src/shared/utils.ts
+++ b/src/shared/utils.ts
@@ -160,4 +160,39 @@ export class Utils {
       return settings && settings.direction === "out";
     });
   }
+
+  /**
+   * Runs an operation with auto retry policy
+   * @param operation The operation to run
+   * @param maxRetries The max number or retreis
+   * @param retryWaitInterval The time to wait between retries
+   */
+  public static async runWithRetry<T>(operation: (retry?: number) => Promise<T>, maxRetries: number = 3, retryWaitInterval: number = 1000) {
+    let retry = 0;
+    let error = null;
+
+    while (retry < maxRetries) {
+      try {
+        retry++;
+        return await operation(retry);
+      }
+      catch (e) {
+        error = e;
+      }
+
+      await Utils.wait(retryWaitInterval);
+    }
+
+    return Promise.reject(error);
+  }
+
+  /**
+   * Waits for the specified amount of time.
+   * @param time The amount of time to wait (default = 1000ms)
+   */
+  public static wait(time: number = 1000) {
+    return new Promise((resolve) => {
+      setTimeout(resolve, time);
+    });
+  }
 }


Test patch:
diff --git a/src/shared/utils.test.ts b/src/shared/utils.test.ts
index 668d6e8..517c60b 100644
--- a/src/shared/utils.test.ts
+++ b/src/shared/utils.test.ts
@@ -105,4 +105,87 @@ describe("utils", () => {
       }
     );
   });
+
+  describe("runWithRetry", () => {
+    it("returns values after 1st run", async () => {
+      const expected = "success";
+      let lastRetry = 0;
+
+      const result = await Utils.runWithRetry((retry) => {
+        lastRetry = retry;
+        return Promise.resolve(expected);
+      });
+
+      expect(lastRetry).toEqual(1);
+      expect(result).toEqual(expected);
+    });
+
+    it("returns values after successfully retry (reject promise)", async () => {
+      const expected = "success";
+      let lastRetry = 0;
+
+      const result = await Utils.runWithRetry((retry) => {
+        lastRetry = retry;
+        if (retry === 1) {
+          return Promise.reject("rejected");
+        }
+
+        return Promise.resolve(expected);
+      });
+
+      expect(lastRetry).toEqual(2);
+      expect(result).toEqual(expected);
+    });
+
+    it("returns values after successfully retry (throw error)", async () => {
+      const expected = "success";
+      let lastRetry = 0;
+
+      const result = await Utils.runWithRetry((retry) => {
+        lastRetry = retry;
+        if (retry === 1) {
+          throw new Error("Ooops!")
+        }
+
+        return Promise.resolve(expected);
+      });
+
+      expect(lastRetry).toEqual(2);
+      expect(result).toEqual(expected);
+    });
+    it("throws error after reties", async () => {
+      const maxRetries = 5;
+      let lastRetry = 0;
+
+      const test = async () => {
+        await Utils.runWithRetry((retry) => {
+          lastRetry = retry;
+          return Promise.reject("rejected");
+        }, maxRetries, 100);
+      };
+
+      await expect(test()).rejects.toEqual("rejected");
+      expect(lastRetry).toEqual(maxRetries);
+    });
+  });
+
+  describe("wait", () => {
+    const setTimeoutMock = jest.fn((resolve) => resolve());
+
+    beforeEach(() => {
+      global.setTimeout = setTimeoutMock;
+    });
+
+    it("waits 1000 by default", async () => {
+      await Utils.wait();
+
+      expect(setTimeoutMock).toBeCalledWith(expect.any(Function), 1000);
+    });
+
+    it("waits specified time", async () => {
+      await Utils.wait(2000);
+
+      expect(setTimeoutMock).toBeCalledWith(expect.any(Function), 2000);
+    });
+  });
 });