Repository: serverless/serverless (JavaScript)
Date: 2017-01-06 21:21:07+00:00
Task type: Bug Fix

Issue:
Serverless metrics outputs incorrect average function duration
<!--
2. Please check if an issue already exists so there are no duplicates
3. Check out and follow our Guidelines: https://github.com/serverless/serverless/blob/master/CONTRIBUTING.md
4. Fill out the whole template so we have a good overview on the issue
5. Do not remove any section of the template. If something is not applicable leave it empty but leave it in the Issue
6. Please follow the template, otherwise we'll have to ask you to update it
-->

# This is a Bug Report

## Description
#### What went wrong?
My functions never executed longer than 800 ms, but serverless says average time is 831 ms.

#### What did you expect should have happened?
I expected to see a real metrics

#### What was the config you used?
Not applicable.

#### What stacktrace or error message from your provider did you see?
No error message.

Similar or dependent issues:
* none

## Additional Data

* ***Serverless Framework Version you're using***: 1.5.0
* ***Operating System***: Ubuntu 16.10
* ***Stack Trace***: none
* ***Provider Error messages***: none

I cloned the latest `master` branch locally to add some `console.log`s to see what's the problem.

**awsMetrics.js**
```diff
      _.forEach(metrics, (metric, index) => {
+        console.log(`--- ${index} function`);
        _.forEach(metric, (funcMetric) => {
          if (funcMetric.Label === 'Invocations') {
            invocations += reduceDatapoints(funcMetric.Datapoints, 'Sum');
          } else if (funcMetric.Label === 'Throttles') {
            throttles += reduceDatapoints(funcMetric.Datapoints, 'Sum');
          } else if (funcMetric.Label === 'Errors') {
            errors += reduceDatapoints(funcMetric.Datapoints, 'Sum');
          } else {
+            funcMetric.Datapoints.forEach(point => {
+              console.log(moment(point.Timestamp).format('YYYY-MM-DD'), '-', point.Average.toFixed(2));
+            });
            duration += reduceDatapoints(funcMetric.Datapoints, 'Average');
          }
        });
      });
```

which produced the output:

```bash
$ ~/serverless/bin/serverless metrics --startTime 10d --stage prod
--- 0 function
2017-01-01 - 247.40
2016-12-29 - 129.54
--- 1 function
2016-12-28 - 575.13
2017-01-01 - 255.61
2016-12-29 - 186.05
2017-01-02 - 284.15
2016-12-31 - 186.15
--- 2 function
2017-01-01 - 102.32
2016-12-29 - 102.07
2017-01-02 - 222.01
2016-12-31 - 97.99
--- 3 function
2016-12-29 - 565.32
2017-01-02 - 371.52
Service wide metrics
December 27, 2016 9:26 PM - January 6, 2017 9:26 PM

Invocations: 725 
Throttles: 0 
Errors: 0 
Duration (avg.): 831.3125ms

```

So it's visible that something is wrong in calculating the average. After looking forward I found that we sum together milliseconds from every day and divide them by `metrics.length` i.e. by number of functions. But we don't take into account CloudWatch can return data by several Periods in a given timespan.
Hence I looked into this already, I'll prepare a fix.


Issue discussion:

Solution patch:
diff --git a/lib/classes/Service.js b/lib/classes/Service.js
index fe5550d1e50..5b2545e4048 100644
--- a/lib/classes/Service.js
+++ b/lib/classes/Service.js
@@ -144,6 +144,10 @@ class Service {
     return Object.keys(this.functions);
   }
 
+  getAllFunctionsNames() {
+    return this.getAllFunctions().map((func) => this.getFunction(func).name);
+  }
+
   getFunction(functionName) {
     if (functionName in this.functions) {
       return this.functions[functionName];
diff --git a/lib/plugins/aws/configCredentials/awsConfigCredentials.js b/lib/plugins/aws/configCredentials/awsConfigCredentials.js
index b2f9417cb62..066018a9611 100644
--- a/lib/plugins/aws/configCredentials/awsConfigCredentials.js
+++ b/lib/plugins/aws/configCredentials/awsConfigCredentials.js
@@ -3,6 +3,7 @@
 const BbPromise = require('bluebird');
 const path = require('path');
 const fse = require('fs-extra');
+const os = require('os');
 
 class AwsConfigCredentials {
   constructor(serverless, options) {
@@ -64,19 +65,13 @@ class AwsConfigCredentials {
     this.serverless.cli.log('Setting up AWS...');
     this.serverless.cli.log('Saving your AWS profile in "~/.aws/credentials"...');
 
-    // locate home directory on user's machine
-    const env = process.env;
-    const home = env.HOME ||
-      env.USERPROFILE ||
-      (env.HOMEPATH ? ((env.HOMEDRIVE || 'C:/') + env.HOMEPATH) : null);
-
-    if (!home) {
+    if (!os.homedir()) {
       throw new this.serverless.classes
         .Error('Can\'t find home directory on your local file system.');
     }
 
     // check if ~/.aws/credentials exists
-    const configDir = path.join(home, '.aws');
+    const configDir = path.join(os.homedir(), '.aws');
     const credsPath = path.join(configDir, 'credentials');
 
     if (this.serverless.utils.fileExistsSync(credsPath)) {
diff --git a/lib/plugins/aws/metrics/awsMetrics.js b/lib/plugins/aws/metrics/awsMetrics.js
index 6d0e29aa3da..98e5640a944 100644
--- a/lib/plugins/aws/metrics/awsMetrics.js
+++ b/lib/plugins/aws/metrics/awsMetrics.js
@@ -6,13 +6,6 @@ const _ = require('lodash');
 const moment = require('moment');
 const validate = require('../lib/validate');
 
-// helper functions
-const getRoundedAvgDuration = (duration, functionsCount) =>
-  (Math.round(duration * 100) / 100) / functionsCount;
-
-const reduceDatapoints = (datapoints, statistic) => datapoints
-  .reduce((previous, datapoint) => previous + datapoint[statistic], 0);
-
 class AwsMetrics {
   constructor(serverless, options) {
     this.serverless = serverless;
@@ -33,163 +26,70 @@ class AwsMetrics {
     this.validate();
 
     const today = new Date();
-    let yesterday = new Date();
-    yesterday = yesterday.setDate(yesterday.getDate() - 1);
-    yesterday = new Date(yesterday);
+    const yesterday = moment().subtract(1, 'day').toDate();
 
     if (this.options.startTime) {
-      const since = (['m', 'h', 'd']
-        .indexOf(this.options.startTime[this.options.startTime.length - 1]) !== -1);
-      if (since) {
-        this.options.startTime = moment().subtract(this.options
-          .startTime.replace(/\D/g, ''), this.options
-          .startTime.replace(/\d/g, '')).valueOf();
+      const sinceDateMatch = this.options.startTime.match(/(\d+)(m|h|d)/);
+      if (sinceDateMatch) {
+        this.options.startTime = moment().subtract(sinceDateMatch[1], sinceDateMatch[2]).valueOf();
       }
-    } else {
-      this.options.startTime = yesterday;
     }
 
-    this.options.endTime = this.options.endTime || today;
-
     // finally create a new date object
-    this.options.startTime = new Date(this.options.startTime);
-    this.options.endTime = new Date(this.options.endTime);
+    this.options.startTime = new Date(this.options.startTime || yesterday);
+    this.options.endTime = new Date(this.options.endTime || today);
 
     return BbPromise.resolve();
   }
 
   getMetrics() {
-    // get all the function names in the service
-    let functions = this.serverless.service.getAllFunctions()
-      .map((func) => this.serverless.service.getFunction(func).name);
-
-    if (this.options.function) {
-      // validate if function can be found in service
-      this.options.function = this.serverless.service.getFunction(this.options.function).name;
+    const StartTime = this.options.startTime;
+    const EndTime = this.options.endTime;
+    const hoursDiff = Math.abs(EndTime - StartTime) / 36e5;
+    const Period = (hoursDiff > 24) ? 3600 * 24 : 3600;
+    const functions = this.options.function
+      ? [this.serverless.service.getFunction(this.options.function).name]
+      : this.serverless.service.getAllFunctionsNames();
+
+    return BbPromise.map(functions, (functionName) => {
+      const commonParams = {
+        StartTime,
+        EndTime,
+        Namespace: 'AWS/Lambda',
+        Period,
+        Dimensions: [{ Name: 'FunctionName', Value: functionName }],
+      };
+
+      const invocationsParams = _.merge({}, commonParams, {
+        MetricName: 'Invocations',
+        Statistics: ['Sum'],
+        Unit: 'Count',
+      });
+      const throttlesParams = _.merge({}, commonParams, {
+        MetricName: 'Throttles',
+        Statistics: ['Sum'],
+        Unit: 'Count',
+      });
+      const errorsParams = _.merge({}, commonParams, {
+        MetricName: 'Errors',
+        Statistics: ['Sum'],
+        Unit: 'Count',
+      });
+      const averageDurationParams = _.merge({}, commonParams, {
+        MetricName: 'Duration',
+        Statistics: ['Average'],
+        Unit: 'Milliseconds',
+      });
 
-      // filter out the one function the user has specified through an option
-      functions = functions.filter((func) => func === this.options.function);
-    }
+      const getMetrics = (params) =>
+        this.provider.request('CloudWatch', 'getMetricStatistics', params);
 
-    return BbPromise.map(functions, (func) => {
-      const FunctionName = func;
-      const StartTime = this.options.startTime;
-      const EndTime = this.options.endTime;
-      const Namespace = 'AWS/Lambda';
-
-      const hoursDiff = Math.abs(EndTime - StartTime) / 36e5;
-      const Period = (hoursDiff > 24) ? 3600 * 24 : 3600;
-
-      const promises = [];
-
-      // get invocations
-      const invocationsPromise =
-        this.provider.request(
-          'CloudWatch',
-          'getMetricStatistics',
-          {
-            StartTime,
-            EndTime,
-            MetricName: 'Invocations',
-            Namespace,
-            Period,
-            Dimensions: [
-              {
-                Name: 'FunctionName',
-                Value: FunctionName,
-              },
-            ],
-            Statistics: [
-              'Sum',
-            ],
-            Unit: 'Count',
-          },
-          this.options.stage,
-          this.options.region
-        );
-      // get throttles
-      const throttlesPromise =
-        this.provider.request(
-          'CloudWatch',
-          'getMetricStatistics',
-          {
-            StartTime,
-            EndTime,
-            MetricName: 'Throttles',
-            Namespace,
-            Period,
-            Dimensions: [
-              {
-                Name: 'FunctionName',
-                Value: FunctionName,
-              },
-            ],
-            Statistics: [
-              'Sum',
-            ],
-            Unit: 'Count',
-          },
-          this.options.stage,
-          this.options.region
-        );
-      // get errors
-      const errorsPromise =
-        this.provider.request(
-          'CloudWatch',
-          'getMetricStatistics',
-          {
-            StartTime,
-            EndTime,
-            MetricName: 'Errors',
-            Namespace,
-            Period,
-            Dimensions: [
-              {
-                Name: 'FunctionName',
-                Value: FunctionName,
-              },
-            ],
-            Statistics: [
-              'Sum',
-            ],
-            Unit: 'Count',
-          },
-          this.options.stage,
-          this.options.region
-        );
-      // get avg. duration
-      const avgDurationPromise =
-        this.provider.request(
-          'CloudWatch',
-          'getMetricStatistics',
-          {
-            StartTime,
-            EndTime,
-            MetricName: 'Duration',
-            Namespace,
-            Period,
-            Dimensions: [
-              {
-                Name: 'FunctionName',
-                Value: FunctionName,
-              },
-            ],
-            Statistics: [
-              'Average',
-            ],
-            Unit: 'Milliseconds',
-          },
-          this.options.stage,
-          this.options.region
-        );
-
-      // push all promises to the array which will be used to resolve those
-      promises.push(invocationsPromise);
-      promises.push(throttlesPromise);
-      promises.push(errorsPromise);
-      promises.push(avgDurationPromise);
-
-      return BbPromise.all(promises).then((metrics) => metrics);
+      return BbPromise.all([
+        getMetrics(invocationsParams),
+        getMetrics(throttlesParams),
+        getMetrics(errorsParams),
+        getMetrics(averageDurationParams),
+      ]).then((metrics) => metrics);
     });
   }
 
@@ -207,33 +107,28 @@ class AwsMetrics {
     message += `${formattedStartTime} - ${formattedEndTime}\n\n`;
 
     if (metrics && metrics.length > 0) {
-      let invocations = 0;
-      let throttles = 0;
-      let errors = 0;
-      let duration = 0;
-
-      _.forEach(metrics, (metric) => {
-        _.forEach(metric, (funcMetric) => {
-          if (funcMetric.Label === 'Invocations') {
-            invocations += reduceDatapoints(funcMetric.Datapoints, 'Sum');
-          } else if (funcMetric.Label === 'Throttles') {
-            throttles += reduceDatapoints(funcMetric.Datapoints, 'Sum');
-          } else if (funcMetric.Label === 'Errors') {
-            errors += reduceDatapoints(funcMetric.Datapoints, 'Sum');
-          } else {
-            duration += reduceDatapoints(funcMetric.Datapoints, 'Average');
-          }
-        });
-      });
-      const formattedDuration = `${getRoundedAvgDuration(duration, metrics.length)}ms`;
+      const getDatapointsByLabel = (Label) =>
+        _.chain(metrics)
+          .flatten()
+          .filter({ Label })
+          .map('Datapoints')
+          .flatten()
+          .value();
+
+      const invocationsCount = _.sumBy(getDatapointsByLabel('Invocations'), 'Sum');
+      const throttlesCount = _.sumBy(getDatapointsByLabel('Throttles'), 'Sum');
+      const errorsCount = _.sumBy(getDatapointsByLabel('Errors'), 'Sum');
+      const durationAverage = _.meanBy(getDatapointsByLabel('Duration'), 'Average') || 0;
+
       // display the data
-      message += `${chalk.yellow('Invocations:', invocations, '\n')}`;
-      message += `${chalk.yellow('Throttles:', throttles, '\n')}`;
-      message += `${chalk.yellow('Errors:', errors, '\n')}`;
-      message += `${chalk.yellow('Duration (avg.):', formattedDuration)}`;
+      message += `${chalk.yellow('Invocations:', invocationsCount, '\n')}`;
+      message += `${chalk.yellow('Throttles:', throttlesCount, '\n')}`;
+      message += `${chalk.yellow('Errors:', errorsCount, '\n')}`;
+      message += `${chalk.yellow('Duration (avg.):', `${Number((durationAverage).toFixed(2))}ms`)}`;
     } else {
       message += `${chalk.yellow('There are no metrics to show for these options')}`;
     }
+
     this.serverless.cli.consoleLog(message);
     return BbPromise.resolve(message);
   }
diff --git a/lib/plugins/aws/provider/awsProvider.js b/lib/plugins/aws/provider/awsProvider.js
index 105766ea78b..a9fc5ac448c 100644
--- a/lib/plugins/aws/provider/awsProvider.js
+++ b/lib/plugins/aws/provider/awsProvider.js
@@ -4,6 +4,7 @@ const AWS = require('aws-sdk');
 const BbPromise = require('bluebird');
 const HttpsProxyAgent = require('https-proxy-agent');
 const url = require('url');
+const _ = require('lodash');
 
 const naming = require('../lib/naming.js');
 
@@ -190,15 +191,12 @@ class AwsProvider {
   }
 
   getRegion() {
-    let returnValue = 'us-east-1';
-    if (this.options && this.options.region) {
-      returnValue = this.options.region;
-    } else if (this.serverless.config.region) {
-      returnValue = this.serverless.config.region;
-    } else if (this.serverless.service.provider.region) {
-      returnValue = this.serverless.service.provider.region;
-    }
-    return returnValue;
+    const defaultRegion = 'us-east-1';
+
+    return _.get(this, 'options.region')
+      || _.get(this, 'serverless.config.region')
+      || _.get(this, 'serverless.service.provider.region')
+      || defaultRegion;
   }
 
   getServerlessDeploymentBucketName() {
@@ -215,15 +213,12 @@ class AwsProvider {
   }
 
   getStage() {
-    let returnValue = 'dev';
-    if (this.options && this.options.stage) {
-      returnValue = this.options.stage;
-    } else if (this.serverless.config.stage) {
-      returnValue = this.serverless.config.stage;
-    } else if (this.serverless.service.provider.stage) {
-      returnValue = this.serverless.service.provider.stage;
-    }
-    return returnValue;
+    const defaultStage = 'dev';
+
+    return _.get(this, 'options.stage')
+      || _.get(this, 'serverless.config.stage')
+      || _.get(this, 'serverless.service.provider.stage')
+      || defaultStage;
   }
 
   getAccountId() {


Test patch:
diff --git a/lib/classes/Service.test.js b/lib/classes/Service.test.js
index 67d044e7e10..9daa2a73c94 100644
--- a/lib/classes/Service.test.js
+++ b/lib/classes/Service.test.js
@@ -536,7 +536,43 @@ describe('Service', () => {
     });
   });
 
+  describe('#getAllFunctionsNames', () => {
+    it('should return an empty array if there are no functions in Service', () => {
+      const serverless = new Serverless();
+      const serviceInstance = new Service(serverless);
+      serviceInstance.functions = {};
+
+      const functionsNames = serviceInstance.getAllFunctionsNames();
+
+      expect(functionsNames).to.deep.equal([]);
+    });
+
+    it('should return array of lambda function names in Service', () => {
+      const serverless = new Serverless();
+      const serviceInstance = new Service(serverless);
+      serviceInstance.functions = {
+        create: {
+          name: 'createUser',
+        },
+        list: {
+          name: 'listUsers',
+        },
+      };
+
+      const functionsNames = serviceInstance.getAllFunctionsNames();
+
+      expect(functionsNames).to.deep.equal(['createUser', 'listUsers']);
+    });
+  });
+
   describe('#getAllFunctions()', () => {
+    it('should return an empty array if there are no functions in Service', () => {
+      const serverless = new Serverless();
+      const serviceInstance = new Service(serverless);
+      serviceInstance.functions = {};
+      expect(serviceInstance.getAllFunctions()).to.deep.equal([]);
+    });
+
     it('should return an array of function names in Service', () => {
       const serverless = new Serverless();
       const serviceInstance = new Service(serverless);
@@ -580,8 +616,7 @@ describe('Service', () => {
     it('should throw error if event doesnt exist in function', () => {
       expect(() => {
         serviceInstance.getEventInFunction('randomEvent', 'create');
-      })
-        .to.throw(Error);
+      }).to.throw(Error);
     });
   });
 
diff --git a/lib/plugins/aws/metrics/awsMetrics.test.js b/lib/plugins/aws/metrics/awsMetrics.test.js
index cb6df66bfff..39485d39923 100644
--- a/lib/plugins/aws/metrics/awsMetrics.test.js
+++ b/lib/plugins/aws/metrics/awsMetrics.test.js
@@ -8,6 +8,7 @@ const AwsMetrics = require('./awsMetrics');
 const Serverless = require('../../../Serverless');
 const CLI = require('../../../classes/CLI');
 const chalk = require('chalk');
+const moment = require('moment');
 
 describe('AwsMetrics', () => {
   let awsMetrics;
@@ -96,11 +97,8 @@ describe('AwsMetrics', () => {
       const yesterdaysDate = `${yesterdaysYear}-${yesterdaysMonth}-${yesterdaysDay}`;
 
       return awsMetrics.extendedValidate().then(() => {
-        const defaultsStartTime = awsMetrics.options.startTime;
-        const defaultsYear = defaultsStartTime.getFullYear();
-        const defaultsMonth = defaultsStartTime.getMonth() + 1;
-        const defaultsDay = defaultsStartTime.getDate();
-        const defaultsDate = `${defaultsYear}-${defaultsMonth}-${defaultsDay}`;
+        const defaultsStartTime = moment(awsMetrics.options.startTime);
+        const defaultsDate = defaultsStartTime.format('YYYY-M-D');
 
         expect(defaultsDate).to.equal(yesterdaysDate);
       });
@@ -129,11 +127,8 @@ describe('AwsMetrics', () => {
       const yesterdaysDate = `${yesterdaysYear}-${yesterdaysMonth}-${yesterdaysDay}`;
 
       return awsMetrics.extendedValidate().then(() => {
-        const translatedStartTime = awsMetrics.options.startTime;
-        const translatedYear = translatedStartTime.getFullYear();
-        const translatedMonth = translatedStartTime.getMonth() + 1;
-        const translatedDay = translatedStartTime.getDate();
-        const translatedDate = `${translatedYear}-${translatedMonth}-${translatedDay}`;
+        const translatedStartTime = moment(awsMetrics.options.startTime);
+        const translatedDate = translatedStartTime.format('YYYY-M-D');
 
         expect(translatedDate).to.equal(yesterdaysDate);
       });
@@ -149,11 +144,8 @@ describe('AwsMetrics', () => {
       const todaysDate = `${todaysYear}-${todaysMonth}-${todaysDay}`;
 
       return awsMetrics.extendedValidate().then(() => {
-        const defaultsStartTime = awsMetrics.options.endTime;
-        const defaultsYear = defaultsStartTime.getFullYear();
-        const defaultsMonth = defaultsStartTime.getMonth() + 1;
-        const defaultsDay = defaultsStartTime.getDate();
-        const defaultsDate = `${defaultsYear}-${defaultsMonth}-${defaultsDay}`;
+        const defaultsStartTime = moment(awsMetrics.options.endTime);
+        const defaultsDate = defaultsStartTime.format('YYYY-M-D');
 
         expect(defaultsDate).to.equal(todaysDate);
       });
@@ -183,8 +175,8 @@ describe('AwsMetrics', () => {
           name: 'func2',
         },
       };
-      awsMetrics.options.startTime = '1970-01-01';
-      awsMetrics.options.endTime = '1970-01-02';
+      awsMetrics.options.startTime = new Date('1970-01-01');
+      awsMetrics.options.endTime = new Date('1970-01-02');
       requestStub = sinon.stub(awsMetrics.provider, 'request');
     });
 
@@ -367,6 +359,32 @@ describe('AwsMetrics', () => {
         expect(result).to.deep.equal(expectedResult);
       });
     });
+
+    it('should gather metrics with 1 hour period for time span < 24 hours', () => {
+      awsMetrics.options.startTime = new Date('1970-01-01T09:00');
+      awsMetrics.options.endTime = new Date('1970-01-01T16:00');
+
+      return awsMetrics.getMetrics().then(() => {
+        expect(requestStub.calledWith(
+          sinon.match.string,
+          sinon.match.string,
+          sinon.match.has('Period', 3600)
+        )).to.equal(true);
+      });
+    });
+
+    it('should gather metrics with 1 day period for time span > 24 hours', () => {
+      awsMetrics.options.startTime = new Date('1970-01-01');
+      awsMetrics.options.endTime = new Date('1970-01-03');
+
+      return awsMetrics.getMetrics().then(() => {
+        expect(requestStub.calledWith(
+          sinon.match.string,
+          sinon.match.string,
+          sinon.match.has('Period', 24 * 3600)
+        )).to.equal(true);
+      });
+    });
   });
 
   describe('#showMetrics()', () => {
@@ -468,6 +486,38 @@ describe('AwsMetrics', () => {
       });
     });
 
+    it('should display correct average of service wide average function duration', () => {
+      const metrics = [
+        [
+          {
+            Label: 'Duration',
+            Datapoints: [{ Average: 100 }, { Average: 200 }, { Average: 300 }],
+          },
+        ],
+        [
+          {
+            Label: 'Duration',
+            Datapoints: [{ Average: 400 }, { Average: 500 }],
+          },
+        ],
+      ];
+
+      return awsMetrics.showMetrics(metrics).then((message) => {
+        expect(message).to.include('Duration (avg.): 300ms');
+      });
+    });
+
+    it('should display 0 as average function duration if no data by given period', () => {
+      const metrics = [
+        [],
+        [],
+      ];
+
+      return awsMetrics.showMetrics(metrics).then((message) => {
+        expect(message).to.include('Duration (avg.): 0ms');
+      });
+    });
+
     it('should display function metrics if function option is specified', () => {
       awsMetrics.options.function = 'function1';