Repository: aliyun/fun
Pull request #176 (2019-04-08 13:06:37)

enhance docker pull and fix docker exec bug on windows
1. 增强了 pull 镜像的效果，现在和 docker pull 的效果一致。
2. docker exec 在 windows 遇到了个问题：
  1. stream.on('end') 无法收到事件，导致 exec 后无法等待到 end 事件，一直 hang 在这里。没有找到很好的解决方法，通过间隔性的 inspect exec 的状态来替代 end 事件。
  2. 通过 exec 的 stdin 传递事件时，每次调用完 stream.close，都会导致脚本直接退出，ExitCode 为 null。这个也没有找到直接的解决方法，从通过 stdin 传事件修改为通过 docker cmd 传事件。可以参见：https://github.com/aliyun/fc-docker/pull/24


Patch:
diff --git a/lib/commands/config.js b/lib/commands/config.js
index 94008c6..e6c9e81 100644
--- a/lib/commands/config.js
+++ b/lib/commands/config.js
@@ -63,7 +63,7 @@ async function config() {
       type: 'input',
       name: 'retries',
       message: 'The maximum number of retries for each SDK client',
-      default: profile.retries || 6,
+      default: profile.retries || 3,
       filter(value) {
         if (typeof value !== 'number') {value = parseInt(value);}
 
diff --git a/lib/deploy/deploy-support.js b/lib/deploy/deploy-support.js
index 08fd1c5..72bfbbe 100644
--- a/lib/deploy/deploy-support.js
+++ b/lib/deploy/deploy-support.js
@@ -522,18 +522,18 @@ function getTriggerConfig(triggerType, triggerProperties) {
       eventFormat: triggerProperties.EventFormat
     };
   } else if (triggerType === 'MNSTopic') {
-    var notifyContentFormat = "STREAM";
+    var notifyContentFormat = 'STREAM';
     if (triggerProperties.NotifyContentFormat !== undefined){
       notifyContentFormat = triggerProperties.NotifyContentFormat;
     }
-    var notifyStrategy = "BACKOFF_RETRY"
+    var notifyStrategy = 'BACKOFF_RETRY';
     if (triggerProperties.NotifyStrategy !== undefined){
       notifyStrategy = triggerProperties.NotifyStrategy;
     }
     var triggerCfg = {
       NotifyContentFormat: notifyContentFormat,
       NotifyStrategy: notifyStrategy,
-    }
+    };
     if (triggerProperties.FilterTag !== undefined){
       triggerCfg.FilterTag = triggerProperties.FilterTag;
     }
diff --git a/lib/docker-opts.js b/lib/docker-opts.js
index 156e809..cbfacfb 100644
--- a/lib/docker-opts.js
+++ b/lib/docker-opts.js
@@ -30,9 +30,9 @@ function resolveRuntimeToDockerImage(runtime, isBuild) {
     const name = runtimeImageMap[runtime];
     var imageName;
     if (isBuild) {
-      imageName = `aliyunfc/runtime-${name}:build-1.5.0`;
+      imageName = `aliyunfc/runtime-${name}:build-1.5.2`;
     } else {
-      imageName = `aliyunfc/runtime-${name}:1.5.0`;
+      imageName = `aliyunfc/runtime-${name}:1.5.2`;
     }
 
     debug('imageName: ' + imageName);
diff --git a/lib/docker.js b/lib/docker.js
index ff7c16d..91c4939 100644
--- a/lib/docker.js
+++ b/lib/docker.js
@@ -2,16 +2,14 @@
 
 const getProfile = require('./profile').getProfile;
 const mkdirp = require('mkdirp-promise');
-const urlencode = require('urlencode');
 const fs = require('fs');
 const path = require('path');
-const { blue, red } = require('colors');
+const { blue, red, yellow } = require('colors');
 const util = require('util');
 const debug = require('debug')('fun:local');
 const lstat = util.promisify(fs.lstat);
 const Docker = require('dockerode');
 const docker = new Docker();
-const http = require('http');
 const dockerOpts = require('./docker-opts');
 
 var containers = new Set();
@@ -59,7 +57,7 @@ function waitingForContainerStopped() {
     console.log(`\nreceived canncel request, stopping running containers.....`);
 
     const jobs = [];
-  
+
     for (let container of containers) {
       try {
         const c = docker.getContainer(container);
@@ -69,7 +67,7 @@ function waitingForContainerStopped() {
         console.log(`stopping container ${container}`);
 
         jobs.push(c.stop());
-      } catch(error) {
+      } catch (error) {
         debug('get container instance error, ignore container to stop, error is', error);
       }
     }
@@ -186,11 +184,18 @@ async function imageExist(imageName) {
   return images.length > 0;
 }
 
-function generateDockerCmd(functionProps, httpMode, invokeInitializer = true) {
+// dockerode exec 在 windows 上有问题，用 exec 的 stdin 传递事件，当调用 stream.end() 时，会直接导致 exec 退出，且 ExitCode 为 null
+function generateDockerCmd(functionProps, httpMode, invokeInitializer = true, event = null) {
   const cmd = ['-h', functionProps.Handler];
 
-  // always pass event using stdin mode
-  cmd.push('--stdin');
+  // 如果提供了 event
+  if (event !== null) {
+    cmd.push('--event', Buffer.from(event).toString('base64'));
+    cmd.push('--event-decode');
+  } else {
+    // always pass event using stdin mode
+    cmd.push('--stdin');
+  }
 
   if (httpMode) {
     cmd.push('--http');
@@ -215,53 +220,40 @@ function generateDockerCmd(functionProps, httpMode, invokeInitializer = true) {
 }
 
 async function pullImage(imageName) {
-
-  // dockerode pull image will occur problem on windows when using pkg packed binary version
-  // error is 'context canceled'
-  // see https://github.com/apocas/dockerode/issues/347
+  // copied from lib/edge/container.js
+  const stream = await docker.pull(imageName);
   return new Promise((resolve, reject) => {
-    const encodedImageName = urlencode(imageName);
+    process.stdout.write(`begin pullling image ${imageName}, you can also use ` + yellow(`'docker pull ${imageName}'`) + ' to pull image');
 
-    const options = {
-      socketPath: docker.modem.socketPath,
-      // https://docs.docker.com/develop/sdk/#api-version-matrix
-      path: `/images/create?fromImage=${encodedImageName}`,
-      method: 'POST',
+    const onFinished = (err) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve(imageName);
     };
-
-    process.stdout.write(`begin pullling image ${imageName}`);
-
-    const callback = res => {
-      res.setEncoding('utf8');
-
-      if (res.statusCode !== 200) {
-        res.on('data', data => {
-          reject(data);
-        });
+    const slog = require('single-line-log').stdout;
+    const statuses = {};
+    const onProgress = (event) => {
+      let status = event.status;
+      if (event.progress) {
+        status = `${event.status} ${event.progress}`;
       }
-
-      res.on('data', data => {
-        debug(data);
-        process.stdout.write('.');
-      });
-
-      res.on('error', data => {
-        reject(data);
-      });
-
-      res.on('end', data => {
-        console.log('\npull image finished');
-        resolve(data);
-      });
-
-      res.on('close', data => {
-        console.log('\npull image finished');
-        resolve(data);
-      });
+      if (event.id) {
+        statuses[event.id] = status;
+      }
+      // Print
+      let output = '';
+      const keys = Object.keys(statuses);
+      for (const key of keys) {
+        output += key + ': ' + statuses[key] + '\n';
+      }
+      if (!event.id) {
+        output += event.status;
+      }
+      slog(output);
     };
-
-    const clientRequest = http.request(options, callback);
-    clientRequest.end();
+    docker.modem.followProgress(stream, onFinished, onProgress);
   });
 }
 
@@ -447,12 +439,12 @@ async function startContainer(opts, outputStream, errorStream) {
       containers.delete(container.id);
     },
 
-    exec: async (cmd, {cwd = '', env = {}, event, outputStream, errorStream, verbose = false} = {}) => {      
+    exec: async (cmd, {cwd = '', env = {}, outputStream, errorStream, verbose = false } = {}) => {
       const options = {
         Cmd: cmd,
         Env: dockerOpts.resolveDockerEnv(env),
         Tty: false,
-        AttachStdin: true,  
+        AttachStdin: false,  
         AttachStdout: true,
         AttachStderr: true,
         WorkingDir: cwd
@@ -463,13 +455,11 @@ async function startContainer(opts, outputStream, errorStream) {
 
       const exec = await container.exec(options);
 
-      const stream = (await exec.start({hijack: true, stdin: true})).output;
+      const stream = (await exec.start({hijack: true, stdin: false})).output;
 
       // todo: have to wait, otherwise stdin may not be readable
       await new Promise(resolve => setTimeout(resolve, 30));
 
-      writeEventToStreamAndClose(stream, event);
-
       if (!outputStream) {
         outputStream = process.stdout;
       }
@@ -485,13 +475,19 @@ async function startContainer(opts, outputStream, errorStream) {
       }
 
       return new Promise((resolve, reject) => {
-        stream.on('end', () => {
-
-          debug('docker exec end event');
 
+        // stream.on('end') could not receive end event on windows.
+        // so use inspect to check exec exit
+        function waitContainerExec() {
           exec.inspect((err, data) => {
+
+            if (data.Running) {
+              setTimeout(waitContainerExec, 100);
+              return ;
+            }
+
             if (err) {
-              console.error(err);
+              debug('docker exec inspect err', err);
               reject(err);
             } else if (data.ExitCode !== 0) {
               reject(red(`${data.ProcessConfig.entrypoint} exited with code ${data.ExitCode}`));
@@ -499,10 +495,10 @@ async function startContainer(opts, outputStream, errorStream) {
               resolve(data.ExitCode);
             }
           });
-        }).on('error', (err) => {
-          console.error(err);
-          reject(err);
-        });
+
+        }
+
+        waitContainerExec(); 
       });
     }
   };  
diff --git a/lib/local/http-invoke.js b/lib/local/http-invoke.js
index 010a433..3afc751 100644
--- a/lib/local/http-invoke.js
+++ b/lib/local/http-invoke.js
@@ -122,7 +122,7 @@ class HttpInvoke extends Invoke {
 
       if (this.debugPort) {
         // don't reuse container
-        const cmd = docker.generateDockerCmd(this.functionProps, true);
+        const cmd = docker.generateDockerCmd(this.functionProps, true, event);
 
         this.containerName = docker.generateRamdomContainerName();
 
@@ -142,7 +142,7 @@ class HttpInvoke extends Invoke {
         // reuse container
         debug('http doInvoke, acquire invoke lock');
 
-        const cmd = [dockerOpts.resolveMockScript(this.runtime), ...docker.generateDockerCmd(this.functionProps, true, this._invokeInitializer)];
+        const cmd = [dockerOpts.resolveMockScript(this.runtime), ...docker.generateDockerCmd(this.functionProps, true, this._invokeInitializer, event)];
 
         debug(`http doInvoke, cmd is : ${cmd}`);
 
@@ -154,7 +154,6 @@ class HttpInvoke extends Invoke {
         try {
           await this.runner.exec(cmd, {
             env: envs,
-            event,
             outputStream,
             errorStream,
             verbose: true
diff --git a/lib/profile.js b/lib/profile.js
index 64c99aa..eb86e59 100644
--- a/lib/profile.js
+++ b/lib/profile.js
@@ -66,7 +66,7 @@ async function getProfileFromFile() {
   }
 
   profile.timeout = profYml.timeout || 10;
-  profile.retries = profYml.retries || 6;
+  profile.retries = profYml.retries || 3;
 
   return profile;
 }
diff --git a/package-lock.json b/package-lock.json
index 4c78a8e..f5bf69a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -147,7 +147,7 @@
     },
     "@sinonjs/formatio": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-2.0.0.tgz",
+      "resolved": "http://registry.npmjs.org/@sinonjs/formatio/-/formatio-2.0.0.tgz",
       "integrity": "sha512-ls6CAMA6/5gG+O/IdsBcblvnd8qcO/l1TYoNeAzp3wcISOxlPXQEus0mLcdwazEkWjaBdaJ3TaxmNgCLWwvWzg==",
       "dev": true,
       "requires": {
@@ -264,9 +264,9 @@
       }
     },
     "argparse": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz",
-      "integrity": "sha1-c9g7wmP4bpf4zE9rrhsOkKfSLIY=",
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "requires": {
         "sprintf-js": "~1.0.2"
       }
@@ -617,7 +617,7 @@
     },
     "buffer": {
       "version": "4.9.1",
-      "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
       "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
       "requires": {
         "base64-js": "^1.0.2",
@@ -1473,9 +1473,9 @@
       }
     },
     "esprima": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
-      "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw=="
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
     },
     "esquery": {
       "version": "1.0.0",
@@ -1981,7 +1981,7 @@
     },
     "git-ignore-parser": {
       "version": "0.0.2",
-      "resolved": "http://registry.npm.taobao.org/git-ignore-parser/download/git-ignore-parser-0.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/git-ignore-parser/-/git-ignore-parser-0.0.2.tgz",
       "integrity": "sha1-fykXBKrv9mlvmHePzLJ2AbtzOTI="
     },
     "glob": {
@@ -2196,9 +2196,9 @@
       "integrity": "sha512-sr1ZQph3UwHTR0XftSbK85OvBbxe/abLGzEnPENCQwmHf7sck8Oyu4ob3LgBxWWxRoM+QszeUyl7jbqapu2TqA=="
     },
     "ieee754": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz",
-      "integrity": "sha512-GguP+DRY+pJ3soyIiGPTvdiVXjZ+DbXOxGpXn3eMvNW4x4irjqXm4wHKscC+TfxSJ0yw/S1F24tqdMNsMZTiLA=="
+      "version": "1.1.13",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
+      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
     },
     "ignore": {
       "version": "5.0.4",
@@ -2366,7 +2366,7 @@
     },
     "int64-buffer": {
       "version": "0.1.9",
-      "resolved": "http://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.9.tgz",
+      "resolved": "https://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.9.tgz",
       "integrity": "sha1-ngOdoEOyT3ixlrKD4EZT716ZD2E="
     },
     "into-stream": {
@@ -2604,9 +2604,9 @@
       "dev": true
     },
     "js-yaml": {
-      "version": "3.9.1",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.9.1.tgz",
-      "integrity": "sha512-CbcG379L1e+mWBnLvHWWeLs8GyV/EMw862uLI3c+GxVyDHWZcjZinwuBd3iW2pgxgIlksW/1vNJa4to+RvDOww==",
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
+      "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
       "requires": {
         "argparse": "^1.0.7",
         "esprima": "^4.0.0"
@@ -4448,9 +4448,9 @@
       }
     },
     "tablestore": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/tablestore/-/tablestore-4.3.0.tgz",
-      "integrity": "sha512-B2XelmC3hOO4qrtxRd5xsv+nbx3L8+xZ2ajPhqHRNCL1BUANFqQOPfQfH4Uhd3obEh4F3sThb2i6s4pG5gwj4g==",
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/tablestore/-/tablestore-4.3.1.tgz",
+      "integrity": "sha512-1inTyCFuDF2DUKqkzkl/fraJlkt6JYj/vFXL5l9HcmM4DV7V6JJmBdqi4qUgk+DBmiiFqThuyq6eJsGUmQG8Ag==",
       "requires": {
         "buffer": "4.9.1",
         "int64-buffer": "0.1.9",
diff --git a/package.json b/package.json
index 6aef69f..112fef9 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "inquirer": "^5.2.0",
     "inquirer-autocomplete-prompt": "^1.0.1",
     "ip": "^1.1.5",
-    "js-yaml": "^3.9.0",
+    "js-yaml": "^3.13.1",
     "jszip": "^3.1.5",
     "kitx": "^1.2.1",
     "lodash": "^4.17.11",
@@ -71,7 +71,7 @@
     "rimraf": "^2.6.2",
     "semver": "^5.5.0",
     "single-line-log": "^1.1.2",
-    "tablestore": "^4.3.0",
+    "tablestore": "^4.3.1",
     "urlencode": "^1.1.0"
   },
   "files": [


Test patch:
diff --git a/test/docker-opts.test.js b/test/docker-opts.test.js
index 70a4622..09d132b 100644
--- a/test/docker-opts.test.js
+++ b/test/docker-opts.test.js
@@ -71,7 +71,7 @@ describe('test generateLocalInvokeOpts', () => {
       'OpenStdin': true,
       'StdinOnce': true,
       'Tty': false,
-      'Image': 'aliyunfc/runtime-nodejs8:1.5.0',
+      'Image': 'aliyunfc/runtime-nodejs8:1.5.2',
       'HostConfig': {
         'AutoRemove': true,
         'Mounts': [
@@ -115,7 +115,7 @@ describe('test generateLocalInvokeOpts', () => {
       'StdinOnce': true,
       'Tty': false,
       'User': null,
-      'Image': 'aliyunfc/runtime-nodejs8:1.5.0',
+      'Image': 'aliyunfc/runtime-nodejs8:1.5.2',
       'Env': [
         'LD_LIBRARY_PATH=/code/.fun/root/usr/lib:/code/.fun/root/usr/lib/x86_64-linux-gnu:/code:/code/lib:/usr/local/lib',
         'PATH=/code/.fun/root/usr/local/bin:/code/.fun/root/usr/local/sbin:/code/.fun/root/usr/bin:/code/.fun/root/usr/sbin:/code/.fun/root/sbin:/code/.fun/root/bin:/code/.fun/python/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin',
diff --git a/test/docker.test.js b/test/docker.test.js
index 2866993..b6afb02 100644
--- a/test/docker.test.js
+++ b/test/docker.test.js
@@ -49,6 +49,23 @@ describe('test generateDockerCmd', () => {
     ]);
   });
 
+  it('test generate docker cmd with event', () => {
+
+    const cmd = docker.generateDockerCmd(functionProps, false, true, 'event');
+
+    expect(cmd).to.eql([
+      '-h',
+      'index.handler',
+      '--event',
+      'ZXZlbnQ=',
+      '--event-decode',
+      '-i',
+      'index.initializer',
+      '--initializationTimeout',
+      '3'
+    ]);
+  });
+
   it('test generate docker http cmd', () => {
     const cmd = docker.generateDockerCmd(functionProps, true);
 
diff --git a/test/local/invoke.test.js b/test/local/invoke.test.js
index b229dde..88b379b 100644
--- a/test/local/invoke.test.js
+++ b/test/local/invoke.test.js
@@ -77,9 +77,9 @@ describe('test invoke construct and init', async () => {
     expect(invoke.codeMount).to.eql(codeMount);
     expect(invoke.mounts).to.eql([codeMount]);
     expect(invoke.containerName).to.contain('fun_local_');
-    expect(invoke.imageName).to.eql('aliyunfc/runtime-python3.6:1.5.0');
+    expect(invoke.imageName).to.contain('aliyunfc/runtime-python3.6:1.5.2');
 
-    assert.calledWith(docker.pullImageIfNeed, 'aliyunfc/runtime-python3.6:1.5.0');
+    assert.calledWith(docker.pullImageIfNeed, 'aliyunfc/runtime-python3.6:1.5.2');
   });
 
   it('test init with nas config', async () => {
@@ -109,9 +109,9 @@ describe('test invoke construct and init', async () => {
     expect(invoke.codeMount).to.eql(codeMount);
     expect(invoke.mounts).to.eql([codeMount, ...nasMounts]);
     expect(invoke.containerName).to.contain('fun_local_');
-    expect(invoke.imageName).to.eql('aliyunfc/runtime-python3.6:1.5.0');
+    expect(invoke.imageName).to.eql('aliyunfc/runtime-python3.6:1.5.2');
 
-    assert.calledWith(docker.pullImageIfNeed, 'aliyunfc/runtime-python3.6:1.5.0');
+    assert.calledWith(docker.pullImageIfNeed, 'aliyunfc/runtime-python3.6:1.5.2');
   });
 });