Repository: iterative/cml (js)

Issue:
runner: Docker socket mounts
Would be nice to auto-mount the Docker socket (in the GitLab runner in particular).
Might circumvent some Docker-in-Docker issues.

Alternatively:

- [x] expose `--docker-volumes` #861
- [ ] document `--docker-volumes` https://github.com/iterative/cml.dev/pull/189
- [x] add `docker_volumes` to TPI https://github.com/iterative/terraform-provider-iterative/pull/367

Pull request:
Runner docker volumes
 - closes #799 
 - ~~adds docker inside images to support dind~~
 - fixes a small warning log of unknown runtime  
 - adds GL macos runner

Usage

```js
cml runner --repo XXX --token XXX  --docker-volumes /test:/test --docker-volumes /test:/testb
```


Patch:
diff --git a/bin/cml/runner.js b/bin/cml/runner.js
index 8de4365..048b0b9 100755
--- a/bin/cml/runner.js
+++ b/bin/cml/runner.js
@@ -115,6 +115,7 @@ const runCloud = async (opts) => {
       idleTimeout,
       name,
       single,
+      dockerVolumes,
       cloud,
       cloudRegion: region,
       cloudType: type,
@@ -163,7 +164,8 @@ const runCloud = async (opts) => {
         spotPrice,
         startupScript,
         awsSecurityGroup,
-        awsSubnet
+        awsSubnet,
+        dockerVolumes
       });
     }
 
@@ -215,14 +217,16 @@ const runCloud = async (opts) => {
 
 const runLocal = async (opts) => {
   winston.info(`Launching ${cml.driver} runner`);
-  const { workdir, name, labels, single, idleTimeout, noRetry } = opts;
+  const { workdir, name, labels, single, idleTimeout, noRetry, dockerVolumes } =
+    opts;
 
   const proc = await cml.startRunner({
     workdir,
     name,
     labels,
     single,
-    idleTimeout
+    idleTimeout,
+    dockerVolumes
   });
 
   const dataHandler = async (data) => {
@@ -331,12 +335,16 @@ const run = async (opts) => {
     labels,
     name,
     reuse,
+    dockerVolumes,
     tfResource,
     workdir
   } = opts;
 
   cml = new CML({ driver, repo, token });
 
+  if (dockerVolumes.length && cml.driver !== 'gitlab')
+    winston.warn('Parameters --docker-volumes is only supported in gitlab');
+
   if (cloud || tfResource) await tf.checkMinVersion();
 
   // prepare tf
@@ -412,6 +420,11 @@ exports.handler = async (opts) => {
 exports.builder = (yargs) =>
   yargs.env('CML_RUNNER').options(
     kebabcaseKeys({
+      dockerVolumes: {
+        type: 'array',
+        default: [],
+        description: 'Docker volumes. This feature is only supported in GitLab'
+      },
       labels: {
         type: 'string',
         default: 'cml',
diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js
index 160d62c..4104379 100644
--- a/src/drivers/gitlab.js
+++ b/src/drivers/gitlab.js
@@ -162,7 +162,14 @@ class Gitlab {
   }
 
   async startRunner(opts) {
-    const { workdir, idleTimeout, single, labels, name } = opts;
+    const {
+      workdir,
+      idleTimeout,
+      single,
+      labels,
+      name,
+      dockerVolumes = []
+    } = opts;
 
     let gpu = true;
     try {
@@ -178,14 +185,19 @@ class Gitlab {
     try {
       const bin = resolve(workdir, 'gitlab-runner');
       if (!(await fse.pathExists(bin))) {
-        const url =
-          'https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64';
+        const arch = process.platform === 'darwin' ? 'darwin' : 'linux';
+        const url = `https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-${arch}-amd64`;
         await download({ url, path: bin });
         await fs.chmod(bin, '777');
       }
 
       const { protocol, host } = new URL(this.repo);
       const { token } = await this.registerRunner({ tags: labels, name });
+
+      let dockerVolumesTpl = '';
+      dockerVolumes.forEach((vol) => {
+        dockerVolumesTpl += `--docker-volumes ${vol} `;
+      });
       const command = `${bin} --log-format="json" run-single \
         --builds-dir "${workdir}" \
         --cache-dir "${workdir}" \
@@ -195,7 +207,8 @@ class Gitlab {
         --wait-timeout ${idleTimeout} \
         --executor "${IN_DOCKER ? 'shell' : 'docker'}" \
         --docker-image "iterativeai/cml:${gpu ? 'latest-gpu' : 'latest'}" \
-        --docker-runtime "${gpu ? 'nvidia' : ''}" \
+        ${gpu ? '--docker-runtime nvidia' : ''} \
+        ${dockerVolumesTpl} \
         ${single ? '--max-builds 1' : ''}`;
 
       return spawn(command, { shell: true });
diff --git a/src/terraform.js b/src/terraform.js
index d157033..4424645 100644
--- a/src/terraform.js
+++ b/src/terraform.js
@@ -86,7 +86,8 @@ const iterativeCmlRunnerTpl = (opts = {}) => {
     spotPrice,
     startupScript,
     awsSecurityGroup,
-    awsSubnet
+    awsSubnet,
+    dockerVolumes
   } = opts;
 
   const template = `
@@ -117,6 +118,7 @@ resource "iterative_cml_runner" "runner" {
       ? `metadata = {\n    ${mapCloudMetadata(metadata).join('\n    ')}\n  }`
       : ''
   }
+  ${dockerVolumes ? `docker_volumes = ${JSON.stringify(dockerVolumes)}` : ''}
 }
 `;
   return template;


Test patch:
diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js
index c0d0aed..496c1f7 100644
--- a/bin/cml/runner.test.js
+++ b/bin/cml/runner.test.js
@@ -14,6 +14,9 @@ describe('CML e2e', () => {
         --version                                 Show version number        [boolean]
         --log                                     Maximum log level
                 [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
+        --docker-volumes                          Docker volumes. This feature is only
+                                                  supported in GitLab
+                                                                 [array] [default: []]
         --labels                                  One or more user-defined labels for
                                                   this runner (delimited with commas)
                                                              [string] [default: \\"cml\\"]
diff --git a/src/terraform.test.js b/src/terraform.test.js
index cd7cae4..0a840b7 100644
--- a/src/terraform.test.js
+++ b/src/terraform.test.js
@@ -4,43 +4,44 @@ describe('Terraform tests', () => {
   test('default options', async () => {
     const output = iterativeCmlRunnerTpl({});
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      }
+      "
+    `);
   });
 
   test('basic settings', async () => {
@@ -63,43 +64,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
       awsSecurityGroup: 'mysg'
     });
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  repo = \\"https://\\"
-  token = \\"abc\\"
-  driver = \\"gitlab\\"
-  labels = \\"mylabel\\"
-  idle_timeout = 300
-  name = \\"myrunner\\"
-  single = \\"true\\"
-  cloud = \\"aws\\"
-  region = \\"west\\"
-  instance_type = \\"mymachinetype\\"
-  instance_gpu = \\"mygputype\\"
-  instance_hdd_size = 50
-  
-  ssh_private = \\"myprivate\\"
-  spot = true
-  spot_price = 0.0001
-  
-  aws_security_group = \\"mysg\\"
-  
-  
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 300
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        
+        aws_security_group = \\"mysg\\"
+        
+        
+        
+      }
+      "
+    `);
   });
 
   test('basic settings with runner forever', async () => {
@@ -121,43 +123,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
       spotPrice: '0.0001'
     });
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  repo = \\"https://\\"
-  token = \\"abc\\"
-  driver = \\"gitlab\\"
-  labels = \\"mylabel\\"
-  idle_timeout = 0
-  name = \\"myrunner\\"
-  single = \\"true\\"
-  cloud = \\"aws\\"
-  region = \\"west\\"
-  instance_type = \\"mymachinetype\\"
-  instance_gpu = \\"mygputype\\"
-  instance_hdd_size = 50
-  
-  ssh_private = \\"myprivate\\"
-  spot = true
-  spot_price = 0.0001
-  
-  
-  
-  
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 0
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        
+        
+        
+        
+        
+      }
+      "
+    `);
   });
 
   test('basic settings with metadata', async () => {
@@ -180,46 +183,107 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
       metadata: { one: 'value', two: null }
     });
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  repo = \\"https://\\"
-  token = \\"abc\\"
-  driver = \\"gitlab\\"
-  labels = \\"mylabel\\"
-  idle_timeout = 300
-  name = \\"myrunner\\"
-  single = \\"true\\"
-  cloud = \\"aws\\"
-  region = \\"west\\"
-  instance_type = \\"mymachinetype\\"
-  instance_gpu = \\"mygputype\\"
-  instance_hdd_size = 50
-  
-  ssh_private = \\"myprivate\\"
-  spot = true
-  spot_price = 0.0001
-  
-  
-  
-  metadata = {
-    one = \\"value\\"
-    two = \\"\\"
-  }
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 300
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        
+        
+        
+        metadata = {
+          one = \\"value\\"
+          two = \\"\\"
+        }
+        
+      }
+      "
+    `);
+  });
+
+  test('basic settings with docker volumes', async () => {
+    const output = iterativeCmlRunnerTpl({
+      repo: 'https://',
+      token: 'abc',
+      driver: 'gitlab',
+      labels: 'mylabel',
+      idleTimeout: 300,
+      name: 'myrunner',
+      single: true,
+      cloud: 'aws',
+      region: 'west',
+      type: 'mymachinetype',
+      gpu: 'mygputype',
+      hddSize: 50,
+      sshPrivate: 'myprivate',
+      spot: true,
+      spotPrice: '0.0001',
+      dockerVolumes: ['/aa:/aa', '/bb:/bb']
+    });
+    expect(output).toMatchInlineSnapshot(`
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 300
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        
+        
+        
+        
+        docker_volumes = [\\"/aa:/aa\\",\\"/bb:/bb\\"]
+      }
+      "
+    `);
   });
 
   test('basic settings with permission set', async () => {
@@ -243,43 +307,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
       awsSecurityGroup: 'mysg'
     });
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  repo = \\"https://\\"
-  token = \\"abc\\"
-  driver = \\"gitlab\\"
-  labels = \\"mylabel\\"
-  idle_timeout = 300
-  name = \\"myrunner\\"
-  single = \\"true\\"
-  cloud = \\"aws\\"
-  region = \\"west\\"
-  instance_type = \\"mymachinetype\\"
-  instance_gpu = \\"mygputype\\"
-  instance_hdd_size = 50
-  instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\"
-  ssh_private = \\"myprivate\\"
-  spot = true
-  spot_price = 0.0001
-  
-  aws_security_group = \\"mysg\\"
-  
-  
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 300
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\"
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        
+        aws_security_group = \\"mysg\\"
+        
+        
+        
+      }
+      "
+    `);
   });
 
   test('Startup script', async () => {
@@ -302,42 +367,43 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
       startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ='
     });
     expect(output).toMatchInlineSnapshot(`
-"
-
-terraform {
-  required_providers {
-    iterative = {
-      source = \\"iterative/iterative\\"
-    }
-  }
-}
-
-provider \\"iterative\\" {}
-
-
-resource \\"iterative_cml_runner\\" \\"runner\\" {
-  repo = \\"https://\\"
-  token = \\"abc\\"
-  driver = \\"gitlab\\"
-  labels = \\"mylabel\\"
-  idle_timeout = 300
-  name = \\"myrunner\\"
-  single = \\"true\\"
-  cloud = \\"aws\\"
-  region = \\"west\\"
-  instance_type = \\"mymachinetype\\"
-  instance_gpu = \\"mygputype\\"
-  instance_hdd_size = 50
-  
-  ssh_private = \\"myprivate\\"
-  spot = true
-  spot_price = 0.0001
-  startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
-  
-  
-  
-}
-"
-`);
+      "
+
+      terraform {
+        required_providers {
+          iterative = {
+            source = \\"iterative/iterative\\"
+          }
+        }
+      }
+
+      provider \\"iterative\\" {}
+
+
+      resource \\"iterative_cml_runner\\" \\"runner\\" {
+        repo = \\"https://\\"
+        token = \\"abc\\"
+        driver = \\"gitlab\\"
+        labels = \\"mylabel\\"
+        idle_timeout = 300
+        name = \\"myrunner\\"
+        single = \\"true\\"
+        cloud = \\"aws\\"
+        region = \\"west\\"
+        instance_type = \\"mymachinetype\\"
+        instance_gpu = \\"mygputype\\"
+        instance_hdd_size = 50
+        
+        ssh_private = \\"myprivate\\"
+        spot = true
+        spot_price = 0.0001
+        startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
+        
+        
+        
+        
+      }
+      "
+    `);
   });
 });