Repository: iterative/cml (js)

Issue:
Missing key/value separator on `cml runner --cloud-metadata`
From https://discord.com/channels/485586884165107732/728693131557732403/997065340724973608, found by running `cml runner --cloud-metadata "Key With Spaces=Value"`

```
│ On main.tf line 30: Expected an equals sign ("=") to mark the beginning of
│ the attribute value. If you intended to given an attribute name containing
│ periods or spaces, write the name in quotes to create a string literal.
```

Proposed workaroud: `cml runner --cloud-metadata '"Key With Spaces"="Value"'`


Pull request:
Use JSON instead of HCL for runner templates
Closes #1099 and several forecasted issues with numbers greater than #1100.

Patch:
diff --git a/bin/cml/runner.js b/bin/cml/runner.js
index 541002b..6a00bab 100755
--- a/bin/cml/runner.js
+++ b/bin/cml/runner.js
@@ -132,7 +132,7 @@ const runCloud = async (opts) => {
       );
 
     const tfPath = workdir;
-    const tfMainPath = join(tfPath, 'main.tf');
+    const tfMainPath = join(tfPath, 'main.tf.json');
 
     const tpl = tf.iterativeCmlRunnerTpl({
       tpiVersion,
@@ -160,7 +160,7 @@ const runCloud = async (opts) => {
       dockerVolumes
     });
 
-    await fs.writeFile(tfMainPath, tpl);
+    await fs.writeFile(tfMainPath, JSON.stringify(tpl));
 
     await tf.init({ dir: tfPath });
     await tf.apply({ dir: tfPath });
@@ -227,9 +227,9 @@ const runLocal = async (opts) => {
 
     const tfPath = workdir;
     await fs.mkdir(tfPath, { recursive: true });
-    const tfMainPath = join(tfPath, 'main.tf');
+    const tfMainPath = join(tfPath, 'main.tf.json');
     const tpl = tf.iterativeProviderTpl({ tpiVersion });
-    await fs.writeFile(tfMainPath, tpl);
+    await fs.writeFile(tfMainPath, JSON.stringify(tpl));
 
     await tf.init({ dir: tfPath });
     await tf.apply({ dir: tfPath });
diff --git a/src/terraform.js b/src/terraform.js
index b9266f8..37dd856 100644
--- a/src/terraform.js
+++ b/src/terraform.js
@@ -63,80 +63,57 @@ const destroy = async (opts = {}) => {
   );
 };
 
-const mapCloudMetadata = (metadata) =>
-  Object.entries(metadata).map(([key, value]) => `${key} = "${value || ''}"`);
-
-const iterativeProviderTpl = (opts = {}) => {
-  const { tpiVersion } = opts;
-  return `terraform {
-  required_providers {
-    iterative = { source = "iterative/iterative"${
-      tpiVersion ? `, version = "${tpiVersion}"` : ''
-    } }
+const iterativeProviderTpl = ({ tpiVersion }) => ({
+  terraform: {
+    required_providers: {
+      iterative: {
+        source: 'iterative/iterative',
+        ...(tpiVersion && { version: tpiVersion })
+      }
+    }
+  },
+  provider: {
+    iterative: {}
   }
-}
-provider "iterative" {}
-`;
-};
+});
 
-const iterativeCmlRunnerTpl = (opts = {}) => {
-  const {
-    repo,
-    token,
-    driver,
-    labels,
-    cmlVersion,
-    idleTimeout,
-    cloud,
-    region,
-    name,
-    single,
-    type,
-    permissionSet,
-    metadata,
-    gpu,
-    hddSize,
-    sshPrivate,
-    spot,
-    spotPrice,
-    startupScript,
-    awsSecurityGroup,
-    awsSubnet,
-    dockerVolumes
-  } = opts;
-
-  const template = `${iterativeProviderTpl(opts)}
-resource "iterative_cml_runner" "runner" {
-  ${repo ? `repo = "${repo}"` : ''}
-  ${token ? `token = "${token}"` : ''}
-  ${driver ? `driver = "${driver}"` : ''}
-  ${labels ? `labels = "${labels}"` : ''}
-  ${cmlVersion ? `cml_version = "${cmlVersion}"` : ''}
-  ${typeof idleTimeout !== 'undefined' ? `idle_timeout = ${idleTimeout}` : ''}
-  ${name ? `name = "${name}"` : ''}
-  ${single ? `single = "${single}"` : ''}
-  ${cloud ? `cloud = "${cloud}"` : ''}
-  ${region ? `region = "${region}"` : ''}
-  ${type ? `instance_type = "${type}"` : ''}
-  ${gpu ? `instance_gpu = "${gpu}"` : ''}
-  ${hddSize ? `instance_hdd_size = ${hddSize}` : ''}
-  ${permissionSet ? `instance_permission_set = "${permissionSet}"` : ''}
-  ${sshPrivate ? `ssh_private = "${sshPrivate}"` : ''}
-  ${spot ? `spot = ${spot}` : ''}
-  ${spotPrice ? `spot_price = ${spotPrice}` : ''}
-  ${startupScript ? `startup_script = "${startupScript}"` : ''}
-  ${awsSecurityGroup ? `aws_security_group = "${awsSecurityGroup}"` : ''}
-  ${awsSubnet ? `aws_subnet_id = "${awsSubnet}"` : ''}
-  ${
-    metadata
-      ? `metadata = {\n    ${mapCloudMetadata(metadata).join('\n    ')}\n  }`
-      : ''
+const iterativeCmlRunnerTpl = (opts = {}) => ({
+  ...iterativeProviderTpl(opts),
+  resource: {
+    iterative_cml_runner: {
+      runner: {
+        ...(opts.awsSecurityGroup && {
+          aws_security_group: opts.awsSecurityGroup
+        }),
+        ...(opts.awsSubnet && { aws_subnet: opts.awsSubnet }),
+        ...(opts.cloud && { cloud: opts.cloud }),
+        ...(opts.cmlVersion && { cml_version: opts.cmlVersion }),
+        ...(opts.dockerVolumes && { docker_volumes: opts.dockerVolumes }),
+        ...(opts.driver && { driver: opts.driver }),
+        ...(opts.gpu && { instance_gpu: opts.gpu }),
+        ...(opts.hddSize && { instance_hdd_size: opts.hddSize }),
+        ...(typeof opts.idleTimeout !== 'undefined' && {
+          idle_timeout: opts.idleTimeout
+        }),
+        ...(opts.labels && { labels: opts.labels }),
+        ...(opts.metadata && { metadata: opts.metadata }),
+        ...(opts.name && { name: opts.name }),
+        ...(opts.permissionSet && {
+          instance_permission_set: opts.permissionSet
+        }),
+        ...(opts.region && { region: opts.region }),
+        ...(opts.repo && { repo: opts.repo }),
+        ...(opts.single && { single: opts.single }),
+        ...(opts.spot && { spot: opts.spot }),
+        ...(opts.spotPrice && { spot_price: opts.spotPrice }),
+        ...(opts.sshPrivate && { ssh_private: opts.sshPrivate }),
+        ...(opts.startupScript && { startup_script: opts.startupScript }),
+        ...(opts.token && { token: opts.token }),
+        ...(opts.type && { instance_type: opts.type })
+      }
+    }
   }
-  ${dockerVolumes ? `docker_volumes = ${JSON.stringify(dockerVolumes)}` : ''}
-}
-`;
-  return template;
-};
+});
 
 const checkMinVersion = async () => {
   const ver = await version();


Test patch:
diff --git a/src/terraform.test.js b/src/terraform.test.js
index 654600a..8299f0b 100644
--- a/src/terraform.test.js
+++ b/src/terraform.test.js
@@ -3,39 +3,24 @@ const { iterativeCmlRunnerTpl } = require('./terraform');
 describe('Terraform tests', () => {
   test('default options', async () => {
     const output = iterativeCmlRunnerTpl({});
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {}
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      }
-      "
+      }"
     `);
   });
 
@@ -58,39 +43,41 @@ describe('Terraform tests', () => {
       spotPrice: '0.0001',
       awsSecurityGroup: 'mysg'
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"aws_security_group\\": \\"mysg\\",
+              \\"cloud\\": \\"aws\\",
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 300,
+              \\"labels\\": \\"mylabel\\",
+              \\"name\\": \\"myrunner\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 300
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        
-        aws_security_group = \\"mysg\\"
-        
-        
-        
-      }
-      "
+      }"
     `);
   });
 
@@ -112,39 +99,40 @@ describe('Terraform tests', () => {
       spot: true,
       spotPrice: '0.0001'
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"cloud\\": \\"aws\\",
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 0,
+              \\"labels\\": \\"mylabel\\",
+              \\"name\\": \\"myrunner\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 0
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        
-        
-        
-        
-        
-      }
-      "
+      }"
     `);
   });
 
@@ -165,44 +153,47 @@ describe('Terraform tests', () => {
       sshPrivate: 'myprivate',
       spot: true,
       spotPrice: '0.0001',
-      metadata: { one: 'value', two: null }
+      metadata: { one: 'value', two: null, 'no problem': 'with spaces' }
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
-        }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 300
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        
-        
-        
-        metadata = {
-          one = \\"value\\"
-          two = \\"\\"
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"cloud\\": \\"aws\\",
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 300,
+              \\"labels\\": \\"mylabel\\",
+              \\"metadata\\": {
+                \\"one\\": \\"value\\",
+                \\"two\\": null,
+                \\"no problem\\": \\"with spaces\\"
+              },
+              \\"name\\": \\"myrunner\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-        
-      }
-      "
+      }"
     `);
   });
 
@@ -225,39 +216,44 @@ describe('Terraform tests', () => {
       spotPrice: '0.0001',
       dockerVolumes: ['/aa:/aa', '/bb:/bb']
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"cloud\\": \\"aws\\",
+              \\"docker_volumes\\": [
+                \\"/aa:/aa\\",
+                \\"/bb:/bb\\"
+              ],
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 300,
+              \\"labels\\": \\"mylabel\\",
+              \\"name\\": \\"myrunner\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 300
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        
-        
-        
-        
-        docker_volumes = [\\"/aa:/aa\\",\\"/bb:/bb\\"]
-      }
-      "
+      }"
     `);
   });
 
@@ -281,39 +277,42 @@ describe('Terraform tests', () => {
       spotPrice: '0.0001',
       awsSecurityGroup: 'mysg'
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"aws_security_group\\": \\"mysg\\",
+              \\"cloud\\": \\"aws\\",
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 300,
+              \\"labels\\": \\"mylabel\\",
+              \\"name\\": \\"myrunner\\",
+              \\"instance_permission_set\\": \\"arn:aws:iam::1:instance-profile/x\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 300
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\"
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        
-        aws_security_group = \\"mysg\\"
-        
-        
-        
-      }
-      "
+      }"
     `);
   });
 
@@ -336,39 +335,41 @@ describe('Terraform tests', () => {
       spotPrice: '0.0001',
       startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ='
     });
-    expect(output).toMatchInlineSnapshot(`
-      "terraform {
-        required_providers {
-          iterative = { source = \\"iterative/iterative\\" }
+    expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(`
+      "{
+        \\"terraform\\": {
+          \\"required_providers\\": {
+            \\"iterative\\": {
+              \\"source\\": \\"iterative/iterative\\"
+            }
+          }
+        },
+        \\"provider\\": {
+          \\"iterative\\": {}
+        },
+        \\"resource\\": {
+          \\"iterative_cml_runner\\": {
+            \\"runner\\": {
+              \\"cloud\\": \\"aws\\",
+              \\"driver\\": \\"gitlab\\",
+              \\"instance_gpu\\": \\"mygputype\\",
+              \\"instance_hdd_size\\": 50,
+              \\"idle_timeout\\": 300,
+              \\"labels\\": \\"mylabel\\",
+              \\"name\\": \\"myrunner\\",
+              \\"region\\": \\"west\\",
+              \\"repo\\": \\"https://\\",
+              \\"single\\": true,
+              \\"spot\\": true,
+              \\"spot_price\\": \\"0.0001\\",
+              \\"ssh_private\\": \\"myprivate\\",
+              \\"startup_script\\": \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\",
+              \\"token\\": \\"abc\\",
+              \\"instance_type\\": \\"mymachinetype\\"
+            }
+          }
         }
-      }
-      provider \\"iterative\\" {}
-
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        repo = \\"https://\\"
-        token = \\"abc\\"
-        driver = \\"gitlab\\"
-        labels = \\"mylabel\\"
-        
-        idle_timeout = 300
-        name = \\"myrunner\\"
-        single = \\"true\\"
-        cloud = \\"aws\\"
-        region = \\"west\\"
-        instance_type = \\"mymachinetype\\"
-        instance_gpu = \\"mygputype\\"
-        instance_hdd_size = 50
-        
-        ssh_private = \\"myprivate\\"
-        spot = true
-        spot_price = 0.0001
-        startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
-        
-        
-        
-        
-      }
-      "
+      }"
     `);
   });
 });