Repository: iterative/cml (js)

Issue:
CML Instance networking options
Using GitHub Actions with AWS in a self-hosted setting.

We have some complicated networking setups in an AWS account that I am trying to migrate projects from a different CI process to GitHub Actions and cml runner.

The Hidden option for an AWS Security Group allows the TF provider to infer which VPC id to use however we have multiple subnets assigned for this virtual network one with public IP addresses and I need to ensure that subnet is not selected (which it has been).

The provider targets the subnet with public IP addresses (to ensure it is reachable for cml I assume?). In this case, cml is running from a self-hosted runner on the internal IP space and would be route-able without a public address.

https://github.com/iterative/terraform-provider-iterative/blob/82d1b6047b728084d8b235e69f181ae608309db8/iterative/aws/provider.go#L201-L213

Pull request:
`cml runner` allow for a user set subnet
Follow up for: https://github.com/iterative/terraform-provider-iterative/pull/323 when complete.

Closes #795 


Patch:
diff --git a/bin/cml/runner.js b/bin/cml/runner.js
index fca7b63..f6cf035 100755
--- a/bin/cml/runner.js
+++ b/bin/cml/runner.js
@@ -127,6 +127,7 @@ const runCloud = async (opts) => {
       cloudSpotPrice: spotPrice,
       cloudStartupScript: startupScript,
       cloudAwsSecurityGroup: awsSecurityGroup,
+      cloudAwsSubnetId: awsSubnetId,
       tfFile,
       workdir
     } = opts;
@@ -161,7 +162,8 @@ const runCloud = async (opts) => {
         spot,
         spotPrice,
         startupScript,
-        awsSecurityGroup
+        awsSecurityGroup,
+        awsSubnetId
       });
     }
 
@@ -183,6 +185,7 @@ const runCloud = async (opts) => {
       for (const { attributes } of resource.instances) {
         const nonSensitiveValues = {
           awsSecurityGroup: attributes.aws_security_group,
+          awsSubnetId: attributes.aws_subnet_id,
           cloud: attributes.cloud,
           driver: attributes.driver,
           id: attributes.id,
@@ -529,6 +532,11 @@ exports.builder = (yargs) =>
         default: '',
         description: 'Specifies the security group in AWS'
       },
+      cloudAwsSubnetId: {
+        type: 'string',
+        default: '',
+        description: 'Specifies the subnet to use within AWS'
+      },
       tfResource: {
         hidden: true,
         alias: 'tf_resource'
diff --git a/src/terraform.js b/src/terraform.js
index cf0b9ba..60be706 100644
--- a/src/terraform.js
+++ b/src/terraform.js
@@ -85,7 +85,8 @@ const iterativeCmlRunnerTpl = (opts = {}) => {
     spot,
     spotPrice,
     startupScript,
-    awsSecurityGroup
+    awsSecurityGroup,
+    awsSubnetId
   } = opts;
 
   const template = `
@@ -114,6 +115,7 @@ resource "iterative_cml_runner" "runner" {
   ${spotPrice ? `spot_price = ${spotPrice}` : ''}
   ${startupScript ? `startup_script = "${startupScript}"` : ''}
   ${awsSecurityGroup ? `aws_security_group = "${awsSecurityGroup}"` : ''}
+  ${awsSubnetId ? `aws_subnet_id = "${awsSubnetId}"` : ''}
   ${
     metadata
       ? `metadata = {\n    ${mapCloudMetadata(metadata).join('\n    ')}\n  }`


Test patch:
diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js
index 76ffe9d..507df22 100644
--- a/bin/cml/runner.test.js
+++ b/bin/cml/runner.test.js
@@ -5,62 +5,64 @@ describe('CML e2e', () => {
     const output = await exec(`echo none | node ./bin/cml.js runner --help`);
 
     expect(output).toMatchInlineSnapshot(`
-      "cml.js runner
+"cml.js runner
 
-      Launch and register a self-hosted runner
+Launch and register a self-hosted runner
 
-      Options:
-        --help                      Show help                                [boolean]
-        --version                   Show version number                      [boolean]
-        --log                       Maximum log level
-                [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
-        --labels                    One or more user-defined labels for this runner
-                                    (delimited with commas)  [string] [default: \\"cml\\"]
-        --idle-timeout              Seconds to wait for jobs before shutting down. Set
-                                    to -1 to disable timeout   [number] [default: 300]
-        --name                      Name displayed in the repository once registered
-                                                          [string] [default: cml-{ID}]
-        --no-retry                  Do not restart workflow terminated due to instance
-                                    disposal or GitHub Actions timeout       [boolean]
-        --single                    Exit after running a single job          [boolean]
-        --reuse                     Don't launch a new runner if an existing one has
-                                    the same name or overlapping labels      [boolean]
-        --driver                    Platform where the repository is hosted. If not
-                                    specified, it will be inferred from the
-                                    environment [string] [choices: \\"github\\", \\"gitlab\\"]
-        --repo                      Repository to be used for registering the runner.
-                                    If not specified, it will be inferred from the
-                                    environment                               [string]
-        --token                     Personal access token to register a self-hosted
-                                    runner on the repository. If not specified, it
-                                    will be inferred from the environment     [string]
-        --cloud                     Cloud to deploy the runner
-                               [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"]
-        --cloud-region              Region where the instance is deployed. Choices:
-                                    [us-east, us-west, eu-west, eu-north]. Also
-                                    accepts native cloud regions
-                                                         [string] [default: \\"us-west\\"]
-        --cloud-type                Instance type. Choices: [m, l, xl]. Also supports
-                                    native types like i.e. t2.micro           [string]
-        --cloud-permission-set      Specifies the instance profile in AWS or instance
-                                    service account in GCP      [string] [default: \\"\\"]
-        --cloud-metadata            Key Value pairs to associate cml-runner instance
-                                    on the provider i.e. tags/labels \\"key=value\\"
-                                                                 [array] [default: []]
-        --cloud-gpu                 GPU type.
-                                   [string] [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"]
-        --cloud-hdd-size            HDD size in GB                            [number]
-        --cloud-ssh-private         Custom private RSA SSH key. If not provided an
-                                    automatically generated throwaway key will be used
-                                                                              [string]
-        --cloud-spot                Request a spot instance                  [boolean]
-        --cloud-spot-price          Maximum spot instance bidding price in USD.
-                                    Defaults to the current spot bidding price
-                                                                [number] [default: -1]
-        --cloud-startup-script      Run the provided Base64-encoded Linux shell script
-                                    during the instance initialization        [string]
-        --cloud-aws-security-group  Specifies the security group in AWS
-                                                                [string] [default: \\"\\"]"
-    `);
+Options:
+  --help                      Show help                                [boolean]
+  --version                   Show version number                      [boolean]
+  --log                       Maximum log level
+          [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
+  --labels                    One or more user-defined labels for this runner
+                              (delimited with commas)  [string] [default: \\"cml\\"]
+  --idle-timeout              Seconds to wait for jobs before shutting down. Set
+                              to -1 to disable timeout   [number] [default: 300]
+  --name                      Name displayed in the repository once registered
+                                                    [string] [default: cml-{ID}]
+  --no-retry                  Do not restart workflow terminated due to instance
+                              disposal or GitHub Actions timeout       [boolean]
+  --single                    Exit after running a single job          [boolean]
+  --reuse                     Don't launch a new runner if an existing one has
+                              the same name or overlapping labels      [boolean]
+  --driver                    Platform where the repository is hosted. If not
+                              specified, it will be inferred from the
+                              environment [string] [choices: \\"github\\", \\"gitlab\\"]
+  --repo                      Repository to be used for registering the runner.
+                              If not specified, it will be inferred from the
+                              environment                               [string]
+  --token                     Personal access token to register a self-hosted
+                              runner on the repository. If not specified, it
+                              will be inferred from the environment     [string]
+  --cloud                     Cloud to deploy the runner
+                         [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"]
+  --cloud-region              Region where the instance is deployed. Choices:
+                              [us-east, us-west, eu-west, eu-north]. Also
+                              accepts native cloud regions
+                                                   [string] [default: \\"us-west\\"]
+  --cloud-type                Instance type. Choices: [m, l, xl]. Also supports
+                              native types like i.e. t2.micro           [string]
+  --cloud-permission-set      Specifies the instance profile in AWS or instance
+                              service account in GCP      [string] [default: \\"\\"]
+  --cloud-metadata            Key Value pairs to associate cml-runner instance
+                              on the provider i.e. tags/labels \\"key=value\\"
+                                                           [array] [default: []]
+  --cloud-gpu                 GPU type.
+                             [string] [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"]
+  --cloud-hdd-size            HDD size in GB                            [number]
+  --cloud-ssh-private         Custom private RSA SSH key. If not provided an
+                              automatically generated throwaway key will be used
+                                                                        [string]
+  --cloud-spot                Request a spot instance                  [boolean]
+  --cloud-spot-price          Maximum spot instance bidding price in USD.
+                              Defaults to the current spot bidding price
+                                                          [number] [default: -1]
+  --cloud-startup-script      Run the provided Base64-encoded Linux shell script
+                              during the instance initialization        [string]
+  --cloud-aws-security-group  Specifies the security group in AWS
+                                                          [string] [default: \\"\\"]
+  --cloud-aws-subnet-id       Specifies the subnet to use within AWS
+                                                          [string] [default: \\"\\"]"
+`);
   });
 });
diff --git a/src/terraform.test.js b/src/terraform.test.js
index 718d7c4..cd7cae4 100644
--- a/src/terraform.test.js
+++ b/src/terraform.test.js
@@ -4,42 +4,43 @@ describe('Terraform tests', () => {
   test('default options', async () => {
     const output = iterativeCmlRunnerTpl({});
     expect(output).toMatchInlineSnapshot(`
-      "
+"
 
-      terraform {
-        required_providers {
-          iterative = {
-            source = \\"iterative/iterative\\"
-          }
-        }
-      }
+terraform {
+  required_providers {
+    iterative = {
+      source = \\"iterative/iterative\\"
+    }
+  }
+}
 
-      provider \\"iterative\\" {}
+provider \\"iterative\\" {}
 
 
-      resource \\"iterative_cml_runner\\" \\"runner\\" {
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-        
-      }
-      "
-    `);
+resource \\"iterative_cml_runner\\" \\"runner\\" {
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+}
+"
+`);
   });
 
   test('basic settings', async () => {
@@ -95,6 +96,7 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
   
   aws_security_group = \\"mysg\\"
   
+  
 }
 "
 `);
@@ -152,6 +154,7 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
   
   
   
+  
 }
 "
 `);
@@ -209,6 +212,7 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
   spot_price = 0.0001
   
   
+  
   metadata = {
     one = \\"value\\"
     two = \\"\\"
@@ -272,6 +276,7 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
   
   aws_security_group = \\"mysg\\"
   
+  
 }
 "
 `);
@@ -330,6 +335,7 @@ resource \\"iterative_cml_runner\\" \\"runner\\" {
   startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
   
   
+  
 }
 "
 `);