Repository: srl-labs/containerlab
Pull request #1663 (2023-10-22 18:47:26)

5 mins is enough for UTM VM

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/srl-labs/containerlab/pull/1663?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=srl-labs) Report
> Merging [#1663](https://app.codecov.io/gh/srl-labs/containerlab/pull/1663?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=srl-labs) (6d94b36) into [main](https://app.codecov.io/gh/srl-labs/containerlab/commit/283aec1d5fa68e37f4d8e90c91f7848007f16553?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=srl-labs) (283aec1) will **not change** coverage.
> The diff coverage is `n/a`.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##             main    #1663   +/-   ##
=======================================
  Coverage   51.12%   51.12%           
=======================================
  Files         137      137           
  Lines       13253    13253           
=======================================
  Hits         6776     6776           
  Misses       5727     5727           
  Partials      750      750           
```

| [Files](https://app.codecov.io/gh/srl-labs/containerlab/pull/1663?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=srl-labs) | Coverage Δ | |
|---|---|---|
| [nodes/srl/srl.go](https://app.codecov.io/gh/srl-labs/containerlab/pull/1663?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=srl-labs#diff-bm9kZXMvc3JsL3NybC5nbw==) | `57.19% <ø> (ø)` | |

</details>

Patch:
diff --git a/cmd/root.go b/cmd/root.go
index 57ad93f4..49ab0df4 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -6,12 +6,14 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/srl-labs/containerlab/utils"
 )
 
 var (
@@ -91,8 +93,10 @@ func preRunFn(cmd *cobra.Command, _ []string) error {
 	return getTopoFilePath(cmd)
 }
 
-// getTopoFilePath finds *.clab.y*ml file in the current working directory if the files was note specified using flags
-// errors if more than one file is found by the glob path.
+// getTopoFilePath finds *.clab.y*ml file in the current working directory
+// if the file was not specified.
+// If the topology file refers to a git repository, it will be cloned to the current directory.
+// Errors if more than one file is found by the glob path.
 func getTopoFilePath(cmd *cobra.Command) error {
 	// set commands which may use topo file find functionality, the rest don't need it
 	if !(cmd.Name() == "deploy" || cmd.Name() == "destroy" || cmd.Name() == "inspect" ||
@@ -106,13 +110,46 @@ func getTopoFilePath(cmd *cobra.Command) error {
 		return nil
 	}
 
+	var err error
+	if utils.IsHttpUri(topo) {
+		switch {
+		case utils.IsGitHubURL(topo):
+			githubURL := utils.NewGithubURL()
+
+			err := githubURL.Parse(topo)
+			if err != nil {
+				return err
+			}
+
+			err = utils.CloneGithubRepo(githubURL)
+			if err != nil {
+				return err
+			}
+
+			err = os.Chdir(githubURL.RepositoryName)
+			if err != nil {
+				return err
+			}
+
+			// once the repo is cloned the topo file is emptied
+			// to ensure that auto find functionality can kick in
+			topo = ""
+
+			// unless the file name is provided in the github url
+			if githubURL.FileName != "" {
+				topo = githubURL.FileName
+			}
+
+		default:
+			return fmt.Errorf("unsupported git repository: %s", topo)
+		}
+	}
+
 	// if topo or name flags have been provided, don't try to derive the topo file
 	if topo != "" || name != "" {
 		return nil
 	}
 
-	var err error
-
 	log.Debugf("trying to find topology files automatically")
 
 	files, err := filepath.Glob("*.clab.y*ml")
diff --git a/docs/cmd/deploy.md b/docs/cmd/deploy.md
index 7bcb2f29..d7bd473a 100644
--- a/docs/cmd/deploy.md
+++ b/docs/cmd/deploy.md
@@ -24,6 +24,25 @@ If more than one file is found for directory-based path or when the flag is omit
 
 It is possible to read the topology file from stdin by passing `-` as a value to the `--topo` flag. See [examples](#deploy-a-lab-from-a-remote-url-with-curl) for more details.
 
+##### Remote topology files
+
+To simplify the deployment of labs that are stored in remote version control systems, containerlab supports the use of remote topology files for Github.
+
+By specifying an URL to a repository or a `.clab.yml` file in a repository, containerlab will automatically clone[^1] the repository in your current directory and deploy it. If the URL points to a `.clab.yml` file, containerlab will clone the repository and deploy the lab defined in the file.
+
+The following URL formats are supported:
+
+| Type                                  | Example                                                            | Which topology file is used                                                        |
+| ------------------------------------- | ------------------------------------------------------------------ | ---------------------------------------------------------------------------------- |
+| Link to github repository             | <https://github.com/hellt/clab-test-repo/>                           | An auto-find procedure will find a `clab.yml` in the repository root and deploy it |
+| Link to a file in a github repository | <https://github.com/hellt/clab-test-repo/blob/main/lab1.clab.yml>    | A file specified in the URL will be deployed                                       |
+| Link to a repo's branch               | <https://github.com/hellt/clab-test-repo/tree/branch1>               | A branch of a repo is cloned and auto-find procedure kicks in                      |
+| Link to a file in a branch of a repo  | <https://github.com/hellt/clab-test-repo/blob/branch1/lab2.clab.yml> | A branch is cloned and a file specified in the URL is used for deployment          |
+
+When the lab is deployed using the URL, the repository is cloned in the current working directory. If the repository is already cloned, the clone operation will fail.
+
+Subsequent lab operations (such as destroy) must use the filesystem path to the topology file and not the URL.
+
 #### name
 
 With the global `--name | -n` flag a user sets a lab name. This value will override the lab name value passed in the topology definition file.
@@ -143,9 +162,12 @@ containerlab deploy
 clab dep -t mylab.clab.yml
 ```
 
+
 #### Deploy a lab from a remote URL with curl
 
 ```bash
 curl -s https://gist.githubusercontent.com/hellt/9baa28d7e3cb8290ade1e1be38a8d12b/raw/03067e242d44c9bbe38afa81131e46bab1fa0c42/test.clab.yml | \
     sudo containerlab deploy -t -
 ```
+
+[^1]: The repository is cloned with `--depth 1` parameter.
diff --git a/nodes/srl/srl.go b/nodes/srl/srl.go
index a9db5fee..d1b8b718 100644
--- a/nodes/srl/srl.go
+++ b/nodes/srl/srl.go
@@ -33,9 +33,9 @@ import (
 )
 
 const (
-	srlDefaultType = "ixrd2"
+	srlDefaultType = "ixrd2" // default srl node type
 
-	readyTimeout = time.Minute * 2 // max wait time for node to boot
+	readyTimeout = time.Minute * 5 // max wait time for node to boot
 	retryTimer   = time.Second
 
 	// defaultCfgPath is a path to a file with default config that clab adds on top of the factory config.
diff --git a/utils/http.go b/utils/http.go
new file mode 100644
index 00000000..cc4c1cd4
--- /dev/null
+++ b/utils/http.go
@@ -0,0 +1,110 @@
+package utils
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/url"
+	"os/exec"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+var errInvalidGithubURL = errors.New("invalid Github URL")
+
+// GithubURL struct holds the parsed github url.
+type GithubURL struct {
+	URLBase        string
+	ProjectOwner   string
+	RepositoryName string
+	GitBranch      string
+	FileName       string
+}
+
+// NewGithubURL returns a pointer to a GithubURI struct
+func NewGithubURL() *GithubURL {
+	return &GithubURL{}
+}
+
+// Parse parses the github.com string url into the GithubURL struct.
+func (u *GithubURL) Parse(ghURL string) error {
+	// strip trailing slash
+	ghURL = strings.TrimSuffix(ghURL, "/")
+
+	parsedURL, err := url.Parse(ghURL)
+	if err != nil {
+		return err
+	}
+
+	splitPath := strings.Split(parsedURL.Path, "/")
+
+	if len(splitPath) < 3 || splitPath[2] == "" {
+		return fmt.Errorf("%w %s", errInvalidGithubURL, ghURL)
+	}
+
+	// github.dev links can be cloned using github.com
+	if parsedURL.Host == "github.dev" {
+		parsedURL.Host = "github.com"
+	}
+
+	u.URLBase = parsedURL.Scheme + "://" + parsedURL.Host
+	u.ProjectOwner = splitPath[1]
+
+	// in case repo url has a trailing .git suffix, trim it
+	splitPath[2] = strings.TrimSuffix(splitPath[2], ".git")
+	u.RepositoryName = splitPath[2]
+
+	switch {
+	// path points to a file at a specific git ref
+	case strings.Contains(ghURL, "/blob/"):
+		if splitPath[len(splitPath)-1] == "" || !(strings.HasSuffix(ghURL, ".yml") || strings.HasSuffix(ghURL, ".yaml")) {
+			return errInvalidGithubURL
+		}
+
+		u.FileName = splitPath[len(splitPath)-1]
+		u.GitBranch = splitPath[len(splitPath)-2]
+
+	// path points to a git ref (branch or tag)
+	case strings.Contains(ghURL, "/tree/"):
+		if splitPath[len(splitPath)-1] == "" {
+			return errInvalidGithubURL
+		}
+
+		u.GitBranch = splitPath[len(splitPath)-1]
+	}
+
+	return nil
+}
+
+// CloneGithubRepo clones the github repo into the current directory.
+func CloneGithubRepo(u *GithubURL) error {
+	cloneArgs := []string{"clone", u.URLBase + "/" + u.ProjectOwner + "/" + u.RepositoryName, "--depth", "1"}
+	if u.GitBranch != "" {
+		cloneArgs = append(cloneArgs, []string{"--branch", u.GitBranch}...)
+	}
+
+	cmd := exec.Command("git", cloneArgs...)
+
+	log.Infof("cloning %s/%s", u.ProjectOwner, u.RepositoryName)
+
+	cmd.Stdout = log.New().Writer()
+
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		log.Errorf("failed to clone %s/%s: %v", u.ProjectOwner, u.RepositoryName, err)
+		log.Error(stderr.String())
+		return err
+	}
+
+	return nil
+}
+
+// IsGitHubURL checks if the url is a github url.
+func IsGitHubURL(url string) bool {
+	return strings.Contains(url, "github.com") ||
+		strings.Contains(url, "github.dev")
+}


Test patch:
diff --git a/tests/01-smoke/12-cloned-lab.robot b/tests/01-smoke/12-cloned-lab.robot
new file mode 100644
index 00000000..793ab879
--- /dev/null
+++ b/tests/01-smoke/12-cloned-lab.robot
@@ -0,0 +1,67 @@
+*** Settings ***
+Library             Process
+Resource            ../common.robot
+
+Suite Setup         Cleanup
+Suite Teardown      Cleanup
+
+
+*** Variables ***
+${lab1-url}     https://github.com/hellt/clab-test-repo
+${lab1-url2}    https://github.com/hellt/clab-test-repo/blob/main/lab1.clab.yml
+${lab2-url}     https://github.com/hellt/clab-test-repo/tree/branch1
+${runtime}      docker
+
+
+*** Test Cases ***
+Test lab1
+    ${output} =    Process.Run Process
+    ...    sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t ${lab1-url}
+    ...    shell=True
+
+    Log    ${output.stdout}
+    Log    ${output.stderr}
+
+    Should Be Equal As Integers    ${output.rc}    0
+
+    # check that node3 was filtered and not present in the lab output
+    Should Contain    ${output.stdout}    clab-lab1-node1
+
+    Cleanup
+
+Test lab2
+    ${output} =    Process.Run Process
+    ...    sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t ${lab1-url2}
+    ...    shell=True
+
+    Log    ${output.stdout}
+    Log    ${output.stderr}
+
+    Should Be Equal As Integers    ${output.rc}    0
+
+    # check that node3 was filtered and not present in the lab output
+    Should Contain    ${output.stdout}    clab-lab1-node1
+
+    Cleanup
+
+Test lab3
+    ${output} =    Process.Run Process
+    ...    sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t ${lab2-url}
+    ...    shell=True
+
+    Log    ${output.stdout}
+    Log    ${output.stderr}
+
+    Should Be Equal As Integers    ${output.rc}    0
+
+    # check that node3 was filtered and not present in the lab output
+    Should Contain    ${output.stdout}    clab-lab2-node1
+
+
+*** Keywords ***
+Cleanup
+    Process.Run Process    sudo -E ${CLAB_BIN} --runtime ${runtime} destroy --all --cleanup
+    ...    shell=True
+
+    Process.Run Process    sudo -E rm -rf clab-test-repo
+    ...    shell=True
diff --git a/utils/http_test.go b/utils/http_test.go
new file mode 100644
index 00000000..af9ca1b9
--- /dev/null
+++ b/utils/http_test.go
@@ -0,0 +1,193 @@
+package utils
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestIsGitHubURL(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		{
+			name:  "github.com",
+			input: "github.com",
+			want:  true,
+		},
+		{
+			name:  "github.com/containers/containerlab/blob/master/README.md",
+			input: "github.com/containers/containerlab/blob/master/README.md",
+			want:  true,
+		},
+		{
+			name:  "google.com/containers",
+			input: "google.com/containers",
+			want:  false,
+		},
+		{
+			name:  "google.com/containers/containerlab/blob/master/README.md",
+			input: "google.com/containers/containerlab/blob/master/README.md",
+			want:  false,
+		},
+		{
+			name:  "gitlab.com/containers",
+			input: "gitlab.com/containers",
+			want:  false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if output := IsGitHubURL(tt.input); output != tt.want {
+				t.Errorf("Test %q failed: want %v, but got %v", tt.name, tt.want, output)
+			}
+		})
+	}
+}
+
+func TestGithubURLParse(t *testing.T) {
+	tests := []struct {
+		name           string
+		ghURL          string
+		expectedResult *GithubURL
+		expectedError  error
+	}{
+		{
+			name:  "bare github url without trailing slash",
+			ghURL: "https://github.com/srl-labs/repo-name",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+			},
+			expectedError: nil,
+		},
+		{
+			name:  "bare github url with trailing slash",
+			ghURL: "https://github.com/srl-labs/repo-name/",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+			},
+			expectedError: nil,
+		},
+		{
+			name:  "bare github.dev url with trailing slash",
+			ghURL: "https://github.dev/srl-labs/repo-name/",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+			},
+			expectedError: nil,
+		},
+		{
+			name:  "bare github url with .git suffix",
+			ghURL: "https://github.com/srl-labs/repo-name.git",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+			},
+			expectedError: nil,
+		},
+		{
+			name:           "invalid url with just org name",
+			ghURL:          "https://github.com/srl-labs/",
+			expectedResult: &GithubURL{},
+			expectedError:  errInvalidGithubURL,
+		},
+		{
+			name:           "invalid url with no owner and no org",
+			ghURL:          "https://github.com/",
+			expectedResult: &GithubURL{},
+			expectedError:  errInvalidGithubURL,
+		},
+		{
+			name:  "github url with a clab file on the main branch",
+			ghURL: "https://github.com/srl-labs/repo-name/blob/main/file.clab.yml",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+				GitBranch:      "main",
+				FileName:       "file.clab.yml",
+			},
+			expectedError: nil,
+		},
+		{
+			name:  "github url with a yaml file on the main branch",
+			ghURL: "https://github.com/srl-labs/repo-name/blob/main/file.yaml",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+				GitBranch:      "main",
+				FileName:       "file.yaml",
+			},
+			expectedError: nil,
+		},
+		{
+			name:           "utl with invalid file on the main branch",
+			ghURL:          "https://github.com/srl-labs/repo-name/blob/main/file.foo",
+			expectedResult: &GithubURL{},
+			expectedError:  errInvalidGithubURL,
+		},
+		{
+			name:  "github url with a specified git ref and no file",
+			ghURL: "https://github.com/srl-labs/repo-name/tree/some-branch",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+				GitBranch:      "some-branch",
+			},
+			expectedError: nil,
+		},
+		{
+			name:  "github url with a specified git ref and no file and trailing slash",
+			ghURL: "https://github.com/srl-labs/repo-name/tree/some-branch/",
+			expectedResult: &GithubURL{
+				URLBase:        "https://github.com",
+				ProjectOwner:   "srl-labs",
+				RepositoryName: "repo-name",
+				GitBranch:      "some-branch",
+			},
+			expectedError: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			u := NewGithubURL()
+			err := u.Parse(tt.ghURL)
+
+			if err != nil && tt.expectedError == nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if err == nil && tt.expectedError != nil {
+				t.Errorf("expected error: %v, but got nil", tt.expectedError)
+			}
+
+			if err != nil && tt.expectedError != nil {
+				if !errors.Is(err, tt.expectedError) {
+					t.Fatalf("expected error: %v, but got %v", err, tt.expectedError)
+				}
+				// exit the test case as we don't want to compare url structs
+				// since when error is available and matches the expected error
+				// we don't care about the state the struct is in
+				return
+			}
+
+			if diff := cmp.Diff(u, tt.expectedResult); diff != "" {
+				t.Errorf("got result: = %v, expected %v, diff:\n%s", u, tt.expectedResult, diff)
+			}
+		})
+	}
+}