Repository: firecow/gitlab-ci-local
Pull request #1203 (2024-05-05 07:43:22)

Implement pull policy

sonarcloud[bot]: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_accepted_issues&view=list)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [81.8% Coverage on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarCloud](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203)


sonarcloud[bot]: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_accepted_issues&view=list)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [100.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarCloud](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203)


sonarcloud[bot]: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_accepted_issues&view=list)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=firecow_gitlab-ci-local&pullRequest=1203&resolved=false&inNewCodePeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [100.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=firecow_gitlab-ci-local&pullRequest=1203&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarCloud](https://sonarcloud.io/dashboard?id=firecow_gitlab-ci-local&pullRequest=1203)



Patch:
diff --git a/src/argv.ts b/src/argv.ts
index d4caae7..d9f5010 100644
--- a/src/argv.ts
+++ b/src/argv.ts
@@ -112,6 +112,10 @@ export class Argv {
         return typeof val == "string" ? val.split(" ") : val;
     }
 
+    get pullPolicy (): string {
+        return this.map.get("pullPolicy") ?? "if-not-present";
+    }
+
     get remoteVariables (): string {
         return this.map.get("remoteVariables");
     }
diff --git a/src/index.ts b/src/index.ts
index 9f4ee02..b17e77a 100755
--- a/src/index.ts
+++ b/src/index.ts
@@ -199,6 +199,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
             description: "Add extra docker host entries",
             requiresArg: false,
         })
+        .option("pull-policy", {
+            type: "string",
+            description: "Set image pull-policy (always or if-not-present)",
+            requiresArg: false,
+        })
         .option("fetch-includes", {
             type: "boolean",
             description: "Fetch all external includes one more time",
diff --git a/src/job.ts b/src/job.ts
index d6081db..d201d32 100644
--- a/src/job.ts
+++ b/src/job.ts
@@ -829,14 +829,23 @@ export class Job {
     }
 
     private async pullImage (writeStreams: WriteStreams, imageToPull: string) {
-        try {
-            await Utils.spawn([this.argv.containerExecutable, "image", "inspect", imageToPull]);
-        } catch (e: any) {
+        const pullPolicy = this.argv.pullPolicy;
+        const actualPull = async () => {
             const time = process.hrtime();
             await Utils.spawn([this.argv.containerExecutable, "pull", imageToPull]);
             const endTime = process.hrtime(time);
             writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright pulled} ${imageToPull} in {magenta ${prettyHrtime(endTime)}}\n`);
             this.refreshLongRunningSilentTimeout(writeStreams);
+        };
+
+        if (pullPolicy === "always") {
+            await actualPull();
+            return;
+        }
+        try {
+            await Utils.spawn([this.argv.containerExecutable, "image", "inspect", imageToPull]);
+        } catch (e: any) {
+            await actualPull();
         }
     }
 
diff --git a/src/parser-includes.ts b/src/parser-includes.ts
index 0675293..911fef0 100644
--- a/src/parser-includes.ts
+++ b/src/parser-includes.ts
@@ -104,6 +104,23 @@ export class ParserIncludes {
 
                     includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
                 }
+            } else if (value["component"]) {
+                const {domain, projectPath, componentName, ref} = this.parseIncludeComponent(value["component"]);
+                // converts component to project
+                const files = [`${componentName}.yml`, `${componentName}/template.yml`, null];
+                for (const f of files) {
+                    assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the file [${files}] must exists in \`${domain}/${projectPath}\``);
+                    if (!(await Utils.remoteFileExist(f, ref, domain, projectPath, gitData.remote.schema))) continue;
+                    const fileDoc = {
+                        include: {
+                            project: projectPath,
+                            file: f,
+                            ref: ref,
+                        },
+                    };
+                    includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
+                    break;
+                }
             } else if (value["template"]) {
                 const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]);
                 const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`);
@@ -164,6 +181,21 @@ export class ParserIncludes {
         };
     }
 
+    static parseIncludeComponent (component: string): {domain: string; projectPath: string; componentName: string; ref: string} {
+        assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
+        // eslint-disable-next-line no-useless-escape
+        const pattern = /(?<domain>[^/\s]+)\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // regexr.com/7v7hm
+        const gitRemoteMatch = pattern.exec(component);
+
+        if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`);
+        return {
+            domain: gitRemoteMatch.groups["domain"],
+            projectPath: gitRemoteMatch.groups["projectPath"],
+            componentName: `templates/${gitRemoteMatch.groups["componentName"]}`,
+            ref: gitRemoteMatch.groups["ref"],
+        };
+    }
+
     static async downloadIncludeRemote (cwd: string, stateDir: string, url: string, fetchIncludes: boolean): Promise<void> {
         const fsUrl = Utils.fsUrl(url);
         try {
diff --git a/src/utils.ts b/src/utils.ts
index 6fe2cd5..badf985 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -7,6 +7,7 @@ import execa from "execa";
 import assert from "assert";
 import {CICDVariable} from "./variables-from-files";
 import globby from "globby";
+import axios from "axios";
 
 type RuleResultOpt = {
     cwd: string;
@@ -259,4 +260,29 @@ export class Utils {
     static isObject (v: any) {
         return Object.getPrototypeOf(v) === Object.prototype;
     }
+
+    static async remoteFileExist (file: string, ref: string, domain: string, projectPath: string, protocol: string) {
+        switch (protocol) {
+            case "git":
+                try {
+                    await Utils.bash(`git archive --remote=ssh://git@${domain}/${projectPath}.git ${ref} ${file} > /dev/null`);
+                    return true;
+                } catch (e: any) {
+                    if (!e.stderr.includes(`remote: fatal: pathspec '${file}' did not match any files`)) throw new Error(e);
+                    return false;
+                }
+
+            case "http":
+            case "https": {
+                try {
+                    const {status} = await axios.get(`${protocol}://${domain}/${projectPath}/-/raw/${ref}/${file}`);
+                    return (status === 200);
+                } catch (e) {
+                    return false;
+                }
+            }
+            default:
+                throw new Error(`${protocol} not supported!`);
+        }
+    }
 }


Test patch:
diff --git a/tests/mocks/utils.mock.ts b/tests/mocks/utils.mock.ts
index 8078a4c..c68835e 100644
--- a/tests/mocks/utils.mock.ts
+++ b/tests/mocks/utils.mock.ts
@@ -1,11 +1,12 @@
 import {when} from "jest-when";
 import {Utils} from "../../src/utils";
 
-export function initBashSpyReject (spyMocks: {cmd: string; rejection: string}[]) {
+export function initBashSpyReject (spyMocks: {cmd: string; rejection: any}[]) {
     const spyOn = jest.spyOn(Utils, "bash");
 
     for (const spyMock of spyMocks) {
-        when(spyOn).calledWith(spyMock.cmd, expect.any(String)).mockRejectedValue(new Error(spyMock.rejection));
+        when(spyOn).calledWith(spyMock.cmd, expect.any(String)).mockRejectedValue(spyMock.rejection);
+        when(spyOn).calledWith(spyMock.cmd).mockRejectedValue(spyMock.rejection);
     }
 }
 
@@ -14,6 +15,7 @@ export function initBashSpy (spyMocks: {cmd: string; returnValue: any}[]) {
 
     for (const spyMock of spyMocks) {
         when(spyOn).calledWith(spyMock.cmd, expect.any(String)).mockResolvedValue(spyMock.returnValue);
+        when(spyOn).calledWith(spyMock.cmd).mockResolvedValue(spyMock.returnValue);
     }
 }
 
@@ -22,5 +24,6 @@ export function initSpawnSpy (spyMocks: {cmdArgs: string[]; returnValue: any}[])
 
     for (const spyMock of spyMocks) {
         when(spyOn).calledWith(spyMock.cmdArgs, expect.any(String)).mockResolvedValue(spyMock.returnValue);
+        when(spyOn).calledWith(spyMock.cmdArgs).mockResolvedValue(spyMock.returnValue);
     }
 }
diff --git a/tests/mocks/when-statics.ts b/tests/mocks/when-statics.ts
index 1436e8c..574aa5a 100644
--- a/tests/mocks/when-statics.ts
+++ b/tests/mocks/when-statics.ts
@@ -44,4 +44,11 @@ export class WhenStatics {
         WhenStatics.mockGitCommitShaShort,
     ];
 
+    public static readonly mockGitRemoteHttp = {
+        cmdArgs: ["git", "remote", "-v"],
+        returnValue: {
+            stdout: `origin https://gitlab.com/gcl/test-project.git (fetch)
+origin https://gitlab.com/gcl/test-project.git (push)`,
+        },
+    };
 }
diff --git a/tests/test-cases/include-component/component/.gitlab-ci.yml b/tests/test-cases/include-component/component/.gitlab-ci.yml
new file mode 100644
index 0000000..b34abff
--- /dev/null
+++ b/tests/test-cases/include-component/component/.gitlab-ci.yml
@@ -0,0 +1,14 @@
+---
+include:
+  - component: gitlab.com/components/go/full-pipeline@0.3.1
+    inputs:
+      stage_format: format
+      stage_build: build
+      stage_test: test
+      go_version: latest
+
+stages:
+  - format
+  - build
+  - test
+  - latest
diff --git a/tests/test-cases/include-component/integration.include-component.test.ts b/tests/test-cases/include-component/integration.include-component.test.ts
new file mode 100644
index 0000000..f832d42
--- /dev/null
+++ b/tests/test-cases/include-component/integration.include-component.test.ts
@@ -0,0 +1,70 @@
+import {WriteStreamsMock} from "../../../src/write-streams";
+import {handler} from "../../../src/handler";
+import assert, {AssertionError} from "assert";
+import {initSpawnSpy} from "../../mocks/utils.mock";
+import {WhenStatics} from "../../mocks/when-statics";
+
+beforeAll(() => {
+    initSpawnSpy(WhenStatics.all);
+});
+
+test.concurrent("include-component no component template file (protocol: https)", async () => {
+    initSpawnSpy([WhenStatics.mockGitRemoteHttp]);
+
+    const writeStreams = new WriteStreamsMock();
+    try {
+        await handler({
+            cwd: "tests/test-cases/include-component/no-component-template-file",
+            preview: true,
+        }, writeStreams);
+        expect(true).toBe(false);
+    } catch (e: any) {
+        assert(e instanceof AssertionError, `Unexpected error thrown:\n ${e}`);
+        expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the file [templates/potato.yml,templates/potato/template.yml,] must exists in `gitlab.com/components/go`");
+    }
+});
+
+test.concurrent("include-component component (protocol: https)", async () => {
+    initSpawnSpy([WhenStatics.mockGitRemoteHttp]);
+
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/include-component/component",
+        preview: true,
+    }, writeStreams);
+
+
+    const expected = `---
+stages:
+  - .pre
+  - format
+  - build
+  - test
+  - latest
+  - .post
+format-latest:
+  image:
+    name: golang:latest
+  stage: format
+  script:
+    - go fmt $(go list ./... | grep -v /vendor/)
+    - go vet $(go list ./... | grep -v /vendor/)
+build-latest:
+  image:
+    name: golang:latest
+  stage: build
+  script:
+    - mkdir -p mybinaries
+    - go build -o mybinaries ./...
+  artifacts:
+    paths:
+      - mybinaries
+test-latest:
+  image:
+    name: golang:latest
+  stage: test
+  script:
+    - go test -race $(go list ./... | grep -v /vendor/)`;
+
+    expect(writeStreams.stdoutLines[0]).toEqual(expected);
+});
diff --git a/tests/test-cases/include-component/no-component-template-file/.gitlab-ci.yml b/tests/test-cases/include-component/no-component-template-file/.gitlab-ci.yml
new file mode 100644
index 0000000..edfcff2
--- /dev/null
+++ b/tests/test-cases/include-component/no-component-template-file/.gitlab-ci.yml
@@ -0,0 +1,5 @@
+---
+include:
+  - component: gitlab.com/components/go/potato@0.3.1
+stages:
+  - format
diff --git a/tests/test-cases/pull-policy/.gitlab-ci.yml b/tests/test-cases/pull-policy/.gitlab-ci.yml
new file mode 100644
index 0000000..4029ddc
--- /dev/null
+++ b/tests/test-cases/pull-policy/.gitlab-ci.yml
@@ -0,0 +1,5 @@
+---
+alpine:
+  image: alpine
+  script:
+    - echo "Hey hey"
diff --git a/tests/test-cases/pull-policy/integration.pull-policy.test.ts b/tests/test-cases/pull-policy/integration.pull-policy.test.ts
new file mode 100644
index 0000000..4f17558
--- /dev/null
+++ b/tests/test-cases/pull-policy/integration.pull-policy.test.ts
@@ -0,0 +1,26 @@
+import {WriteStreamsMock} from "../../../src/write-streams";
+import {handler} from "../../../src/handler";
+import {initSpawnSpy} from "../../mocks/utils.mock";
+import {WhenStatics} from "../../mocks/when-statics";
+import {Utils} from "../../../src/utils";
+
+jest.setTimeout(60000);
+
+beforeAll(() => {
+    initSpawnSpy(WhenStatics.all);
+});
+
+test.concurrent("pull-policy <alpine-guest> --pull-policy=always", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        pullPolicy: "always",
+        cwd: "tests/test-cases/pull-policy/",
+        job: ["alpine"],
+    }, writeStreams);
+
+    // Pre pull image
+    await Utils.spawn(["docker", "pull", "alpine"]);
+
+    // Expect its getting pulld again
+    expect(writeStreams.stdoutLines.join("\n")).toMatch(/pulled/);
+});
diff --git a/tests/utils.test.ts b/tests/utils.test.ts
new file mode 100644
index 0000000..23f6c36
--- /dev/null
+++ b/tests/utils.test.ts
@@ -0,0 +1,45 @@
+import {Utils} from "../src/utils";
+import {initBashSpyReject, initBashSpy} from "./mocks/utils.mock";
+
+import {isSshDirFound} from "./utils";
+
+test("remoteFileExist protocol: git", async () => {
+    const file = "templates/test.yml";
+    const ref = "0.3.1";
+    const domain = "gitlab.com";
+    const projectPath = "components/go";
+
+    // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found
+    if (!isSshDirFound()) {
+        const spyGitArchive = {
+            cmd: `git archive --remote=ssh://git@${domain}/${projectPath}.git ${ref} ${file} > /dev/null`,
+            returnValue: {output: ""},
+        };
+        initBashSpy([spyGitArchive]);
+    }
+
+    const fileExist = await Utils.remoteFileExist(file, ref, domain, projectPath, "git");
+    expect(fileExist).toBe(true);
+});
+
+test("remoteFileDoesNotExist protocol: git", async () => {
+    const file = "templates/potato.yml";
+    const ref = "0.3.1";
+    const domain = "gitlab.com";
+    const projectPath = "components/go";
+
+    // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found
+    if (!isSshDirFound()) {
+        const spyGitArchive = {
+            cmd: `git archive --remote=ssh://git@${domain}/${projectPath}.git ${ref} ${file} > /dev/null`,
+            rejection: {
+                stderr: `fatal: sent error to the client: git upload-archive: archiver died with error
+remote: fatal: pathspec 'templates/potato.yml' did not match any files
+remote: git upload-archive: archiver died with error`,
+            },
+        };
+        initBashSpyReject([spyGitArchive]);
+    }
+    const fileExist = await Utils.remoteFileExist(file, ref, domain, projectPath, "git");
+    expect(fileExist).toBe(false);
+});
diff --git a/tests/utils.ts b/tests/utils.ts
new file mode 100644
index 0000000..fcb7dc3
--- /dev/null
+++ b/tests/utils.ts
@@ -0,0 +1,10 @@
+import * as fs from "fs-extra";
+
+export function isSshDirFound () {
+    try {
+        fs.statSync(`${process.env.HOME}/.ssh`);
+        return true;
+    } catch {
+        return false;
+    }
+}