Repository: firecow/gitlab-ci-local
Pull request #1219 (2024-05-14 14:09:59)

[FEATURE] #1217 Added feature to emulate gitlab hosted runner using the `--container-emulate` option with the name of one of the gitlab preset.
### summary
Added an option to prefill `memory`, `kernel-memory` and  `cpus` docker options based on the configuration of Gitlab's hosted runner. This is useful when attempting to debug performance issue. By default, no limit is applied, the host resources are all available. 

When passing the option `--container-emulate=` with the name of a gitlab hosted runner, the docker resources is limited to the resource available for this class of hosted runner. As of this PR, those are the values

| name | memory | CPU | 
| --- | --- | --- |
| `saas-linux-small` | 8 GB | 2 |
| `saas-linux-medium` | 16 GB | 4 |
| `saas-linux-large` | 32 GB | 8 |
| `saas-linux-xlarge` | 64 GB | 16 |
| `saas-linux-2xlarge` | 128 GB | 32 |

[see the gitlab documentation here](https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html#machine-types-available-for-linux---x86-64)

Of course, you are also limited to the host resources. You cannot have 128 GB of RAM on a host that only has 16 GB.

As of right now, I've only added support for the base linux machine. This does not include runner with a GPUs. This could be a nice amelioration in the futur. The runners names does not contains the architecture, as this is based on the host CPU, and cannot be emulated.

### Usage

You can pass an additional option `--containter-emulate` with one of the previously mentionned runner name. For example, using the name `saas-linux-small` will limit the container resource to 8 GB or RAM and 2 CPUs. 

```
gitlab-ci-local --cwd /path/to/.gitlab.yml --container-emulate=saas-linux-small
```

You can check the memory limit and CPU usage using the `docker stats {container_id}` command.

![image](https://github.com/firecow/gitlab-ci-local/assets/3102811/a1e78d00-956a-49e0-baf9-11bfe8dc68f3)

This PR Closes https://github.com/firecow/gitlab-ci-local/issues/1217
  


Patch:
diff --git a/src/argv.ts b/src/argv.ts
index d9f5010..1a433e0 100644
--- a/src/argv.ts
+++ b/src/argv.ts
@@ -240,6 +240,10 @@ export class Argv {
         return this.map.get("containerMacAddress") ?? null;
     }
 
+    get containerEmulate (): string | null {
+        return this.map.get("containerEmulate") ?? null;
+    }
+
     get concurrency (): number | null {
         const concurrency = this.map.get("concurrency");
         if (!concurrency) return null;
diff --git a/src/git-data.ts b/src/git-data.ts
index 6f94b0c..35d34bc 100644
--- a/src/git-data.ts
+++ b/src/git-data.ts
@@ -117,7 +117,7 @@ export class GitData {
                 this.remote.schema = gitRemoteMatch.groups.schema;
                 this.remote.port = gitRemoteMatch.groups.port ?? "22";
             } else {
-                gitRemoteMatch = /(?<username>\S+)@(?<host>[^:]+):(?<group>\S+)\/(?<project>[^ .]+)\.git/.exec(gitRemote); // regexr.com/7vjoq
+                gitRemoteMatch = /(?<username>\S+)@(?<host>[^:]+):(?<group>\S+)\/(?<project>\S+)\.git/.exec(gitRemote); // regexr.com/7vjoq
                 assert(gitRemoteMatch?.groups != null, "git remote get-url origin didn't provide valid matches");
 
                 const {stdout} = await Utils.spawn(["ssh", "-G", `${gitRemoteMatch.groups.username}@${gitRemoteMatch.groups.host}`]);
diff --git a/src/gitlab-preset.ts b/src/gitlab-preset.ts
new file mode 100644
index 0000000..bea2eea
--- /dev/null
+++ b/src/gitlab-preset.ts
@@ -0,0 +1,25 @@
+export enum GitlabRunnerPreset {
+    SMALL = "saas-linux-small",
+    MEDIUM = "saas-linux-medium",
+    LARGE = "saas-linux-large",
+    XLARGE = "saas-linux-xlarge",
+    TWO_XLARGE = "saas-linux-2xlarge",
+}
+
+export const GitlabRunnerPresetValues: string[] = Object.values(GitlabRunnerPreset);
+
+export const GitlabRunnerMemoryPresetValue: Record<string, number> = {
+    [GitlabRunnerPreset.SMALL]: 8 * 1024,
+    [GitlabRunnerPreset.MEDIUM]: 16 * 1024,
+    [GitlabRunnerPreset.LARGE]: 32 * 1024,
+    [GitlabRunnerPreset.XLARGE]: 64 * 1024,
+    [GitlabRunnerPreset.TWO_XLARGE]: 128 * 1024,
+};
+
+export const GitlabRunnerCPUsPresetValue: Record<string, number> = {
+    [GitlabRunnerPreset.SMALL]: 2,
+    [GitlabRunnerPreset.MEDIUM]: 4,
+    [GitlabRunnerPreset.LARGE]: 8,
+    [GitlabRunnerPreset.XLARGE]: 16,
+    [GitlabRunnerPreset.TWO_XLARGE]: 32,
+};
diff --git a/src/index.ts b/src/index.ts
index b17e77a..658dba8 100755
--- a/src/index.ts
+++ b/src/index.ts
@@ -12,6 +12,7 @@ import {Executor} from "./executor";
 import {Argv} from "./argv";
 import {AssertionError} from "assert";
 import {Job, cleanupJobResources} from "./job";
+import {GitlabRunnerPresetValues} from "./gitlab-preset";
 
 const jobs: Job[] = [];
 
@@ -257,6 +258,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
             description: "Container MAC address (e.g., aa:bb:cc:dd:ee:ff)",
             requiresArg: false,
         })
+        .option("container-emulate", {
+            type: "string",
+            description: "The name, without the architecture, of a gitlab hosted runner to emulate. See here: https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html#machine-types-available-for-linux---x86-64",
+            choices: GitlabRunnerPresetValues,
+        })
         .completion("completion", false, (current: string, yargsArgv: any, completionFilter: any, done: (completions: string[]) => any) => {
             try {
                 if (current.startsWith("-")) {
diff --git a/src/job.ts b/src/job.ts
index 06fe5a8..3ff6732 100644
--- a/src/job.ts
+++ b/src/job.ts
@@ -11,6 +11,7 @@ import {Mutex} from "./mutex";
 import {Argv} from "./argv";
 import execa from "execa";
 import {CICDVariable} from "./variables-from-files";
+import {GitlabRunnerCPUsPresetValue, GitlabRunnerMemoryPresetValue, GitlabRunnerPresetValues} from "./gitlab-preset";
 
 const CI_PROJECT_DIR = "/gcl-builds";
 interface JobOptions {
@@ -677,6 +678,21 @@ export class Job {
                 dockerCmd += "--network-alias=build ";
             }
 
+            if (this.argv.containerEmulate) {
+                const runnerName: string = this.argv.containerEmulate;
+
+                if (!GitlabRunnerPresetValues.includes(runnerName)) {
+                    throw new Error("Invalid gitlab runner to emulate.");
+                }
+
+                const memoryConfig = GitlabRunnerMemoryPresetValue[runnerName];
+                const cpuConfig = GitlabRunnerCPUsPresetValue[runnerName];
+
+                dockerCmd += `--memory=${memoryConfig}m `;
+                dockerCmd += `--kernel-memory=${memoryConfig}m `;
+                dockerCmd += `--cpus=${cpuConfig} `;
+            }
+
             dockerCmd += `--volume ${buildVolumeName}:/gcl-builds `;
             dockerCmd += `--volume ${tmpVolumeName}:${this.fileVariablesDir} `;
             dockerCmd += "--workdir /gcl-builds ";


Test patch:
diff --git a/tests/mocks/utils.mock.ts b/tests/mocks/utils.mock.ts
index 93d0535..e75cb48 100644
--- a/tests/mocks/utils.mock.ts
+++ b/tests/mocks/utils.mock.ts
@@ -17,6 +17,8 @@ export function initBashSpy (spyMocks: {cmd: string; returnValue: any}[]) {
         when(spyOn).calledWith(spyMock.cmd, expect.any(String)).mockResolvedValue(spyMock.returnValue);
         when(spyOn).calledWith(spyMock.cmd).mockResolvedValue(spyMock.returnValue);
     }
+
+    return spyOn;
 }
 
 export function initSpawnSpy (spyMocks: {cmdArgs: string[]; returnValue: any}[]) {
diff --git a/tests/test-cases/gitlab-runner-emulation/.gitlab-ci.yml b/tests/test-cases/gitlab-runner-emulation/.gitlab-ci.yml
new file mode 100644
index 0000000..30259b9
--- /dev/null
+++ b/tests/test-cases/gitlab-runner-emulation/.gitlab-ci.yml
@@ -0,0 +1,5 @@
+---
+somejob:
+  image: docker.io/library/alpine
+  script:
+    - echo Hello
diff --git a/tests/test-cases/gitlab-runner-emulation/integration.emulate-runner.test.ts b/tests/test-cases/gitlab-runner-emulation/integration.emulate-runner.test.ts
new file mode 100644
index 0000000..dd11a4d
--- /dev/null
+++ b/tests/test-cases/gitlab-runner-emulation/integration.emulate-runner.test.ts
@@ -0,0 +1,112 @@
+import {WriteStreamsMock} from "../../../src/write-streams";
+import {handler} from "../../../src/handler";
+import {initBashSpy, initSpawnSpy} from "../../mocks/utils.mock";
+import {WhenStatics} from "../../mocks/when-statics";
+import {
+    GitlabRunnerCPUsPresetValue,
+    GitlabRunnerMemoryPresetValue,
+    GitlabRunnerPresetValues,
+} from "../../../src/gitlab-preset";
+
+beforeAll(() => {
+    initSpawnSpy([...WhenStatics.all, {
+        cmdArgs: ["docker", "cp", expect.any(String), expect.any(String)],
+        returnValue: {stdout: "Ok"},
+    }]);
+});
+
+test("--container-emulate some_unexisting_runner", async () => {
+    try {
+        const writeStreams = new WriteStreamsMock();
+        await handler({
+            cwd: "tests/test-cases/container-executable",
+            containerEmulate: "some_unexisting_runner",
+        }, writeStreams);
+
+        expect(true).toBe(false);
+    } catch (e: unknown) {
+        expect((e as Error).message).toContain("Invalid gitlab runner to emulate.");
+    }
+});
+
+test("should contains memory config when emulating valid runner", async () => {
+    // given a spy of the bash utils
+    const bashSpy = initBashSpy([{cmd: expect.any(String), returnValue: {stdout: "abcde12345", stderr: "", exitCode: 0}}]);
+
+    // and a WriteStreamsMock
+    const writeStreams = new WriteStreamsMock();
+
+    // when calling the handler with a valid emulate value
+    await handler({
+        cwd: "tests/test-cases/container-executable",
+        containerEmulate: "saas-linux-small",
+    }, writeStreams);
+
+    // then the bash spy should have been called with the memory option
+    expect(bashSpy).toHaveBeenLastCalledWith(expect.stringMatching(/ --memory=\d{1,6}m /), expect.any(String));
+});
+
+test("should contains kernel memory config when emulating valid runner", async () => {
+    // given a spy of the bash utils
+    const bashSpy = initBashSpy([{cmd: expect.any(String), returnValue: {stdout: "abcde12345", stderr: "", exitCode: 0}}]);
+
+    // and a WriteStreamsMock
+    const writeStreams = new WriteStreamsMock();
+
+    // when calling the handler with a valid emulate value
+    await handler({
+        cwd: "tests/test-cases/container-executable",
+        containerEmulate: "saas-linux-small",
+    }, writeStreams);
+
+    // then the bash spy should have been called with the kernel option
+    expect(bashSpy).toHaveBeenLastCalledWith(expect.stringMatching(/ --kernel-memory=\d{1,6}m /), expect.any(String));
+});
+
+test("should contains cpus config when emulating valid runner", async () => {
+    // given a spy of the bash utils
+    const bashSpy = initBashSpy([{cmd: expect.any(String), returnValue: {stdout: "abcde12345", stderr: "", exitCode: 0}}]);
+
+    // and a WriteStreamsMock
+    const writeStreams = new WriteStreamsMock();
+
+    // when calling the handler with a valid emulate value
+    await handler({
+        cwd: "tests/test-cases/container-executable",
+        containerEmulate: "saas-linux-small",
+    }, writeStreams);
+
+    // then the bash spy should have been called with the cpus option
+    expect(bashSpy).toHaveBeenLastCalledWith(expect.stringMatching(/ --cpus=\d{1,3}(\.\d{1,3})? /), expect.any(String));
+});
+
+const gitlabRunnerDataProvider = GitlabRunnerPresetValues.map(name => ({
+    name,
+    memory: GitlabRunnerMemoryPresetValue[name],
+    cpus: GitlabRunnerCPUsPresetValue[name],
+}));
+
+describe.each(gitlabRunnerDataProvider)("gitlab runner configuration", (data) => {
+    test(`should set the proper values when emulating ${data.name}`, async () => {
+        // given a spy of the bash utils
+        const bashSpy = initBashSpy([{cmd: expect.any(String), returnValue: {stdout: "abcde12345", stderr: "", exitCode: 0}}]);
+
+        // and a WriteStreamsMock
+        const writeStreams = new WriteStreamsMock();
+
+        // when calling the handler with the given emulated runner
+        await handler({
+            cwd: "tests/test-cases/container-executable",
+            containerEmulate: data.name,
+        }, writeStreams);
+
+        // then the bash spy should have been called with the memory option
+        expect(bashSpy).toHaveBeenLastCalledWith(expect.stringContaining(`--memory=${data.memory}`), expect.any(String));
+
+        // and the bash spy should have been called with the kernel memory option
+        expect(bashSpy).toHaveBeenLastCalledWith(expect.stringContaining(`--kernel-memory=${data.memory}`), expect.any(String));
+
+        // and the bash spy should have been called with the cpus option
+        expect(bashSpy).toHaveBeenLastCalledWith(expect.stringContaining(`--cpus=${data.cpus}`), expect.any(String));
+    });
+});