Repository: modelix/modelix.core
Pull request #1550: feat(git-import): new docker image for running the Git importer
Author: login: slisson
Labels: name: released
Created: 2025-04-16T19:31:52Z | Merged: 2025-04-16T19:55:31Z

For running the importer as a task in a Kestra flow.


Diff:
diff --git a/commitlint.config.js b/commitlint.config.js
index 0776709ab6..3a1afd8088 100644
--- a/commitlint.config.js
+++ b/commitlint.config.js
@@ -10,6 +10,7 @@ module.exports = {
         "bulk-model-sync-gradle",
         "datastructures",
         "deps",
+        "git-import",
         "metamodel-export",
         "model-api-gen",
         "model-api-gen-gradle",
@@ -19,7 +20,6 @@ module.exports = {
         "model-server",
         "model-server-openapi",
         "modelql",
-        "mps-git-import-plugin",
         "mps-model-adapters",
         "mps-model-server",
         "mps-sync-plugin",
diff --git a/docker-build-gitimport.sh b/docker-build-gitimport.sh
new file mode 100755
index 0000000000..eecbef3375
--- /dev/null
+++ b/docker-build-gitimport.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+TAG=$( ./modelix-version.sh )
+
+(
+  cd mps-git-import-cli
+  if [ "${CI}" = "true" ]; then
+    docker buildx build --platform linux/amd64,linux/arm64 --push \
+    -t modelix/mps-git-import:latest -t "modelix/mps-git-import:${TAG}" .
+  else
+    docker build \
+    -t modelix/mps-git-import:latest -t "modelix/mps-git-import:${TAG}" .
+  fi
+)
diff --git a/docker-ci.sh b/docker-ci.sh
index c883ae3e5d..c3ca4e6d7a 100755
--- a/docker-ci.sh
+++ b/docker-ci.sh
@@ -5,3 +5,4 @@ set -e
 echo "$DOCKER_HUB_KEY" | docker login -u "$DOCKER_HUB_USER" --password-stdin
 
 ./docker-build-model.sh
+./docker-build-gitimport.sh
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index e6a09143d1..83273fffda 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -32,7 +32,7 @@ kotlinSerialization="1.8.1"
 ignite="2.17.0"
 apacheCxf="3.6.6"
 node="22.14.0"
-modelixBuildtools="1.8.1"
+modelixBuildtools="1.9.0"
 openapi = "7.12.0"
 micrometer = "1.14.6"
 dokka = "2.0.0"
diff --git a/mps-git-import-cli/Dockerfile b/mps-git-import-cli/Dockerfile
new file mode 100644
index 0000000000..ca583a6183
--- /dev/null
+++ b/mps-git-import-cli/Dockerfile
@@ -0,0 +1,23 @@
+FROM eclipse-temurin:21
+ARG MPS_VERSION="2024.1.3"
+
+RUN apt update && apt install wget ant -y
+
+# download MPS and extract into /mps
+RUN set -e \
+    && mkdir /mpstmp \
+    && cd /mpstmp \
+    && MPS_MAJOR_VERSION=`echo "$MPS_VERSION" | grep -oE '20[0-9]{2}\.[0-9]+'` \
+    && wget "https://download.jetbrains.com/mps/${MPS_MAJOR_VERSION}/MPS-${MPS_VERSION}.tar.gz" \
+    && tar -xf $(ls | head -n 1) \
+    && mv "MPS $MPS_MAJOR_VERSION" "/mps" \
+    && cd .. \
+    && rm -rf /mpstmp \
+    && rm -rf /mps/jbr
+
+COPY build/install/mps-git-import-cli /mps-git-import-cli
+
+#RUN /mps-git-import-cli/bin/mps-git-import-cli self-test
+
+ENTRYPOINT ["/mps-git-import-cli/bin/mps-git-import-cli"]
+CMD ["self-test"]
diff --git a/mps-git-import-cli/build.gradle.kts b/mps-git-import-cli/build.gradle.kts
new file mode 100644
index 0000000000..f1cddac937
--- /dev/null
+++ b/mps-git-import-cli/build.gradle.kts
@@ -0,0 +1,27 @@
+import org.modelix.mpsHomeDir
+
+plugins {
+    `modelix-kotlin-jvm-with-junit`
+    `maven-publish`
+    application
+    alias(libs.plugins.modelix.mps.buildtools)
+}
+
+dependencies {
+    implementation(project(":mps-git-import"))
+    implementation(libs.modelix.buildtools.lib)
+
+    compileOnly(
+        fileTree(mpsHomeDir).matching {
+            include("lib/**/*.jar")
+        },
+    )
+}
+
+application {
+    mainClass.set("org.modelix.mps.gitimport.cli.MainKt")
+}
+
+tasks.assemble {
+    dependsOn(tasks.installDist)
+}
diff --git a/mps-git-import-cli/src/main/kotlin/org/modelix/mps/gitimport/cli/Main.kt b/mps-git-import-cli/src/main/kotlin/org/modelix/mps/gitimport/cli/Main.kt
new file mode 100644
index 0000000000..2df4439bd3
--- /dev/null
+++ b/mps-git-import-cli/src/main/kotlin/org/modelix/mps/gitimport/cli/Main.kt
@@ -0,0 +1,25 @@
+package org.modelix.mps.gitimport.cli
+
+import org.modelix.buildtools.runner.MPSRunner
+import org.modelix.buildtools.runner.MPSRunnerConfig
+import java.io.File
+import java.util.UUID
+
+fun main(args: Array<String>) {
+    val buildDir = File("/tmp/git-import-build")
+    val workDir = File("/tmp/git-import-work")
+    buildDir.mkdirs()
+    workDir.mkdirs()
+    val config = MPSRunnerConfig(
+        mainClassName = "org.modelix.mps.gitimport.MainKt",
+        mainMethodName = "runFromAnt",
+        mpsHome = File("/mps"),
+        jarFolders = listOf(File("/mps-git-import-cli/lib")),
+        jvmArgs = args.mapIndexed { index, arg -> "-Dmodelix.git.import.args.$index=$arg" },
+        moduleId = UUID.randomUUID(),
+        buildDir = buildDir,
+        workDir = workDir,
+    )
+    val runner = MPSRunner(config)
+    runner.run()
+}
diff --git a/mps-git-import-plugin/build.gradle.kts b/mps-git-import-plugin/build.gradle.kts
index 9ff6346a7c..4298357f07 100644
--- a/mps-git-import-plugin/build.gradle.kts
+++ b/mps-git-import-plugin/build.gradle.kts
@@ -19,6 +19,7 @@ dependencies {
         exclude("org.slf4j", "slf4j-api")
     }
 
+    implementation(project(":mps-git-import"), excludeMPSLibraries)
     implementation(project(":bulk-model-sync-lib"), excludeMPSLibraries)
     implementation(project(":bulk-model-sync-mps"), excludeMPSLibraries)
     implementation(project(":mps-model-adapters"), excludeMPSLibraries)
@@ -26,7 +27,6 @@ dependencies {
     implementation(project(":datastructures"), excludeMPSLibraries)
     implementation(libs.modelix.mpsApi, excludeMPSLibraries)
     implementation(libs.kotlin.logging, excludeMPSLibraries)
-    implementation(libs.kotlin.html, excludeMPSLibraries)
     implementation(libs.kotlin.datetime, excludeMPSLibraries)
     implementation("org.eclipse.jgit:org.eclipse.jgit:7.2.0.202503040940-r")
     implementation("com.github.ajalt.clikt:clikt:5.0.3")
diff --git a/mps-git-import/build.gradle.kts b/mps-git-import/build.gradle.kts
new file mode 100644
index 0000000000..b16fb5a3c4
--- /dev/null
+++ b/mps-git-import/build.gradle.kts
@@ -0,0 +1,40 @@
+import org.modelix.mpsHomeDir
+
+plugins {
+    `modelix-kotlin-jvm-with-junit`
+    `maven-publish`
+}
+
+dependencies {
+    implementation(project(":bulk-model-sync-lib"))
+    implementation(project(":bulk-model-sync-mps"))
+    implementation(project(":mps-model-adapters"))
+    implementation(project(":model-client"))
+    implementation(project(":datastructures"))
+    implementation(libs.modelix.mpsApi)
+    implementation(libs.kotlin.logging)
+    implementation(libs.kotlin.datetime)
+    implementation("org.eclipse.jgit:org.eclipse.jgit:7.2.0.202503040940-r")
+    api("com.github.ajalt.clikt:clikt:5.0.3")
+
+    compileOnly(
+        fileTree(mpsHomeDir).matching {
+            include("lib/**/*.jar")
+        },
+    )
+}
+
+group = "org.modelix.mps"
+
+java {
+    withSourcesJar()
+}
+
+publishing {
+    publications {
+        create<MavenPublication>("maven") {
+            artifactId = "git-import"
+            from(components["java"])
+        }
+    }
+}
diff --git a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/DummyRepo.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/DummyRepo.kt
similarity index 100%
rename from mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/DummyRepo.kt
rename to mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/DummyRepo.kt
diff --git a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitDataSourceFactoryRule.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitDataSourceFactoryRule.kt
similarity index 100%
rename from mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitDataSourceFactoryRule.kt
rename to mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitDataSourceFactoryRule.kt
diff --git a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
similarity index 99%
rename from mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
rename to mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
index dd23412d51..417127aa5f 100644
--- a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
+++ b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
@@ -48,6 +48,7 @@ class GitImporter(
     val baseBranch: BranchReference,
     val targetBranchName: String?,
     val gitRevision: String,
+    val token: String? = null,
 ) {
     @OptIn(ExperimentalStdlibApi::class)
     suspend fun runSuspending() {
@@ -62,7 +63,9 @@ class GitImporter(
         val resolvedCommit = git.repository.parseCommit(git.repository.resolve(gitRevision))
         val targetBranch = repositoryId.getBranchReference(targetBranchName ?: "git-import-${resolvedCommit.name}")
 
-        val client = ModelClientV2.builder().url(modelServerUrl).build()
+        val client = ModelClientV2.builder().url(modelServerUrl).also {
+            if (token != null) it.authToken { token }
+        }.build()
         client.init()
 
         println("Finding existing imports")
diff --git a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitObjectAsVirtualFile.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitObjectAsVirtualFile.kt
similarity index 100%
rename from mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/GitObjectAsVirtualFile.kt
rename to mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitObjectAsVirtualFile.kt
diff --git a/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/Main.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/Main.kt
new file mode 100644
index 0000000000..3b7bbc95ca
--- /dev/null
+++ b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/Main.kt
@@ -0,0 +1,15 @@
+package org.modelix.mps.gitimport
+
+import com.github.ajalt.clikt.core.main
+
+fun runFromAnt() {
+    val args = sequence {
+        var i = 0
+        while (true) {
+            yield(System.getProperty("modelix.git.import.args.$i") ?: return@sequence)
+            i++
+        }
+    }.toList()
+
+    ModelixSyncCommand().main(args)
+}
diff --git a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
similarity index 86%
rename from mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
rename to mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
index 46bbbecc0b..6777f0b865 100644
--- a/mps-git-import-plugin/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
+++ b/mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
@@ -15,7 +15,7 @@ import org.modelix.model.lazy.RepositoryId
 
 class ModelixSyncCommand : CliktCommand() {
     init {
-        subcommands(GitImport())
+        subcommands(GitImport(), SelfTest())
     }
 
     override fun run() {}
@@ -23,6 +23,7 @@ class ModelixSyncCommand : CliktCommand() {
     class GitImport : CliktCommand(name = "git-import") {
         val gitDir by argument().file(mustExist = true, canBeFile = false, canBeDir = true).help("Path to a Git repository")
         val modelServer by option().default("http://localhost:28101").help("URL of the Modelix model server")
+        val token by option().help("JWT token for the model server access")
         val repository by option().required().help("Name of the target repository on the model server")
         val branch by option().default("git-import").help("Name of the target branch on the model server")
         val rev by option().default("HEAD").help("Git revision, which can a branch name, a tag or a commit ID")
@@ -35,8 +36,15 @@ class ModelixSyncCommand : CliktCommand() {
                 baseBranch = RepositoryId(repository).getBranchReference(branch),
                 targetBranchName = branch,
                 gitRevision = rev,
+                token = token,
             )
             runBlocking { importer.runSuspending() }
         }
     }
+
+    class SelfTest : CliktCommand(name = "self-test") {
+        override fun run() {
+            println("OK")
+        }
+    }
 }
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 53ab45b400..64cce74395 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -32,6 +32,8 @@ include("modelql-html")
 include("modelql-server")
 include("modelql-typed")
 include("modelql-untyped")
+include("mps-git-import")
+include("mps-git-import-cli")
 include("mps-git-import-plugin")
 include("mps-model-adapters")
 include("mps-model-adapters-plugin")


Reviews:
user: login: github-advanced-security[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-16T19:55:25Z

Review comments:
diff_hunk: @@ -0,0 +1,25 @@
+package org.modelix.mps.gitimport.cli
+
+import org.modelix.buildtools.runner.MPSRunner
+import org.modelix.buildtools.runner.MPSRunnerConfig
+import java.io.File
+import java.util.UUID
+
+fun main(args: Array<String>) {
path: mps-git-import-cli/src/main/kotlin/org/modelix/mps/gitimport/cli/Main.kt
user: login: github-advanced-security[bot]
body: The function main is missing documentation.

[Show more details](https://github.com/modelix/modelix.core/security/code-scanning/7122)
created_at: 2025-04-16T19:55:24Z
author_association: NONE
diff_hunk: @@ -48,6 +48,7 @@
     val baseBranch: BranchReference,
     val targetBranchName: String?,
     val gitRevision: String,
+    val token: String? = null,
path: mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/GitImporter.kt
user: login: github-advanced-security[bot]
body: The property token is missing documentation.

[Show more details](https://github.com/modelix/modelix.core/security/code-scanning/7073)
created_at: 2025-04-16T19:55:24Z
author_association: NONE
diff_hunk: @@ -0,0 +1,15 @@
+package org.modelix.mps.gitimport
+
+import com.github.ajalt.clikt.core.main
+
+fun runFromAnt() {
path: mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/Main.kt
user: login: github-advanced-security[bot]
body: The function runFromAnt is missing documentation.

[Show more details](https://github.com/modelix/modelix.core/security/code-scanning/7100)
created_at: 2025-04-16T19:55:24Z
author_association: NONE
diff_hunk: @@ -15,14 +15,15 @@
 
 class ModelixSyncCommand : CliktCommand() {
     init {
-        subcommands(GitImport())
+        subcommands(GitImport(), SelfTest())
     }
 
     override fun run() {}
 
     class GitImport : CliktCommand(name = "git-import") {
         val gitDir by argument().file(mustExist = true, canBeFile = false, canBeDir = true).help("Path to a Git repository")
         val modelServer by option().default("http://localhost:28101").help("URL of the Modelix model server")
+        val token by option().help("JWT token for the model server access")
path: mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
user: login: github-advanced-security[bot]
body: The property token is missing documentation.

[Show more details](https://github.com/modelix/modelix.core/security/code-scanning/7090)
created_at: 2025-04-16T19:55:24Z
author_association: NONE
diff_hunk: @@ -35,8 +36,15 @@
                 baseBranch = RepositoryId(repository).getBranchReference(branch),
                 targetBranchName = branch,
                 gitRevision = rev,
+                token = token,
             )
             runBlocking { importer.runSuspending() }
         }
     }
+
+    class SelfTest : CliktCommand(name = "self-test") {
path: mps-git-import/src/main/kotlin/org/modelix/mps/gitimport/ModelixSyncCommand.kt
user: login: github-advanced-security[bot]
body: SelfTest is missing required documentation.

[Show more details](https://github.com/modelix/modelix.core/security/code-scanning/7087)
created_at: 2025-04-16T19:55:25Z
author_association: NONE

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-16T19:51:13Z
body: ### JVM coverage report
|Overall Project|56.22% **`-0.11%`**|:x:|
|:-|:-|:-:|
|Files changed|10.67%|:x:|
<br>

|File|Coverage||
|:-|:-|:-:|
|[GitImporter.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FGitImporter.kt)|91.34% **`-1.1%`**|:x:|
|[GitDataSourceFactoryRule.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FGitDataSourceFactoryRule.kt)|80.69%|:green_apple:|
|[GitObjectAsVirtualFile.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FGitObjectAsVirtualFile.kt)|36.58%|:green_apple:|
|[DummyRepo.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FDummyRepo.kt)|24%|:green_apple:|
|[ModelixSyncCommand.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FModelixSyncCommand.kt)|0% **`-17.08%`**|:x:|
|[Main.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2FMain.kt)|0%|:x:|
|[Main.kt](https://github.com/modelix/modelix.core/blob/9b32a1df6794f0eb94e232e743e73ee596e434aa/mps-git-import-cli%2Fsrc%2Fmain%2Fkotlin%2Forg%2Fmodelix%2Fmps%2Fgitimport%2Fcli%2FMain.kt)|0%|:x:|
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T19:55:05Z
body: ## Test Results
  218 files    218 suites   47m 49s ⏱️
1 162 tests 1 152 ✅ 10 💤 0 ❌
1 172 runs  1 162 ✅ 10 💤 0 ❌

Results for commit 9b32a1df.

[test-results]:data:application/gzip;base64,H4sIABYLAGgC/03MSw6DIBSF4a0Yxh3IQ5BupkG8JKQqDY+R6d4L1BaG5zvJfyJjNwjoPhA83wYUko3dXJNX0bqjwMxllvzG8mPMyW8+QtK62tTZ076KjX8xym5ZGoD3zl/i0/HNCnKtVuWNumiFrll3n9Ru323MA8mFEoVXw4VkZoRFMiCUgGAUBAWYJAdGmVLo/QGVNuylEAEAAA==

author_association: CONTRIBUTOR
user: login: slisson
created_at: 2025-04-17T02:57:49Z
body: :tada: This PR is included in version 14.4.0 :tada:

The release is available on [GitHub release](https://github.com/modelix/modelix.core/releases/tag/14.4.0)

Your **[semantic-release](https://github.com/semantic-release/semantic-release)** bot :package::rocket:
author_association: MEMBER