Repository: frankframework/frankframework
Pull request #8904: Run LarvaTest scenarios individually in the Integration Test build
Author: login: tnleeuw
Created: 2025-04-29T22:12:57Z | Merged: 2025-05-06T07:11:53Z



Diff:
diff --git a/pom.xml b/pom.xml
index a8454f2ed25..3ba48b202d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -811,7 +811,8 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<!-- Surefire 3.5.3 has a bug and does not report properly on dynamic larva tests -->
+					<version>3.5.2</version>
 					<configuration>
 						<redirectTestOutputToFile>true</redirectTestOutputToFile>
 						<reuseForks>true</reuseForks>
diff --git a/test/src/test/java/org/frankframework/runner/RunLarvaTests.java b/test/src/test/java/org/frankframework/runner/RunLarvaTests.java
index 1736e7f1f54..8e7a2d826e4 100644
--- a/test/src/test/java/org/frankframework/runner/RunLarvaTests.java
+++ b/test/src/test/java/org/frankframework/runner/RunLarvaTests.java
@@ -3,6 +3,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -17,7 +18,9 @@
 import jakarta.annotation.Nonnull;
 import jakarta.servlet.ServletContext;
 
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.custommonkey.xmlunit.XMLUnit;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
@@ -54,6 +57,59 @@ public class RunLarvaTests {
 
 	public static final LarvaLogLevel LARVA_LOG_LEVEL = LarvaLogLevel.WRONG_PIPELINE_MESSAGES;
 
+	public static final Set<String> IGNORED_SCENARIOS = Set.of(
+			"ApiListener/scenario01.properties",
+			"ApiListener/scenario02.properties",
+			"ApiListener/scenario03.properties",
+			"ApiListener/scenario04.properties",
+			"ApiListener/scenario05.properties",
+			"ApiListener/scenario06.properties",
+			"ApiListener/scenario07.properties",
+			"ApiListener/scenario08.properties",
+			"Authentication/scenario03.properties",
+			"Authentication/scenario04.properties",
+			"Base64Pipe/scenario01.properties",
+			"Base64Pipe/scenario02.properties",
+			"CorrelationMessageId/scenario04.properties",
+			"CorrelationMessageId/scenario05.properties",
+			"Exits/api/scenario01.properties",
+			"Exits/api/scenario01b.properties",
+			"Exits/api/scenario03.properties",
+			"Exits/soap/scenario01.properties",
+			"Exits/soap/scenario02.properties",
+			"JSON/DataSonnet/scenario01.properties",
+			"JSON/JsonPipe/scenario01.properties",
+			"FileSender/scenario01.properties",
+			"ForwardNameProvidingSenders/scenario10.properties",
+			"ForwardNameProvidingSenders/scenario11.properties",
+			"ForwardNameProvidingSenders/scenario12.properties",
+			"ForwardNameProvidingSenders/scenario13.properties",
+			"FrankSender/scenario05.properties",
+			"ManagedFileHandler/scenario01.properties",
+			"ManagedFileHandler/scenario02.properties",
+			"LocalFileSystemPipe/scenario07.properties",
+			"LocalFileSystemPipe/scenario08.properties",
+			"MoveFiles/scenario01.properties",
+			"MoveFiles/scenario04.properties",
+			"MoveFiles/scenario09.properties",
+			"Receivers/NonTransacted/NoInProcess/scenario01.properties",
+			"Receivers/NonTransacted/NoInProcess/scenario05.properties",
+			"Receivers/NonTransacted/NoInProcess/scenario06.properties",
+			"Receivers/Transacted/WithInProcess/scenario03.properties",
+			"RestListener/scenario01.properties",
+			"RestListener/scenario02.properties",
+			"Validators/SoapValidator/scenario07.properties",
+			"WebServiceListenerSender/scenario11b.properties",
+			"WebServiceListenerSender/scenario11c.properties",
+			"WebServiceListenerSender/scenario11d.properties",
+			"WsdlGeneratorPipe/scenario01.properties",
+			"WsdlGeneratorPipe/scenario02.properties",
+			"WsdlGeneratorPipe/scenario03.properties",
+			"XsltProviderListener/scenario04.properties",
+			"Zip/ZipWriter/scenario 01.properties",
+			"Zip/ZipWriter/scenario 02.properties"
+	);
+
 	private static ConfigurableApplicationContext applicationContext;
 	private static LarvaTool larvaTool;
 	private static ScenarioRunner scenarioRunner;
@@ -82,6 +138,8 @@ static void setup() throws IOException {
 
 		scenarioRunner = new ScenarioRunner(larvaTool, ibisContext, testConfig, appConstants, 100, LARVA_LOG_LEVEL);
 		scenarioRootDir = larvaTool.initScenariosRootDirectories(null, new ArrayList<>(), new ArrayList<>());
+
+		XMLUnit.setIgnoreWhitespace(true);
 	}
 
 	@AfterAll
@@ -99,7 +157,6 @@ static void tearDown() {
 	 *
 	 */
 	@TestFactory
-	@Disabled("Not yet working properly, reasons not yet known.")
 	Stream<DynamicNode> larvaTests() {
 		List<File> allScenarioFiles = larvaTool.readScenarioFiles(appConstants, scenarioRootDir);
 		assertFalse(allScenarioFiles.isEmpty(), () -> "Did not find any scenario-files in scenarioRootDir [%s]!".formatted(scenarioRootDir));
@@ -132,18 +189,20 @@ Stream<DynamicNode> larvaTests() {
 
 	private DynamicTest convertLarvaScenarioToTest(File scenarioFile) {
 		// Scenario name always computed from the scenario root dir to be understandable without context of immediate parent
-		String scenarioName = scenarioFile.getAbsolutePath().substring(scenarioRootDir.length());
+		String scenarioName = FilenameUtils.normalize(scenarioFile.getAbsolutePath().substring(scenarioRootDir.length()), true);
 		return DynamicTest.dynamicTest(
 				scenarioName, scenarioFile.toURI(), () -> {
 					System.out.println("Running scenario: [" + scenarioName + "]");
 					int scenarioPassed = scenarioRunner.runOneFile(scenarioFile, scenarioRootDir, true);
 
+					assumeTrue(scenarioPassed != LarvaTool.RESULT_ERROR || !IGNORED_SCENARIOS.contains(scenarioName), () -> "Ignoring Blacklisted Scenario: [" + scenarioName + "]");
 					assertNotEquals(LarvaTool.RESULT_ERROR, scenarioPassed, () -> "Scenario failed: [" + scenarioName + "]");
 				}
 		);
 	}
 
 	@Test
+	@Disabled("Run Larva test scenarios individually now")
 	void runLarvaTests() throws IOException {
 		assertTrue(applicationContext.isRunning());
 		List<File> allScenarioFiles = larvaTool.readScenarioFiles(appConstants, scenarioRootDir);


Reviews:
user: login: evandongen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T11:19:52Z
user: login: tnleeuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T12:08:26Z
user: login: evandongen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T13:22:24Z
user: login: nielsm5
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-02T11:54:35Z
user: login: nielsm5
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-02T11:55:16Z
user: login: tnleeuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-04T16:41:06Z
user: login: evandongen
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T06:28:50Z
user: login: nielsm5
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-06T07:11:47Z

Review comments:
diff_hunk: @@ -811,7 +811,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<version>3.5.2</version>
path: pom.xml
user: login: evandongen
body: Waarom?
created_at: 2025-04-30T11:19:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -811,7 +811,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<version>3.5.2</version>
path: pom.xml
user: login: tnleeuw
body: Bug in Surefire 3.5.3 waardoor de dynamische tests niet worden meegenomen in het report (en dus ook de build niet failen).
created_at: 2025-04-30T12:08:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -811,7 +811,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<version>3.5.2</version>
path: pom.xml
user: login: evandongen
body: Aha, moeten we dat dan niet ergens vastleggen?
created_at: 2025-04-30T13:22:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -132,18 +188,24 @@ Stream<DynamicNode> larvaTests() {
 
 	private DynamicTest convertLarvaScenarioToTest(File scenarioFile) {
 		// Scenario name always computed from the scenario root dir to be understandable without context of immediate parent
-		String scenarioName = scenarioFile.getAbsolutePath().substring(scenarioRootDir.length());
+		String scenarioName = normalizeName(scenarioFile.getAbsolutePath().substring(scenarioRootDir.length()));
 		return DynamicTest.dynamicTest(
 				scenarioName, scenarioFile.toURI(), () -> {
 					System.out.println("Running scenario: [" + scenarioName + "]");
 					int scenarioPassed = scenarioRunner.runOneFile(scenarioFile, scenarioRootDir, true);
 
+					assumeTrue(scenarioPassed != LarvaTool.RESULT_ERROR || !IGNORED_SCENARIOS.contains(scenarioName), () -> "Ignoring Blacklisted Scenario: [" + scenarioName + "]");
 					assertNotEquals(LarvaTool.RESULT_ERROR, scenarioPassed, () -> "Scenario failed: [" + scenarioName + "]");
 				}
 		);
 	}
 
+	private static String normalizeName(String name) {
+		return name.replace('\\', '/');
path: test/src/test/java/org/frankframework/runner/RunLarvaTests.java
user: login: nielsm5
body: FilenameUtils.normalize(name, true);
created_at: 2025-05-02T11:54:35Z
author_association: MEMBER
diff_hunk: @@ -811,7 +811,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<version>3.5.2</version>
path: pom.xml
user: login: nielsm5
body: Ja igg een comment er boven, anders gaat dependabot dingen suggeren en mergen wij het blind...
created_at: 2025-05-02T11:55:16Z
author_association: MEMBER
diff_hunk: @@ -811,7 +811,7 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.5.3</version>
+					<version>3.5.2</version>
path: pom.xml
user: login: tnleeuw
body: Dat comment staat er ondertussen. 😃 
created_at: 2025-05-04T16:41:06Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-05-04T14:43:05Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=ibissource_iaf&pullRequest=8904) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=ibissource_iaf&pullRequest=8904&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=ibissource_iaf&pullRequest=8904&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=ibissource_iaf&pullRequest=8904&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=ibissource_iaf&pullRequest=8904&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=ibissource_iaf&pullRequest=8904&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=ibissource_iaf&pullRequest=8904)


author_association: NONE