Repository: runningcode/fladle (kotlin)

Issue:
Support for Robo Tests without Robo script
This is an enhancement request to support robo tests without a robo script.  Firebase Test Lab supports this, although it appears fladle does not. In the fladle source, the decision to do a robo test is based on the roboscript file path being non-null.

Issues sometimes come up with R8, ProGuard, or DexGuard that make robo testing the my release APK helpful in catching issues that might occur during first launch.  However, DexGuard also obfuscates resource IDs which makes creating a roboscript for a release build much more difficult.

Being able to launch a robo test without a roboscript will enable my use case, but would also lower the barrier for others who want to run a robo test without recording a script.

Pull request description:
Fixes #165 

Flank supports roboTest without roboScript (sanityRobo) https://github.com/Flank/flank/pull/1108
> Sanity robo should be ONLY launched when there is NO, robo-script, test, additional-app-test-apks. This reflects gcloud's behavior


# ***TO BE UPDATED***

To run sanity robo test add `-PsanityRobo` to gradle command
```
./gradlew runFlank -PsanityRobo
```

With this property, `fladle` should skip appending config yaml file with `robo-directives`, `robo-script`, `test`, `additional-app-test-apks`
Example:
1) Run `./gradlew -p sample-kotlin writeConfigProps -PsanityRobo` from project root
2) generated config yaml should look (no `test`)
```yml
gcloud:
  app: [path]
  device:
  - model: Pixel2
    version: 26
  - model: Nexus5
    version: 23

  use-orchestrator: true
  auto-google-login: false
  record-video: true
  performance-metrics: true
  timeout: 15m
  environment-variables:
    clearPackageData: true
  test-targets:
  - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView
  num-flaky-test-attempts: 1

flank:
  smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
  keep-file-path: false
  ignore-failed-tests: false
  disable-sharding: false
  smart-flank-disable-upload: false
  legacy-junit-result: false
  full-junit-result: false
  output-style: single
```

Patch:
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt
index 1edbdb67..d845b5df 100644
--- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt
@@ -18,6 +18,19 @@ interface FladleConfig {
   @get:Optional
   val serviceAccountCredentials: RegularFileProperty
 
+  /**
+   * debugApk and instrumentationApk are [Property<String>] and not [RegularFileProperty] because we support wildcard characters.
+   */
+  @get:Input
+  @get:Optional
+  val debugApk: Property<String>
+  @get:Input
+  @get:Optional
+  val instrumentationApk: Property<String>
+
+  @get:Input
+  val sanityRobo: Property<Boolean>
+
   @get:Input
   val useOrchestrator: Property<Boolean>
 
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt
index 61a86922..b8c21d3f 100644
--- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt
@@ -9,6 +9,9 @@ data class FladleConfigImpl(
   internal val name: String,
   override val projectId: Property<String>,
   override val serviceAccountCredentials: RegularFileProperty,
+  override val debugApk: Property<String>,
+  override val instrumentationApk: Property<String>,
+  override val sanityRobo: Property<Boolean>,
   override val useOrchestrator: Property<Boolean>,
   override val autoGoogleLogin: Property<Boolean>,
   override val devices: ListProperty<Map<String, String>>,
@@ -45,4 +48,19 @@ data class FladleConfigImpl(
   override val outputStyle: Property<String>,
   override val legacyJunitResult: Property<Boolean>,
   override val fullJunitResult: Property<Boolean>
-) : FladleConfig
+) : FladleConfig {
+  /**
+   * Prepare config to run sanity robo.
+   *
+   * Sets [sanityRobo] property as `true`.
+   *
+   * Cleans [instrumentationApk], [additionalTestApks], [roboDirectives], [roboScript] properties.
+   */
+  fun clearPropertiesForSanityRobo() {
+    sanityRobo.set(true)
+    additionalTestApks.empty()
+    instrumentationApk.set("")
+    roboDirectives.empty()
+    roboScript.set("")
+  }
+}
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt
index 15415501..b5f5ce9c 100644
--- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt
@@ -40,6 +40,9 @@ class FladlePluginDelegate {
       // Must be done afterEvaluate otherwise extension values will not be set.
       project.dependencies.add(FLADLE_CONFIG, "${base.flankCoordinates.get()}:${base.flankVersion.get()}")
 
+      checkIfSanityAndValidateConfigs(base)
+      base.configs.forEach(::checkIfSanityAndValidateConfigs)
+
       // Only use automatic apk path detection for 'com.android.application' projects.
       project.pluginManager.withPlugin("com.android.application") {
         if (!base.debugApk.isPresent || !base.instrumentationApk.isPresent) {
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt
index 0196ab75..7cc93eee 100644
--- a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt
@@ -19,6 +19,8 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl
   @get:Input
   val flankCoordinates: Property<String> = objects.property(String::class.java).convention("com.github.flank:flank")
 
+  override val sanityRobo: Property<Boolean> = objects.property<Boolean>().convention(false)
+
   @get:Input
   val flankVersion: Property<String> = objects.property(String::class.java).convention("20.09.3")
   // Project id is automatically discovered by default. Use this to override the project id.
@@ -50,12 +52,9 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl
   /**
    * debugApk and instrumentationApk are [Property<String>] and not [RegularFileProperty] because we support wildcard characters.
    */
-  @get:Input
-  @get:Optional
-  val debugApk: Property<String> = objects.property()
-  @get:Input
-  @get:Optional
-  val instrumentationApk: Property<String> = objects.property()
+  override val debugApk: Property<String> = objects.property()
+
+  override val instrumentationApk: Property<String> = objects.property()
 
   override val directoriesToPull: ListProperty<String> = objects.listProperty()
 
@@ -115,6 +114,9 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl
       name = it,
       projectId = objects.property<String>().convention(projectId),
       serviceAccountCredentials = objects.fileProperty().convention(serviceAccountCredentials),
+      debugApk = objects.property<String>().convention(debugApk),
+      instrumentationApk = objects.property<String>().convention(instrumentationApk),
+      sanityRobo = objects.property<Boolean>().convention(false),
       useOrchestrator = objects.property<Boolean>().convention(useOrchestrator),
       autoGoogleLogin = objects.property<Boolean>().convention(autoGoogleLogin),
       devices = objects.listProperty<Map<String, String>>().convention(devices),
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt
new file mode 100644
index 00000000..5539bbb7
--- /dev/null
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt
@@ -0,0 +1,32 @@
+package com.osacky.flank.gradle
+
+import org.gradle.api.GradleException
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
+
+@Throws(GradleException::class)
+fun checkIfSanityAndValidateConfigs(config: FladleConfig) = when (config) {
+  is FlankGradleExtension -> config.checkAndValidateConfig() { option, _ ->
+    "Incorrect [base] configuration. [$option] can't be used together with sanityRobo."
+  }
+  is FladleConfigImpl -> config.checkAndValidateConfig(config.name) { option, name ->
+    "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. " +
+      "If you want to launch robo test run without robo script place only clearPropertiesForSanityRobo() into [$name] configuration"
+  }
+  else -> throw GradleException("Unable to check for sanity, check config type")
+}
+
+private fun FladleConfig.checkAndValidateConfig(name: String = "base", message: (String, String) -> String) {
+  if (sanityRobo.getOrElse(false)) when {
+    roboDirectives.isNotPresentOrEmpty -> throw GradleException(message("roboDirectives", name))
+    roboScript.isNotPresentOrBlank -> throw GradleException(message("roboScript", name))
+    instrumentationApk.isNotPresentOrBlank -> throw GradleException(message("instrumentationApk", name))
+    additionalTestApks.isNotPresentOrEmpty -> throw GradleException(message("additionalTestApks", name))
+  }
+}
+
+private val Property<String>.isNotPresentOrBlank
+  get() = orNull.isNullOrBlank().not()
+
+private val <T> ListProperty<T>.isNotPresentOrEmpty
+  get() = getOrElse(emptyList()).isEmpty().not()
diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt
index dcd183d2..dd3f27d4 100644
--- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt
+++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt
@@ -12,27 +12,31 @@ internal class YamlWriter {
       check(base.serviceAccountCredentials.isPresent) { "ServiceAccountCredentials in fladle extension not set. https://github.com/runningcode/fladle#serviceaccountcredentials" }
     }
     check(base.debugApk.isPresent) { "debugApk must be specified" }
-    check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) {
-      val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) {
-        "Both instrumentationApk file and roboScript file were specified, but only one is expected."
-      } else {
-        "Must specify either a instrumentationApk file or a roboScript file."
-      }
-      """
+    if (config.sanityRobo.get() == false) {
+      check(config.instrumentationApk.isPresent xor !config.roboScript.orNull.isNullOrBlank()) {
+        val prefix = if (base.instrumentationApk.isPresent && !config.roboScript.orNull.isNullOrBlank()) {
+          "Both instrumentationApk file and roboScript file were specified, but only one is expected."
+        } else {
+          "Must specify either a instrumentationApk file or a roboScript file."
+        }
+        """
       $prefix
-      instrumentationApk=${base.instrumentationApk.orNull}
-      roboScript=${base.roboScript.orNull}
-      """.trimIndent()
+      instrumentationApk=${config.instrumentationApk.orNull}
+      roboScript=${config.roboScript.orNull}
+        """.trimIndent()
+      }
     }
 
+    val shouldPrintTestAndRobo = config.sanityRobo.get().not()
     val additionalProperties = writeAdditionalProperties(config)
     val flankProperties = writeFlankProperties(config)
 
     return buildString {
       appendln("gcloud:")
-      appendln("  app: ${base.debugApk.get()}")
-      if (base.instrumentationApk.isPresent) {
-        appendln("  test: ${base.instrumentationApk.get()}")
+      appendln("  app: ${config.debugApk.get()}")
+      // We don't want to print instrumentation apks if sanityRobo == true
+      if (shouldPrintTestAndRobo && config.instrumentationApk.isPresent) {
+        appendln("  test: ${config.instrumentationApk.get()}")
       }
       if (config.devices.isPresentAndNotEmpty) appendln(createDeviceString(config.devices.get()))
       appendln(additionalProperties)
@@ -50,7 +54,8 @@ internal class YamlWriter {
     appendProperty(config.projectId, name = "project")
     appendProperty(config.keepFilePath, name = "keep-file-path")
     appendListProperty(config.filesToDownload, name = "files-to-download") { appendln("  - $it") }
-    appendListProperty(config.additionalTestApks, name = "additional-app-test-apks") { appendln("    $it") }
+    if (!config.sanityRobo.get())
+      appendListProperty(config.additionalTestApks, name = "additional-app-test-apks") { appendln("    $it") }
     appendProperty(config.runTimeout, name = "run-timeout")
     appendProperty(config.ignoreFailedTests, name = "ignore-failed-tests")
     appendProperty(config.disableSharding, name = "disable-sharding")
@@ -82,10 +87,12 @@ internal class YamlWriter {
     appendMapProperty(config.clientDetails, name = "client-details") { appendln("    ${it.key}: ${it.value}") }
     appendMapProperty(config.otherFiles, name = "other-files") { appendln("    ${it.key}: ${it.value}") }
     appendProperty(config.networkProfile, name = "network-profile")
-    appendProperty(config.roboScript, name = "robo-script")
-    appendListProperty(config.roboDirectives, name = "robo-directives") {
-      val value = it.getOrElse(2) { "" }.let { stringValue -> if (stringValue.isBlank()) "\"\"" else stringValue }
-      appendln("    ${it[0]}:${it[1]}: $value")
+    if (!config.sanityRobo.get()) {
+      appendProperty(config.roboScript, name = "robo-script")
+      appendListProperty(config.roboDirectives, name = "robo-directives") {
+        val value = it.getOrElse(2) { "" }.let { stringValue -> if (stringValue.isBlank()) "\"\"" else stringValue }
+        appendln("    ${it[0]}:${it[1]}: $value")
+      }
     }
   }
 


Test patch:
diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt
new file mode 100644
index 00000000..e1127cff
--- /dev/null
+++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt
@@ -0,0 +1,395 @@
+package com.osacky.flank.gradle.integration
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+
+private fun baseConfigMessage(option: String) = "Incorrect [base] configuration. [$option] can't be used together with sanityRobo."
+private fun additionalConfigMessage(option: String, name: String) = "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. If you want to launch robo test run without robo script place only clearPropertiesForSanityRobo() into [$name] configuration"
+
+class SanityRoboTest {
+  @get:Rule
+  var testProjectRoot = TemporaryFolder()
+
+  @Before
+  fun setUp() = testProjectRoot.newFile("flank-gradle-service.json").writeText("{}")
+
+  @Test
+  fun `sanityRobo - should throw an error if instrumentationApk set`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  debugApk = "debug.apk"
+      |  sanityRobo = true
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  instrumentationApk = "test.apk"
+      |  configs {
+      |    sanity {
+      |      makeSanityRun()
+      |    }
+      |  }
+      |}
+    """
+    )
+
+    val result = failedGradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("FAILED")
+    assertThat(result.output).contains(baseConfigMessage("instrumentationApk"))
+
+    val resultSanity = failedGradleRun(
+      arguments = listOf("printYmlSanity"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(resultSanity.output).contains("FAILED")
+    assertThat(result.output).contains(baseConfigMessage("instrumentationApk"))
+  }
+
+  @Test
+  fun `sanityRobo - should throw an error if roboScript set`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  debugApk = "debug.apk"
+      |  sanityRobo = true
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  roboScript = "some/path/script.json"
+      |}
+    """
+    )
+
+    val result = failedGradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("FAILED")
+    assertThat(result.output).contains(baseConfigMessage("roboScript"))
+  }
+
+  @Test
+  fun `sanityRobo - should throw an error if roboDirectives set`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  debugApk = "debug.apk"
+      |  sanityRobo = true
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  roboDirectives = [
+      |    ["click", "button1", ""],
+      |    ["ignore", "button2"],
+      |    ["text", "field1", "my text"],
+      |  ]
+      |}
+    """
+    )
+
+    val result = failedGradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("FAILED")
+    assertThat(result.output).contains(baseConfigMessage("roboDirectives"))
+  }
+
+  @Test
+  fun `sanityRobo - should throw an error if additionalTestApks set`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  debugApk = "debug.apk"
+      |  sanityRobo = true
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  additionalTestApks = [
+      |    "- app: debug2.apk",
+      |    "  test: test2.apk",
+      |    "- test: test3.apk"
+      |  ]
+      |}
+    """
+    )
+
+    val result = failedGradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("FAILED")
+    assertThat(result.output).contains(baseConfigMessage("additionalTestApks"))
+  }
+
+  @Test
+  fun `sanityRobo - should throw an error if roboScript set (multiple config)`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  debugApk = "foo.apk"
+      |  instrumentationApk = "test.apk"
+      |  additionalTestApks = [
+      |    "- app: debug2.apk",
+      |    "  test: test2.apk",
+      |    "- test: test3.apk"
+      |  ]
+      |  configs {
+      |    sanity {
+      |      clearPropertiesForSanityRobo()
+      |      roboScript.set("path/to/script.json")
+      |    }
+      |  }
+      |}
+    """
+    )
+
+    val expectedMessage = additionalConfigMessage("roboScript", "sanity")
+
+    val result = failedGradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("FAILED")
+    assertThat(result.output).contains(expectedMessage)
+
+    val resultOrange = failedGradleRun(
+      arguments = listOf("printYmlSanity"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(resultOrange.output).contains("FAILED")
+    assertThat(resultOrange.output).contains(expectedMessage)
+  }
+
+  @Test
+  fun `sanityRobo - should print correct config yamls (inner config is sanity run)`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  debugApk = "foo.apk"
+      |  instrumentationApk = "test.apk"
+      |  additionalTestApks = [
+      |    "- app: debug2.apk",
+      |    "  test: test2.apk",
+      |    "- test: test3.apk"
+      |  ]
+      |  configs {
+      |    orange {
+      |      clearPropertiesForSanityRobo()
+      |    }
+      |  }
+      |}
+    """
+    )
+
+    val result = gradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("SUCCESS")
+    assertThat(result.output).contains(
+      """
+      |gcloud:
+      |  app: foo.apk
+      |  test: test.apk
+      |  device:
+      |  - model: NexusLowRes
+      |    version: 28
+      |
+      |  use-orchestrator: false
+      |  auto-google-login: false
+      |  record-video: true
+      |  performance-metrics: true
+      |  timeout: 15m
+      |  num-flaky-test-attempts: 0
+      |
+      |flank:
+      |  keep-file-path: false
+      |  additional-app-test-apks:
+      |    - app: debug2.apk
+      |      test: test2.apk
+      |    - test: test3.apk
+      |  ignore-failed-tests: false
+      |  disable-sharding: false
+      |  smart-flank-disable-upload: false
+      |  legacy-junit-result: false
+      |  full-junit-result: false
+      |  output-style: single
+    """.trimMargin()
+    )
+
+    val resultOrange = gradleRun(
+      arguments = listOf("printYmlOrange"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(resultOrange.output).contains("SUCCESS")
+    assertThat(resultOrange.output).contains(
+      """
+      |gcloud:
+      |  app: foo.apk
+      |  device:
+      |  - model: NexusLowRes
+      |    version: 28
+      |
+      |  use-orchestrator: false
+      |  auto-google-login: false
+      |  record-video: true
+      |  performance-metrics: true
+      |  timeout: 15m
+      |  num-flaky-test-attempts: 0
+      |
+      |flank:
+      |  keep-file-path: false
+      |  ignore-failed-tests: false
+      |  disable-sharding: false
+      |  smart-flank-disable-upload: false
+      |  legacy-junit-result: false
+      |  full-junit-result: false
+      |  output-style: single
+    """.trimMargin()
+    )
+  }
+
+  @Test
+  fun `sanityRobo - should print correct config yamls (base config is sanity run)`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+      |plugins {
+      |  id "com.osacky.fladle"
+      |}
+      |
+      |fladle {
+      |  serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json")
+      |  debugApk = "foo.apk"
+      |  sanityRobo = true
+      |  configs {
+      |    orange {
+      |      instrumentationApk.set("test.apk")
+      |      additionalTestApks.set(project.provider { [
+      |        "- app: debug2.apk",
+      |        "  test: test2.apk",
+      |        "- test: test3.apk"
+      |      ] })
+      |    }
+      |  }
+      |}
+    """
+    )
+
+    val result = gradleRun(
+      arguments = listOf("printYml"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(result.output).contains("SUCCESS")
+    assertThat(result.output).contains(
+      """
+      |gcloud:
+      |  app: foo.apk
+      |  device:
+      |  - model: NexusLowRes
+      |    version: 28
+      |
+      |  use-orchestrator: false
+      |  auto-google-login: false
+      |  record-video: true
+      |  performance-metrics: true
+      |  timeout: 15m
+      |  num-flaky-test-attempts: 0
+      |
+      |flank:
+      |  keep-file-path: false
+      |  ignore-failed-tests: false
+      |  disable-sharding: false
+      |  smart-flank-disable-upload: false
+      |  legacy-junit-result: false
+      |  full-junit-result: false
+      |  output-style: single
+    """.trimMargin()
+    )
+
+    val resultOrange = gradleRun(
+      arguments = listOf("printYmlOrange"),
+      projectDir = testProjectRoot.root
+    )
+
+    assertThat(resultOrange.output).contains("SUCCESS")
+    assertThat(resultOrange.output).contains(
+      """
+      |gcloud:
+      |  app: foo.apk
+      |  test: test.apk
+      |  device:
+      |  - model: NexusLowRes
+      |    version: 28
+      |
+      |  use-orchestrator: false
+      |  auto-google-login: false
+      |  record-video: true
+      |  performance-metrics: true
+      |  timeout: 15m
+      |  num-flaky-test-attempts: 0
+      |
+      |flank:
+      |  keep-file-path: false
+      |  additional-app-test-apks:
+      |    - app: debug2.apk
+      |      test: test2.apk
+      |    - test: test3.apk
+      |  ignore-failed-tests: false
+      |  disable-sharding: false
+      |  smart-flank-disable-upload: false
+      |  legacy-junit-result: false
+      |  full-junit-result: false
+      |  output-style: single
+    """.trimMargin()
+    )
+  }
+}
diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt
new file mode 100644
index 00000000..f9edfb2a
--- /dev/null
+++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt
@@ -0,0 +1,321 @@
+package com.osacky.flank.gradle.integration
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+
+private const val commonScriptPart = """
+  plugins {
+    id 'com.android.application'
+    id 'com.osacky.fladle'
+  }
+
+  android {
+    compileSdkVersion 29
+    defaultConfig {
+      applicationId "com.osacky.flank.gradle.sample"
+      minSdkVersion 23
+      targetSdkVersion 29
+      versionCode 1
+      versionName "1.0"
+      testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+    testOptions {
+      execution 'ANDROIDX_TEST_ORCHESTRATOR'
+    }
+  }
+"""
+
+class SanityWithAutoConfigureTest {
+
+  @get:Rule
+  var testProjectRoot = TemporaryFolder()
+
+  @Before
+  fun setUp() {
+    testProjectRoot.newFile("flank-gradle-service.json").writeText("{}")
+    testProjectRoot.newFile("local.properties").writeText("sdk.dir=${androidHome()}\n")
+    testProjectRoot.newFile("gradle.properties").writeText("android.useAndroidX=true")
+  }
+
+  @Test
+  fun `test auto configuration with sanityRobo set (inner config)`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+            $commonScriptPart
+
+            fladle {
+              serviceAccountCredentials = project.layout.projectDirectory.file("flank-gradle-service.json")
+              useOrchestrator = true
+              environmentVariables = [
+                "clearPackageData": "true"
+              ]
+              testTargets = [
+                "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView"
+              ]
+              devices = [
+                [ "model": "Pixel2", "version": "26" ],
+                [ "model": "Nexus5", "version": "23" ]
+              ]
+              smartFlankGcsPath = "gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml"
+              configs {
+                sanity {
+                  clearPropertiesForSanityRobo()
+                  useOrchestrator.set(false)
+                  testTargets.set(project.provider { [
+                    "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail"
+                  ] })
+                  flakyTestAttempts.set(3)
+                }
+                oranges {
+                  useOrchestrator.set(false)
+                  testTargets.set(project.provider { [
+                    "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail"
+                  ] })
+                  flakyTestAttempts.set(6)
+                }
+              }
+            }
+      """
+    )
+
+    val baseResult = gradleRun(
+      projectDir = testProjectRoot.root,
+      arguments = listOf("printYml")
+    )
+
+    assertThat(baseResult.output).contains("BUILD SUCCESSFUL")
+    assertThat(baseResult.output).containsMatch(
+      """
+        gcloud:
+          app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk
+          test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk
+          device:
+          - model: Pixel2
+            version: 26
+          - model: Nexus5
+            version: 23
+
+          use-orchestrator: true
+          auto-google-login: false
+          record-video: true
+          performance-metrics: true
+          timeout: 15m
+          environment-variables:
+            clearPackageData: true
+          test-targets:
+          - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView
+          num-flaky-test-attempts: 0
+
+        flank:
+          smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
+          keep-file-path: false
+          ignore-failed-tests: false
+          disable-sharding: false
+          smart-flank-disable-upload: false
+          legacy-junit-result: false
+          full-junit-result: false
+          output-style: single
+      """.trimIndent()
+    )
+
+    val sanityResult = gradleRun(
+      projectDir = testProjectRoot.root,
+      arguments = listOf("printYmlSanity")
+    )
+
+    assertThat(sanityResult.output).contains("BUILD SUCCESSFUL")
+    assertThat(sanityResult.output).containsMatch(
+      """
+        gcloud:
+          app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk
+          device:
+          - model: Pixel2
+            version: 26
+          - model: Nexus5
+            version: 23
+
+          use-orchestrator: false
+          auto-google-login: false
+          record-video: true
+          performance-metrics: true
+          timeout: 15m
+          environment-variables:
+            clearPackageData: true
+          test-targets:
+          - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail
+          num-flaky-test-attempts: 3
+
+        flank:
+          smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
+          keep-file-path: false
+          ignore-failed-tests: false
+          disable-sharding: false
+          smart-flank-disable-upload: false
+          legacy-junit-result: false
+          full-junit-result: false
+          output-style: single
+      """.trimIndent()
+    )
+
+    val orangesResult = gradleRun(
+      projectDir = testProjectRoot.root,
+      arguments = listOf("printYmlOranges")
+    )
+
+    assertThat(orangesResult.output).contains("BUILD SUCCESSFUL")
+    assertThat(orangesResult.output).containsMatch(
+      """
+        gcloud:
+          app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk
+          test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk
+          device:
+          - model: Pixel2
+            version: 26
+          - model: Nexus5
+            version: 23
+
+          use-orchestrator: false
+          auto-google-login: false
+          record-video: true
+          performance-metrics: true
+          timeout: 15m
+          environment-variables:
+            clearPackageData: true
+          test-targets:
+          - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail
+          num-flaky-test-attempts: 6
+
+        flank:
+          smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
+          keep-file-path: false
+          ignore-failed-tests: false
+          disable-sharding: false
+          smart-flank-disable-upload: false
+          legacy-junit-result: false
+          full-junit-result: false
+          output-style: single
+      """.trimIndent()
+    )
+  }
+
+  @Test
+  fun `test auto configuration with sanityRobo set (base config)`() {
+    makeBuildDotGradle(
+      where = testProjectRoot,
+      buildScript =
+        """
+            $commonScriptPart
+
+            fladle {
+              sanityRobo = true
+              serviceAccountCredentials = project.layout.projectDirectory.file("flank-gradle-service.json")
+              useOrchestrator = true
+              environmentVariables = [
+                "clearPackageData": "true"
+              ]
+              testTargets = [
+                "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView"
+              ]
+              devices = [
+                [ "model": "Pixel2", "version": "26" ],
+                [ "model": "Nexus5", "version": "23" ]
+              ]
+              smartFlankGcsPath = "gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml"
+              configs {
+                oranges {
+                  useOrchestrator.set(false)
+                  testTargets.set(project.provider { [
+                    "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail"
+                  ] })
+                  flakyTestAttempts.set(3)
+                }
+              }
+            }
+      """
+    )
+
+    val baseResult = gradleRun(
+      projectDir = testProjectRoot.root,
+      arguments = listOf("printYml")
+    )
+
+    assertThat(baseResult.output).contains("BUILD SUCCESSFUL")
+    assertThat(baseResult.output).containsMatch(
+      """
+        gcloud:
+          app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk
+          device:
+          - model: Pixel2
+            version: 26
+          - model: Nexus5
+            version: 23
+
+          use-orchestrator: true
+          auto-google-login: false
+          record-video: true
+          performance-metrics: true
+          timeout: 15m
+          environment-variables:
+            clearPackageData: true
+          test-targets:
+          - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView
+          num-flaky-test-attempts: 0
+
+        flank:
+          smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
+          keep-file-path: false
+          ignore-failed-tests: false
+          disable-sharding: false
+          smart-flank-disable-upload: false
+          legacy-junit-result: false
+          full-junit-result: false
+          output-style: single
+      """.trimIndent()
+    )
+
+    val orangesResult = gradleRun(
+      projectDir = testProjectRoot.root,
+      arguments = listOf("printYmlOranges")
+    )
+
+    assertThat(orangesResult.output).contains("BUILD SUCCESSFUL")
+    assertThat(orangesResult.output).containsMatch(
+      """
+        gcloud:
+          app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk
+          test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk
+          device:
+          - model: Pixel2
+            version: 26
+          - model: Nexus5
+            version: 23
+
+          use-orchestrator: false
+          auto-google-login: false
+          record-video: true
+          performance-metrics: true
+          timeout: 15m
+          environment-variables:
+            clearPackageData: true
+          test-targets:
+          - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail
+          num-flaky-test-attempts: 3
+
+        flank:
+          smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml
+          keep-file-path: false
+          ignore-failed-tests: false
+          disable-sharding: false
+          smart-flank-disable-upload: false
+          legacy-junit-result: false
+          full-junit-result: false
+          output-style: single
+      """.trimIndent()
+    )
+  }
+}
diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt
index 1d65a757..175d5531 100644
--- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt
+++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt
@@ -1,8 +1,26 @@
 package com.osacky.flank.gradle.integration
 
+import org.gradle.testkit.runner.GradleRunner
 import org.junit.rules.TemporaryFolder
 import java.io.File
 
 fun TemporaryFolder.setupFixture(fixtureName: String) {
   File(this::class.java.classLoader.getResource(fixtureName)!!.file).copyRecursively(newFile(fixtureName), true)
 }
+
+internal fun makeBuildDotGradle(where: TemporaryFolder, buildScript: String) = where
+  .newFile("build.gradle")
+  .writeText(buildScript.trimMargin())
+
+internal fun gradleRun(projectDir: File, arguments: List<String> = emptyList()) =
+  commonGradleRunConfig(projectDir, arguments).build()
+
+internal fun failedGradleRun(projectDir: File, arguments: List<String> = emptyList()) =
+  commonGradleRunConfig(projectDir, arguments).buildAndFail()
+
+private fun commonGradleRunConfig(projectDir: File, arguments: List<String>) =
+  GradleRunner.create()
+    .withPluginClasspath()
+    .withArguments(arguments)
+    .forwardOutput()
+    .withProjectDir(projectDir)