Repository: serpro69/kotlin-faker (kotlin)

Issue:
Add support for using <lang> dict when <lang-COUNTRY> dict is missing
This was removed in #129 ( https://github.com/serpro69/kotlin-faker/commit/3a9dde0a95017299fecbbdd1a45148ce555db5b8 ) due to changes in `fr` and `ja` dicts - they became directories rather than single files.
Therefore it got a bit complicated to handle cases like:
```
faker {
  fakerConfig {
    locale = "fr-FR"
  }
}
```

where `fr-FR.yml` does not exist and would default to using `fr.yml`

Pull request:
Also allows adding single-category localizations for languages besides English, French and Japanese.

This code change should provide a solution for #131.
The idea is that, say, all variants of French, like `fr-CA`, would inherit the base `fr` dictionary, meaning that in case there is no locale file for the actual variant, at least a user will get French from the `fr` base locale.

It also allows a user to create a custom dictionary and _only_ provide translations for the new category. (#271)

The code still has one or two oddities related to French and Japanese because I didn't want to break existing behaviour.

Patch:
diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index ff272dec23..3990b18e18 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -4,6 +4,7 @@
 [discrete]
 === Breaking Change
 
+* https://github.com/serpro69/kotlin-faker/pull/274[#274] (:core) Add support for language variants extending their base locale
 * https://github.com/serpro69/kotlin-faker/pull/259[#259] (:core) Replace java.util.Random with kotlin.random.Random
 * https://github.com/serpro69/kotlin-faker/pull/246[#246] (:extension) Remove KSP from kotest-property extension
 * https://github.com/serpro69/kotlin-faker/pull/219[#219] (:core) Extract faker's providers into several submodules of their own
@@ -11,6 +12,7 @@
 [discrete]
 === Added
 
+* https://github.com/serpro69/kotlin-faker/issues/131[#131] (:core) Add support for using `lang` dict when `lang-COUNTRY` dict is missing (via #274)
 * https://github.com/serpro69/kotlin-faker/pull/268[#268] (:core) Add randomClassInstance top-level function
 * https://github.com/serpro69/kotlin-faker/pull/267[#267] (:core) Add support for sealed classes in randomClassInstance
 * https://github.com/serpro69/kotlin-faker/pull/266[#266] (:core) Add support for internal constructors in randomClassInstance
diff --git a/core/src/integration/kotlin/io/github/serpro69/kfaker/UniqueDataProvierIT.kt b/core/src/integration/kotlin/io/github/serpro69/kfaker/UniqueDataProviderIT.kt
similarity index 100%
rename from core/src/integration/kotlin/io/github/serpro69/kfaker/UniqueDataProvierIT.kt
rename to core/src/integration/kotlin/io/github/serpro69/kfaker/UniqueDataProviderIT.kt
diff --git a/core/src/main/kotlin/io/github/serpro69/kfaker/FakerService.kt b/core/src/main/kotlin/io/github/serpro69/kfaker/FakerService.kt
index a6f4a4562b..9bc547a9ac 100644
--- a/core/src/main/kotlin/io/github/serpro69/kfaker/FakerService.kt
+++ b/core/src/main/kotlin/io/github/serpro69/kfaker/FakerService.kt
@@ -59,6 +59,26 @@ class FakerService {
         } ?: requireNotNull(javaClass.classLoader.getResourceAsStream("locales/$locale/${category.lowercase()}.json"))
     }
 
+    private fun getAllLocaleOrCategoryFileStreams(
+        locale: String,
+        category: YamlCategory,
+        secondaryCategory: Category?,
+    ): List<InputStream> {
+        val baseLocale = locale.replace('_', '-').substringBefore("-")
+
+        val resourceFilenames = listOfNotNull(
+            "locales/$baseLocale.json",
+            "locales/$baseLocale/${category.lowercase()}.json",
+            secondaryCategory?.let { "locales/$baseLocale/${it.lowercase()}.json" },
+            "locales/$locale.json",
+            "locales/$locale/${category.lowercase()}.json",
+            secondaryCategory?.let { "locales/$locale/${it.lowercase()}.json" },
+        ).distinct()
+        val localeStreams = resourceFilenames.mapNotNull { javaClass.classLoader.getResourceAsStream(it) }
+        require(localeStreams.isNotEmpty() || locale in listOf("fr", "ja")) { "Dictionary file not found for locale values: '$locale' or '$baseLocale'" }
+        return localeStreams
+    }
+
     private fun getCategoryFileStreamOrNull(
         locale: String,
         category: YamlCategory,
@@ -220,26 +240,22 @@ class FakerService {
                         defaultValues.putAll(readCategory(instr, "en", category))
                     }
 
-                    // merge localized values
-                    val input = when (locale) {
-                        // these have multiple files per directory, as opposed to other localizations
-                        "fr", "ja" -> getCategoryFileStreamOrNull(locale, category, secondaryCategory)
-                        else -> if (locale != "en") /*'en' is already processed at this point*/ {
-                            getLocaleFileStream(locale) ?: throw IllegalArgumentException(
-                                "Dictionary file not found for locale values: '$locale' or '${locale.substringBefore("-")}'"
-                            )
-                        } else null
-                    }
-                    input?.use { instr ->
-                        readCategoryOrNull(instr, locale, category)?.let {
-                            val localized = with(category.lowercase()) {
-                                val merged = merge(
-                                    hashMapOf(this to defaultValues),
-                                    hashMapOf(this to it)
-                                )
-                                merged[this]
+                    if (locale != "en") { // 'en' is already processed at this point
+                        val inputs = getAllLocaleOrCategoryFileStreams(locale, category, secondaryCategory)
+
+                        inputs.forEach { input ->
+                            input.use { instr ->
+                                readCategoryOrNull(instr, locale, category)?.let {
+                                    val localized = with(category.lowercase()) {
+                                        val merged = merge(
+                                            hashMapOf(this to defaultValues),
+                                            hashMapOf(this to it)
+                                        )
+                                        merged[this]
+                                    }
+                                    localized?.forEach { m -> defaultValues.merge(m.key, m.value as Any) { orig, loc -> loc } }
+                                }
                             }
-                            localized?.forEach { m -> defaultValues.merge(m.key, m.value as Any) { orig, loc -> loc } }
                         }
                     }
                 }
diff --git a/docs/src/orchid/resources/data.yml b/docs/src/orchid/resources/data.yml
index 7a2e74b00a..d47df2fb79 100644
--- a/docs/src/orchid/resources/data.yml
+++ b/docs/src/orchid/resources/data.yml
@@ -33,6 +33,8 @@ contributors:
   link: https://github.com/efenderbosch-atg
 - name: Mathieu Boisnard
   link: https://github.com/mboisnard
+- name: Carsten Brachem
+  link: https://github.com/cbrachem
 
 homepageSections:
   - title: '…a plethora of real-looking data in various domains:'
diff --git a/docs/src/orchid/resources/wiki/custom-data-provider.md b/docs/src/orchid/resources/wiki/custom-data-provider.md
index a44f2f7429..ee9b859bd1 100644
--- a/docs/src/orchid/resources/wiki/custom-data-provider.md
+++ b/docs/src/orchid/resources/wiki/custom-data-provider.md
@@ -33,7 +33,7 @@ First, we need to create a json file that will serve as your "data dictionary",
 - After that you have your data parameters with values as key-value pairs
 
 {% tip %}
-If you want to support a different locale, place the file in `resources/locales/<locale>.json` instead
+If you want to support a different locale, place the file in `resources/locales/<locale>/<category>.json` instead.
 {% endtip %}
 
 Then, we need to implement the `YamlFakeDataProvider` class:


Test patch:
diff --git a/core/src/test/kotlin/io/github/serpro69/kfaker/FakerServiceTest.kt b/core/src/test/kotlin/io/github/serpro69/kfaker/FakerServiceTest.kt
index 72817f4194..110726f38a 100644
--- a/core/src/test/kotlin/io/github/serpro69/kfaker/FakerServiceTest.kt
+++ b/core/src/test/kotlin/io/github/serpro69/kfaker/FakerServiceTest.kt
@@ -15,6 +15,7 @@ import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
 import io.kotest.matchers.collections.shouldHaveAtLeastSize
 import io.kotest.matchers.collections.shouldHaveAtMostSize
 import io.kotest.matchers.collections.shouldHaveSize
+import io.kotest.matchers.collections.shouldNotContain
 import io.kotest.matchers.maps.shouldContainExactly
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNotBe
@@ -549,28 +550,52 @@ internal class FakerServiceTest : DescribeSpec({
             }
         }
 
-//        context("fr locale") { // also applicable for ja and any other multi-file localized data
-//            // TODO not supported since 1.11.0
-//            //  see: https://github.com/serpro69/kotlin-faker/issues/131
-//            //  Note: when implemented also add tests to check that existing `lang-COUNTRY` locale is actually loaded, e.g. `fr-CA`
-//            context("it is set as `lang-COUNTRY` but dictionary file exists only for `lang`") {
-//                val frFRDict = FakerService(Faker(), Locale.forLanguageTag("fr-FR")).dictionary
-//
-//                it("localized dictionary for `lang` should be loaded") {
-//                    frFRDict shouldNotBe null
-//                    frFRDict shouldNotBe emptyMap<YamlCategory, YamlCategoryData>()
-//                }
-//            }
-//
-//            context("it is set as `lang_COUNTRY` String") {
-//                val frFRDict = FakerService(Faker(), Locale.forLanguageTag("fr_FR")).dictionary
-//
-//                it("it should be set as `lang-COUNTRY` String") {
-//                    frFRDict shouldNotBe null
-//                    frFRDict shouldNotBe emptyMap<YamlCategory, YamlCategoryData>()
-//                }
-//            }
-//        }
+        context("fr locale") { // also applicable for ja and any other multi-file localized data
+            context("it is set as `lang-COUNTRY` but dictionary file exists only for `lang`") {
+                val frFRDict = fakerService("fr-FR", YamlCategory.ADDRESS).dictionary // FakerService(Faker(), Locale.forLanguageTag("fr-FR")).dictionary
+
+                it("localized dictionary for `lang` should be loaded") {
+                    frFRDict shouldNotBe null
+                    frFRDict shouldNotBe emptyMap<YamlCategory, YamlCategoryData>()
+                }
+            }
+
+            context("it is set as `lang_COUNTRY` String") {
+                val frFRDict = fakerService("fr_FR", YamlCategory.ADDRESS).dictionary // FakerService(Faker(), Locale.forLanguageTag("fr-FR")).dictionary
+
+                it("it should be set as `lang-COUNTRY` String") {
+                    frFRDict shouldNotBe null
+                    frFRDict shouldNotBe emptyMap<YamlCategory, YamlCategoryData>()
+                }
+            }
+
+            context("existing lang-COUNTRY locale is actually loaded") {
+                val faker = fakerService("fr-CA")
+                faker.load(YamlCategory.ADDRESS)
+                faker.load(YamlCategory.ANCIENT)
+                val frCAdict = faker.dictionary
+
+                it("should actually load data from the specified locale") {
+                    val states = frCAdict.getEntryByCategory("address")["state"] as List<String>
+                    states shouldContain "Ontario"
+                    states shouldNotContain "Paris"
+                }
+
+                it("should use values from the base locale as a fallback") {
+                    val gods = frCAdict.getEntryByCategory("ancient")["god"] as List<String>
+                    gods shouldContain "Zeus"
+                }
+            }
+        }
+
+        context("locales other than en, fr or ja") {
+            it("should be possible to localize values for a single category") {
+                val foodstuff = fakerService("de", YamlCategory.FOOD)
+                    .dictionary
+                    .getEntryByCategory("food")["ingredients"] as List<*>
+                foodstuff shouldContainExactly listOf("Brot")
+            }
+        }
     }
 
     describe("regexify a string") {
diff --git a/core/src/test/resources/locales/de/food.json b/core/src/test/resources/locales/de/food.json
new file mode 100644
index 0000000000..5f83da1fc7
--- /dev/null
+++ b/core/src/test/resources/locales/de/food.json
@@ -0,0 +1,11 @@
+{
+  "de": {
+    "faker": {
+      "food": {
+        "ingredients": [
+          "Brot"
+        ]
+      }
+    }
+  }
+}