Repository: microsoft/data-factory-testing-framework (language: python)

Issue:
Framework does not allow to read files encoded in UTF-16 LE
Fabric's git integration stores `pipeline-content.json` currently as UTF-16-LE.
The testing framework fails to read these fails as:
https://github.com/microsoft/data-factory-testing-framework/blob/b228f1db59c07c24287cd3d2666cd30b16da3904/src/data_factory_testing_framework/_repositories/_factories/fabric_repository_factory.py#L18

opens file with `r` and thus tries to utf-8 encode.


Discussion / hints:

Fix patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 17b1b374..3e8ba0ae 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -19,6 +19,10 @@ repos:
     rev: v4.5.0
     hooks:
       - id: end-of-file-fixer
+        exclude: |
+          (?x)^(
+            tests/functional/utf_16_encoding/pipeline-content.json
+          )$
       - id: trailing-whitespace
       - id: mixed-line-ending
         args: [--fix=auto]


Test patch:
diff --git a/src/data_factory_testing_framework/_repositories/_factories/fabric_repository_factory.py b/src/data_factory_testing_framework/_repositories/_factories/fabric_repository_factory.py
index 19702459..5e7f4405 100644
--- a/src/data_factory_testing_framework/_repositories/_factories/fabric_repository_factory.py
+++ b/src/data_factory_testing_framework/_repositories/_factories/fabric_repository_factory.py
@@ -1,3 +1,4 @@
+import codecs
 import os
 from typing import List
 
@@ -15,7 +16,10 @@ def _get_data_factory_pipelines_by_folder_path(self, folder_path: str) -> list[P
         for pipeline_folder in pipeline_folders:
             pipeline_file = os.path.join(pipeline_folder, "pipeline-content.json")
             pipeline_metadata_file = os.path.join(pipeline_folder, "item.metadata.json")
-            with open(pipeline_file, "r") as pipeline_file, open(pipeline_metadata_file, "r") as pipeline_metadata_file:
+            pipeline_content_encoding = FabricRepositoryFactory._detect_encoding(pipeline_file)
+            with open(pipeline_file, "r", encoding=pipeline_content_encoding) as pipeline_file, open(
+                pipeline_metadata_file, "r"
+            ) as pipeline_metadata_file:
                 pipelines.append(
                     parse_fabric_pipeline_from_pipeline_json_files(pipeline_metadata_file.read(), pipeline_file.read())
                 )
@@ -37,3 +41,20 @@ def _find_folders_containing_pipeline(search_path: str) -> List[str]:
                 raise FileNotFoundError(f"Pipeline folder {pipeline_folder} does not contain metadata file")
 
         return pipeline_folders
+
+    @staticmethod
+    def _detect_encoding(file_path: str) -> str:
+        """Detects the encoding of the file and returns it as a string.
+
+        Args:
+            file_path (str): The path to the file.
+
+        Returns:
+            str: The encoding of the file. Possible values are "utf-8" and "utf-16" (little or big endian).
+        """
+        with open(file_path, "rb") as file:
+            bom = file.read(2)
+            if bom == codecs.BOM_UTF16_LE or bom == codecs.BOM_UTF16_BE:
+                return "utf-16"
+            else:
+                return "utf-8"
diff --git a/tests/functional/utf_16_encoding/item.metadata.json b/tests/functional/utf_16_encoding/item.metadata.json
new file mode 100644
index 00000000..56bb0003
--- /dev/null
+++ b/tests/functional/utf_16_encoding/item.metadata.json
@@ -0,0 +1,4 @@
+{
+  "type": "Pipeline",
+  "displayName": "utf_16_encoding"
+}
diff --git a/tests/functional/utf_16_encoding/pipeline-content.json b/tests/functional/utf_16_encoding/pipeline-content.json
new file mode 100644
index 00000000..8e3e15cf
Binary files /dev/null and b/tests/functional/utf_16_encoding/pipeline-content.json differ
diff --git a/tests/functional/utf_16_encoding/test_encoding_support.py b/tests/functional/utf_16_encoding/test_encoding_support.py
new file mode 100644
index 00000000..561513dd
--- /dev/null
+++ b/tests/functional/utf_16_encoding/test_encoding_support.py
@@ -0,0 +1,18 @@
+import pytest
+from data_factory_testing_framework import TestFramework, TestFrameworkType
+
+
+def test_fabric_supports_utf_16_le_encoded_pipeline_content(request: pytest.FixtureRequest) -> None:
+    # Arrange
+    test_framework = TestFramework(
+        framework_type=TestFrameworkType.Fabric,
+        root_folder_path=request.fspath.dirname,
+        should_evaluate_child_pipelines=True,
+    )
+
+    # Act
+    pipeline = test_framework.get_pipeline_by_name("utf_16_encoding")
+
+    # Assert
+    assert pipeline is not None
+    assert pipeline.name == "utf_16_encoding"