Repository: Viatorus/quom

Issue:
Open files with UTF-8 encoding
On Windows, Python uses ANSI encoding as its default text encoding when opening files. This is fine if you only use ASCII characters since ANSI is compatible with ASCII. But otherwise you will get errors like the following:
```py
  File "C:\Users\Username\AppData\Roaming\Python\Python39\site-packages\quom\__main__.py", line 54, in run
    main(sys.argv[1:])
  File "C:\Users\Username\AppData\Roaming\Python\Python39\site-packages\quom\__main__.py", line 49, in main
    Quom(args.input_path, file, args.stitch, args.include_guard, args.trim, args.include_directory,
  File "C:\Users\Username\AppData\Roaming\Python\Python39\site-packages\quom\quom.py", line 49, in __init__
    self.__process_file(Path(), src_file_path, False, True)
  File "C:\Users\Username\AppData\Roaming\Python\Python39\site-packages\quom\quom.py", line 70, in __process_file
    tokens = tokenize(file.read())
UnicodeDecodeError: 'gbk' codec can't decode byte 0xbf in position 2: illegal multibyte sequence
```
(GBK, or CP936, is the ANSI encoding under Simplified Chinese)

My suggestion is to change to using UTF-8 encoding to open files, which is more appropriate for most programmers:
```py
file_path.open(encoding='utf-8')
```

Discussion:

Patch:
diff --git a/src/quom/__main__.py b/src/quom/__main__.py
index 77d99b3..bb62485 100644
--- a/src/quom/__main__.py
+++ b/src/quom/__main__.py
@@ -29,6 +29,8 @@ def main(args: List[str]):
     parser.add_argument('--source_directory', '-S', type=str, action='append', default=['.'],
                         help='Set the source directories for source files. '
                              'Use ./ or .\\ in front of a path to mark as relative to the header file.')
+    parser.add_argument('--encoding', '-e', type=str, default='utf-8',
+                        help='The encoding used to read and write all files.')
 
     args = parser.parse_args(args)
 
@@ -45,9 +47,9 @@ def main(args: List[str]):
         else:
             source_directories.append(path.resolve())
 
-    with args.output_path.open('w+') as file:
+    with args.output_path.open('w+', encoding=args.encoding) as file:
         Quom(args.input_path, file, args.stitch, args.include_guard, args.trim, args.include_directory,
-             relative_source_directories, source_directories)
+             relative_source_directories, source_directories, args.encoding)
 
 
 def run():
diff --git a/src/quom/quom.py b/src/quom/quom.py
index 5f7d51c..3cf4e13 100644
--- a/src/quom/quom.py
+++ b/src/quom/quom.py
@@ -31,7 +31,8 @@ class Quom:
                  include_guard_format: str = None, trim: bool = True,
                  include_directories: List[Union[Path, str]] = None,
                  relative_source_directories: List[Union[Path]] = None,
-                 source_directories: List[Union[Path]] = None):
+                 source_directories: List[Union[Path]] = None,
+                 encoding: str = 'utf-8'):
         self.__dst = dst
         self.__stitch_format = stitch_format
         self.__include_guard_format = re.compile('^{}$'.format(include_guard_format)) if include_guard_format else None
@@ -40,6 +41,7 @@ class Quom:
         self.__relative_source_directories = relative_source_directories if relative_source_directories else [] \
             if source_directories else [Path('.')]
         self.__source_directories = source_directories if source_directories else [Path('.')]
+        self.__encoding = encoding
 
         self.__processed_files = set()
         self.__source_files = Queue()
@@ -65,16 +67,11 @@ class Quom:
                        is_main_header=False):
         # First check if file exists relative.
         file_path = relative_path / include_path
-        if file_path.exists():
-            with file_path.open() as file:
-                tokens = tokenize(file.read())
-        else:
+        if not file_path.exists():
             # Otherwise search in include directories.
             for include_directory in self.__include_directories:
                 file_path = include_directory / include_path
                 if file_path.exists():
-                    with file_path.open() as file:
-                        tokens = tokenize(file.read())
                     break
             else:
                 raise QuomError('Include not found: "{}"'.format(include_path))
@@ -85,6 +82,9 @@ class Quom:
             return
         self.__processed_files.add(file_path)
 
+        # Tokenize the file.
+        tokens = tokenize(file_path.read_text(encoding=self.__encoding))
+
         for token in tokens:
             # Find local includes.
             token = self.__scan_for_include(file_path, token, is_source_file)


Test patch:
diff --git a/tests/test_quom/same_file_different_include.py b/tests/test_quom/same_file_different_include.py
index 8ef3b9d..a913dbc 100644
--- a/tests/test_quom/same_file_different_include.py
+++ b/tests/test_quom/same_file_different_include.py
@@ -26,13 +26,13 @@ def test_same_file_different_include(fs):
     os.makedirs('a')
     os.makedirs('b')
 
-    with open('main.cpp', 'w+') as file:
+    with open('main.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_CPP)
 
-    with open('a/foo.hpp', 'w+') as file:
+    with open('a/foo.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_HPP)
 
-    with open('b/bar.hpp', 'w+') as file:
+    with open('b/bar.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_BAR_HPP)
 
     dst = StringIO()
diff --git a/tests/test_quom/test_file_encoding.py b/tests/test_quom/test_file_encoding.py
new file mode 100644
index 0000000..416a993
--- /dev/null
+++ b/tests/test_quom/test_file_encoding.py
@@ -0,0 +1,40 @@
+from pathlib import Path
+
+import pytest
+
+from quom.__main__ import main
+
+FILE_MAIN_HPP = """
+int foo(); // qθομ"""
+
+
+def test_file_encoding_default_encoding(fs):
+    with open('main.hpp', 'w+', encoding='utf-8') as file:
+        file.write(FILE_MAIN_HPP)
+
+    main(['main.hpp', 'result.hpp'])
+    assert Path('result.hpp').read_text('utf-8') == FILE_MAIN_HPP
+
+    with pytest.raises(UnicodeDecodeError):
+        Path('result.hpp').read_text('ascii')
+
+    with pytest.raises(UnicodeDecodeError):
+        Path('result.hpp').read_text('utf-32')
+
+
+def test_file_encoding_custom_encoding(fs):
+    with open('main.hpp', 'w+', encoding='utf-32') as file:
+        file.write(FILE_MAIN_HPP)
+
+    main(['main.hpp', 'result.hpp', '--encoding=utf-32'])
+
+    assert Path('result.hpp').read_text('utf-32') == FILE_MAIN_HPP
+
+    with pytest.raises(UnicodeDecodeError):
+        Path('result.hpp').read_text('utf-8')
+
+    with pytest.raises(UnicodeDecodeError):
+        main(['main.hpp', 'result.hpp'])
+
+    with pytest.raises(UnicodeDecodeError):
+        main(['main.hpp', 'result.hpp', '--encoding=utf-8'])
diff --git a/tests/test_quom/test_include_directory.py b/tests/test_quom/test_include_directory.py
index 1d8e572..1666ecf 100644
--- a/tests/test_quom/test_include_directory.py
+++ b/tests/test_quom/test_include_directory.py
@@ -79,28 +79,28 @@ def test_include_directory(fs):
     os.makedirs('include/my_lib/util')
     os.makedirs('include/my_other_lib/')
 
-    with open('include/my_lib/main.hpp', 'w+') as file:
+    with open('include/my_lib/main.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_HPP)
 
-    with open('include/my_lib/core/core.hpp', 'w+') as file:
+    with open('include/my_lib/core/core.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_CORE_HPP)
 
-    with open('include/my_lib/core/core.cpp', 'w+') as file:
+    with open('include/my_lib/core/core.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_CORE_CPP)
 
-    with open('include/my_lib/util/foo.hpp', 'w+') as file:
+    with open('include/my_lib/util/foo.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_HPP)
 
-    with open('include/my_lib/util/foo.cpp', 'w+') as file:
+    with open('include/my_lib/util/foo.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_CPP)
 
-    with open('include/my_other_lib/bar.hpp', 'w+') as file:
+    with open('include/my_other_lib/bar.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_BAR_HPP)
 
-    with open('include/my_other_lib/bar.cpp', 'w+') as file:
+    with open('include/my_other_lib/bar.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_BAR_CPP)
 
-    with open('include/my_other_lib/info.hpp', 'w+') as file:
+    with open('include/my_other_lib/info.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_INFO_HPP)
 
     dst = StringIO()
diff --git a/tests/test_quom/test_last_source_file.py b/tests/test_quom/test_last_source_file.py
index 96e2c98..c227c21 100644
--- a/tests/test_quom/test_last_source_file.py
+++ b/tests/test_quom/test_last_source_file.py
@@ -82,13 +82,13 @@ int foo = 42;"""
 
 
 def init():
-    with open('main.hpp', 'w+') as file:
+    with open('main.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_HPP)
 
-    with open('foo.hpp', 'w+') as file:
+    with open('foo.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_HPP)
 
-    with open('foo.cpp', 'w+') as file:
+    with open('foo.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_CPP)
 
 
@@ -111,7 +111,7 @@ def test_normal_without_trim(fs):
 
 
 def test_without_newline_at_end(fs):
-    with open('main.hpp', 'w+') as file:
+    with open('main.hpp', 'w+', encoding='utf-8') as file:
         file.write('int a;')
 
     dst = StringIO()
diff --git a/tests/test_quom/test_normal.py b/tests/test_quom/test_normal.py
index a718b87..48dd016 100644
--- a/tests/test_quom/test_normal.py
+++ b/tests/test_quom/test_normal.py
@@ -166,13 +166,13 @@ int foo = 42;
 
 
 def init():
-    with open('main.hpp', 'w+') as file:
+    with open('main.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_HPP)
 
-    with open('foo.hpp', 'w+') as file:
+    with open('foo.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_HPP)
 
-    with open('foo.cpp', 'w+') as file:
+    with open('foo.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_FOO_CPP)
 
 
diff --git a/tests/test_quom/test_source_directory.py b/tests/test_quom/test_source_directory.py
index 790d417..9bd63bc 100644
--- a/tests/test_quom/test_source_directory.py
+++ b/tests/test_quom/test_source_directory.py
@@ -30,10 +30,10 @@ def test_source_directory(fs):
     os.makedirs('include/')
     os.makedirs('src/')
 
-    with open('include/main.hpp', 'w+') as file:
+    with open('include/main.hpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_HPP)
 
-    with open('src/main.cpp', 'w+') as file:
+    with open('src/main.cpp', 'w+', encoding='utf-8') as file:
         file.write(FILE_MAIN_CPP)
 
     dst = StringIO()