Repository: mindsdb/mindsdb

Issue:
Handle CSV in non utf-8 encoding
At the moment only .csv in utf-8 is supported


Discussion:

Resolving patch:
diff --git a/mindsdb/integrations/handlers/file_handler/file_handler.py b/mindsdb/integrations/handlers/file_handler/file_handler.py
--- a/mindsdb/integrations/handlers/file_handler/file_handler.py
+++ b/mindsdb/integrations/handlers/file_handler/file_handler.py
@@ -9,6 +9,7 @@
 
 import requests
 import pandas as pd
+from charset_normalizer import from_bytes
 
 from mindsdb_sql import parse_sql
 from mindsdb_sql.parser.ast.base import ASTNode
@@ -195,8 +196,18 @@ def _get_data_io(file_path):
             if byte_str.startswith(codecs.BOM_UTF8):
                 data = StringIO(byte_str.decode('utf-8-sig'))
             else:
-                data = StringIO(byte_str.decode('utf-8'))
-
+                file_encoding_meta = from_bytes(
+                    byte_str[:32 * 1024],
+                    steps=32,           # Number of steps/block to extract from my_byte_str
+                    chunk_size=1024,    # Set block size of each extraction)
+                    explain=False
+                )
+                best_meta = file_encoding_meta.best()
+                if best_meta is not None:
+                    encoding = file_encoding_meta.best().encoding
+                else:
+                    encoding = 'utf-8'
+                data = StringIO(byte_str.decode(encoding))
         except Exception:
             print(traceback.format_exc())
             print('Could not load into string')
@@ -220,7 +231,7 @@ def _get_data_io(file_path):
 
         # lets try to figure out if its a csv
         try:
-            dialect = FileHandler._get_csv_dialect(file_path)
+            dialect = FileHandler._get_csv_dialect(data)
             if dialect:
                 return data, 'csv', dialect
             return data, None, dialect
@@ -242,13 +253,14 @@ def _get_file_path(path) -> str:
         return path
 
     @staticmethod
-    def _get_csv_dialect(file_path) -> csv.Dialect:
-        with open(file_path, 'rt') as f:
-            try:
-                accepted_csv_delimiters = [',', '\t', ';']
-                dialect = csv.Sniffer().sniff(f.read(128 * 1024), delimiters=accepted_csv_delimiters)
-            except csv.Error:
-                dialect = None
+    def _get_csv_dialect(buffer) -> csv.Dialect:
+        sample = buffer.read(128 * 1024)
+        buffer.seek(0)
+        try:
+            accepted_csv_delimiters = [',', '\t', ';']
+            dialect = csv.Sniffer().sniff(sample, delimiters=accepted_csv_delimiters)
+        except csv.Error:
+            dialect = None
         return dialect
 
     @staticmethod
diff --git a/mindsdb/interfaces/file/file_controller.py b/mindsdb/interfaces/file/file_controller.py
--- a/mindsdb/interfaces/file/file_controller.py
+++ b/mindsdb/interfaces/file/file_controller.py
@@ -66,6 +66,7 @@ def save_file(self, name, file_path, file_name=None, company_id=None):
         if file_name is None:
             file_name = Path(file_path).name
 
+        file_dir = None
         try:
             df, _col_map = FileHandler._handle_source(file_path)
 
@@ -99,7 +100,8 @@ def save_file(self, name, file_path, file_name=None, company_id=None):
             log.error(e)
             raise
         finally:
-            shutil.rmtree(file_dir)
+            if file_dir is not None:
+                shutil.rmtree(file_dir)
 
         return file_record.id
 


Test patch: