Repository: mindsdb/lightwood

Issue:
Analyze takes too much time
## Your Environment
* Lightwood version: 22.8.1.0 stable
* Old cpu i5-4460, 16Gb ram

## Describe your issue

Analysis of tiny dataset take more that 2sec

## How can we replicate it?
```
from time import time
import lightwood
from pandas import DataFrame

df = DataFrame([[1]], columns=['a'])
start = time()
analysis = lightwood.analyze_dataset(df)
print(f'analysis {time() - start}')
```


Discussion:
@StpMax can you share the log when running the above? I get:


```
INFO:lightwood-22833:Analyzing a sample of 1
INFO:lightwood-22833:from a total population of 1, this is equivalent to 100.0% of your data.
INFO:lightwood-22833:Using 5 processes to deduct types.
INFO:lightwood-22833:Infering type for: a
INFO:lightwood-22833:Column a has data type binary
WARNING:lightwood-22833:Column a is an identifier of type "No Information"
INFO:lightwood-22833:Starting statistical analysis
INFO:lightwood-22833:Finished statistical analysis
analysis 0.07892560958862305
```

Suspected a high number of processes at first, but that CPU has only 4, so am now less certain what may be the cause.
@paxcema that is strange, but now it takes for me about 0.12sec. That is still big time, but not critical. So if you think it is fine - close the issue
```
INFO:lightwood-100668:Analyzing a sample of 1
INFO:lightwood-100668:from a total population of 1, this is equivalent to 100.0% of your data.
INFO:lightwood-100668:Using 3 processes to deduct types.
INFO:lightwood-100668:Infering type for: a
INFO:lightwood-100668:Column a has data type binary
WARNING:lightwood-100668:Column a is an identifier of type "No Information"
INFO:lightwood-100668:Starting statistical analysis
INFO:lightwood-100668:Finished statistical analysis
analysis 0.1256120204925537
```

Resolving patch:
diff --git a/lightwood/data/infer_types.py b/lightwood/data/infer_types.py
--- a/lightwood/data/infer_types.py
+++ b/lightwood/data/infer_types.py
@@ -132,8 +132,7 @@ def type_check_date(element: object) -> str:
     try:
         dt = dateutil.parser.parse(str(element))
 
-        # Not accurate 100% for a single datetime str,
-        # but should work in aggregate
+        # Not accurate 100% for a single datetime str, but should work in aggregate
         if dt.hour == 0 and dt.minute == 0 and dt.second == 0 and len(str(element)) <= 16:
             return dtype.date
         else:
@@ -367,7 +366,7 @@ def sample_data(df: pd.DataFrame):
     return df.iloc[input_data_sample_indexes]
 
 
-def infer_types(data: pd.DataFrame, pct_invalid: float, seed_nr: int = 420) -> TypeInformation:
+def infer_types(data: pd.DataFrame, pct_invalid: float, seed_nr: int = 420, mp_cutoff: int = 1e4) -> TypeInformation:
     seed(seed_nr)
     type_information = TypeInformation()
     sample_df = sample_data(data)
@@ -378,7 +377,7 @@ def infer_types(data: pd.DataFrame, pct_invalid: float, seed_nr: int = 420) -> T
         f'from a total population of {population_size}, this is equivalent to {round(sample_size*100/population_size, 1)}% of your data.') # noqa
 
     nr_procs = get_nr_procs(data)
-    if nr_procs > 1:
+    if data.size > mp_cutoff and nr_procs > 1:
         log.info(f'Using {nr_procs} processes to deduct types.')
         pool = mp.Pool(processes=nr_procs)
         # Make type `object` so that dataframe cells can be python lists
@@ -408,7 +407,7 @@ def infer_types(data: pd.DataFrame, pct_invalid: float, seed_nr: int = 420) -> T
             'dtype_dist': data_dtype_dist
         }
 
-    if nr_procs > 1:
+    if data.size > mp_cutoff and nr_procs > 1:
         pool = mp.Pool(processes=nr_procs)
         answer_arr = pool.map(get_identifier_description_mp, [
             (data[x], x, type_information.dtypes[x])
diff --git a/lightwood/helpers/text.py b/lightwood/helpers/text.py
--- a/lightwood/helpers/text.py
+++ b/lightwood/helpers/text.py
@@ -16,7 +16,8 @@
 from typing import Iterable
 import numpy as np
 import scipy.stats as st
-import langdetect
+from langid.langid import LanguageIdentifier
+from langid.langid import model as langid_model
 import nltk
 from lightwood.api.dtype import dtype
 
@@ -34,23 +35,23 @@
 
 
 def get_language_dist(data):
-    langdetect.DetectorFactory.seed = 0
     lang_dist = defaultdict(lambda: 0)
     lang_dist['Unknown'] = 0
     lang_probs_cache = dict()
+    identifier = LanguageIdentifier.from_modelstring(langid_model, norm_probs=True)
     for text in data:
         text = str(text)
         text = ''.join([c for c in text if c not in string.punctuation])
         if text not in lang_probs_cache:
             try:
-                lang_probs = langdetect.detect_langs(text)
-            except langdetect.lang_detect_exception.LangDetectException:
+                lang_probs = identifier.classify(text)
+            except Exception:
                 lang_probs = []
             lang_probs_cache[text] = lang_probs
 
         lang_probs = lang_probs_cache[text]
-        if len(lang_probs) > 0 and lang_probs[0].prob > 0.90:
-            lang_dist[lang_probs[0].lang] += 1
+        if len(lang_probs) > 0 and lang_probs[1] > 10 * (1 / len(identifier.nb_classes)):
+            lang_dist[lang_probs[0]] += 1
         else:
             lang_dist['Unknown'] += 1
 


Test patch:
diff --git a/tests/unit_tests/data/test_infer_types.py b/tests/unit_tests/data/test_infer_types.py
new file mode 100644
--- /dev/null
+++ b/tests/unit_tests/data/test_infer_types.py
@@ -0,0 +1,9 @@
+import unittest
+
+from lightwood.api.dtype import dtype
+from lightwood.data.infer_types import type_check_date
+
+
+class TestTypeInference(unittest.TestCase):
+    def test_0_type_check_dates(self):
+        self.assertEqual(type_check_date('31/12/2010'), dtype.date)