Repository: mindsdb/mindsdb

Issue:
remove ludwig/tesorflow warnings, only show errors
its distracting to see all those warnigns and we dont have any control over this, so lets see if we can hide them


Discussion:

Resolving patch:
diff --git a/mindsdb/libs/backends/ludwig.py b/mindsdb/libs/backends/ludwig.py
--- a/mindsdb/libs/backends/ludwig.py
+++ b/mindsdb/libs/backends/ludwig.py
@@ -1,7 +1,10 @@
 from mindsdb.libs.constants.mindsdb import *
 from mindsdb.config import *
+from mindsdb.libs.helpers.general_helpers import disable_ludwig_output
+
 from dateutil.parser import parse as parse_datetime
 from scipy.misc import imread
+import os, sys
 
 from ludwig import LudwigModel
 import pandas as pd
@@ -353,16 +356,17 @@ def train(self):
         if len(timeseries_cols) > 0:
             training_dataframe, model_definition =  self._translate_df_to_timeseries_format(training_dataframe, model_definition, timeseries_cols, 'train')
 
-        model = LudwigModel(model_definition)
+        with disable_ludwig_output():
+            model = LudwigModel(model_definition)
 
-        # Figure out how to pass `model_load_path`
-        train_stats = model.train(data_df=training_dataframe, model_name=self.transaction.persistent_model_metadata.model_name)
+            # Figure out how to pass `model_load_path`
+            train_stats = model.train(data_df=training_dataframe, model_name=self.transaction.persistent_model_metadata.model_name)
 
-        #model.model.weights_save_path.rstrip('/model_weights_progress') + '/model'
-        ludwig_model_savepath = Config.LOCALSTORE_PATH.rstrip('local_jsondb_store') + self.transaction.persistent_model_metadata.model_name
+            #model.model.weights_save_path.rstrip('/model_weights_progress') + '/model'
+            ludwig_model_savepath = Config.LOCALSTORE_PATH.rstrip('local_jsondb_store') + self.transaction.persistent_model_metadata.model_name
 
-        model.save(ludwig_model_savepath)
-        model.close()
+            model.save(ludwig_model_savepath)
+            model.close()
 
         self.transaction.persistent_model_metadata.ludwig_data = {'ludwig_save_path': ludwig_model_savepath, 'model_definition': model_definition}
 
@@ -371,8 +375,6 @@ def predict(self, mode='predict', ignore_columns=[]):
         predict_dataframe, model_definition = self._create_ludwig_dataframe(mode)
         model_definition = self.transaction.persistent_model_metadata.ludwig_data['model_definition']
 
-        model = LudwigModel.load(self.transaction.persistent_model_metadata.ludwig_data['ludwig_save_path'])
-
         if self.transaction.persistent_model_metadata.model_order_by is None:
             timeseries_cols = []
         else:
@@ -388,7 +390,9 @@ def predict(self, mode='predict', ignore_columns=[]):
                 for date_appendage in ['_year', '_month','_day']:
                     predict_dataframe[ignore_col + date_appendage] = [None] * len(predict_dataframe[ignore_col + date_appendage])
 
-        predictions = model.predict(data_df=predict_dataframe)
+        with disable_ludwig_output():
+            model = LudwigModel.load(self.transaction.persistent_model_metadata.ludwig_data['ludwig_save_path'])
+            predictions = model.predict(data_df=predict_dataframe)
 
         for col_name in predictions:
             col_name_normalized = col_name.replace('_predictions', '')
diff --git a/mindsdb/libs/helpers/general_helpers.py b/mindsdb/libs/helpers/general_helpers.py
--- a/mindsdb/libs/helpers/general_helpers.py
+++ b/mindsdb/libs/helpers/general_helpers.py
@@ -5,6 +5,12 @@
 from pathlib import Path
 import pickle
 import requests
+from contextlib import contextmanager
+import ctypes
+import io
+import os, sys
+import tempfile
+
 
 from mindsdb.__about__ import __version__
 from mindsdb.config import CONFIG
@@ -197,3 +203,42 @@ def evaluate_accuracy(predictions, real_values, col_stats, output_columns):
     if score == 0:
         score = 0.00000001
     return score
+
+
+class suppress_stdout_stderr(object):
+    def __init__(self):
+        # Open a pair of null files
+        self.null_fds =  [os.open(os.devnull,os.O_RDWR) for x in range(2)]
+        # Save the actual stdout (1) and stderr (2) file descriptors.
+        self.c_stdout = sys.stdout.fileno() #int(ctypes.c_void_p.in_dll(ctypes.CDLL(None), 'stdout'))
+        self.c_stderr = sys.stderr.fileno() #int(ctypes.c_void_p.in_dll(ctypes.CDLL(None), 'stderr'))
+
+        self.save_fds = [os.dup(self.c_stdout), os.dup(self.c_stderr)]
+
+    def __enter__(self):
+        # Assign the null pointers to stdout and stderr.
+        os.dup2(self.null_fds[0],self.c_stdout)
+        os.dup2(self.null_fds[1],self.c_stderr)
+
+    def __exit__(self, *_):
+        # Re-assign the real stdout/stderr back to (1) and (2)
+        os.dup2(self.save_fds[0],self.c_stdout)
+        os.dup2(self.save_fds[1],self.c_stderr)
+        # Close all file descriptors
+        for fd in self.null_fds + self.save_fds:
+            os.close(fd)
+
+@contextmanager
+# @TODO: Make it work with mindsdb logger/log levels... maybe
+def disable_ludwig_output():
+    try:
+        try:
+            old_tf_loglevel = os.environ['TF_CPP_MIN_LOG_LEVEL']
+        except:
+            old_tf_loglevel = '2'
+        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+        # Maybe get rid of this to not supress all errors and stdout
+        with suppress_stdout_stderr():
+            yield
+    finally:
+        os.environ['TF_CPP_MIN_LOG_LEVEL'] = old_tf_loglevel


Test patch: