Repository: ludwig-ai/ludwig

Issue:
Preload fails with ludwig 
I am using ludwig on cpu only mode on multicore system. I run as below

horovodrun -np 24 -H localhost:24 --binding-args="-bind-to socket -map-by socket" ludwig train --training_set train.csv --validation_set valid.csv --random_seed 42 -cf ${MODELFILE} --model_name ${MODELNAME} 

Now I am trying to preload the model file for a different run (all else being the same. Infact, exact same data file even).
horovodrun -np 24 -H localhost:24 --binding-args="-bind-to socket -map-by socket" ludwig train --training_set train.csv --validation_set valid.csv --random_seed 42 -cf ${MODELFILE} --model_name ${MODELNAME} -mlp results/experiment_train_model/model

I am assuming this is the correct way to make multicore (otherwise ludwig just runs on single core)

It fails with error
```
[1,2]<stderr>:    "Intra op parallelism cannot be modified after initialization.")
[1,2]<stderr>:RuntimeError: Intra op parallelism cannot be modified after initialization.
```

Any idea why preload fails ?

Will build a reproducer if needed, but want to eliminative trivial issues if I am missing something.

 - OS:  18.04.1-Ubuntu
- Python version 3.7.6
- Ludwig version
- Tensorflow 2.3.1




Discussion:
Hey @mangleddata, thanks for reporting this.  This does look like a bug on our side when specifying the `model_load_path`.  The root of the problem, I suspect, is that we are not plumbing the parameters [here](https://github.com/uber/ludwig/blob/master/ludwig/train.py#L162), resulting in attempts to initialize TensorFlow with different parameters.  I'll take a look at this today.

Resolving patch:
diff --git a/ludwig/api.py b/ludwig/api.py
--- a/ludwig/api.py
+++ b/ludwig/api.py
@@ -1292,7 +1292,11 @@ def load(
         ```
 
         """
+        # Initialize Horovod and TensorFlow before calling `broadcast()` to prevent initializing
+        # TensorFlow with default parameters
         horovod = configure_horovod(use_horovod)
+        initialize_tensorflow(gpus, gpu_memory_limit, allow_parallel_threads, horovod)
+
         config = broadcast_return(lambda: load_json(os.path.join(
             model_dir,
             MODEL_HYPERPARAMETERS_FILE_NAME
diff --git a/ludwig/train.py b/ludwig/train.py
--- a/ludwig/train.py
+++ b/ludwig/train.py
@@ -159,7 +159,14 @@ def train_cli(
                                 config_file)
 
     if model_load_path:
-        model = LudwigModel.load(model_load_path)
+        model = LudwigModel.load(
+            model_load_path,
+            logging_level=logging_level,
+            use_horovod=use_horovod,
+            gpus=gpus,
+            gpu_memory_limit=gpu_memory_limit,
+            allow_parallel_threads=allow_parallel_threads,
+        )
     else:
         model = LudwigModel(
             config=config,


Test patch:
diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py
--- a/tests/integration_tests/test_cli.py
+++ b/tests/integration_tests/test_cli.py
@@ -28,13 +28,13 @@
 from tests.integration_tests.utils import sequence_feature
 
 
-def _run_ludwig(command, **ludwig_kwargs):
-    commands = ['ludwig', command]
+def _run_commands(commands, **ludwig_kwargs):
     for arg_name, value in ludwig_kwargs.items():
         commands += ['--' + arg_name, value]
     cmdline = ' '.join(commands)
     print(cmdline)
-    completed_process = subprocess.run(cmdline, shell=True, encoding='utf-8',
+    completed_process = subprocess.run(cmdline,
+                                       shell=True,
                                        stdout=subprocess.PIPE,
                                        env=os.environ.copy())
     assert completed_process.returncode == 0
@@ -42,6 +42,16 @@ def _run_ludwig(command, **ludwig_kwargs):
     return completed_process
 
 
+def _run_ludwig(command, **ludwig_kwargs):
+    commands = ['ludwig', command]
+    return _run_commands(commands, **ludwig_kwargs)
+
+
+def _run_ludwig_horovod(command, **ludwig_kwargs):
+    commands = ['horovodrun', '-np', '2', 'ludwig', command]
+    return _run_commands(commands, **ludwig_kwargs)
+
+
 def _prepare_data(csv_filename, config_filename):
     # Single sequence input, single category output
     input_features = [sequence_feature(reduce_output='sum')]
@@ -136,6 +146,33 @@ def test_train_cli_training_set(csv_filename):
                     output_directory=tmpdir)
 
 
+def test_train_cli_horovod(csv_filename):
+    """Test training using `horovodrun -np 2 ludwig train --dataset`."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        config_filename = os.path.join(tmpdir,
+                                       'config.yaml')
+        dataset_filename = _prepare_data(csv_filename,
+                                         config_filename)
+        _run_ludwig_horovod(
+            'train',
+            dataset=dataset_filename,
+            config_file=config_filename,
+            output_directory=tmpdir,
+            experiment_name='horovod_experiment',
+        )
+
+        # Check that `model_load_path` works correctly
+        _run_ludwig_horovod(
+            'train',
+            dataset=dataset_filename,
+            config_file=config_filename,
+            output_directory=tmpdir,
+            model_load_path=os.path.join(
+                tmpdir, 'horovod_experiment_run', 'model'
+            ),
+        )
+
+
 def test_export_savedmodel_cli(csv_filename):
     """Test exporting Ludwig model to Tensorflows savedmodel format."""
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -268,6 +305,7 @@ def test_collect_summary_activations_weights_cli(csv_filename):
                                             'experiment_run',
                                             'model')
                                         )
+        stdout = completed_process.stdout.decode('utf-8')
 
         # parse output of collect_summary to find tensor names to use
         # in the collect_wights and collect_activations.
@@ -275,9 +313,9 @@ def test_collect_summary_activations_weights_cli(csv_filename):
         # search for substring with layer names
         layers = re.search(
             "Layers(\w|\d|\:|\/|\n)*Weights",
-            completed_process.stdout
+            stdout
         )
-        substring = completed_process.stdout[layers.start(): layers.end()]
+        substring = stdout[layers.start(): layers.end()]
 
         # extract layer names
         layers_list = []
@@ -290,9 +328,9 @@ def test_collect_summary_activations_weights_cli(csv_filename):
         # search for substring with weights names
         weights = re.search(
             "Weights(\w|\d|\:|\/|\n)*",
-            completed_process.stdout
+            stdout
         )
-        substring = completed_process.stdout[weights.start(): weights.end()]
+        substring = stdout[weights.start(): weights.end()]
 
         # extract weights names
         weights_list = []