Repository: iterative/dvc

Issue:
DVC fails tu launch tensorflow
I have test python script:
`import tensorflow as tf
print(tf.__version__)
`
It runs well when launched from bash `python3 test.py`. But when I launch it with `dvc run python3 test.py` I got the following out:

```
Running command:
        python3 test.py
/bin/bash: /tmp/_MEIACeYFM/libtinfo.so.5: no version information available (required by /bin/bash)
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow.py", line 58, in <module>
    from tensorflow.python.pywrap_tensorflow_internal import *
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow_internal.py", line 28, in <module>
    _pywrap_tensorflow_internal = swig_import_helper()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow_internal.py", line 24, in swig_import_helper
    _mod = imp.load_module('_pywrap_tensorflow_internal', fp, pathname, description)
  File "/usr/lib/python3.5/imp.py", line 242, in load_module
    return load_dynamic(name, filename, file)
  File "/usr/lib/python3.5/imp.py", line 342, in load_dynamic
    return _load(spec)
ImportError: /tmp/_MEIACeYFM/libstdc++.so.6: version `GLIBCXX_3.4.20' not found (required by /usr/local/lib/python3.5/dist-packages/tensorflow/python/_pywrap_tensorflow_internal.so)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "test.py", line 1, in <module>
    import tensorflow as tf
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/__init__.py", line 22, in <module>
    from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/__init__.py", line 49, in <module>
    from tensorflow.python import pywrap_tensorflow
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow.py", line 74, in <module>
    raise ImportError(msg)
ImportError: Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow.py", line 58, in <module>
    from tensorflow.python.pywrap_tensorflow_internal import *
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow_internal.py", line 28, in <module>
    _pywrap_tensorflow_internal = swig_import_helper()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/pywrap_tensorflow_internal.py", line 24, in swig_import_helper
    _mod = imp.load_module('_pywrap_tensorflow_internal', fp, pathname, description)
  File "/usr/lib/python3.5/imp.py", line 242, in load_module
    return load_dynamic(name, filename, file)
  File "/usr/lib/python3.5/imp.py", line 342, in load_dynamic
    return _load(spec)
ImportError: /tmp/_MEIACeYFM/libstdc++.so.6: version `GLIBCXX_3.4.20' not found (required by /usr/local/lib/python3.5/dist-packages/tensorflow/python/_pywrap_tensorflow_internal.so)


Failed to load the native TensorFlow runtime.

See https://www.tensorflow.org/install/install_sources#common_installation_problems

for some common reasons and solutions.  Include the entire stack trace
above this error message when asking for help.
Failed to run command: Stage 'Dvcfile' cmd python3 test.py failed
```


Discussion:
Hi @stratomaster31 !

Which version of dvc are you using(`dvc --version`)?
Hi @efiop !
dvc 0.15.1

And could you please show us the output of the following two commands?
```
$ ps -p $$ -oargs=
$ grep $USER /etc/passwd
```

Thank you,
Ruslan
For sure, thx!

```
bash
igjano:x:1002:1003:Ingacio,,,:/home/igjano:/bin/bash
```

Thank you! Looks like there is something wrong with the environment, need more info. Could you also run these commands in your repository as well and show us the output?

```
$ printenv
$ dvc run -f printenv.dvc printenv
```

```
$ echo $0
$ dvc run -f sh.dvc 'echo $0'
```

You can then safely remove printenv.dvc and sh.dvc files.

```
$ printenv
LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
LD_LIBRARY_PATH=/usr/local/cuda-9.1/lib64:/usr/local/cuda/extras/CUPTI/lib64
XDG_MENU_PREFIX=gnome-
LANG=es_ES.UTF-8
GDM_LANG=es_ES.UTF-8
PROFILEHOME=
DISPLAY=:2
SHELL_SESSION_ID=13fb9723d61546d190ae84a6b319b188
COLORTERM=truecolor
USERNAME=igjano
XDG_VTNR=3
GIO_LAUNCHED_DESKTOP_FILE_PID=92395
SSH_AUTH_SOCK=/run/user/1002/keyring/ssh
XDG_SESSION_ID=76
USER=igjano
DESKTOP_SESSION=default
QT_QPA_PLATFORMTHEME=qgnomeplatform
PWD=/srv/repositories/igjano/SqueezeSeg_ML
HOME=/home/igjano
JOURNAL_STREAM=8:98314
SSH_AGENT_PID=9483
QT_ACCESSIBILITY=1
XDG_SESSION_TYPE=x11
XDG_DATA_DIRS=/usr/share/gnome:/usr/share/gnome:/usr/local/share/:/usr/share/
KONSOLE_DBUS_SESSION=/Sessions/1
XDG_SESSION_DESKTOP=default
GJS_DEBUG_OUTPUT=stderr
KONSOLE_DBUS_WINDOW=/Windows/1
GTK_MODULES=gail:atk-bridge
WINDOWPATH=3
SHELL=/bin/bash
TERM=xterm-256color
KONSOLE_DBUS_SERVICE=:1.293
XDG_CURRENT_DESKTOP=GNOME
GPG_AGENT_INFO=/run/user/1002/gnupg/S.gpg-agent:0:1
QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
KONSOLE_PROFILE_NAME=Default
GIO_LAUNCHED_DESKTOP_FILE=/usr/share/applications/org.kde.konsole.desktop
SHLVL=1
COLORFGBG=15;0
XDG_SEAT=seat0
LANGUAGE=
WINDOWID=37748742
GDMSESSION=default
GNOME_DESKTOP_SESSION_ID=this-is-deprecated
LOGNAME=igjano
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1002/bus
XDG_RUNTIME_DIR=/run/user/1002
XAUTHORITY=/run/user/1002/gdm/Xauthority
PATH=/usr/local/cuda-9.1/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
GJS_DEBUG_TOPICS=JS ERROR;JS LOG
SESSION_MANAGER=local/INN0447:@/tmp/.ICE-unix/11701,unix/INN0447:/tmp/.ICE-unix/11701
OLDPWD=/srv/repositories/igjano
_=/usr/bin/printenv
```


```
$ dvc run -f printenv.dvc printenv
LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
LD_LIBRARY_PATH=/usr/local/cuda-9.1/lib64:/usr/local/cuda/extras/CUPTI/lib64
XDG_MENU_PREFIX=gnome-
LANG=es_ES.UTF-8
GDM_LANG=es_ES.UTF-8
PROFILEHOME=
DISPLAY=:2
SHELL_SESSION_ID=13fb9723d61546d190ae84a6b319b188
COLORTERM=truecolor
USERNAME=igjano
XDG_VTNR=3
GIO_LAUNCHED_DESKTOP_FILE_PID=92395
SSH_AUTH_SOCK=/run/user/1002/keyring/ssh
XDG_SESSION_ID=76
USER=igjano
DESKTOP_SESSION=default
QT_QPA_PLATFORMTHEME=qgnomeplatform
PWD=/srv/repositories/igjano/SqueezeSeg_ML
HOME=/home/igjano
JOURNAL_STREAM=8:98314
SSH_AGENT_PID=9483
QT_ACCESSIBILITY=1
XDG_SESSION_TYPE=x11
XDG_DATA_DIRS=/usr/share/gnome:/usr/share/gnome:/usr/local/share/:/usr/share/
KONSOLE_DBUS_SESSION=/Sessions/1
XDG_SESSION_DESKTOP=default
GJS_DEBUG_OUTPUT=stderr
KONSOLE_DBUS_WINDOW=/Windows/1
GTK_MODULES=gail:atk-bridge
WINDOWPATH=3
SHELL=/bin/bash
TERM=xterm-256color
KONSOLE_DBUS_SERVICE=:1.293
XDG_CURRENT_DESKTOP=GNOME
GPG_AGENT_INFO=/run/user/1002/gnupg/S.gpg-agent:0:1
QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
KONSOLE_PROFILE_NAME=Default
GIO_LAUNCHED_DESKTOP_FILE=/usr/share/applications/org.kde.konsole.desktop
SHLVL=1
COLORFGBG=15;0
XDG_SEAT=seat0
LANGUAGE=
WINDOWID=37748742
GDMSESSION=default
GNOME_DESKTOP_SESSION_ID=this-is-deprecated
LOGNAME=igjano
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1002/bus
XDG_RUNTIME_DIR=/run/user/1002
XAUTHORITY=/run/user/1002/gdm/Xauthority
PATH=/usr/local/cuda-9.1/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
GJS_DEBUG_TOPICS=JS ERROR;JS LOG
SESSION_MANAGER=local/INN0447:@/tmp/.ICE-unix/11701,unix/INN0447:/tmp/.ICE-unix/11701
OLDPWD=/srv/repositories/igjano
_=/usr/bin/printenv
```
```
$ echo $0
/bin/bash
$ dvc run -f sh.dvc 'echo $0'
Running command:
        echo $0
/bin/bash: /tmp/_MEIiG8tpu/libtinfo.so.5: no version information available (required by /bin/bash)
/bin/bash
```

I am not able to reproduce this issue. Your environment seems identical with and without `dvc run`, which is how it is supposed to be. Could you also try these commands please?

```
$ python3 -c 'import os; import tensorflow as tf; print(str(os.environ)); print(tf.__version__)'
$ dvc run -f environ.dvc python3 -c 'import os; import tensorflow as tf; print(str(os.environ)); print(tf.__version__)'
```


Ah, I see what probably happened in your case. You are using a binary dvc package, right?(not from pip) If so this looks like an issue with the binary built by pyinstaller. Already investigating it.
Yes, it is a binary package!

Resolving patch:
diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py
--- a/dvc/remote/hdfs.py
+++ b/dvc/remote/hdfs.py
@@ -9,6 +9,7 @@
 from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class RemoteHDFS(RemoteBase):
@@ -32,6 +33,7 @@ def hadoop_fs(self, cmd, user=None):
                   shell=True,
                   close_fds=True,
                   executable=os.getenv('SHELL'),
+                  env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,
                   stderr=PIPE)
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -1,16 +1,16 @@
 import os
 import yaml
 import itertools
+import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
-import posixpath
 
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
 from dvc.prompt import prompt
 from dvc.logger import Logger
-from dvc.utils import dict_md5
+from dvc.utils import dict_md5, fix_env
 
 
 class StageCmdFailedError(DvcException):
@@ -331,7 +331,7 @@ def run(self, dry=False):
                 p = subprocess.Popen(self.cmd,
                                      cwd=self.cwd,
                                      shell=True,
-                                     env=os.environ,
+                                     env=fix_env(os.environ),
                                      executable=os.getenv('SHELL'))
                 p.communicate()
                 if p.returncode != 0:
diff --git a/dvc/utils.py b/dvc/utils.py
--- a/dvc/utils.py
+++ b/dvc/utils.py
@@ -134,3 +134,21 @@ def to_chunks(l, jobs):
         n = 1
 
     return [l[x:x+n] for x in range(0, len(l), n)]
+
+
+# NOTE: Fix env variables modified by PyInstaller
+# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+def fix_env(env):
+    if env is None:
+        env = os.environ.copy()
+    else:
+        env = env.copy()
+
+    lp_key = 'LD_LIBRARY_PATH'
+    lp_orig = env.get(lp_key + '_ORIG')
+    if lp_orig is not None:
+        env[lp_key] = lp_orig
+    else:
+        env.pop(lp_key, None)
+
+    return env


Test patch: