Repository: conan-io/conan

Issue:
Using "?" in a tools.get() URL will fail, while using it in a tools.download() will succeed.
To help us debug your issue please explain:

- [x] I've read the [CONTRIBUTING guide](https://raw.githubusercontent.com/conan-io/conan/develop/.github/CONTRIBUTING.md).
- [x] I've specified the Conan version, operating system version and any tool that can be relevant.
- [x] I've explained the steps to reproduce the error or the motivation/use case of the question/suggestion.

Example:
```
from conans import ConanFile, tools

class LibraryConan(ConanFile):
    name = "Library"
    version = "1.0"
    license = ""
    url = "<Package recipe repository url here, for issues about the package>"
    description = ""
    settings = "os", "compiler", "build_type", "arch"
    
    def source(self):
        tools.get("http://example.com/?file=1")
```
```
conan create . Company/beta
ERROR: Library/1.0@Company/beta: Error in source() method, line 12
        tools.get("http://example.com/?file=1")
        ConanConnectionError: Download failed, check server, possibly try again
[Errno 22] Invalid argument: 'C:\\Users\\User\\.conan\\data\\Library\\1.0\\Company\\beta\\source\\?file=1'
```


Discussion:

Patch:
diff --git a/.ci/jenkins/conf.py b/.ci/jenkins/conf.py
index f59d4441e..02c9bc118 100644
--- a/.ci/jenkins/conf.py
+++ b/.ci/jenkins/conf.py
@@ -9,7 +9,7 @@ winpylocation = {"py27": "C:\\Python27\\python.exe",
 
 macpylocation = {"py27": "/usr/bin/python",  # /Users/jenkins_ci/.pyenv/versions/2.7.11/bin/python",
                  "py34": "/Users/jenkins_ci/.pyenv/versions/3.4.7/bin/python",
-                 "py36": "/Users/jenkins_ci/.pyenv/versions/3.6.3/bin/python"}
+                 "py36": "/Users/jenkins_ci/.pyenv/versions/3.6.5/bin/python"}
 
 linuxpylocation = {"py27": "/usr/bin/python2.7",
                    "py34": "/usr/bin/python3.4",
diff --git a/conans/client/generators/visualstudio.py b/conans/client/generators/visualstudio.py
index 1b91d7a0e..37c7597bd 100644
--- a/conans/client/generators/visualstudio.py
+++ b/conans/client/generators/visualstudio.py
@@ -1,5 +1,8 @@
+import os
+
 from conans.model import Generator
 from conans.paths import BUILD_INFO_VISUAL_STUDIO
+import re
 
 
 class VisualStudioGenerator(Generator):
@@ -76,4 +79,8 @@ class VisualStudioGenerator(Generator):
             'exe_flags': " ".join(self._deps_build_info.exelinkflags)
         }
         formatted_template = self.template.format(**fields)
+        userprofile = os.getenv("USERPROFILE")
+        if userprofile:
+            userprofile = userprofile.replace("\\", "/")
+            formatted_template = re.sub(userprofile, "$(USERPROFILE)", formatted_template, flags=re.I)
         return formatted_template
diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py
index bd37186bf..929174655 100644
--- a/conans/client/remote_manager.py
+++ b/conans/client/remote_manager.py
@@ -12,7 +12,7 @@ from conans.model.manifest import gather_files
 from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME, \
     rm_conandir, EXPORT_SOURCES_TGZ_NAME, EXPORT_SOURCES_DIR_OLD
 from conans.util.files import gzopen_without_timestamps, is_dirty,\
-    make_read_only
+    make_read_only, set_dirty, clean_dirty
 from conans.util.files import tar_extract, rmdir, exception_message_safe, mkdir
 from conans.util.files import touch_folder
 from conans.util.log import logger
@@ -23,6 +23,7 @@ from conans.util.tracer import (log_package_upload, log_recipe_upload,
                                 log_package_download)
 from conans.client.source import merge_directories
 from conans.util.env_reader import get_env
+from conans.search.search import filter_packages
 
 
 class RemoteManager(object):
@@ -39,6 +40,14 @@ class RemoteManager(object):
 
         t1 = time.time()
         export_folder = self._client_cache.export(conan_reference)
+
+        for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
+            tgz_path = os.path.join(export_folder, f)
+            if is_dirty(tgz_path):
+                self._output.warn("%s: Removing %s, marked as dirty" % (str(conan_reference), f))
+                os.remove(tgz_path)
+                clean_dirty(tgz_path)
+
         files, symlinks = gather_files(export_folder)
         if CONANFILE not in files or CONAN_MANIFEST not in files:
             raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
@@ -99,6 +108,11 @@ class RemoteManager(object):
                                  "Remove it with 'conan remove %s -p=%s'" % (package_reference,
                                                                              package_reference.conan,
                                                                              package_reference.package_id))
+        tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
+        if is_dirty(tgz_path):
+            self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME))
+            os.remove(tgz_path)
+            clean_dirty(tgz_path)
         # Get all the files in that directory
         files, symlinks = gather_files(package_folder)
 
@@ -188,7 +202,8 @@ class RemoteManager(object):
         t1 = time.time()
 
         def filter_function(urls):
-            file_url = urls.get(EXPORT_SOURCES_TGZ_NAME)
+            file_url = urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
+            check_compressed_files(EXPORT_SOURCES_TGZ_NAME, urls)
             if file_url:
                 urls = {EXPORT_SOURCES_TGZ_NAME: file_url}
             else:
@@ -254,7 +269,9 @@ class RemoteManager(object):
         return self._call_remote(remote, "search", pattern, ignorecase)
 
     def search_packages(self, remote, reference, query):
-        return self._call_remote(remote, "search_packages", reference, query)
+        packages = self._call_remote(remote, "search_packages", reference, query)
+        packages = filter_packages(query, packages)
+        return packages
 
     def remove(self, conan_ref, remote):
         """
@@ -324,11 +341,10 @@ def compress_package_files(files, symlinks, dest_folder, output):
 
 
 def compress_files(files, symlinks, name, dest_dir):
-    """Compress the package and returns the new dict (name => content) of files,
-    only with the conanXX files and the compressed file"""
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
+    set_dirty(tgz_path)
     with open(tgz_path, "wb") as tgz_handle:
         # tgz_contents = BytesIO()
         tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle)
@@ -354,17 +370,27 @@ def compress_files(files, symlinks, name, dest_dir):
 
         tgz.close()
 
+    clean_dirty(tgz_path)
     duration = time.time() - t1
     log_compressed_files(files, duration, tgz_path)
 
     return tgz_path
 
 
+def check_compressed_files(tgz_name, files):
+    bare_name = os.path.splitext(tgz_name)[0]
+    for f in files:
+        if bare_name == os.path.splitext(f)[0]:
+            raise ConanException("This Conan version is not prepared to handle '%s' file format. "
+                                 "Please upgrade conan client." % f)
+
+
 def unzip_and_get_files(files, destination_dir, tgz_name):
     """Moves all files from package_files, {relative_name: tmp_abs_path}
     to destination_dir, unzipping the "tgz_name" if found"""
 
     tgz_file = files.pop(tgz_name, None)
+    check_compressed_files(tgz_name, files)
     if tgz_file:
         uncompress_file(tgz_file, destination_dir)
         os.remove(tgz_file)
diff --git a/conans/client/runner.py b/conans/client/runner.py
index a598e9721..a71a8f942 100644
--- a/conans/client/runner.py
+++ b/conans/client/runner.py
@@ -1,6 +1,7 @@
 import io
 import os
 import sys
+from contextlib import contextmanager
 from subprocess import Popen, PIPE, STDOUT
 from conans.util.files import decode_text
 from conans.errors import ConanException
@@ -37,18 +38,19 @@ class ConanRunner(object):
         if self._print_commands_to_output and stream_output and self._log_run_to_output:
             stream_output.write(call_message)
 
-        # No output has to be redirected to logs or buffer or omitted
-        if output is True and not log_filepath and self._log_run_to_output and not subprocess:
-            return self._simple_os_call(command, cwd)
-        elif log_filepath:
-            if stream_output:
-                stream_output.write("Logging command output to file '%s'\n" % log_filepath)
-            with open(log_filepath, "a+") as log_handler:
-                if self._print_commands_to_output:
-                    log_handler.write(call_message)
-                return self._pipe_os_call(command, stream_output, log_handler, cwd)
-        else:
-            return self._pipe_os_call(command, stream_output, None, cwd)
+        with pyinstaller_bundle_env_cleaned():
+            # No output has to be redirected to logs or buffer or omitted
+            if output is True and not log_filepath and self._log_run_to_output and not subprocess:
+                return self._simple_os_call(command, cwd)
+            elif log_filepath:
+                if stream_output:
+                    stream_output.write("Logging command output to file '%s'\n" % log_filepath)
+                with open(log_filepath, "a+") as log_handler:
+                    if self._print_commands_to_output:
+                        log_handler.write(call_message)
+                    return self._pipe_os_call(command, stream_output, log_handler, cwd)
+            else:
+                return self._pipe_os_call(command, stream_output, None, cwd)
 
     def _pipe_os_call(self, command, stream_output, log_handler, cwd):
 
@@ -69,7 +71,7 @@ class ConanRunner(object):
                 if stream_output and self._log_run_to_output:
                     try:
                         stream_output.write(decoded_line)
-                    except UnicodeEncodeError:  # be agressive on text encoding
+                    except UnicodeEncodeError:  # be aggressive on text encoding
                         decoded_line = decoded_line.encode("latin-1", "ignore").decode("latin-1",
                                                                                        "ignore")
                         stream_output.write(decoded_line)
@@ -100,3 +102,27 @@ class ConanRunner(object):
             finally:
                 os.chdir(old_dir)
             return result
+
+
+if getattr(sys, 'frozen', False) and 'LD_LIBRARY_PATH' in os.environ:
+
+    # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
+    pyinstaller_bundle_dir = os.environ['LD_LIBRARY_PATH'].replace(
+        os.environ.get('LD_LIBRARY_PATH_ORIG', ''), ''
+    ).strip(';:')
+
+    @contextmanager
+    def pyinstaller_bundle_env_cleaned():
+        """Removes the pyinstaller bundle directory from LD_LIBRARY_PATH
+
+        :return: None
+        """
+        ld_library_path = os.environ['LD_LIBRARY_PATH']
+        os.environ['LD_LIBRARY_PATH'] = ld_library_path.replace(pyinstaller_bundle_dir, '')
+        yield
+        os.environ['LD_LIBRARY_PATH'] = ld_library_path
+
+else:
+    @contextmanager
+    def pyinstaller_bundle_env_cleaned():
+        yield
diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py
index 5d6ddef97..0d42fe368 100644
--- a/conans/client/tools/files.py
+++ b/conans/client/tools/files.py
@@ -11,6 +11,7 @@ from conans.client.output import ConanOutput
 from conans.errors import ConanException
 from conans.util.files import (load, save, _generic_algorithm_sum)
 from conans.unicode import get_cwd
+import six
 
 
 _global_output = None
@@ -68,6 +69,11 @@ def unzip(filename, destination=".", keep_permissions=False, pattern=None):
             filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or
             filename.endswith(".tar")):
         return untargz(filename, destination, pattern)
+    if filename.endswith(".tar.xz") or filename.endswith(".txz"):
+        if six.PY2:
+            raise ConanException("XZ format not supported in Python 2. Use Python 3 instead")
+        return untargz(filename, destination, pattern)
+
     import zipfile
     full_path = os.path.normpath(os.path.join(get_cwd(), destination))
 
diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py
index 235fc69b5..59293e293 100644
--- a/conans/client/tools/net.py
+++ b/conans/client/tools/net.py
@@ -9,10 +9,13 @@ from conans.errors import ConanException
 _global_requester = None
 
 
-def get(url, md5='', sha1='', sha256='', destination="."):
+def get(url, md5='', sha1='', sha256='', destination=".", filename=""):
     """ high level downloader + unzipper + (optional hash checker) + delete temporary zip
     """
-    filename = os.path.basename(url)
+    if not filename and ("?" in url or "=" in url):
+        raise ConanException("Cannot deduce file name form url. Use 'filename' parameter.")
+
+    filename = filename or os.path.basename(url)
     download(url, filename)
 
     if md5:
diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py
index 455397546..17557672b 100644
--- a/conans/model/conan_file.py
+++ b/conans/model/conan_file.py
@@ -12,6 +12,8 @@ from conans.model.user_info import DepsUserInfo
 from conans.paths import RUN_LOG_NAME
 from conans.tools import environment_append, no_op
 from conans.client.output import Color
+from conans.client.run_environment import RunEnvironment
+from conans.client.tools.oss import os_info
 
 
 def create_options(conanfile):
@@ -255,15 +257,23 @@ class ConanFile(object):
         """ define cpp_build_info, flags, etc
         """
 
-    def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True):
-        if not win_bash:
-            retcode = self._runner(command, output, os.path.abspath(RUN_LOG_NAME), cwd)
-        else:
+    def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True,
+            ignore_errors=False, run_environment=False):
+        def _run():
+            if not win_bash:
+                return self._runner(command, output, os.path.abspath(RUN_LOG_NAME), cwd)
             # FIXME: run in windows bash is not using output
-            retcode = tools.run_in_windows_bash(self, bashcmd=command, cwd=cwd, subsystem=subsystem,
-                                                msys_mingw=msys_mingw)
+            return tools.run_in_windows_bash(self, bashcmd=command, cwd=cwd, subsystem=subsystem,
+                                             msys_mingw=msys_mingw)
+        if run_environment:
+            with tools.environment_append(RunEnvironment(self).vars):
+                if os_info.is_macos:
+                    command = 'DYLD_LIBRARY_PATH="%s" %s' % (os.environ.get('DYLD_LIBRARY_PATH', ''), command)
+                retcode = _run()
+        else:
+            retcode = _run()
 
-        if retcode != 0:
+        if not ignore_errors and retcode != 0:
             raise ConanException("Error %d while executing %s" % (retcode, command))
 
         return retcode
diff --git a/conans/search/search.py b/conans/search/search.py
index 975a97403..2e189bb45 100644
--- a/conans/search/search.py
+++ b/conans/search/search.py
@@ -61,7 +61,7 @@ def evaluate(prop_name, prop_value, conan_vars_info):
     """
 
     def compatible_prop(setting_value, prop_value):
-        return setting_value is None or prop_value == setting_value
+        return (prop_value == setting_value) or (prop_value == "None" and setting_value is None)
 
     info_settings = conan_vars_info.get("settings", [])
     info_options = conan_vars_info.get("options", [])
diff --git a/conans/server/rest/controllers/file_upload_download_controller.py b/conans/server/rest/controllers/file_upload_download_controller.py
index b49091a07..fe8ec36ab 100644
--- a/conans/server/rest/controllers/file_upload_download_controller.py
+++ b/conans/server/rest/controllers/file_upload_download_controller.py
@@ -20,7 +20,12 @@ class FileUploadDownloadController(Controller):
             token = request.query.get("signature", None)
             file_path = service.get_file_path(filepath, token)
             # https://github.com/kennethreitz/requests/issues/1586
-            mimetype = "x-gzip" if filepath.endswith(".tgz") else "auto"
+            if filepath.endswith(".tgz"):
+                mimetype = "x-gzip"
+            elif filepath.endswith(".txz"):
+                mimetype = "x-xz"
+            else:
+                mimetype = "auto"
             return static_file(os.path.basename(file_path),
                                root=os.path.dirname(file_path),
                                mimetype=mimetype)


Test patch:
diff --git a/conans/test/command/remove_test.py b/conans/test/command/remove_test.py
index 093fddfb6..5cddca144 100644
--- a/conans/test/command/remove_test.py
+++ b/conans/test/command/remove_test.py
@@ -15,6 +15,35 @@ from conans.test.utils.test_files import temp_folder
 
 
 class RemoveOutdatedTest(unittest.TestCase):
+
+    def remove_query_test(self):
+        test_server = TestServer(users={"lasote": "password"})  # exported users and passwords
+        servers = {"default": test_server}
+        client = TestClient(servers=servers, users={"default": [("lasote", "password")]})
+        conanfile = """from conans import ConanFile
+class Test(ConanFile):
+    settings = "os"
+    """
+        client.save({"conanfile.py": conanfile})
+        client.run("create . Test/0.1@lasote/testing -s os=Windows")
+        client.run("create . Test/0.1@lasote/testing -s os=Linux")
+        client.save({"conanfile.py": conanfile.replace("settings", "pass #")})
+        client.run("create . Test2/0.1@lasote/testing")
+        client.run("upload * --all --confirm")
+        for remote in ("", "-r=default"):
+            client.run("remove Test/0.1@lasote/testing -q=os=Windows -f %s" % remote)
+            client.run("search Test/0.1@lasote/testing %s" % remote)
+            self.assertNotIn("os: Windows", client.out)
+            self.assertIn("os: Linux", client.out)
+
+            client.run("remove Test2/0.1@lasote/testing -q=os=Windows -f %s" % remote)
+            client.run("search Test2/0.1@lasote/testing %s" % remote)
+            self.assertIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", client.out)
+            client.run("remove Test2/0.1@lasote/testing -q=os=None -f %s" % remote)
+            client.run("search Test2/0.1@lasote/testing %s" % remote)
+            self.assertNotIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", client.out)
+            self.assertIn("There are no packages", client.out)
+
     def remove_outdated_test(self):
         test_server = TestServer(users={"lasote": "password"})  # exported users and passwords
         servers = {"default": test_server}
diff --git a/conans/test/command/search_test.py b/conans/test/command/search_test.py
index e94731ce9..457217c15 100644
--- a/conans/test/command/search_test.py
+++ b/conans/test/command/search_test.py
@@ -387,8 +387,7 @@ helloTest/1.4.10@fenix/stable""".format(remote)
 
             q = 'compiler="gcc" OR compiler.libcxx=libstdc++11'
             # Should find Visual because of the OR, visual doesn't care about libcxx
-            self._assert_pkg_q(q, ["LinuxPackageSHA", "PlatformIndependantSHA",
-                                   "WindowsPackageSHA"], remote)
+            self._assert_pkg_q(q, ["LinuxPackageSHA", "PlatformIndependantSHA"], remote)
 
             q = '(compiler="gcc" AND compiler.libcxx=libstdc++11) OR compiler.version=4.5'
             self._assert_pkg_q(q, ["LinuxPackageSHA"], remote)
@@ -407,18 +406,25 @@ helloTest/1.4.10@fenix/stable""".format(remote)
             self._assert_pkg_q(q, ["PlatformIndependantSHA", "WindowsPackageSHA"], remote)
 
             q = '(os="Linux" OR os=Windows)'
-            self._assert_pkg_q(q, ["PlatformIndependantSHA", "LinuxPackageSHA",
-                                   "WindowsPackageSHA"], remote)
+            self._assert_pkg_q(q, ["LinuxPackageSHA", "WindowsPackageSHA"], remote)
+
+            q = '(os="Linux" OR os=None)'
+            self._assert_pkg_q(q, ["LinuxPackageSHA", "PlatformIndependantSHA"], remote)
+
+            q = '(os=None)'
+            self._assert_pkg_q(q, ["PlatformIndependantSHA"], remote)
 
             q = '(os="Linux" OR os=Windows) AND use_Qt=True'
+            self._assert_pkg_q(q, ["WindowsPackageSHA"], remote)
+
+            q = '(os=None OR os=Windows) AND use_Qt=True'
             self._assert_pkg_q(q, ["PlatformIndependantSHA", "WindowsPackageSHA"], remote)
 
             q = '(os="Linux" OR os=Windows) AND use_Qt=True AND nonexistant_option=3'
-            self._assert_pkg_q(q, ["PlatformIndependantSHA", "WindowsPackageSHA"], remote)
+            self._assert_pkg_q(q, [], remote)
 
             q = '(os="Linux" OR os=Windows) AND use_Qt=True OR nonexistant_option=3'
-            self._assert_pkg_q(q, ["PlatformIndependantSHA",
-                                   "WindowsPackageSHA", "LinuxPackageSHA"], remote)
+            self._assert_pkg_q(q, ["WindowsPackageSHA", "LinuxPackageSHA"], remote)
 
         # test in local
         test_cases()
@@ -474,9 +480,19 @@ helloTest/1.4.10@fenix/stable""".format(remote)
 
         self.client.run('search Hello/1.4.10/fenix/testing -q os=Windows')
         self.assertIn("WindowsPackageSHA", self.client.out)
+        self.assertNotIn("PlatformIndependantSHA", self.client.out)
+        self.assertNotIn("LinuxPackageSHA", self.client.out)
+
+        self.client.run('search Hello/1.4.10/fenix/testing -q "os=Windows or os=None"')
+        self.assertIn("WindowsPackageSHA", self.client.out)
         self.assertIn("PlatformIndependantSHA", self.client.out)
         self.assertNotIn("LinuxPackageSHA", self.client.out)
 
+        self.client.run('search Hello/1.4.10/fenix/testing -q "os=Windows or os=Linux"')
+        self.assertIn("WindowsPackageSHA", self.client.out)
+        self.assertNotIn("PlatformIndependantSHA", self.client.out)
+        self.assertIn("LinuxPackageSHA", self.client.out)
+
         self.client.run('search Hello/1.4.10/fenix/testing -q "os=Windows AND compiler.version=4.5"')
         self.assertIn("There are no packages for reference 'Hello/1.4.10@fenix/testing' "
                       "matching the query 'os=Windows AND compiler.version=4.5'", self.client.out)
diff --git a/conans/test/command/upload_test.py b/conans/test/command/upload_test.py
index 661bf9d8e..193024241 100644
--- a/conans/test/command/upload_test.py
+++ b/conans/test/command/upload_test.py
@@ -2,9 +2,12 @@ import unittest
 from conans.tools import environment_append
 from conans.test.utils.tools import TestClient, TestServer
 from conans.test.utils.cpp_test_files import cpp_hello_conan_files
-from conans.model.ref import ConanFileReference
-from conans.util.files import save
+from conans.model.ref import ConanFileReference, PackageReference
+from conans.util.files import save, is_dirty, gzopen_without_timestamps
 import os
+from mock import mock
+from conans.errors import ConanException
+from conans.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME
 
 
 conanfile = """from conans import ConanFile
@@ -75,6 +78,62 @@ class UploadTest(unittest.TestCase):
         self.assertIn("Uploading conan_package.tgz", client.user_io.out)
         self.assertIn("Uploading conanfile.py", client.user_io.out)
 
+    def broken_sources_tgz_test(self):
+        # https://github.com/conan-io/conan/issues/2854
+        client = self._client()
+        client.save({"conanfile.py": conanfile,
+                     "source.h": "my source"})
+        client.run("create . user/testing")
+        ref = ConanFileReference.loads("Hello0/1.2.1@user/testing")
+
+        def gzopen_patched(name, mode="r", fileobj=None, compresslevel=None, **kwargs):
+            raise ConanException("Error gzopen %s" % name)
+        with mock.patch('conans.client.remote_manager.gzopen_without_timestamps', new=gzopen_patched):
+            error = client.run("upload * --confirm", ignore_error=True)
+            self.assertTrue(error)
+            self.assertIn("ERROR: Error gzopen conan_sources.tgz", client.out)
+
+            export_folder = client.client_cache.export(ref)
+            tgz = os.path.join(export_folder, EXPORT_SOURCES_TGZ_NAME)
+            self.assertTrue(os.path.exists(tgz))
+            self.assertTrue(is_dirty(tgz))
+
+        client.run("upload * --confirm")
+        self.assertIn("WARN: Hello0/1.2.1@user/testing: Removing conan_sources.tgz, marked as dirty",
+                      client.out)
+        self.assertTrue(os.path.exists(tgz))
+        self.assertFalse(is_dirty(tgz))
+
+    def broken_package_tgz_test(self):
+        # https://github.com/conan-io/conan/issues/2854
+        client = self._client()
+        client.save({"conanfile.py": conanfile,
+                     "source.h": "my source"})
+        client.run("create . user/testing")
+        package_ref = PackageReference.loads("Hello0/1.2.1@user/testing:"
+                                             "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
+
+        def gzopen_patched(name, mode="r", fileobj=None, compresslevel=None, **kwargs):
+            if name == PACKAGE_TGZ_NAME:
+                raise ConanException("Error gzopen %s" % name)
+            return gzopen_without_timestamps(name, mode, fileobj, compresslevel, **kwargs)
+        with mock.patch('conans.client.remote_manager.gzopen_without_timestamps', new=gzopen_patched):
+            error = client.run("upload * --confirm --all", ignore_error=True)
+            self.assertTrue(error)
+            self.assertIn("ERROR: Error gzopen conan_package.tgz", client.out)
+
+            export_folder = client.client_cache.package(package_ref)
+            tgz = os.path.join(export_folder, PACKAGE_TGZ_NAME)
+            self.assertTrue(os.path.exists(tgz))
+            self.assertTrue(is_dirty(tgz))
+
+        client.run("upload * --confirm --all")
+        self.assertIn("WARN: Hello0/1.2.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: "
+                      "Removing conan_package.tgz, marked as dirty",
+                      client.out)
+        self.assertTrue(os.path.exists(tgz))
+        self.assertFalse(is_dirty(tgz))
+
     def corrupt_upload_test(self):
         client = self._client()
 
diff --git a/conans/test/functional/runner_test.py b/conans/test/functional/runner_test.py
index f0dc4fda8..d00f9eb71 100644
--- a/conans/test/functional/runner_test.py
+++ b/conans/test/functional/runner_test.py
@@ -2,7 +2,6 @@ import os
 import six
 import unittest
 
-from io import StringIO
 from conans.client.runner import ConanRunner
 from conans.test.utils.tools import TestClient
 
@@ -19,6 +18,18 @@ class RunnerTest(unittest.TestCase):
         client.run("build .")
         return client
 
+    def ignore_error_test(self):
+        conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    def source(self):
+        ret = self.run("not_a_command", ignore_errors=True)
+        self.output.info("RETCODE %s" % (ret!=0))
+"""
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.run("source .")
+        self.assertIn("RETCODE True", client.out)
+
     def basic_test(self):
         conanfile = '''
 from conans import ConanFile
diff --git a/conans/test/generators/visual_studio_test.py b/conans/test/generators/visual_studio_test.py
index 0f43200ec..fce6ead8c 100644
--- a/conans/test/generators/visual_studio_test.py
+++ b/conans/test/generators/visual_studio_test.py
@@ -7,6 +7,10 @@ from conans.model.settings import Settings
 from conans.model.conan_file import ConanFile
 from conans.model.build_info import CppInfo
 from conans.model.ref import ConanFileReference
+from conans.test.utils.test_files import temp_folder
+from conans.util.files import save
+import os
+from conans import tools
 
 
 class VisualStudioGeneratorTest(unittest.TestCase):
@@ -27,3 +31,32 @@ class VisualStudioGeneratorTest(unittest.TestCase):
         self.assertIn('<PropertyGroup Label="Conan-RootDirs">', content)
         self.assertIn("<Conan-MyPkg-Root>dummy_root_folder1</Conan-MyPkg-Root>", content)
         self.assertIn("<Conan-My-Fancy-Pkg_2-Root>dummy_root_folder2</Conan-My-Fancy-Pkg_2-Root>", content)
+
+    def user_profile_test(self):
+        conanfile = ConanFile(None, None, Settings({}), None)
+        ref = ConanFileReference.loads("MyPkg/0.1@user/testing")
+        tmp_folder = temp_folder()
+        pkg1 = os.path.join(tmp_folder, "pkg1")
+        cpp_info = CppInfo(pkg1)
+        cpp_info.includedirs = ["include"]
+        save(os.path.join(pkg1, "include/file.h"), "")
+        conanfile.deps_cpp_info.update(cpp_info, ref.name)
+        ref = ConanFileReference.loads("My.Fancy-Pkg_2/0.1@user/testing")
+        pkg2 = os.path.join(tmp_folder, "pkg2")
+        cpp_info = CppInfo(pkg2)
+        cpp_info.includedirs = ["include"]
+        save(os.path.join(pkg2, "include/file.h"), "")
+        conanfile.deps_cpp_info.update(cpp_info, ref.name)
+        generator = VisualStudioGenerator(conanfile)
+
+        with tools.environment_append({"USERPROFILE": tmp_folder}):
+            content = generator.content
+            xml.etree.ElementTree.fromstring(content)
+            self.assertIn("<AdditionalIncludeDirectories>$(USERPROFILE)/pkg1/include;"
+                          "$(USERPROFILE)/pkg2/include;", content)
+
+        with tools.environment_append({"USERPROFILE": tmp_folder.upper()}):
+            content = generator.content
+            xml.etree.ElementTree.fromstring(content)
+            self.assertIn("<AdditionalIncludeDirectories>$(USERPROFILE)/pkg1/include;"
+                          "$(USERPROFILE)/pkg2/include;", content)
diff --git a/conans/test/integration/run_envronment_test.py b/conans/test/integration/run_envronment_test.py
index 48a3638b9..efd78257b 100644
--- a/conans/test/integration/run_envronment_test.py
+++ b/conans/test/integration/run_envronment_test.py
@@ -34,3 +34,68 @@ class HelloConan(ConanFile):
         client.save({"conanfile.py": reuse}, clean_first=True)
         client.run("install . --build missing")
         client.run("build .")
+        self.assertIn("Hello Hello0", client.out)
+
+    def test_shared_run_environment(self):
+        client = TestClient()
+        cmake = """set(CMAKE_CXX_COMPILER_WORKS 1)
+set(CMAKE_CXX_ABI_COMPILED 1)
+project(MyHello CXX)
+cmake_minimum_required(VERSION 2.8.12)
+
+add_library(hello SHARED hello.cpp)
+add_executable(say_hello main.cpp)
+target_link_libraries(say_hello hello)"""
+        hello_h = """#ifdef WIN32
+  #define HELLO_EXPORT __declspec(dllexport)
+#else
+  #define HELLO_EXPORT
+#endif
+
+HELLO_EXPORT void hello();
+"""
+        hello_cpp = r"""#include "hello.h"
+#include <iostream>
+void hello(){
+    std::cout<<"Hello Tool!\n";
+}
+"""
+        main = """#include "hello.h"
+        int main(){
+            hello();
+        }
+        """
+        conanfile = """from conans import ConanFile, CMake
+class Pkg(ConanFile):
+    exports_sources = "*"
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        self.copy("*say_hello.exe", dst="bin", keep_path=False)
+        self.copy("*say_hello", dst="bin", keep_path=False)
+        self.copy(pattern="*.dll", dst="bin", keep_path=False)
+        self.copy(pattern="*.dylib", dst="lib", keep_path=False)
+        self.copy(pattern="*.so", dst="lib", keep_path=False)
+"""
+        client.save({"conanfile.py": conanfile,
+                     "CMakeLists.txt": cmake,
+                     "main.cpp": main,
+                     "hello.cpp": hello_cpp,
+                     "hello.h": hello_h})
+        client.run("create . Pkg/0.1@user/testing")
+
+        reuse = '''from conans import ConanFile
+class HelloConan(ConanFile):
+    requires = "Pkg/0.1@user/testing"
+
+    def build(self):
+        self.run("say_hello", run_environment=True)
+'''
+
+        client.save({"conanfile.py": reuse}, clean_first=True)
+        client.run("install .")
+        client.run("build .")
+        self.assertIn("Hello Tool!", client.out)
diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py
index 557095f54..6927347c6 100644
--- a/conans/test/util/tools_test.py
+++ b/conans/test/util/tools_test.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from bottle import static_file, request
 import mock
 import os
 import platform
@@ -25,7 +25,8 @@ from conans.model.settings import Settings
 
 from conans.test.utils.runner import TestRunner
 from conans.test.utils.test_files import temp_folder
-from conans.test.utils.tools import TestClient, TestBufferConanOutput, create_local_git_repo
+from conans.test.utils.tools import TestClient, TestBufferConanOutput, create_local_git_repo, \
+    StoppableThreadBottle
 
 from conans.tools import which
 from conans.tools import OSInfo, SystemPackageTool, replace_in_file, AptTool, ChocolateyTool,\
@@ -33,6 +34,8 @@ from conans.tools import OSInfo, SystemPackageTool, replace_in_file, AptTool, Ch
 from conans.util.files import save, load, md5
 import requests
 
+from nose.plugins.attrib import attr
+
 
 class SystemPackageToolTest(unittest.TestCase):
     def setUp(self):
@@ -993,6 +996,51 @@ ProgramFiles(x86)=C:\Program Files (x86)
         else:
             self.assertEqual(str, type(result))
 
+    @attr('slow')
+    def get_filename_download_test(self):
+        # Create a tar file to be downloaded from server
+        with tools.chdir(tools.mkdir_tmp()):
+            import tarfile
+            tar_file = tarfile.open("sample.tar.gz", "w:gz")
+            tools.mkdir("test_folder")
+            tar_file.add(os.path.abspath("test_folder"), "test_folder")
+            tar_file.close()
+            file_path = os.path.abspath("sample.tar.gz")
+            assert(os.path.exists(file_path))
+
+        # Instance stoppable thread server and add endpoints
+        thread = StoppableThreadBottle()
+
+        @thread.server.get("/this_is_not_the_file_name")
+        def get_file():
+            return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
+
+        @thread.server.get("/")
+        def get_file2():
+            self.assertEquals(request.query["file"], "1")
+            return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
+
+        thread.run_server()
+
+        # Test: File name cannot be deduced from '?file=1'
+        with self.assertRaisesRegexp(ConanException,
+                                     "Cannot deduce file name form url. Use 'filename' parameter."):
+            tools.get("http://localhost:8266/?file=1")
+
+        # Test: Works with filename parameter instead of '?file=1'
+        with tools.chdir(tools.mkdir_tmp()):
+            tools.get("http://localhost:8266/?file=1", filename="sample.tar.gz")
+            self.assertTrue(os.path.exists("test_folder"))
+
+        # Test: Use a different endpoint but still not the filename one
+        with tools.chdir(tools.mkdir_tmp()):
+            from zipfile import BadZipfile
+            with self.assertRaises(BadZipfile):
+                tools.get("http://localhost:8266/this_is_not_the_file_name")
+            tools.get("http://localhost:8266/this_is_not_the_file_name", filename="sample.tar.gz")
+            self.assertTrue(os.path.exists("test_folder"))
+        thread.stop()
+
 
 class GitToolTest(unittest.TestCase):
 
@@ -1059,10 +1107,10 @@ class GitToolTest(unittest.TestCase):
         def _create_paths():
             tmp = temp_folder()
             submodule_path = os.path.join(
-                tmp, 
+                tmp,
                 os.path.basename(os.path.normpath(submodule)))
             subsubmodule_path = os.path.join(
-                submodule_path, 
+                submodule_path,
                 os.path.basename(os.path.normpath(subsubmodule)))
             return tmp, submodule_path, subsubmodule_path
 
@@ -1079,7 +1127,7 @@ class GitToolTest(unittest.TestCase):
         with self.assertRaisesRegexp(ConanException, "Invalid 'submodule' attribute value in the 'scm'."):
             git.clone(path, submodule="invalid")
 
-        # Check shallow 
+        # Check shallow
         tmp, submodule_path, subsubmodule_path = _create_paths()
         git = Git(tmp)
         git.clone(path, submodule="shallow")
diff --git a/conans/test/util/xz_test.py b/conans/test/util/xz_test.py
new file mode 100644
index 000000000..262549736
--- /dev/null
+++ b/conans/test/util/xz_test.py
@@ -0,0 +1,88 @@
+import os
+from unittest import TestCase
+import six
+import unittest
+import tarfile
+
+from conans.test.utils.test_files import temp_folder
+from conans.tools import unzip, save
+from conans.util.files import load, save_files
+from conans.errors import ConanException
+from conans.test.utils.tools import TestClient, TestServer
+from conans.model.ref import ConanFileReference, PackageReference
+
+
+class XZTest(TestCase):
+    def test_error_xz(self):
+        server = TestServer()
+        ref = ConanFileReference.loads("Pkg/0.1@user/channel")
+        export = server.paths.export(ref)
+        save_files(export, {"conanfile.py": "#",
+                            "conanmanifest.txt": "#",
+                            "conan_export.txz": "#"})
+        client = TestClient(servers={"default": server},
+                            users={"default": [("lasote", "mypass")]})
+        error = client.run("install Pkg/0.1@user/channel", ignore_error=True)
+        self.assertTrue(error)
+        self.assertIn("ERROR: This Conan version is not prepared to handle "
+                      "'conan_export.txz' file format", client.out)
+
+    def test_error_sources_xz(self):
+        server = TestServer()
+        ref = ConanFileReference.loads("Pkg/0.1@user/channel")
+        client = TestClient(servers={"default": server},
+                            users={"default": [("lasote", "mypass")]})
+        export = server.paths.export(ref)
+        conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    exports_sources = "*"
+"""
+        save_files(export, {"conanfile.py": conanfile,
+                            "conanmanifest.txt": "1",
+                            "conan_sources.txz": "#"})
+        error = client.run("install Pkg/0.1@user/channel --build", ignore_error=True)
+        self.assertTrue(error)
+        self.assertIn("ERROR: This Conan version is not prepared to handle "
+                      "'conan_sources.txz' file format", client.out)
+
+    def test_error_package_xz(self):
+        server = TestServer()
+        ref = ConanFileReference.loads("Pkg/0.1@user/channel")
+        client = TestClient(servers={"default": server},
+                            users={"default": [("lasote", "mypass")]})
+        export = server.paths.export(ref)
+        conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    exports_sources = "*"
+"""
+        save_files(export, {"conanfile.py": conanfile,
+                            "conanmanifest.txt": "1"})
+        pkg_ref = PackageReference(ref, "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
+        package = server.paths.package(pkg_ref)
+        save_files(package, {"conaninfo.txt": "#",
+                             "conanmanifest.txt": "1",
+                             "conan_package.txz": "#"})
+        error = client.run("install Pkg/0.1@user/channel", ignore_error=True)
+        self.assertTrue(error)
+        self.assertIn("ERROR: This Conan version is not prepared to handle "
+                      "'conan_package.txz' file format", client.out)
+
+    @unittest.skipUnless(six.PY3, "only Py3")
+    def test(self):
+        tmp_dir = temp_folder()
+        file_path = os.path.join(tmp_dir, "a_file.txt")
+        save(file_path, "my content!")
+        txz = os.path.join(tmp_dir, "sample.tar.xz")
+        with tarfile.open(txz, "w:xz") as tar:
+            tar.add(file_path, "a_file.txt")
+
+        dest_folder = temp_folder()
+        unzip(txz, dest_folder)
+        content = load(os.path.join(dest_folder, "a_file.txt"))
+        self.assertEqual(content, "my content!")
+
+    @unittest.skipUnless(six.PY2, "only Py2")
+    def test_error_python2(self):
+        with self.assertRaisesRegexp(ConanException, "XZ format not supported in Python 2"):
+            dest_folder = temp_folder()
+            unzip("somefile.tar.xz", dest_folder)
diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py
index 824cbb68e..ee5ff714b 100644
--- a/conans/test/utils/tools.py
+++ b/conans/test/utils/tools.py
@@ -2,13 +2,16 @@ import os
 import shlex
 import shutil
 import sys
+import threading
 import uuid
 from collections import Counter
 from contextlib import contextmanager
 from io import StringIO
 
+import bottle
 import requests
 import six
+import time
 from mock import Mock
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from webtest.app import TestApp
@@ -507,3 +510,24 @@ class TestClient(object):
         save_files(path, files)
         if not files:
             mkdir(self.current_folder)
+
+
+class StoppableThreadBottle(threading.Thread):
+    """
+    Real server to test download endpoints
+    """
+    server = None
+
+    def __init__(self, host="127.0.0.1", port=8266):
+        self.server = bottle.Bottle()
+        super(StoppableThreadBottle, self).__init__(target=self.server.run, kwargs={"host": host,
+                                                                                    "port": port})
+        self.daemon = True
+        self._stop = threading.Event()
+
+    def stop(self):
+        self._stop.set()
+
+    def run_server(self):
+        self.start()
+        time.sleep(1)