internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: iterative/dvc

Issue:
dvc remove does not remove persistent outputs
https://github.com/iterative/dvc/issues/1214#issuecomment-476524131

Newly introduced --outs-persist and --outs-persist-no-cache make stage files so persistent, that even `dvc remove` is not removing them. 


Discussion:

Resolving patch:
diff --git a/dvc/repo/remove.py b/dvc/repo/remove.py
--- a/dvc/repo/remove.py
+++ b/dvc/repo/remove.py
@@ -3,8 +3,8 @@ def remove(self, target, outs_only=False):
 
     stage = Stage.load(self, target)
     if outs_only:
-        stage.remove_outs()
+        stage.remove_outs(force=True)
     else:
-        stage.remove()
+        stage.remove(force=True)
 
     return stage
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -259,10 +259,10 @@ def changed(self):
 
         return ret
 
-    def remove_outs(self, ignore_remove=False):
+    def remove_outs(self, ignore_remove=False, force=False):
         """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
         for out in self.outs:
-            if out.persist:
+            if out.persist and not force:
                 out.unprotect()
             else:
                 logger.debug(
@@ -276,8 +276,8 @@ def unprotect_outs(self):
         for out in self.outs:
             out.unprotect()
 
-    def remove(self):
-        self.remove_outs(ignore_remove=True)
+    def remove(self, force=False):
+        self.remove_outs(ignore_remove=True, force=force)
         os.unlink(self.path)
 
     def reproduce(


Test patch:
diff --git a/tests/test_run.py b/tests/test_run.py
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -777,6 +777,14 @@ def _test(self):
         file_content = "content"
         stage_file = file + Stage.STAGE_FILE_SUFFIX
 
+        self.run_command(file, file_content)
+        self.stage_should_contain_persist_flag(stage_file)
+
+        self.should_append_upon_repro(file, stage_file)
+
+        self.should_remove_persistent_outs(file, stage_file)
+
+    def run_command(self, file, file_content):
         ret = main(
             [
                 "run",
@@ -787,11 +795,13 @@ def _test(self):
         )
         self.assertEqual(0, ret)
 
+    def stage_should_contain_persist_flag(self, stage_file):
         stage_file_content = load_stage_file(stage_file)
         self.assertEqual(
             True, stage_file_content["outs"][0][OutputBase.PARAM_PERSIST]
         )
 
+    def should_append_upon_repro(self, file, stage_file):
         ret = main(["repro", stage_file])
         self.assertEqual(0, ret)
 
@@ -799,6 +809,12 @@ def _test(self):
             lines = fobj.readlines()
         self.assertEqual(2, len(lines))
 
+    def should_remove_persistent_outs(self, file, stage_file):
+        ret = main(["remove", stage_file])
+        self.assertEqual(0, ret)
+
+        self.assertFalse(os.path.exists(file))
+
 
 class TestRunPersistOuts(TestRunPersist):
     @property
Repository: iterative/dvc

Issue:
`dvc remote add` replaces the existing remote silently
The `dvc remote add` command ignores the existing remote and overwrites it silently. 
```
→ dvc --version
0.32.1+7d7ed4
```

### To reproduce

```bash
dvc remote add s3 s3://bucket/subdir
dvc remote add s3 s3://bucket/subdir2
```

### Expected behavior

The second command `dvc remote add s3 s3://bucket/subdir2` should fail with the `Remote with name "s3" already exists` message.

### Current behavior

Remote URL is silently overwriten:
```bash
> cat .dvc/config
['remote "s3"']
url = s3://bucket/subdir2
```


Discussion:

Resolving patch:
diff --git a/dvc/command/remote.py b/dvc/command/remote.py
--- a/dvc/command/remote.py
+++ b/dvc/command/remote.py
@@ -5,9 +5,9 @@
 import re
 
 import dvc.logger as logger
-from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.config import Config
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import Config
 
 
 class CmdRemoteAdd(CmdConfig):
@@ -39,6 +39,14 @@ def run(self):
             )
 
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        if (section in self.configobj.keys()) and not self.args.force:
+            logger.error(
+                "Remote with name {} already exists. "
+                "Use -f (--force) to overwrite remote "
+                "with new value".format(self.args.name)
+            )
+            return 1
+
         ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
@@ -156,6 +164,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Set as default remote.",
     )
+    remote_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
     REMOTE_DEFAULT_HELP = "Set/unset default remote."


Test patch:
diff --git a/tests/test_remote.py b/tests/test_remote.py
--- a/tests/test_remote.py
+++ b/tests/test_remote.py
@@ -57,6 +57,15 @@ def test_relative_path(self):
         config = configobj.ConfigObj(self.dvc.config.config_file)
         self.assertEqual(config['remote "mylocal"']["url"], rel)
 
+    def test_overwrite(self):
+        remote_name = "a"
+        remote_url = "s3://bucket/name"
+        self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0)
+        self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1)
+        self.assertEqual(
+            main(["remote", "add", "-f", remote_name, remote_url]), 0
+        )
+
 
 class TestRemoteRemoveDefault(TestDvc):
     def test(self):
Repository: iterative/dvc

Issue:
pull fails with azure SAS connection strings
Azure SAS connection strings can be used to allow read-only access to specific container, which is useful in the context of CI pipelines and automation.

However, when using such limited-credentials connection string, `dvc pull` abends with the error below.

Some digging reveals that this is because `remote/azure.py` always tries to create the bucket (which already exists). If we check for existence before trying to create - this should work...
(will send PR)

```
[#########                     ] 30% Collecting informationClient-Request-ID=19129d6a-5393-11e9-80ab-5800e34ea0d9 Retry policy did not allow for a retry: Server-Timestamp=Sun, 31 Mar 2019 08:58:06 GMT, Server-Request-ID=c72fbeda-501e-0089-3c9f-e78298000000, HTTP status code=403, Exception=This request is not authorized to perform this operation. ErrorCode: AuthorizationFailure&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;&lt;Error&gt;&lt;Code&gt;AuthorizationFailure&lt;/Code&gt;&lt;Message&gt;This request is not authorized to perform this operation.RequestId:c72fbeda-501e-0089-3c9f-e78298000000Time:2019-03-31T08:58:06.2059267Z&lt;/Message&gt;&lt;/Error&gt;.

<font color="#CC0000">Error</font>: failed to pull data from the cloud - This request is not authorized to perform this operation. ErrorCode: AuthorizationFailure
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;&lt;Error&gt;&lt;Code&gt;AuthorizationFailure&lt;/Code&gt;&lt;Message&gt;This request is not authorized to perform this operation.
RequestId:c72fbeda-501e-0089-3c9f-e78298000000
Time:2019-03-31T08:58:06.2059267Z&lt;/Message&gt;&lt;/Error&gt;

<font color="#C4A000">Having any troubles?</font> Hit us up at <font color="#3465A4">https://dvc.org/support</font>, we are always happy to help!
```

**Please provide information about your setup**
DVC version: 0.29.0+220b4b.mod
linux x86_64 py3.6 pip



Discussion:

Resolving patch:
diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py
--- a/dvc/remote/azure.py
+++ b/dvc/remote/azure.py
@@ -6,6 +6,7 @@
 
 try:
     from azure.storage.blob import BlockBlobService
+    from azure.common import AzureMissingResourceHttpError
 except ImportError:
     BlockBlobService = None
 
@@ -77,7 +78,12 @@ def blob_service(self):
                 connection_string=self.connection_string
             )
             logger.debug("Container name {}".format(self.bucket))
-            self.__blob_service.create_container(self.bucket)
+            try:  # verify that container exists
+                self.__blob_service.list_blobs(
+                    self.bucket, delimiter="/", num_results=1
+                )
+            except AzureMissingResourceHttpError:
+                self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
     def remove(self, path_info):


Test patch:
Repository: iterative/dvc

Issue:
pull: incorrect bar
```
(3.7.0-dvc) ➜  dvc git:(dvctags) ✗ dvc pull
Preparing to download data from 's3://dvc-share/dvc/'
Preparing to collect status from s3://dvc-share/dvc/
[##############################] 100% Collecting information
[##############################] 100% Analysing status.
(1/3): [##############################] 100% dvc_up.bmp
(2/3): [##############################] 100% dvc.ico
(3/3): [##############################] 100% dvc_left.bmp
(4/3): [##############################] 100% Checkout finished!
```

looks like checkout didn't reset progress counter


Discussion:

Resolving patch:
diff --git a/dvc/progress.py b/dvc/progress.py
--- a/dvc/progress.py
+++ b/dvc/progress.py
@@ -144,6 +144,7 @@ class ProgressCallback(object):
     def __init__(self, total):
         self.total = total
         self.current = 0
+        progress.reset()
 
     def update(self, name, progress_to_add=1):
         self.current += progress_to_add


Test patch:
diff --git a/tests/unit/test_progress.py b/tests/unit/test_progress.py
--- a/tests/unit/test_progress.py
+++ b/tests/unit/test_progress.py
@@ -1,7 +1,7 @@
 import mock
 from unittest import TestCase
 
-from dvc.progress import progress, progress_aware
+from dvc.progress import progress, progress_aware, ProgressCallback
 
 
 class TestProgressAware(TestCase):
@@ -25,3 +25,13 @@ def test(self, mock_print, _):
         self.assertEqual(4, mock_print.call_count)
 
         progress.finish_target("testing")
+
+
+class TestProgressCallback(TestCase):
+    @mock.patch("dvc.progress.progress")
+    def test_should_init_reset_progress(self, progress_mock):
+        total_files_num = 1
+
+        ProgressCallback(total_files_num)
+
+        self.assertEqual([mock.call.reset()], progress_mock.method_calls)
Repository: iterative/dvc

Issue:
Can't push data on Windows machine
**Please provide information about your setup**
DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))

I'm running DVC version 0.30.1 on Windows 10. DVC was installed via binary exe file. 

I've run "pip install google.cloud.storage" and "pip install dvc[all]", but still can't push data to remote Google Could Storage bucket. 

The exception message is:

> Error: failed to push data to the cloud - URL 'gs://cylera-data-science/device_port_for_device_classification' is supported but requires these missing dependencies: [u'google.cloud.storage']. If you have installed dvc using pip, choose one of these options to proceed:

> 
>     1) Install specific missing dependencies:
>         pip install google.cloud.storage
>     2) Install dvc package that includes those missing dependencies:
>         pip install dvc[gs]
>     3) Install dvc package with all possible dependencies included:
>         pip install dvc[all]
> 

> If you have installed dvc from a binary package and you are still seeing this message, please report it to us using https://github.com/iterative/dvc/issues. Thank you!
 


Discussion:
Problem solved by uninstalling DVC installed via binary.
@richardxy , I'll leave this open, as we need to check "binary" compatibility with external packages (`pip install ...`). Thanks for the report!
@mroutis Binary is not compatible with anything since it comes as an all-in-one package. Looks like pyinstaller was not able to include gcloud in our package. Will take a look.
Thanks, @efiop ! 
Can confirm this bug also on linux when using binary packages. Investigating...

Resolving patch:
diff --git a/scripts/hooks/hook-checksumdir.py b/scripts/hooks/hook-checksumdir.py
deleted file mode 100644
--- a/scripts/hooks/hook-checksumdir.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("checksumdir")
diff --git a/scripts/hooks/hook-google.cloud.storage.py b/scripts/hooks/hook-google.cloud.storage.py
--- a/scripts/hooks/hook-google.cloud.storage.py
+++ b/scripts/hooks/hook-google.cloud.storage.py
@@ -1,3 +1,9 @@
 from PyInstaller.utils.hooks import copy_metadata
 
 datas = copy_metadata("google-cloud-storage")
+
+# NOTE: including "requests" metadata because of google-resumable-media
+# bug. See [1] for more info.
+#
+# [1] https://github.com/googleapis/google-resumable-media-python/issues/59
+datas += copy_metadata("requests")


Test patch:
Repository: iterative/dvc

Issue:
dvc status regression after 0.33.1 release
I'm using OSX/Darwin.

All the checksums in my pipelines have changed starting from version 0.33.1.

Bug very similar to https://github.com/iterative/dvc/issues/1680

E.g: With a single file added with `dvc add helico_val.h5.dvc`:

###### dvc 0.32.1: OK ✅
```
$ pip install dvc==0.32.1
$ dvc status --show-checksums helico_val.h5.dvc
+-------------------------------------------+
|                                           |
|     Update available 0.32.1 -> 0.34.1     |
|       Run pip install dvc --upgrade       |
|                                           |
+-------------------------------------------+

Pipeline is up to date. Nothing to reproduce.
```

##### dvc 0.33.1: NOK ❌
```
$ pip install dvc==0.33.1
$ dvc status --show-checksums helico_val.h5.dvc
+-------------------------------------------+
|                                           |
|     Update available 0.33.1 -> 0.34.1     |
|       Run pip install dvc --upgrade       |
|                                           |
+-------------------------------------------+

helico_val.h5.dvc:
	changed checksum
```

##### dvc 0.34.1: NOK ❌
```
$ pip install dvc==0.34.1
$ dvc status --show-checksums helico_val.h5.dvc
helico_val.h5.dvc:
	changed checksum
```

I also did the check by computing the md5 sum with `md5 helico_val.h5` and it seems that the right version of the md5 is for version <= 0.32.1






Discussion:
Hi @qchenevier !

Could you show `helico_val.h5.dvc` please?
@qchenevier Also, mind showing `dvc status helico_val.h5.dvc -v` logs for those versions?
Ok, was able to reproduce
```
#!/bin/bash

set -e
set -x

rm -rf myrepo
mkdir myrepo
cd myrepo
git init

pip uninstall -y dvc
pip install dvc==0.32.1

dvc init

echo foo > foo

dvc add foo

dvc status

pip install dvc==0.33.1

dvc status
```

Resolving patch:
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -621,7 +621,7 @@ def dump(self):
         self.repo.scm.track_file(os.path.relpath(fname))
 
     def _compute_md5(self):
-        from dvc.output.local import OutputLOCAL
+        from dvc.output.base import OutputBase
 
         d = self.dumpd()
 
@@ -643,8 +643,9 @@ def _compute_md5(self):
             d,
             exclude=[
                 self.PARAM_LOCKED,
-                OutputLOCAL.PARAM_METRIC,
-                OutputLOCAL.PARAM_TAGS,
+                OutputBase.PARAM_METRIC,
+                OutputBase.PARAM_TAGS,
+                OutputBase.PARAM_PERSIST,
             ],
         )
         logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))


Test patch:
diff --git a/tests/test_add.py b/tests/test_add.py
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -39,6 +39,7 @@ def test(self):
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(stage.cmd, None)
         self.assertEqual(stage.outs[0].info["md5"], md5)
+        self.assertEqual(stage.md5, "ee343f2482f53efffc109be83cc976ac")
 
     def test_unicode(self):
         fname = "\xe1"
Repository: iterative/dvc

Issue:
dvc: drop support for python 3.4
See https://github.com/iterative/dvc/pull/1845 . Need to not forget to remove it from our test matrixes as well.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -87,13 +87,12 @@ def run(self):
         ':python_version=="2.7"': ["futures"],
     },
     keywords="data science, data version control, machine learning",
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
+    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",


Test patch:
Repository: iterative/dvc

Issue:
Repro: logger doesn't work correctly on exception
DVC version: 0.35.5+d80137,
Platform: Linux
Method of installation: pip install from git

https://github.com/iterative/dvc/blob/54072d70b542115a78a374fa702129b6959a1d02/dvc/command/repro.py#L50-L51 

This lines should be:
```
except DvcException, msg:
            logger.exception(msg)
```




Discussion:
Great catch @olveirap ! :tada: 

@mroutis This seems to be related to the logger redesign. We used to have logger.error(), that was able to work without any message by simply printing the last exception. Should we support the same with new logger or should we simply pass something like an empty string there?
@efiop , I was using empty strings, I'll submit a patch fixing any instance.

Resolving patch:
diff --git a/dvc/command/repro.py b/dvc/command/repro.py
--- a/dvc/command/repro.py
+++ b/dvc/command/repro.py
@@ -48,7 +48,7 @@ def run(self):
                     metrics = self.repo.metrics.show()
                     show_metrics(metrics)
             except DvcException:
-                logger.exception()
+                logger.exception("")
                 ret = 1
                 break
 


Test patch:
Repository: iterative/dvc

Issue:
Logging: stderr not being catched when using subprocess
DVC version: 0.35.6+d766d3, 
Platform: Linux
Installation: pip -e .

It prints to stdout this kind of message with any command: `fatal: no tag exactly matches 'd766d32d6e82c6952571eeb7ef76d48e8eec5f98'`


Logging: stderr not being catched when using subprocess
DVC version: 0.35.6+d766d3, 
Platform: Linux
Installation: pip -e .

It prints to stdout this kind of message with any command: `fatal: no tag exactly matches 'd766d32d6e82c6952571eeb7ef76d48e8eec5f98'`




Discussion:

Resolving patch:
diff --git a/dvc/version.py b/dvc/version.py
--- a/dvc/version.py
+++ b/dvc/version.py
@@ -17,7 +17,7 @@ def _generate_version(base_version):
     if not _is_git_repo(pkg_dir) or not _have_git():
         return base_version
 
-    if _is_release(pkg_dir, base_version):
+    if _is_release(pkg_dir, base_version) and not _is_dirty(pkg_dir):
         return base_version
 
     return "{base_version}+{short_sha}{dirty}".format(
@@ -46,7 +46,9 @@ def _have_git():
 def _is_release(dir_path, base_version):
     try:
         output = subprocess.check_output(
-            ["git", "describe", "--tags", "--exact-match"], cwd=dir_path
+            ["git", "describe", "--tags", "--exact-match"],
+            cwd=dir_path,
+            stderr=subprocess.STDOUT,
         ).decode("utf-8")
         tag = output.strip()
         return tag == base_version


Test patch:
Repository: iterative/dvc

Issue:
run: deprecate `--remove-outs`
We've switched from a potentially dangerous default of not removing outputs before `dvc run` and so `--remove-outs` is now effectively enabled by default (see `Stage.run()`). So effectively, `--remove-outs` is now simply an equivalent of `--ignore-build-cache`(see `Stage.create()`). Need to reflect that in our help and our docs.


Discussion:

Resolving patch:
diff --git a/dvc/command/run.py b/dvc/command/run.py
--- a/dvc/command/run.py
+++ b/dvc/command/run.py
@@ -175,7 +175,7 @@ def add_parser(subparsers, parent_parser):
         "--remove-outs",
         action="store_true",
         default=False,
-        help="Remove outputs before running the command.",
+        help="Deprecated, this is now the default behavior",
     )
     run_parser.add_argument(
         "--no-commit",
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -471,6 +471,11 @@ def create(
 
         # NOTE: remove outs before we check build cache
         if remove_outs:
+            logger.warning(
+                "--remove-outs is deprecated."
+                " It is now the default behavior,"
+                " so there's no need to use this option anymore."
+            )
             stage.remove_outs(ignore_remove=False)
             logger.warning("Build cache is ignored when using --remove-outs.")
             ignore_build_cache = True


Test patch:
Repository: iterative/dvc

Issue:
run: make sure `persist` is ignoring build cache
We should not use build cache if there are any persistent outputs in the stage. Don't forget to put a warning on about it. See `Stage.is_cached`. Also need to not forget to reflect that in cmd help and our docs.


Discussion:

Resolving patch:
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -502,6 +502,10 @@ def create(
         else:
             stage.unprotect_outs()
 
+        if os.path.exists(path) and any(out.persist for out in stage.outs):
+            logger.warning("Build cache is ignored when persisting outputs.")
+            ignore_build_cache = True
+
         if validate_state:
             if os.path.exists(path):
                 if not ignore_build_cache and stage.is_cached:


Test patch:
diff --git a/tests/func/test_run.py b/tests/func/test_run.py
--- a/tests/func/test_run.py
+++ b/tests/func/test_run.py
@@ -1,6 +1,7 @@
 import os
 import uuid
 
+import logging
 import mock
 import shutil
 import filecmp
@@ -948,3 +949,33 @@ def test(self):
 
                     mock_run.assert_called_once()
                     mock_checkout.assert_not_called()
+
+
+class TestPersistentOutput(TestDvc):
+    def test_ignore_build_cache(self):
+        warning = "Build cache is ignored when persisting outputs."
+
+        with open("immutable", "w") as fobj:
+            fobj.write("1")
+
+        cmd = [
+            "run",
+            "--overwrite-dvcfile",
+            "--deps",
+            "immutable",
+            "--outs-persist",
+            "greetings",
+            "echo hello>>greetings",
+        ]
+
+        with self._caplog.at_level(logging.WARNING, logger="dvc"):
+            assert main(cmd) == 0
+            assert warning not in self._caplog.text
+
+            assert main(cmd) == 0
+            assert warning in self._caplog.text
+
+        # Even if the "immutable" dependency didn't change
+        # it should run the command again, as it is "ignoring build cache"
+        with open("greetings", "r") as fobj:
+            assert "hello\nhello\n" == fobj.read()