anon-iclr-submission/benchname-bug-localization huggingface dataset:
Repository: iterative/dvc
Issue: dvc run: regular output files should not be in .gitignore

Please add a corresponded unit-test. See examples in class `TestGitIgnoreWhenCheckout` file `test_checkout.py`.

Changed files:
tests/test_checkout.py, dvc/stage.py

diff --git a/dvc/stage.py b/dvc/stage.py
index 4bd12d16e..3a49544ba 100644
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -293,7 +293,8 @@ class Stage(object):
     def remove_outs(self):
         for out in self.outs:
             out.remove()
-            self.project.scm.ignore_remove(out.path)
+            if out.use_cache:
+                self.project.scm.ignore_remove(out.path)
 
     def remove(self):
         self.remove_outs()
@@ -353,7 +354,8 @@ class Stage(object):
         for out in self.outs:
             out.update()
             out.save()
-            self.project.scm.ignore(out.path)
+            if out.use_cache:
+                self.project.scm.ignore(out.path)
 
     def run(self):
         if not self.is_data_source:
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 9a079cbde..8528dd7b4 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -75,6 +75,7 @@ class TestGitIgnoreBasic(CheckoutBase):
     def test(self):
         fname1 = 'file_1'
         fname2 = 'file_2'
+        fname3 = 'file_3'
 
         self.dvc.scm.add(self.dvc.scm.untracked_files())
         self.dvc.scm.commit('add all files')
@@ -83,15 +84,18 @@ class TestGitIgnoreBasic(CheckoutBase):
 
         self.commit_data_file(fname1)
         self.commit_data_file(fname2)
+        self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, fname3),
+                     deps=[self.CODE, self.FOO],
+                     outs_no_cache=[fname3])
 
         self.assertTrue(os.path.exists(self.GIT_IGNORE))
 
         ignored = self.read_ignored()
 
-        self.assertEqual(len(ignored), 2)   # FAILS!!!
+        self.assertEqual(len(ignored), 2)
 
         self.assertIn(fname1, ignored)
-        self.assertIn(fname2, ignored)      # FAILS!!!
+        self.assertIn(fname2, ignored)
 
 
 class TestGitIgnoreWhenCheckout(CheckoutBase):
@@ -120,4 +124,4 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.dvc.scm.checkout(branch_1)
         main(['checkout'])
         ignored = self.read_ignored()
-        self.assertIn(fname_branch, ignored)    # FAILS!!!
+        self.assertIn(fname_branch, ignored)
Repository: iterative/dvc
Issue: dvc add dir: file exist

First, take a look at #447 

If a directory was already added then I got an error message:
```
$ cp -r data/SO raw  # <-- from #447
$ rm data/SO
$ dvc add raw
Unexpected error: [Errno 17] File exists: '/Users/dmitry/src/myrepo_1/.dvc/cache/ee3887acb78d695e2ac68dc85e05e2a1'
```

Is it because a directory 

Also, two minor issues:
1. Full file path.
2. Should be "Directory was already added", not "File exists".

One more related error:

```
$ rm raw/Tags.xml
$ dvc add raw

Unexpected error: [Errno 17] File exists
```

Changed files:
dvc/project.py, dvc/stage.py, dvc/output.py

diff --git a/dvc/output.py b/dvc/output.py
index 58034769d..f516085af 100644
--- a/dvc/output.py
+++ b/dvc/output.py
@@ -51,7 +51,7 @@ class Dependency(object):
         self.path = os.path.abspath(os.path.realpath(path))
 
         if not self.path.startswith(self.project.root_dir):
-            raise CmdOutputOutsideOfRepoError(self.path)
+            raise CmdOutputOutsideOfRepoError(self.rel_path)
 
         self.md5 = md5
 
diff --git a/dvc/project.py b/dvc/project.py
index 96f6c35de..8050cc30d 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -152,7 +152,7 @@ class Project(object):
             try:
                 result += self._reproduce_stage(stages, n, force)
             except Exception as ex:
-                raise ReproductionError(n, ex)
+                raise ReproductionError(stages[n].relpath, ex)
         return result
 
     def checkout(self):
diff --git a/dvc/stage.py b/dvc/stage.py
index 5eba993f1..e60a85a1d 100644
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -9,7 +9,7 @@ from dvc.output import Output, Dependency, OutputError
 
 class StageCmdFailedError(DvcException):
     def __init__(self, stage):
-        msg = 'Stage {} cmd {} failed'.format(stage.path, stage.cmd)
+        msg = 'Stage {} cmd {} failed'.format(stage.relpath, stage.cmd)
         super(StageCmdFailedError, self).__init__(msg)
Repository: iterative/dvc
Issue: repro: missing cache should not break repo

1) One directory in dvc cache was removed. As a result, I cannot reproduce the directory. It basically breaks the entire repository.

```
$ dvc repro
Failed to reproduce 'Dvcfile': Failed to reproduce 'train.dvc': [Errno 2] No such file or directory: '/home/ubuntu/src/dvc_tutorial/.dvc/cache/0d/d437f2f04fbf54355beab021494baa'
```

2) The same issue for dvc checkout. Not sure about right behaviour - needs to be discussed.

```
$ dvc checkout
'data/plot_loss.jpeg': cache file not found
'data/plot_acc.jpeg': cache file not found
'data/cats_and_dogs_small': cache file not found
Unexpected error: [Errno 2] No such file or directory: '/home/ubuntu/src/dvc_tutorial/.dvc/cache/da/49cf17236a5f2b4cf7046518b272d8'
```

3) (minor) It is better to replace the full path to the relative in the error message.

Changed files:
dvc/output.py

diff --git a/dvc/output.py b/dvc/output.py
index f86070e17..815c28304 100644
--- a/dvc/output.py
+++ b/dvc/output.py
@@ -237,12 +237,22 @@ class Output(Dependency):
 
     @staticmethod
     def load_dir_cache(path):
-        with open(path, 'r') as fd:
-            d = yaml.safe_load(fd)
+        if os.path.isabs(path):
+            relpath = os.path.relpath(path)
+        else:
+            relpath = path
+
+        try:
+            with open(path, 'r') as fd:
+                d = yaml.safe_load(fd)
+        except Exception as exc:
+            msg = u'Failed to load dir cache \\'{}\\''
+            Logger.error(msg.format(relpath), exc)
+            return []
 
         if not isinstance(d, list):
             msg = u'Dir cache file format error \\'{}\\': skipping the file'
-            Logger.error(msg.format(path))
+            Logger.error(msg.format(relpath))
             return []
 
         return d
Repository: iterative/dvc
Issue: File not found in cloud

Hi,

I am facing strange issue where dvc is unable to pull file from cloud when configured in jenkins. Below are the logs. Not sure, How can i debug this.

aws s3 ls s3://rahul-dvc/data/xyz/
2018-04-01 14:11:55          0 
2018-04-03 10:15:53      69474 18\\4b9aee2f5df48f1ed2cf22d3c33d07
2018-04-03 10:15:49     128425 22\\7fd4cfec8ca7afff7115041d1d23d0
2018-04-03 10:15:50    3304188 41\\8721169c96ea6dee8e13709e0fd4c4
2018-04-03 10:15:50      32763 42\\4e1fc9ad54266a6246850286b75b45
2018-04-03 10:15:53      97861 5f\\b9105c342df2de330913ae8e3767a7
+ dvc pull -v
File '/var/jenkins_home/workspace/dvcbuild/.dvc/cache/22/7fd4cfec8ca7afff7115041d1d23d0' does not exist in the cloud
File '/var/jenkins_home/workspace/dvcbuild/.dvc/cache/41/8721169c96ea6dee8e13709e0fd4c4' does not exist in the cloud
File '/var/jenkins_home/workspace/dvcbuild/.dvc/cache/18/4b9aee2f5df48f1ed2cf22d3c33d07' does not exist in the cloud
File '/var/jenkins_home/workspace/dvcbuild/.dvc/cache/42/4e1fc9ad54266a6246850286b75b45' does not exist in the cloud
File '/var/jenkins_home/workspace/dvcbuild/.dvc/cache/5f/b9105c342df2de330913ae8e3767a7' does not exist in the cloud

Changed files:
dvc/cloud/base.py, dvc/__init__.py

diff --git a/dvc/__init__.py b/dvc/__init__.py
index 72769e611..7288e58f9 100644
--- a/dvc/__init__.py
+++ b/dvc/__init__.py
@@ -5,7 +5,7 @@ Make your data science projects reproducible and shareable.
 """
 import os
 
-VERSION = '0.9.3'
+VERSION = '0.9.4'
 
 if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' and os.getenv('TRAVIS_TAG', '') == '':
     # Dynamically update version
diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py
index 9e983753d..3a23a23e6 100644
--- a/dvc/cloud/base.py
+++ b/dvc/cloud/base.py
@@ -75,7 +75,7 @@ class DataCloudBase(object):
     def cache_file_key(self, fname):
         """ Key of a file within the bucket """
         relpath = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
-        relpath.replace('\\\\', '/')
+        relpath = relpath.replace('\\\\', '/')
         return '{}/{}'.format(self.storage_prefix, relpath).strip('/')
 
     @staticmethod
Repository: iterative/dvc
Issue: dvc status error with S3 output

DVC version 0.17.1, installed with pip.

Running `dvc status` shows
```
Failed to obtain data status: 'OutputS3' object has no attribute 'rel_path'
```
and nothing else (e.g. files that are not output to S3)

My .dvc/config:
```
['remote "data"']
url = s3://xxxxx/data
[core]
remote = data
['remote "s3cache"']
url = s3://xxxxx/cache
[cache]
s3 = s3cache
```

Changed files:
tests/test_repro.py, dvc/dependency/base.py, dvc/config.py

diff --git a/dvc/config.py b/dvc/config.py
index 18f5ddd61..352065fe3 100644
--- a/dvc/config.py
+++ b/dvc/config.py
@@ -10,8 +10,9 @@ from dvc.exceptions import DvcException
 
 class ConfigError(DvcException):
     """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
+                                          ex)
 
 
 def supported_url(url):
diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py
index 96f37a60f..970fc540a 100644
--- a/dvc/dependency/base.py
+++ b/dvc/dependency/base.py
@@ -59,7 +59,11 @@ class DependencyBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            return {self.rel_path: 'changed'}
+            if self.path_info['scheme'] == 'local':
+                p = self.rel_path
+            else:
+                p = self.path
+            return {p: 'changed'}
         return {}
 
     def save(self):
diff --git a/tests/test_repro.py b/tests/test_repro.py
index ad5147cea..f5af30043 100644
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -380,6 +380,8 @@ class TestReproExternalBase(TestDvc):
 
         sleep()
 
+        self.dvc.status()
+
         stages = self.dvc.reproduce(import_stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(os.path.exists('import'))
Repository: iterative/dvc
Issue: Feature request: Better support for folder dependencies

Say I have the following structure:

```
script.py
module/submodule/foo.py
```

Then:

```
dvc run script.py -d module -o bar.npy
```

If I modify foo.py, `dvc repro` does not notice. The reason is that neither the `inode` nor the `mtime` of `module` is updated when a modification is made to `foo.py`.

For now it is possible to add all the files in `module` as dependencies but it would surely be better to have a way to handle folders recursively.

Changed files:
tests/test_repro.py, dvc/state.py

diff --git a/dvc/state.py b/dvc/state.py
index efcf142d8..56d6fadb5 100644
--- a/dvc/state.py
+++ b/dvc/state.py
@@ -96,18 +96,14 @@ class State(object):
 
     @staticmethod
     def mtime(path):
+        mtime = os.path.getmtime(path)
         if os.path.isdir(path):
-            mtime = -1
             for root, dirs, files in os.walk(path):
                 for fname in files:
                     f = os.path.join(root, fname)
                     m = os.path.getmtime(f)
                     if m > mtime:
                         mtime = m
-            if mtime == -1:
-                mtime = os.path.getmtime(path)
-        else:
-            mtime = os.path.getmtime(path)
 
         return str(int(nanotime.timestamp(mtime)))
 
diff --git a/tests/test_repro.py b/tests/test_repro.py
index 827ee8102..01f361a0c 100644
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -281,6 +281,12 @@ class TestReproChangedDirData(TestDvc):
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
+        # Check that dvc indeed registers changed output dir
+        shutil.move(self.BAR, dir_name)
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
 
 class TestReproMissingMd5InStageFile(TestRepro):
     def test(self):
Repository: iterative/dvc
Issue: Possible bug related to re-computing md5 for directories

DVC 0.29.0 / ubuntu / pip

I believe you recently fixed a bug related to re-computing md5s for large files.  There might be something similar happening again — or maybe I just need to better understand what triggers md5 to be computed.

$ dvc status
Computing md5 for a large directory project/data/images. This is only done once.
[##############################] 100% project/data/images

This happens not every time I run dvc status, but at least every time I reboot the machine — I haven't 100% narrowed down what triggers it.  Is this expected?

It's not super high priority — it only takes ~30 seconds to re-compute md5s for these directories, which is kind of surprisingly fast.  Could it be caching file-level (image-level) md5s and then simply recomputing the directory-level md5?


Changed files:
dvc/remote/local.py, tests/test_add.py

diff --git a/dvc/remote/local.py b/dvc/remote/local.py
index 3d0c9a3bf..7a81b9704 100644
--- a/dvc/remote/local.py
+++ b/dvc/remote/local.py
@@ -422,6 +422,7 @@ class RemoteLOCAL(RemoteBase):
 
         cache = self.get(md5)
         self.state.update(cache)
+        self.state.update(path, md5)
 
         if bar:
             progress.finish_target(dir_relpath)
diff --git a/tests/test_add.py b/tests/test_add.py
index cc66ffb6c..7e5a0002f 100644
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -299,3 +299,35 @@ class TestShouldNotCheckCacheForDirIfCacheMetadataDidNotChange(TestDvc):
             ret = main(["status", "{}.dvc".format(self.DATA_DIR)])
             self.assertEqual(ret, 0)
             self.assertEqual(1, remote_local_loader_spy.mock.call_count)
+
+
+class TestShouldCollectDirCacheOnlyOnce(TestDvc):
+    NEW_LARGE_DIR_SIZE = 1
+
+    @patch("dvc.remote.local.LARGE_DIR_SIZE", NEW_LARGE_DIR_SIZE)
+    def test(self):
+        from dvc.remote.local import RemoteLOCAL
+
+        collect_dir_counter = spy(RemoteLOCAL.collect_dir_cache)
+        with patch.object(
+            RemoteLOCAL, "collect_dir_cache", collect_dir_counter
+        ):
+
+            LARGE_DIR_FILES_NUM = self.NEW_LARGE_DIR_SIZE + 1
+            data_dir = "dir"
+
+            os.makedirs(data_dir)
+
+            for i in range(LARGE_DIR_FILES_NUM):
+                with open(os.path.join(data_dir, str(i)), "w+") as f:
+                    f.write(str(i))
+
+            ret = main(["add", data_dir])
+            self.assertEqual(0, ret)
+
+            ret = main(["status"])
+            self.assertEqual(0, ret)
+
+            ret = main(["status"])
+            self.assertEqual(0, ret)
+        self.assertEqual(1, collect_dir_counter.mock.call_count)
Repository: iterative/dvc
Issue: Exception is raised when adding the same data again

**DVC version:** 0.24.3

**Steps to reproduce:**

```
echo 'test' > foo
dvc add foo
dvc add foo
```

**Output:**
```Debug: UPDATE state_info SET count = 13 WHERE rowid = 1
Error: unexpected error - 'NoneType' object has no attribute 'dump'
------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/ivan/Projects/cookiecutter-data-science/dvc/.venv/lib/python2.7/site-packages/dvc/main.py", line 30, in main
    ret = cmd.run_cmd()
  File "/Users/ivan/Projects/cookiecutter-data-science/dvc/.venv/lib/python2.7/site-packages/dvc/command/base.py", line 50, in run_cmd
    return self.run()
  File "/Users/ivan/Projects/cookiecutter-data-science/dvc/.venv/lib/python2.7/site-packages/dvc/command/add.py", line 10, in run
    self.project.add(target, recursive=self.args.recursive)
  File "/Users/ivan/Projects/cookiecutter-data-science/dvc/.venv/lib/python2.7/site-packages/dvc/project.py", line 226, in add
    stage.dump()
AttributeError: 'NoneType' object has no attribute 'dump'
------------------------------------------------------------

Having any troubles? Hit us up at https://dvc.org/support, we are always happy to help!
```


Changed files:
dvc/project.py, tests/test_add.py

diff --git a/dvc/project.py b/dvc/project.py
index 57b586c9f..e198d3499 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -229,7 +229,8 @@ class Project(object):
         self._check_dag(self.stages() + stages)
 
         for stage in stages:
-            stage.dump()
+            if stage is not None:
+                stage.dump()
 
         self._remind_to_git_add()
 
diff --git a/tests/test_add.py b/tests/test_add.py
index 7ccc109a7..085c752c8 100644
--- a/tests/test_add.py
+++ b/tests/test_add.py
@@ -188,3 +188,19 @@ class TestCmdAdd(TestDvc):
 
         ret = main(["add", "non-existing-file"])
         self.assertNotEqual(ret, 0)
+
+
+class TestDoubleAddUnchanged(TestDvc):
+    def test_file(self):
+        ret = main(["add", self.FOO])
+        self.assertEqual(ret, 0)
+
+        ret = main(["add", self.FOO])
+        self.assertEqual(ret, 0)
+
+    def test_dir(self):
+        ret = main(["add", self.DATA_DIR])
+        self.assertEqual(ret, 0)
+
+        ret = main(["add", self.DATA_DIR])
+        self.assertEqual(ret, 0)
Repository: iterative/dvc
Issue: cache: dir option is not working correctly

version: `0.32.1`

-----

It raises an error when using the command:
```bash
❯ dvc cache dir
Error: unexpected error - 'NoneType' object has no attribute 'startswith'

Having any troubles? Hit us up at https://dvc.org/support, we are always happy to help!
```

Changed files:
dvc/command/cache.py, tests/test_cache.py

diff --git a/dvc/command/cache.py b/dvc/command/cache.py
index 3a3a974d9..42edab5cd 100644
--- a/dvc/command/cache.py
+++ b/dvc/command/cache.py
@@ -53,7 +53,6 @@ def add_parser(subparsers, parent_parser):
     )
     cache_dir_parser.add_argument(
         "value",
-        nargs="?",
         default=None,
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 5fb620225..26a896aff 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -128,6 +128,10 @@ class TestCacheLinkType(TestDvc):
 
 
 class TestCmdCacheDir(TestDvc):
+    def test(self):
+        ret = main(["cache", "dir"])
+        self.assertEqual(ret, 254)
+
     def test_abs_path(self):
         dname = os.path.join(os.path.dirname(self._root_dir), "dir")
         ret = main(["cache", "dir", dname])
Repository: iterative/dvc
Issue: typo in docs

super minor typo:

$dvc repro --help
 -c CWD, --cwd CWD     Directory within your repo to **reroduce** from.

dvc --version
0.30.1



Changed files:
dvc/command/repro.py

diff --git a/dvc/command/repro.py b/dvc/command/repro.py
index 05b82e879..8e582196f 100644
--- a/dvc/command/repro.py
+++ b/dvc/command/repro.py
@@ -80,7 +80,7 @@ def add_parser(subparsers, parent_parser):
         "-c",
         "--cwd",
         default=os.path.curdir,
-        help="Directory within your repo to reroduce from.",
+        help="Directory within your repo to reproduce from.",
     )
     repro_parser.add_argument(
         "-m",
Repository: iterative/dvc
Issue: dvc remove does not remove persistent outputs

https://github.com/iterative/dvc/issues/1214#issuecomment-476524131

Newly introduced --outs-persist and --outs-persist-no-cache make stage files so persistent, that even `dvc remove` is not removing them. 

Changed files:
tests/test_run.py, dvc/stage.py, dvc/repo/remove.py

diff --git a/dvc/repo/remove.py b/dvc/repo/remove.py
index 2977ee82c..80792e497 100644
--- a/dvc/repo/remove.py
+++ b/dvc/repo/remove.py
@@ -3,8 +3,8 @@ def remove(self, target, outs_only=False):
 
     stage = Stage.load(self, target)
     if outs_only:
-        stage.remove_outs()
+        stage.remove_outs(force=True)
     else:
-        stage.remove()
+        stage.remove(force=True)
 
     return stage
diff --git a/dvc/stage.py b/dvc/stage.py
index ca338bd68..0a6f9ccc8 100644
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -259,10 +259,10 @@ class Stage(object):
 
         return ret
 
-    def remove_outs(self, ignore_remove=False):
+    def remove_outs(self, ignore_remove=False, force=False):
         """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
         for out in self.outs:
-            if out.persist:
+            if out.persist and not force:
                 out.unprotect()
             else:
                 logger.debug(
@@ -276,8 +276,8 @@ class Stage(object):
         for out in self.outs:
             out.unprotect()
 
-    def remove(self):
-        self.remove_outs(ignore_remove=True)
+    def remove(self, force=False):
+        self.remove_outs(ignore_remove=True, force=force)
         os.unlink(self.path)
 
     def reproduce(
diff --git a/tests/test_run.py b/tests/test_run.py
index 86ae2f0dd..5b0694abc 100644
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -777,6 +777,14 @@ class TestRunPersist(TestDvc):
         file_content = "content"
         stage_file = file + Stage.STAGE_FILE_SUFFIX
 
+        self.run_command(file, file_content)
+        self.stage_should_contain_persist_flag(stage_file)
+
+        self.should_append_upon_repro(file, stage_file)
+
+        self.should_remove_persistent_outs(file, stage_file)
+
+    def run_command(self, file, file_content):
         ret = main(
             [
                 "run",
@@ -787,11 +795,13 @@ class TestRunPersist(TestDvc):
         )
         self.assertEqual(0, ret)
 
+    def stage_should_contain_persist_flag(self, stage_file):
         stage_file_content = load_stage_file(stage_file)
         self.assertEqual(
             True, stage_file_content["outs"][0][OutputBase.PARAM_PERSIST]
         )
 
+    def should_append_upon_repro(self, file, stage_file):
         ret = main(["repro", stage_file])
         self.assertEqual(0, ret)
 
@@ -799,6 +809,12 @@ class TestRunPersist(TestDvc):
             lines = fobj.readlines()
         self.assertEqual(2, len(lines))
 
+    def should_remove_persistent_outs(self, file, stage_file):
+        ret = main(["remove", stage_file])
+        self.assertEqual(0, ret)
+
+        self.assertFalse(os.path.exists(file))
+
 
 class TestRunPersistOuts(TestRunPersist):
     @property
Repository: iterative/dvc
Issue: move: *.dvc file not moved with target file

0.23.2

using move command does not move .dvc file, and it should.

Test:
```
class TestMoveFileToDirectory(TestDvc):
    def test(self):
        foo_dvc_file = self.FOO + ".dvc"
        ret = main(["add", self.FOO])
        self.assertEqual(ret, 0)
        self.assertTrue(os.path.exists(foo_dvc_file))

        new_foo_path = self.DATA_DIR+"/"+self.FOO
        new_foo_dvc_path = new_foo_path + ".dvc"
        ret = main(["move", self.FOO, new_foo_path])
        self.assertEqual(ret, 0)

        self.assertTrue(os.path.exists(new_foo_path))
        self.assertTrue(os.path.exists(new_foo_dvc_path))
```
last assert fails.



Changed files:
dvc/project.py, tests/test_move.py

diff --git a/dvc/project.py b/dvc/project.py
index c9242551f..90110200f 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -308,7 +308,7 @@ class Project(object):
             os.unlink(stage.path)
 
             stage.path = os.path.join(
-                os.path.dirname(stage.path),
+                os.path.dirname(to_out.path),
                 os.path.basename(to_path) + Stage.STAGE_FILE_SUFFIX
             )
 
diff --git a/tests/test_move.py b/tests/test_move.py
index ab43f623e..088eaa07c 100644
--- a/tests/test_move.py
+++ b/tests/test_move.py
@@ -2,6 +2,7 @@ import os
 
 from dvc.main import main
 from dvc.exceptions import DvcException, MoveNotDataSourceError
+from dvc.stage import Stage
 
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro
@@ -76,3 +77,21 @@ class TestMoveFileWithExtension(TestDvc):
         self.assertFalse(os.path.exists('file.csv.dvc'))
         self.assertTrue(os.path.exists('other_name.csv'))
         self.assertTrue(os.path.exists('other_name.csv.dvc'))
+
+
+class TestMoveFileToDirectory(TestDvc):
+    def test(self):
+        foo_dvc_file = self.FOO + Stage.STAGE_FILE_SUFFIX
+        ret = main(["add", self.FOO])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.exists(foo_dvc_file))
+
+        new_foo_path = self.DATA_DIR+"/"+self.FOO
+        new_foo_dvc_path = new_foo_path + Stage.STAGE_FILE_SUFFIX
+        ret = main(["move", self.FOO, new_foo_path])
+        self.assertEqual(ret, 0)
+
+        self.assertFalse(os.path.exists(self.FOO))
+        self.assertFalse(os.path.exists(foo_dvc_file))
+        self.assertTrue(os.path.exists(new_foo_path))
+        self.assertTrue(os.path.exists(new_foo_dvc_path))
Repository: iterative/dvc
Issue: How is the sync direction controlled when both local and remote files exist?

I noticed that the data files are being downloaded even though they should have been a local copy (can't verify it now)

What happens if both local and remote files exists (with the same hash)?


Changed files:
dvc/data_cloud.py

diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py
index c99e194d0..1e245f3f7 100644
--- a/dvc/data_cloud.py
+++ b/dvc/data_cloud.py
@@ -3,6 +3,7 @@ import hashlib
 import os
 import threading
 import configparser
+import tempfile
 
 from boto.s3.connection import S3Connection
 from google.cloud import storage as gc
@@ -251,10 +252,14 @@ class DataCloudAWS(DataCloudBase):
         Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name))
 
         try:
-            key.get_contents_to_filename(item.resolved_cache.relative,
+            temp_file = tempfile.NamedTemporaryFile(dir=item.resolved_cache.dirname, delete=False)
+            key.get_contents_to_filename(temp_file.name,
                                      cb=create_cb(item.resolved_cache.relative))
+            os.rename(temp_file.name, item.resolved_cache.relative)
         except Exception as exc:
             Logger.error('Failed to download "{}": {}'.format(key_name, exc))
+            if temp_file and os.path.exists(temp_file.name):
+                os.remove(temp_file.name)
             return
 
         progress.finish_target(os.path.basename(item.resolved_cache.relative))
Repository: iterative/dvc
Issue: checkout: doesn't remove excess files in directories

```
#!/bin/bash

set -e
set -x

rm -rf myrepo
mkdir myrepo
cd myrepo
git init
dvc init
git commit -s -m init

mkdir dir
echo 1 > dir/1

dvc add dir

echo 2 > dir/2

dvc checkout

ls dir/
```

dir/2 should've been removed by checkout. Related to https://github.com/iterative/dvc/commit/fc7545b6cf9579916ef0f1a0e269c76e7d4add0e


Changed files:
dvc/remote/local.py, tests/test_checkout.py

diff --git a/dvc/remote/local.py b/dvc/remote/local.py
index 317c47dbf..8816b1307 100644
--- a/dvc/remote/local.py
+++ b/dvc/remote/local.py
@@ -337,11 +337,30 @@ class RemoteLOCAL(RemoteBase):
             if bar:
                 progress.update_target(dir_relpath, processed, dir_size)
 
+        self._discard_working_directory_changes(path, dir_info)
+
         self.state.update_link(path)
 
         if bar:
             progress.finish_target(dir_relpath)
 
+    def _discard_working_directory_changes(self, path, dir_info):
+        working_dir_files = set(
+            os.path.join(root, file)
+            for root, _, files in os.walk(path)
+            for file in files
+        )
+
+        cached_files = set(
+            os.path.join(path, file['relpath'])
+            for file in dir_info
+        )
+
+        delta = working_dir_files - cached_files
+
+        for file in delta:
+            remove(file)
+
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
         # case inp and outp are in different filesystems
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 4e46fcde0..efd4c7e28 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -71,7 +71,7 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
 class TestCheckoutCorruptedCacheDir(TestDvc):
     def test(self):
         time.sleep(1)
-        
+
         # NOTE: using 'copy' so that cache and link don't have same inode
         ret = main(['config', 'cache.type', 'copy'])
         self.assertEqual(ret, 0)
@@ -159,6 +159,20 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
         self.assertFalse(os.path.exists(fname))
 
 
+class TestCheckoutCleanWorkingDir(CheckoutBase):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR)
+        stage = stages[0]
+
+        working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
+        with open(working_dir_change, 'w') as f:
+            f.write('not_cached')
+
+        ret = main(['checkout', stage.relpath])
+        self.assertEqual(ret, 0)
+        self.assertFalse(os.path.exists(working_dir_change))
+
+
 class TestCheckoutSelectiveRemove(CheckoutBase):
     def test(self):
         # Use copy to test for changes in the inodes
Repository: iterative/dvc
Issue: run/repro: check that --cwd is inside the repository

macOS High Sierra 10.13.6 (17G65)
```
$ python --version
Python 3.6.6
```

Reproduce MWE:

Clean installation:

```
$ virtualenv venv --python=python3.6
$ source venv/bin/activate
$ pip install dvc==0.19.13
$ git init
$ dvc init
```

Minimal example that works:

```
$ dvc run -o /tmp/input.txt `echo "Test" > /tmp/input.txt`
$ dvc run -d /tmp/input.txt -o /tmp/output.txt `wc /tmp/input.txt > /tmp/output.txt`
$ dvc repro output.txt.dvc
```

Minimal example that crashes:

```
$ dvc run --cwd /tmp -o /tmp/input.txt `echo "Test" > /tmp/input.txt`
$ dvc run --cwd /tmp -d /tmp/input.txt -o /tmp/output.txt `wc /tmp/input.txt > /tmp/output.txt`
$ dvc repro --cwd /tmp /tmp/output.txt.dvc
Error: Unexpected error: '../../../../tmp/output.txt.dvc'
```

Changed files:
tests/test_run.py, dvc/stage.py

diff --git a/dvc/stage.py b/dvc/stage.py
index 9020b18f9..ff2dacde0 100644
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -40,6 +40,12 @@ class StageFileBadNameError(DvcException):
         super(StageFileBadNameError, self).__init__(msg)
 
 
+class StageBadCwdError(DvcException):
+    def __init__(self, cwd):
+        msg = "Stage cwd '{}' is outside of the current dvc project"
+        super(StageBadCwdError, self).__init__(msg.format(cwd))
+
+
 class MissingDep(DvcException):
     def __init__(self, deps):
         assert len(deps) > 0
@@ -275,6 +281,13 @@ class Stage(object):
 
         return (fname, cwd)
 
+    @staticmethod
+    def _check_inside_project(project, cwd):
+        assert project is not None
+        proj_dir = os.path.realpath(project.root_dir)
+        if not os.path.realpath(cwd).startswith(proj_dir):
+            raise StageBadCwdError(cwd)
+
     @staticmethod
     def loads(project=None,
               cmd=None,
@@ -305,6 +318,8 @@ class Stage(object):
 
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
+        Stage._check_inside_project(project, cwd)
+
         cwd = os.path.abspath(cwd)
         path = os.path.join(cwd, fname)
 
diff --git a/tests/test_run.py b/tests/test_run.py
index 4c14940fd..b42c90605 100644
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -4,6 +4,7 @@ import filecmp
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage, StageFileBadNameError, MissingDep
+from dvc.stage import StageBadCwdError
 from dvc.command.run import CmdRun
 from dvc.exceptions import OutputDuplicationError, CircularDependencyError
 
@@ -101,6 +102,12 @@ class TestRunCircularDependency(TestDvc):
                          outs=[self.FOO],
                          fname='circular-dependency.dvc')
 
+class TestRunBadCwd(TestDvc):
+    def test(self):
+        with self.assertRaises(StageBadCwdError):
+            self.dvc.run(cmd='',
+                         cwd=self.mkdtemp())
+
 
 class TestCmdRun(TestDvc):
     def test_run(self):
Repository: iterative/dvc
Issue: dvc move complains about 

**Please provide information about your setup**
DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))

DVC version: 0.19.2
Method of installation: DEB

When I run `dvc move`, it throws me this error:
```
Error: Unexpected error: 'Stage' object has no attribute 'rel_path'
```

Changed files:
dvc/project.py, tests/test_move.py, dvc/exceptions.py

diff --git a/dvc/exceptions.py b/dvc/exceptions.py
index e489a45a2..552326428 100644
--- a/dvc/exceptions.py
+++ b/dvc/exceptions.py
@@ -29,3 +29,12 @@ class OutputDuplicationError(DvcException):
         for stage in stages:
             s += "\\n    " + stage
         super(OutputDuplicationError, self).__init__(msg.format(output, s))
+
+
+class MoveNotDataSourceError(DvcException):
+    def __init__(self, path):
+        msg = "Move is not permitted for stages that are not data sources. " \\
+              "You need to either move '{path}' to a new location and edit " \\
+              "it by hand, or remove '{path}' and create a new one at the " \\
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
diff --git a/dvc/project.py b/dvc/project.py
index 10691e1eb..e1fbb2f0c 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -1,6 +1,6 @@
 import os
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 from dvc.stage import Stage
 
 
@@ -209,8 +209,7 @@ class Project(object):
                         continue
 
                     if not stage.is_data_source:
-                        msg = 'Dvcfile \\'{}\\' is not a data source.'
-                        raise DvcException(msg.format(stage.rel_path))
+                        raise MoveNotDataSourceError(stage.relpath)
 
                     found = True
                     to_out = Output.loads_from(out.stage,
diff --git a/tests/test_move.py b/tests/test_move.py
index 9b81b7bb6..d12440068 100644
--- a/tests/test_move.py
+++ b/tests/test_move.py
@@ -1,9 +1,10 @@
 import os
 
 from dvc.main import main
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 
 from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro
 
 
 class TestMove(TestDvc):
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
 
         ret = main(['move', 'non-existing-file', 'dst'])
         self.assertNotEqual(ret, 0)
+
+
+class TestMoveNotDataSource(TestRepro):
+    def test(self):
+        from dvc.project import Project
+
+        self.dvc = Project(self._root_dir)
+        with self.assertRaises(MoveNotDataSourceError):
+            self.dvc.move(self.file1, 'dst')
+
+        ret = main(['move', self.file1, 'dst'])
+        self.assertNotEqual(ret, 0)
Repository: iterative/dvc
Issue: SSH: Fail to push binary data

## Description
I get an error when trying to push generated model files to my remote over ssh.

## General steps to reproduce:
* Init dvc repository
* Add an ssh remote
* Train the model using `dvc run`
* DVC push
The error I get is : `Error: Failed to push data to the cloud: sequence item 0: expected str instance, bytes found`

## More details
* First, I am just beginning to use DVC, so I may have missed a step.
* The output of `dvc run...` is a directory, which contains several files. Some are text files, but others are binary files.



Changed files:
dvc/remote/ssh.py

diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py
index a04f45419..8c5ebde07 100644
--- a/dvc/remote/ssh.py
+++ b/dvc/remote/ssh.py
@@ -152,7 +152,7 @@ class RemoteSSH(RemoteBase):
         stderr.close()
 
         if stdout.channel.recv_exit_status() != 0:
-            err = ''.join(stderr_chunks)
+            err = b''.join(stderr_chunks).decode('utf-8')
             msg = 'SSH command \\'{}\\' failed: {}'.format(cmd, err)
             raise DvcException(msg)
Repository: iterative/dvc
Issue: dvc: speedup stage collection by not visiting output dirs of the stages

Right now dvc will walk through all directories and files in the repo to find dvc files. In cases when there are some giant directories, it vastly slows down every dvc command. Need to stop visiting those directories.

Changed files:
dvc/project.py, dvc/scm.py

diff --git a/dvc/project.py b/dvc/project.py
index 98e58f59a..eb5dc80ff 100644
--- a/dvc/project.py
+++ b/dvc/project.py
@@ -746,12 +746,28 @@ class Project(object):
 
     def stages(self):
         stages = []
+        outs = []
         for root, dirs, files in os.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_stage_file(path):
                     continue
-                stages.append(Stage.load(self, path))
+                stage = Stage.load(self, path)
+                for out in stage.outs:
+                    outs.append(out.path + out.sep)
+                stages.append(stage)
+
+            def filter_dirs(dname):
+                path = os.path.join(root, dname)
+                if path == self.dvc_dir or path == self.scm.dir:
+                    return False
+                for out in outs:
+                    if path == os.path.normpath(out) or path.startswith(out):
+                        return False
+                return True
+
+            dirs[:] = list(filter(filter_dirs, dirs))
+
         return stages
 
     def active_stages(self):
diff --git a/dvc/scm.py b/dvc/scm.py
index eadaf9943..5ac1d16ca 100644
--- a/dvc/scm.py
+++ b/dvc/scm.py
@@ -99,6 +99,10 @@ class Git(Base):
         git_dir = os.path.join(root_dir, Git.GIT_DIR)
         return os.path.isdir(git_dir)
 
+    @property
+    def dir(self):
+        return self.repo.git_dir
+
     def ignore_file(self):
         return self.GITIGNORE