internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: iterative/dvc

Issue:
Fix GCP and test
GCP doesn't currently work out-of-the-box, need to fix it and add a test so this doesn't happen ever again.


Discussion:

Resolving patch:
diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py
--- a/dvc/command/data_sync.py
+++ b/dvc/command/data_sync.py
@@ -52,9 +52,6 @@ def run(self):
         with DvcLock(self.is_locker, self.git):
             status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
 
-            if len(list(status)) != len(self.parsed_args.targets):
-                return 1
-
             self._show(status)
 
             return 0
diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py
--- a/dvc/data_cloud.py
+++ b/dvc/data_cloud.py
@@ -579,7 +579,7 @@ class DataCloudGCP(DataCloudBase):
     """ DataCloud class for Google Cloud Platform """
     @property
     def gc_project_name(self):
-        return self._cloud_config.get('ProjectName', None)
+        return self._cloud_settings.cloud_config.get('ProjectName', None)
 
     def sanity_check(self):
         project = self.gc_project_name
@@ -614,7 +614,11 @@ def _import(self, bucket_name, key, fname, data_item):
             Logger.error('File "{}" does not exist in the cloud'.format(key))
             return None
 
-        Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))
+        if self._cmp_checksum(blob, fname):
+            Logger.debug('File "{}" matches with "{}".'.format(fname, key))
+            return data_item
+
+        Logger.debug('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))
 
         # percent_cb is not available for download_to_filename, so
         # lets at least update progress at keypoints(start, finish)
@@ -629,7 +633,7 @@ def _import(self, bucket_name, key, fname, data_item):
 
         progress.finish_target(name)
 
-        Logger.info('Downloading completed')
+        Logger.debug('Downloading completed')
 
         return data_item
 
@@ -654,7 +658,7 @@ def push(self, data_item):
         blob.upload_from_filename(data_item.resolved_cache.relative)
 
         progress.finish_target(name)
-        Logger.info('uploading %s completed' % data_item.resolved_cache.relative)
+        Logger.debug('uploading %s completed' % data_item.resolved_cache.relative)
 
         return data_item
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -94,6 +94,7 @@
     'google-compute-engine==2.4.1',
     'configobj==5.0.6',
     'networkx==1.11',
+    'gcs-oauth2-boto-plugin==1.14',
 ]
 
 if platform.system() == 'Darwin':


Test patch:
Repository: iterative/dvc

Issue:
dvc repro error
```$ dvc repro
Exception caught in CmdRepro
Traceback (most recent call last):
  File "/Users/dmitry/src/dvc/dvc/main.py", line 18, in main
    ret = instance.run()
  File "/Users/dmitry/src/dvc/dvc/command/repro.py", line 50, in run
    target = self.settings.config.get('Target', None)
AttributeError: 'Config' object has no attribute 'get'```


Discussion:
A similar issue with 
```$ dvc show workflow
Exception caught in CmdShowWorkflow
Traceback (most recent call last):
  File "/Users/dmitry/src/dvc/dvc/main.py", line 18, in main
    ret = instance.run()
  File "/Users/dmitry/src/dvc/dvc/command/show_workflow.py", line 14, in run
    target = self.settings.config.target_file
AttributeError: 'Config' object has no attribute 'target_file'
```

Resolving patch:
diff --git a/dvc/command/repro.py b/dvc/command/repro.py
--- a/dvc/command/repro.py
+++ b/dvc/command/repro.py
@@ -47,7 +47,7 @@ def run(self):
             if self.parsed_args.target:
                 targets += self.parsed_args.target
             else:
-                target = self.settings.config.get('Target', None)
+                target = self.settings.config._config['Global'].get('Target', None)
                 if not target or len(target) == 0:
                     Logger.error('Reproduction target is not defined. ' +
                                  'Specify data file or set target by ' +
diff --git a/dvc/command/show_workflow.py b/dvc/command/show_workflow.py
--- a/dvc/command/show_workflow.py
+++ b/dvc/command/show_workflow.py
@@ -11,14 +11,8 @@ def __init__(self, settings):
     def run(self):
         target = self.settings.parsed_args.target
         if not target:
-            target = self.settings.config.target_file
-
-            if not os.path.exists(target):
-                Logger.warn(u'Target is not defined: use empty target')
-                target = ''
-            else:
-                target = open(target).read()
-                Logger.debug(u'Set show workflow target as {}'.format(target))
+            target = self._settings.config._config['Global'].get('Target', '')
+            Logger.debug(u'Set show workflow target as {}'.format(target))
 
         wf = self.git.get_all_commits(target, self.settings)
         wf.build_graph(self.settings.parsed_args.dvc_commits, self.settings.parsed_args.all_commits)


Test patch:
Repository: iterative/dvc

Issue:
Workflow error
```
(dvc) ➜  myrepo git:(master) dvc show workflow data/evaluation.txtException caught in CmdShowWorkflow
Traceback (most recent call last):
  File "/home/efiop/virtenvs/dvc/lib/python2.7/site-packages/dvc/main.py", line 18, in main
    ret = instance.run()
  File "/home/efiop/virtenvs/dvc/lib/python2.7/site-packages/dvc/command/show_workflow.py", line 17, in run
    wf = self.git.get_all_commits(target, self.settings)
  File "/home/efiop/virtenvs/dvc/lib/python2.7/site-packages/dvc/git_wrapper.py", line 258, in get_all_commits
    branch_tips=branches_multimap.get(hash))
  File "/home/efiop/virtenvs/dvc/lib/python2.7/site-packages/dvc/git_wrapper.py", line 272, in is_target
    metric = self.target_metric_from_git_history(hash, symlink_content, target, settings)
  File "/home/efiop/virtenvs/dvc/lib/python2.7/site-packages/dvc/git_wrapper.py", line 318, in target_metric_from_git_history
    return float(lines[0])
ValueError: could not convert string to float: AUC: 0.596182

```


Discussion:

Resolving patch:
diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py
--- a/dvc/git_wrapper.py
+++ b/dvc/git_wrapper.py
@@ -309,15 +309,23 @@ def target_metric_from_git_history(self, hash, symlink_content, target, settings
         file_name = os.path.join(settings.config.cache_dir, cache_file_name)
         full_file_name = os.path.join(self.git_dir_abs, file_name)
 
-        if os.path.exists(full_file_name):
-            lines = open(full_file_name).readlines(2)
-            if len(lines) != 1:
-                msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.'
-                Logger.warn(msg.format(target, hash, len(lines)))
-            else:
-                return float(lines[0])
+        if not os.path.exists(full_file_name):
+            return None
+
+        lines = open(full_file_name).readlines(2)
+        if len(lines) != 1:
+            msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.'
+            Logger.warn(msg.format(target, hash, len(lines)))
+            return None
+
+        # Extract float from string. I.e. from 'AUC: 0.596182'
+        num = re.findall("\d+\.\d+", lines[0])
+        if len(num) != 1:
+            msg = '[dvc-git] Unable to parse metrics from \'{}\' file {}'
+            Logger.warn(msg.format(lines[0], target))
+            return None
 
-        return None
+        return float(num[0])
 
     @staticmethod
     def get_merges_map():


Test patch:
Repository: iterative/dvc

Issue:
Get rid of bits and pieces of 'dvc test' command
It is old and not working. The duties of it are already spread across 'sanity_check' in data_cloud and other places.


Discussion:

Resolving patch:
diff --git a/dvc/cli.py b/dvc/cli.py
--- a/dvc/cli.py
+++ b/dvc/cli.py
@@ -15,7 +15,6 @@
 from dvc.command.instance_create import CmdInstanceCreate
 from dvc.command.config import CmdConfig
 from dvc.command.show_pipeline import CmdShowPipeline
-from dvc.command.test import CmdTest
 from dvc.config import Config
 from dvc import VERSION
 


Test patch:
diff --git a/dvc/command/test.py b/dvc/command/test.py
deleted file mode 100644
--- a/dvc/command/test.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-import sys
-
-from google.cloud import storage
-
-import dvc
-from dvc.command.base import CmdBase
-from dvc.logger import Logger
-from dvc.config import Config
-
-class CmdTest(CmdBase):
-    def __init__(self, settings):
-        super(CmdTest, self).__init__(settings)
-
-    def define_args(self, parser):
-        self.set_no_git_actions(parser)
-
-    def run(self):
-
-        cloud = self.settings._config.cloud
-        assert cloud in ['amazon', 'google'], 'unknown cloud: %s' % cloud
-        if cloud == 'amazon':
-            print('TODO')
-            sys.exit(-1)
-        elif cloud == 'google':
-            Logger.info('testing gcloud')
-
-            if self.settings._config.gc_project_name == '':
-                Logger.error('Please specify the google cloud project name in dvc.conf')
-                raise dvc.settings.SettingsError('must specify GC ProjectName')
-
-            try:
-                client = storage.Client(project=self.settings._config.gc_project_name)
-            except:
-                Logger.error('initializing storage.Client raised')
-                raise
-
-            sb = self.config.storage_bucket
-            if sb == '':
-                Logger.error('Please specificy a bucket for google cloud in dvc.conf')
-                sys.exit(-1)
-
-            bucket = client.bucket(sb)
-            if not bucket.exists():
-                Logger.error('google cloud storage: bucket %s doesn\'t exist' % sb)
-                sys.exit(-1)
-            Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb)
-            sys.exit(0)
Repository: iterative/dvc

Issue:
Print exceptions only in verbose mode



Discussion:

Resolving patch:
diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py
--- a/dvc/git_wrapper.py
+++ b/dvc/git_wrapper.py
@@ -103,7 +103,16 @@ class GitWrapper(GitWrapperI):
     def __init__(self):
         super(GitWrapper, self).__init__()
 
+    def is_inside_work_tree(self):
+        code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree'])
+        if code != 0:
+            Logger.error('[dvc-git] Not inside git tree');
+        return code == 0
+
     def is_ready_to_go(self):
+        if not self.is_inside_work_tree():
+            return False
+
         statuses = self.git_file_statuses()
         if len(statuses) > 0:
             Logger.error('[dvc-git] Commit all changed files before running '
@@ -112,6 +121,9 @@ def is_ready_to_go(self):
                 Logger.error("{} {}".format(status, file))
             return False
 
+        return self.check_config()
+
+    def check_config(self):
         # Sanity check to make sure we will be able to commit
         name = self.git_config_get('user.name')
         if name == None:
diff --git a/dvc/main.py b/dvc/main.py
--- a/dvc/main.py
+++ b/dvc/main.py
@@ -5,19 +5,23 @@
 import sys
 from dvc.settings import Settings
 from dvc.logger import Logger
+from dvc.config import ConfigError
 
 def main():
     try:
         settings = Settings(sys.argv[1:])
         instance = settings._parsed_args.func(settings)
     except Exception as e:
-        Logger.error("Exception caught while parsing settings", exc_info=True)
+        # In case we didn't even manage to parse options
+        exc_info = '-v' in sys.argv or '--verbose' in sys.argv
+        Logger.error("Settings error: {}".format(e), exc_info=exc_info)
         return 255
 
     try:
         ret = instance.run()
     except Exception as e:
-        Logger.error("Exception caught in " + instance.__class__.__name__, exc_info=True)
+        exc_info = settings.parsed_args.verbose
+        Logger.error("{} error: {}".format(instance.__class__.__name__, e), exc_info=exc_info)
         return 254
 
     return ret


Test patch:
Repository: iterative/dvc

Issue:
Deprecate 'dvc sync' command
Per the discussion with Dmitry P., **dvc sync** should be deprecated in the system.

In particular, it means that

- the new versions of dvc should not support it anymore
- the output of **dvc -h** should not list the information on _sync_ command


Discussion:

Resolving patch:
diff --git a/dvc/cli.py b/dvc/cli.py
--- a/dvc/cli.py
+++ b/dvc/cli.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import sys
 import argparse
 from multiprocessing import cpu_count
 
@@ -7,7 +8,7 @@
 from dvc.command.remove import CmdRemove
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
-from dvc.command.data_sync import CmdDataSync, CmdDataPush, CmdDataPull, CmdDataStatus
+from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataStatus
 from dvc.command.lock import CmdLock
 from dvc.command.gc import CmdGC
 from dvc.command.import_file import CmdImportFile
@@ -55,6 +56,11 @@ def parse_args(argv=None):
                         dest='cmd',
                         help='Use dvc CMD --help for command-specific help')
 
+    # NOTE: Workaround for bug in Python 3
+    if sys.version_info[0] == 3:
+        subparsers.required = True
+        subparsers.dest = 'cmd'
+
     # Init
     init_parser = subparsers.add_parser(
                         'init',
@@ -122,13 +128,6 @@ def parse_args(argv=None):
                         default=cpu_count(),
                         help='Number of jobs to run simultaneously.')
 
-    # Sync
-    sync_parser = subparsers.add_parser(
-                        'sync',
-                        parents=[parent_sync_parser],
-                        help='Synchronize data file with cloud (cloud settings already setup.')
-    sync_parser.set_defaults(func=CmdDataSync)
-
     # Pull
     pull_parser = subparsers.add_parser(
                         'pull',
diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py
--- a/dvc/command/data_sync.py
+++ b/dvc/command/data_sync.py
@@ -3,14 +3,6 @@
 
 import dvc.data_cloud as cloud
 
-class CmdDataSync(CmdBase):
-    def __init__(self, settings):
-        super(CmdDataSync, self).__init__(settings)
-
-    def run(self):
-        with DvcLock(self.is_locker, self.git):
-            self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs)
-
 
 class CmdDataPull(CmdBase):
     def __init__(self, settings):


Test patch:
Repository: iterative/dvc

Issue:
Read only hardlinks
Should we make the hardlinks\data-files read-only?


Discussion:

Resolving patch:
diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py
--- a/dvc/path/data_item.py
+++ b/dvc/path/data_item.py
@@ -1,4 +1,5 @@
 import os
+import stat
 import shutil
 
 from dvc.path.path import Path
@@ -104,6 +105,6 @@ def move_data_to_cache(self):
         md5 = file_md5(self.data.relative)[0]
         self._cache_file = os.path.join(self.cache_dir, md5)
         self._git.modify_gitignore([self.data.relative])
-        if os.path.isfile(self.cache.relative):
-            return
-        System.hardlink(self.data.relative, self.cache.relative)
+        if not os.path.isfile(self.cache.relative):
+            System.hardlink(self.data.relative, self.cache.relative)
+        os.chmod(self.data.relative, stat.S_IREAD)
diff --git a/dvc/path/factory.py b/dvc/path/factory.py
--- a/dvc/path/factory.py
+++ b/dvc/path/factory.py
@@ -4,7 +4,7 @@
 from dvc.path.path import Path
 from dvc.path.stated_data_item import StatedDataItem
 from dvc.system import System
-
+from dvc.logger import Logger
 
 class PathFactory(object):
     def __init__(self, git, config):
diff --git a/dvc/repository_change.py b/dvc/repository_change.py
--- a/dvc/repository_change.py
+++ b/dvc/repository_change.py
@@ -7,7 +7,7 @@
 from dvc.path.data_item import DataDirError, DataItemInStatusDirError
 from dvc.path.stated_data_item import StatedDataItem
 from dvc.utils import cached_property
-
+from dvc.config import ConfigI
 
 class RepositoryChangeError(DvcException):
     def __init__(self, msg):
@@ -116,7 +116,10 @@ def created_status_files(self):
 
     def data_file_timesteps(self):
         res = set()
-        for root, dirs, files in os.walk('.'):
+        for root, dirs, files in os.walk(self._settings.git.git_dir_abs):
+            if root.startswith(os.path.join(self._settings.git.git_dir_abs, ConfigI.CONFIG_DIR)):
+                continue
+
             for file in files:
                 filename = os.path.join(root, file)
                 if os.path.exists(filename):


Test patch:
Repository: iterative/dvc

Issue:
dvc repro after merge
`dvc repro` rerun command after merge if data files were generated in different branches.

Please take a look at `functest/tutorial_branches.sh` from beginning to line 61. In theory, in this line only steps after `Posts.tsv` should be reproduced.


Discussion:
The issue still exists with the new hash sum logic. We still use `git diff --name-only HEAD 34ef5a` command when looking for changes (see `globally_changed_files` in repro logic).

To reproduce go to line 66 in the `functest/tutorial_branches.sh`. This repro command re-run all the steps, including the first two (tar and run) which are not actually needed.
And it is not directly related to the merge command.
The issue is related to `dvc merge` which changes state files for data sources\imports. All data source files become "state-file changed" and require reproduction.

Possible fix - ignore state file changes for data sources. It can potentially break some scenarios.

Resolving patch:
diff --git a/dvc/command/repro.py b/dvc/command/repro.py
--- a/dvc/command/repro.py
+++ b/dvc/command/repro.py
@@ -85,6 +85,7 @@ def repro_data_items(self, data_item_list, recursive, force):
                     continue
 
                 globally_changed_files = self.git.get_changed_files(target_commit)
+                Logger.debug('Repro globally changed files: {}'.format(', '.join(globally_changed_files)))
                 changed_files = set()
                 change = ReproChange(data_item, self, globally_changed_files, recursive, force)
                 if change.reproduce(changed_files):
@@ -244,9 +245,13 @@ def were_dependencies_changed(self, changed_files, data_item_dvc):
                 Logger.debug(u'Repro data item {}: dependency {} was changed'.format(
                     data_item_dvc, data_item.data.dvc))
             elif data_item.state.dvc in self._globally_changed_files:
-                msg = u'Repro data item {}: dependency {} was not changed but the state file was changed'
-                Logger.debug(msg.format(data_item_dvc, data_item.data.dvc))
-                result = True
+                dep_state = StateFile.load(data_item, self._settings)
+                if dep_state.command != StateFile.COMMAND_IMPORT_FILE:
+                    msg = u'Repro data item {}: dependency {} was not changed but the state file was changed'
+                    Logger.debug(msg.format(data_item_dvc, data_item.data.dvc))
+                    result = True
+                else:
+                    Logger.debug(u'Repro data item {}: skip dependency state file change {} due to data source command')
             else:
                 msg = u'Repro data item {}: dependency {} was not changed'
                 Logger.debug(msg.format(data_item_dvc, data_item.data.dvc))


Test patch:
Repository: iterative/dvc

Issue:
Add a test for partial repro, when first few steps do not require reproduction



Discussion:

Resolving patch:
diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py
--- a/dvc/path/data_item.py
+++ b/dvc/path/data_item.py
@@ -20,9 +20,9 @@ def __init__(self, msg):
         super(DataDirError, self).__init__(msg)
 
 
-class DataItemInStatusDirError(DataDirError):
+class DataItemInConfigDirError(DataDirError):
     def __init__(self, file):
-        msg = 'File "{}" is in state directory'.format(file)
+        msg = 'File "{}" is in config directory'.format(file)
         super(DataItemInStatusDirError, self).__init__(msg)
 
 
@@ -48,10 +48,8 @@ def __init__(self, data_file, git, config, cache_file=None):
         if not self._data.abs.startswith(self._git.git_dir_abs):
             raise NotInGitDirError(data_file, self._git.git_dir_abs)
 
-        if self._data.abs.startswith(self.state_dir_abs):
-            raise DataItemInStatusDirError(data_file)
-
-        pass
+        if self._data.abs.startswith(os.path.join(self._git.git_dir_abs, self._config.CONFIG_DIR)):
+            raise DataItemInConfigDirError(data_file)
 
     def copy(self, cache_file=None):
         if not cache_file:
diff --git a/dvc/repository_change.py b/dvc/repository_change.py
--- a/dvc/repository_change.py
+++ b/dvc/repository_change.py
@@ -4,7 +4,7 @@
 from dvc.git_wrapper import GitWrapper
 from dvc.executor import Executor
 from dvc.logger import Logger
-from dvc.path.data_item import DataDirError, DataItemInStatusDirError
+from dvc.path.data_item import DataDirError, DataItemInConfigDirError
 from dvc.path.stated_data_item import StatedDataItem
 from dvc.utils import cached_property
 from dvc.config import ConfigI
@@ -74,7 +74,7 @@ def _add_stated_data_item(self, state, file):
             Logger.debug('[Repository change] Add status: {} {}'.format(
                          item.status,
                          item.data.dvc))
-        except DataItemInStatusDirError:
+        except DataItemInConfigDirError:
             self._created_status_files.append(file)
         except DataDirError:
             self._externally_created_files.append(file)


Test patch:
Repository: iterative/dvc

Issue:
local_state files are not in .gitignore after dvc checkout
```
# A few data files and corresponded local_state files were created.
$ git checkout master
$ git status
...
.dvc/state/     # <-- something is wrong
... 

$ git status .dvc/state/data/
...
	.dvc/state/data/Posts.tsv.local_state
	.dvc/state/data/Posts.xml.local_state
	.dvc/state/data/Posts.xml.tgz.local_state
...
```

This happens because of `.dvc/state/data/.gitignore` was not "checkouted".

Possible solutions:
1. Check out `.gitignore` as part of `dvc checkout`
2. Keep all cache files in a separate dir like `.dvc/local_state` (should we clean up this dir after `checkout`)

UPDATE:
3. Add `*.local_state` into the global `.gitignore` diring `dvc init`


Discussion:
It looks like solution global .gitignore (solution 3) works fine.

Resolving patch:
diff --git a/dvc/command/init.py b/dvc/command/init.py
--- a/dvc/command/init.py
+++ b/dvc/command/init.py
@@ -5,6 +5,7 @@
 from dvc.logger import Logger
 from dvc.config import Config
 from dvc.exceptions import DvcException
+from dvc.path.data_item import DataItem
 
 
 class InitError(DvcException):
@@ -107,7 +108,8 @@ def run(self):
         conf_file.close()
 
         self.git.modify_gitignore([os.path.join(config_dir_path.name, cache_dir_path.name),
-                                   os.path.join(config_dir_path.name, os.path.basename(self.git.lock_file))])
+                                   os.path.join(config_dir_path.name, os.path.basename(self.git.lock_file)),
+                                   os.path.join(config_dir_path.name, state_dir_path.name, '*' + DataItem.LOCAL_STATE_FILE_SUFFIX)])
 
         message = 'DVC init. cache dir {}, state dir {}, '.format(cache_dir_path.name, state_dir_path.name)
         return self.commit_if_needed(message)
diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py
--- a/dvc/path/data_item.py
+++ b/dvc/path/data_item.py
@@ -125,4 +125,3 @@ def move_data_to_cache(self):
         cache_state = CacheStateFile(self).save()
 
         local_state = LocalStateFile(self).save()
-        self._git.modify_gitignore([self.local_state.relative])
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -95,6 +95,7 @@
     'configobj==5.0.6',
     'networkx==1.11',
     'gcs-oauth2-boto-plugin==1.14',
+    'pyyaml==3.12',
 ]
 
 if platform.system() == 'Darwin':


Test patch: