Repository: codalab/codalab-worksheets

Issue:
[USER ISSUE] cl wadd across instances doesn't seem to work
**Describe the bug**

Apologies if I'm just messing up something very simple here:

```
$ cl alias
localhost: http://localhost:2900
main: https://worksheets.codalab.org
stanford: http://codalab.stanford.edu

$ cl work main::home-nfliu
Switched to worksheet: https://worksheets.codalab.org/worksheets/0x6c26647dad70404eb6ea6a01ed8b6a99 (home-nfliu)

$ cl ls
### Worksheet: https://worksheets.codalab.org::home-nfliu(0x6c26647dad70404eb6ea6a01ed8b6a99)
### Title: <none>
### Tags:
### Owner: nfliu(0x23c379b450cb4552b314268c413e2468)
### Permissions: public(0xc573c2):read
ref  uuid      name          summary  owner  created              data_size  state
----------------------------------------------------------------------------------
 ^2  0xd9fb67  run-allennlp           nfliu  2020-02-19 16:54:43       1.4g  ready
 ^1  0xb3c03f  run-echo               nfliu  2020-04-01 15:12:55       4.0k  ready

$ cl work stanford::
Switched to worksheet: http://codalab.stanford.edu/worksheets/0x8e694eac468f4763aa23b112561447ce (home-nfliu)

$ cl new nfliu-test
0xf74aaee3663c4281b8ed0e7258c41954

$ cl wadd main::home-nfliu stanford::nfliu-test
Copying run-allennlp(0xd9fb67a5bdc4483f833ed8a66062f8e5)...
Traceback (most recent call last):
  File "/Users/nfliu/miniconda3/envs/ood-variance/bin/cl", line 11, in <module>
    load_entry_point('codalab', 'console_scripts', 'cl')()
  File "/Users/nfliu/git/codalab-worksheets/codalab/bin/cl.py", line 10, in main
    cli.do_command(sys.argv[1:])
  File "/Users/nfliu/git/codalab-worksheets/codalab/lib/bundle_cli.py", line 864, in do_command
    structured_result = command_fn()
  File "/Users/nfliu/git/codalab-worksheets/codalab/lib/bundle_cli.py", line 858, in <lambda>
    command_fn = lambda: args.function(self, args)
  File "/Users/nfliu/git/codalab-worksheets/codalab/lib/bundle_cli.py", line 3297, in do_wadd_command
    add_to_worksheet=False,
  File "/Users/nfliu/git/codalab-worksheets/codalab/lib/bundle_cli.py", line 1433, in copy_bundle
    target_info = source_client.fetch_contents_info((source_bundle_uuid, ''))
  File "/Users/nfliu/git/codalab-worksheets/codalab/client/json_api_client.py", line 16, in wrapper
    return f(*args, **kwargs)
  File "/Users/nfliu/git/codalab-worksheets/codalab/client/json_api_client.py", line 594, in fetch_contents_info
    target.bundle_uuid,
AttributeError: 'tuple' object has no attribute 'bundle_uuid'
```


**To Reproduce**
Steps to reproduce the behavior :
1. Try to wadd from one instance to another.


Discussion:
candicegjing: Ran tests locally and seemed to be working. 

Patch:
diff --git a/codalab/lib/bundle_cli.py b/codalab/lib/bundle_cli.py
index e0bb8135..ae698044 100644
--- a/codalab/lib/bundle_cli.py
+++ b/codalab/lib/bundle_cli.py
@@ -1276,6 +1276,12 @@ class BundleCLI(object):
                 '--output-path',
                 help='Path to download bundle to.  By default, the bundle or subpath name in the current directory is used.',
             ),
+            Commands.Argument(
+                '-f',
+                '--force',
+                action='store_true',
+                help='Overwrite the output path if a file already exists.',
+            ),
             Commands.Argument(
                 '-w',
                 '--worksheet-spec',
@@ -1306,8 +1312,11 @@ class BundleCLI(object):
             )
         final_path = os.path.join(os.getcwd(), local_path)
         if os.path.exists(final_path):
-            print('Local file/directory \'%s\' already exists.' % local_path, file=self.stdout)
-            return
+            if args.force:
+                shutil.rmtree(final_path)
+            else:
+                print('Local file/directory \'%s\' already exists.' % local_path, file=self.stdout)
+                return
 
         # Do the download.
         target_info = client.fetch_contents_info(target, 0)
@@ -1421,7 +1430,7 @@ class BundleCLI(object):
 
         # If bundle contents don't exist, finish after just copying metadata
         try:
-            target_info = source_client.fetch_contents_info((source_bundle_uuid, ''))
+            target_info = source_client.fetch_contents_info(BundleTarget(source_bundle_uuid, ''))
         except NotFoundError:
             return
 
diff --git a/codalab/server/bundle_manager.py b/codalab/server/bundle_manager.py
index 83909a22..ee614379 100644
--- a/codalab/server/bundle_manager.py
+++ b/codalab/server/bundle_manager.py
@@ -475,7 +475,7 @@ class BundleManager(object):
             # 6. break ties randomly by a random seed.
             return (
                 not worker['tag_exclusive'],
-                worker['gpus'],
+                worker['gpus'] or worker['has_gpus'],
                 -num_available_deps,
                 worker['cpus'],
                 len(worker['run_uuids']),
diff --git a/codalab/server/worker_info_accessor.py b/codalab/server/worker_info_accessor.py
index 2ab38754..6285fc4a 100644
--- a/codalab/server/worker_info_accessor.py
+++ b/codalab/server/worker_info_accessor.py
@@ -33,6 +33,9 @@ class WorkerInfoAccessor(object):
             for uuid in worker['run_uuids']:
                 self._uuid_to_worker[uuid] = worker
             self._user_id_to_workers[worker['user_id']].append(worker)
+            # 'gpus' field contains the number of free GPUs that comes with each worker. Adding an additional
+            # 'has_gpus' flag here to indicate if the current worker has GPUs or not.
+            worker['has_gpus'] = True if worker['gpus'] > 0 else False
 
     @refresh_cache
     def workers(self):
diff --git a/docs/CLI-Reference.md b/docs/CLI-Reference.md
index 41ddc4f0..fa98e14e 100644
--- a/docs/CLI-Reference.md
+++ b/docs/CLI-Reference.md
@@ -196,6 +196,7 @@ Usage: `cl <command> <arguments>`
     Arguments:
       target_spec           [[(<alias>|<address>)::](<uuid>|<name>)//](<uuid>|<name>|^<index>)[/<subpath within bundle>]
       -o, --output-path     Path to download bundle to.  By default, the bundle or subpath name in the current directory is used.
+      -f, --force           Overwrite the output path if a file already exists.
       -w, --worksheet-spec  Operate on this worksheet ([(<alias>|<address>)::](<uuid>|<name>)).
 
 ### mimic:


Test patch:
diff --git a/tests/server/bundle_manager_test.py b/tests/server/bundle_manager_test.py
index c753cd1e..9ea1ace0 100644
--- a/tests/server/bundle_manager_test.py
+++ b/tests/server/bundle_manager_test.py
@@ -49,6 +49,8 @@ class BundleManagerTest(unittest.TestCase):
                 'dependencies': [(1, ''), (2, '')],
                 'shared_file_system': False,
                 'tag_exclusive': False,
+                # An additional flag that is generated from WorkerInfoAccessor class
+                'has_gpus': True,
             },
             {
                 'worker_id': 1,
@@ -60,17 +62,20 @@ class BundleManagerTest(unittest.TestCase):
                 'dependencies': [(1, ''), (2, ''), (3, ''), (4, '')],
                 'shared_file_system': False,
                 'tag_exclusive': False,
+                'has_gpus': True,
             },
+            # the value of GPUs has been deducted to 0 at this point even though this worker has GPUs
             {
                 'worker_id': 2,
                 'cpus': 4,
-                'gpus': 1,
+                'gpus': 0,
                 'memory_bytes': 4 * 1000,
                 'tag': None,
                 'run_uuids': [],
                 'dependencies': [(1, ''), (2, ''), (3, ''), (4, '')],
                 'shared_file_system': False,
                 'tag_exclusive': False,
+                'has_gpus': True,
             },
             {
                 'worker_id': 3,
@@ -82,20 +87,35 @@ class BundleManagerTest(unittest.TestCase):
                 'dependencies': [(1, '')],
                 'shared_file_system': False,
                 'tag_exclusive': False,
+                'has_gpus': False,
             },
             {
                 'worker_id': 4,
                 'cpus': 6,
                 'gpus': 0,
+                'memory_bytes': 4 * 1000,
+                'tag': None,
+                'run_uuids': [1, 2],
+                'dependencies': [(1, '')],
+                'shared_file_system': False,
+                'tag_exclusive': False,
+                'has_gpus': False,
+            },
+            # Tagged workers
+            {
+                'worker_id': 5,
+                'cpus': 6,
+                'gpus': 0,
                 'memory_bytes': 2 * 1000,
                 'tag': 'worker_X',
                 'run_uuids': [],
                 'dependencies': [],
                 'shared_file_system': False,
                 'tag_exclusive': False,
+                'has_gpus': False,
             },
             {
-                'worker_id': 5,
+                'worker_id': 6,
                 'cpus': 6,
                 'gpus': 0,
                 'memory_bytes': 2 * 1000,
@@ -104,31 +124,42 @@ class BundleManagerTest(unittest.TestCase):
                 'dependencies': [],
                 'shared_file_system': False,
                 'tag_exclusive': True,
+                'has_gpus': False,
             },
         ]
         return workers_list
 
-    def test__filter_and_sort_workers_gpus(self):
+    def test_filter_and_sort_workers_gpus(self):
         # Only GPU workers should appear from the returning sorted worker list
         self.bundle_resources.gpus = 1
         sorted_workers_list = self.bundle_manager._filter_and_sort_workers(
             self.workers_list, self.bundle, self.bundle_resources
         )
-        self.assertEqual(len(sorted_workers_list), 3)
-        self.assertEqual(sorted_workers_list[0]['worker_id'], 2)
-        self.assertEqual(sorted_workers_list[1]['worker_id'], 1)
+        self.assertEqual(len(sorted_workers_list), 2)
+        self.assertEqual(sorted_workers_list[0]['worker_id'], 1)
+        self.assertEqual(sorted_workers_list[1]['worker_id'], 0)
+
+    def test_filter_and_sort_workers_cpus(self):
+        # CPU workers should appear on the top of the returning sorted worker list
+        self.bundle_resources.cpus = 1
+        sorted_workers_list = self.bundle_manager._filter_and_sort_workers(
+            self.workers_list, self.bundle, self.bundle_resources
+        )
+        self.assertEqual(len(sorted_workers_list), 6)
+        self.assertEqual(sorted_workers_list[0]['worker_id'], 3)
+        self.assertEqual(sorted_workers_list[1]['worker_id'], 4)
         self.assertEqual(sorted_workers_list[-1]['worker_id'], 0)
 
-    def test__filter_and_sort_workers_tag_exclusive(self):
+    def test_filter_and_sort_workers_tag_exclusive(self):
         # Only non-tag_exclusive workers should appear in the returned sorted worker list.
         sorted_workers_list = self.bundle_manager._filter_and_sort_workers(
             self.workers_list, self.bundle, self.bundle_resources
         )
-        self.assertEqual(len(sorted_workers_list), 5)
+        self.assertEqual(len(sorted_workers_list), 6)
         for worker in sorted_workers_list:
             self.assertEqual(worker['tag_exclusive'], False)
 
-    def test__filter_and_sort_workers_tag_exclusive_priority(self):
+    def test_filter_and_sort_workers_tag_exclusive_priority(self):
         # All other things being equal, tag_exclusive workers
         # should appear in the top from the returned sorted workers list.
         self.bundle.metadata.request_queue = "tag=worker_X"
@@ -136,51 +167,42 @@ class BundleManagerTest(unittest.TestCase):
             self.workers_list, self.bundle, self.bundle_resources
         )
         self.assertEqual(len(sorted_workers_list), 2)
-        self.assertEqual(sorted_workers_list[0]['worker_id'], 5)
-        self.assertEqual(sorted_workers_list[1]['worker_id'], 4)
-
-    def test__filter_and_sort_workers_cpus(self):
-        # CPU workers should appear in the top from the returning sorted worker list
-        self.bundle_resources.cpus = 1
-        sorted_workers_list = self.bundle_manager._filter_and_sort_workers(
-            self.workers_list, self.bundle, self.bundle_resources
-        )
-        self.assertEqual(len(sorted_workers_list), 5)
-        self.assertEqual(sorted_workers_list[0]['worker_id'], 3)
+        self.assertEqual(sorted_workers_list[0]['worker_id'], 6)
+        self.assertEqual(sorted_workers_list[1]['worker_id'], 5)
 
-    def test__get_matched_workers_with_tag(self):
+    def test_get_matched_workers_with_tag(self):
         self.bundle.metadata.request_queue = "tag=worker_X"
         matched_workers = BundleManager._get_matched_workers(
             self.bundle.metadata.request_queue, self.workers_list
         )
         self.assertEqual(len(matched_workers), 2)
-        self.assertEqual(matched_workers[0]['worker_id'], 4)
-        self.assertEqual(matched_workers[1]['worker_id'], 5)
+        self.assertEqual(matched_workers[0]['worker_id'], 5)
+        self.assertEqual(matched_workers[1]['worker_id'], 6)
 
-    def test__get_matched_workers_with_bad_formatted_tag(self):
+    def test_get_matched_workers_with_bad_formatted_tag(self):
         self.bundle.metadata.request_queue = "tag="
         matched_workers = BundleManager._get_matched_workers(
             self.bundle.metadata.request_queue, self.workers_list
         )
         self.assertEqual(len(matched_workers), 0)
 
-    def test__get_matched_workers_without_tag_prefix(self):
+    def test_get_matched_workers_without_tag_prefix(self):
         self.bundle.metadata.request_queue = "worker_X"
         matched_workers = BundleManager._get_matched_workers(
             self.bundle.metadata.request_queue, self.workers_list
         )
         self.assertEqual(len(matched_workers), 2)
-        self.assertEqual(matched_workers[0]['worker_id'], 4)
-        self.assertEqual(matched_workers[1]['worker_id'], 5)
+        self.assertEqual(matched_workers[0]['worker_id'], 5)
+        self.assertEqual(matched_workers[1]['worker_id'], 6)
 
-    def test__get_matched_workers_not_exist_tag(self):
+    def test_get_matched_workers_not_exist_tag(self):
         self.bundle.metadata.request_queue = "worker_Y"
         matched_workers = BundleManager._get_matched_workers(
             self.bundle.metadata.request_queue, self.workers_list
         )
         self.assertEqual(len(matched_workers), 0)
 
-    def test__get_matched_workers_empty_tag(self):
+    def test_get_matched_workers_empty_tag(self):
         self.bundle.metadata.request_queue = ""
         matched_workers = BundleManager._get_matched_workers(
             self.bundle.metadata.request_queue, self.workers_list