Repository: aws/aws-cli

Issue:
Unknown options: --recursive
As is documented [here](http://docs.aws.amazon.com/cli/latest/reference/s3/) and [here](http://aws.amazon.com/cli/), I would expect the following command to succeed: `aws s3 ls s3://BUCKETNAME --recursive`, but instead I receive the following error message: `Unknown options: --recursive`. I have tried the `--recursive` in every relevant position in this command, and still get this error message every time.

I'm not sure how to reconcile the documentation with this error.

Version: `aws-cli/1.2.3 Python/2.7.5 Darwin/12.5.0` (installed from pip)



Discussion:
+1
Same problem.

Looking, though I wonder if we should **not** allow `aws s3 ls --recursive` and only allow `aws s3 ls s3://bucket/ --recursive`.  In theory, wouldn't the first option list every single object in every single one of your buckets?


Resolving patch:
diff --git a/awscli/customizations/s3/s3.py b/awscli/customizations/s3/s3.py
--- a/awscli/customizations/s3/s3.py
+++ b/awscli/customizations/s3/s3.py
@@ -31,8 +31,7 @@
 from awscli.customizations.s3.s3handler import S3Handler
 from awscli.customizations.s3.description import add_command_descriptions, \
     add_param_descriptions
-from awscli.customizations.s3.utils import find_bucket_key, check_error, \
-        uni_print
+from awscli.customizations.s3.utils import find_bucket_key, uni_print
 from awscli.customizations.s3.dochandler import S3DocumentEventHandler
 
 
@@ -68,8 +67,6 @@ def awscli_initialize(cli):
     """
     cli.register("building-command-table.main", add_s3)
     cli.register("doc-examples.S3.*", add_s3_examples)
-    for cmd in CMD_DICT.keys():
-        cli.register("building-parameter-table.s3.%s" % cmd, add_cmd_params)
 
 
 def s3_plugin_initialize(event_handlers):
@@ -89,17 +86,6 @@ def add_s3(command_table, session, **kwargs):
     command_table['s3'] = S3('s3', session)
 
 
-def add_cmd_params(parameter_table, command, **kwargs):
-    """
-    This creates the ParameterArgument object for each possible parameter
-    in a specified command
-    """
-    for param in CMD_DICT[command]['params']:
-        parameter_table[param] = S3Parameter(param,
-                                             PARAMS_DICT[param]['options'],
-                                             PARAMS_DICT[param]['documents'])
-
-
 def add_s3_examples(help_command, **kwargs):
     """
     This function is used to add examples for each command.  It reads in
@@ -304,8 +290,7 @@ def create_help_command(self):
         table.  This command is necessary for generating html docs for
         the specified command.
         """
-        arg_table = {}
-        add_cmd_params(arg_table, self._name)
+        arg_table = self._populate_parameter_table()
         return S3HelpCommand(self._session, self,
                              command_table=None,
                              arg_table=arg_table)
@@ -316,13 +301,21 @@ def _create_parameter_table(self):
         S3Parameter objects corresponding to the specified command when
         the event is emitted.
         """
-        parameter_table = {}
+        parameter_table = self._populate_parameter_table()
         self._session.emit('building-parameter-table.s3.%s' % self._name,
                            parameter_table=parameter_table,
                            command=self._name)
 
         return parameter_table
 
+    def _populate_parameter_table(self):
+        parameter_table = {}
+        for param in CMD_DICT[self._name]['params']:
+            parameter_table[param] = S3Parameter(param,
+                                                 PARAMS_DICT[param]['options'],
+                                                 PARAMS_DICT[param]['documents'])
+        return parameter_table
+
     def _build_call_parameters(self, args, service_params):
         """
         This takes all of the commands in the name space and puts them
@@ -354,6 +347,9 @@ def _do_command(self, parsed_args, parsed_globals):
         self.endpoint = self._get_endpoint(self.service, parsed_globals)
         if not bucket:
             self._list_all_buckets()
+        elif parsed_args.dir_op:
+            # Then --recursive was specified.
+            self._list_all_objects_recursive(bucket, key)
         else:
             self._list_all_objects(bucket, key)
         return 0
@@ -363,24 +359,30 @@ def _list_all_objects(self, bucket, key):
         iterator = operation.paginate(self.endpoint, bucket=bucket,
                                       prefix=key, delimiter='/')
         for _, response_data in iterator:
-            common_prefixes = response_data['CommonPrefixes']
-            contents = response_data['Contents']
-            for common_prefix in common_prefixes:
-                prefix_components = common_prefix['Prefix'].split('/')
-                prefix = prefix_components[-2]
-                pre_string = "PRE".rjust(30, " ")
-                print_str = pre_string + ' ' + prefix + '/\n'
-                uni_print(print_str)
-                sys.stdout.flush()
-            for content in contents:
-                last_mod_str = self._make_last_mod_str(content['LastModified'])
-                size_str = self._make_size_str(content['Size'])
+            self._display_page(response_data)
+
+    def _display_page(self, response_data, use_basename=True):
+        common_prefixes = response_data['CommonPrefixes']
+        contents = response_data['Contents']
+        for common_prefix in common_prefixes:
+            prefix_components = common_prefix['Prefix'].split('/')
+            prefix = prefix_components[-2]
+            pre_string = "PRE".rjust(30, " ")
+            print_str = pre_string + ' ' + prefix + '/\n'
+            uni_print(print_str)
+            sys.stdout.flush()
+        for content in contents:
+            last_mod_str = self._make_last_mod_str(content['LastModified'])
+            size_str = self._make_size_str(content['Size'])
+            if use_basename:
                 filename_components = content['Key'].split('/')
                 filename = filename_components[-1]
-                print_str = last_mod_str + ' ' + size_str + ' ' + \
-                    filename + '\n'
-                uni_print(print_str)
-                sys.stdout.flush()
+            else:
+                filename = content['Key']
+            print_str = last_mod_str + ' ' + size_str + ' ' + \
+                filename + '\n'
+            uni_print(print_str)
+            sys.stdout.flush()
 
     def _list_all_buckets(self):
         operation = self.service.get_operation('ListBuckets')
@@ -392,6 +394,13 @@ def _list_all_buckets(self):
             uni_print(print_str)
             sys.stdout.flush()
 
+    def _list_all_objects_recursive(self, bucket, key):
+        operation = self.service.get_operation('ListObjects')
+        iterator = operation.paginate(self.endpoint, bucket=bucket,
+                                      prefix=key)
+        for _, response_data in iterator:
+            self._display_page(response_data, use_basename=False)
+
     def _make_last_mod_str(self, last_mod):
         """
         This function creates the last modified time string whenever objects
@@ -810,7 +819,7 @@ def check_endpoint_url(self, parsed_globals):
                                 'content-encoding', 'content-language',
                                 'expires']},
             'ls': {'options': {'nargs': '?', 'default': 's3://'},
-                   'params': [], 'default': 's3://',
+                   'params': ['recursive'], 'default': 's3://',
                    'command_class': ListCommand},
             'mb': {'options': {'nargs': 1}, 'params': []},
             'rb': {'options': {'nargs': 1}, 'params': ['force']},


Test patch:
diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py
--- a/tests/integration/customizations/s3/test_plugin.py
+++ b/tests/integration/customizations/s3/test_plugin.py
@@ -508,6 +508,18 @@ def test_ls_with_prefix(self):
         self.assertIn('8 foo', p.stdout)
         self.assertIn('8 bar.txt', p.stdout)
 
+    def test_ls_recursive(self):
+        bucket_name = self.create_bucket()
+        self.put_object(bucket_name, 'foo.txt', 'contents')
+        self.put_object(bucket_name, 'foo', 'contents')
+        self.put_object(bucket_name, 'bar.txt', 'contents')
+        self.put_object(bucket_name, 'subdir/foo.txt', 'contents')
+        p = aws('s3 ls s3://%s --recursive' % bucket_name)
+        self.assertIn('8 foo.txt', p.stdout)
+        self.assertIn('8 foo', p.stdout)
+        self.assertIn('8 bar.txt', p.stdout)
+        self.assertIn('8 subdir/foo.txt', p.stdout)
+
 
 class TestMbRb(BaseS3CLICommand):
     """
diff --git a/tests/unit/customizations/s3/test_ls_command.py b/tests/unit/customizations/s3/test_ls_command.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/customizations/s3/test_ls_command.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from tests.unit import BaseAWSCommandParamsTest
+
+
+class TestLSCommand(BaseAWSCommandParamsTest):
+
+    def test_operations_used_in_recursive_list(self):
+        self.parsed_responses = [{"CommonPrefixes": [], "Contents": [
+            {"Key": "foo/bar.txt", "Size": 100,
+             "LastModified": "2014-01-09T20:45:49.000Z"}]}]
+        stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --recursive', expected_rc=0)
+        call_args = self.operations_called[0][1]
+        # We should not be calling the args with any delimiter because we
+        # want a recursive listing.
+        self.assertEqual(call_args['prefix'], '')
+        self.assertEqual(call_args['bucket'], 'bucket')
+        self.assertNotIn('delimiter', call_args)
+        # Using assertRegexpMatches because the actual time displayed
+        # is specific to your tzinfo.
+        self.assertRegexpMatches(
+            stdout, '2014-01-09 \d{2}:\d{2}:\d{2}        100 foo/bar.txt\n')
diff --git a/tests/unit/customizations/s3/test_s3.py b/tests/unit/customizations/s3/test_s3.py
--- a/tests/unit/customizations/s3/test_s3.py
+++ b/tests/unit/customizations/s3/test_s3.py
@@ -21,7 +21,7 @@
 from mock import Mock, MagicMock, patch
 
 from awscli.customizations.s3.s3 import AppendFilter, \
-    awscli_initialize, add_s3, add_cmd_params, \
+    awscli_initialize, add_s3, \
     S3, S3SubCommand, S3Parameter, CommandArchitecture, CommandParameters, \
     ListCommand
 from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \
@@ -81,24 +81,6 @@ def test_s3(self):
         for service in self.services.keys():
             self.assertIn(service, ['s3'])
 
-    def test_parameters(self):
-        """
-        Ensures that all of the parameters generated for a specified
-        command are an elgible parameter.
-        """
-        commands_list = ['cp', 'mv', 'rm', 'sync', 'ls', 'mb', 'rb']
-        params_list = ['dryrun', 'delete', 'quiet', 'recursive', 'exclude',
-                       'include', 'acl', 'force', 'no-guess-mime-type',
-                       'sse', 'storage-class', 'website-redirect',
-                       'content-type', 'cache-control', 'content-disposition',
-                       'content-language', 'content-encoding', 'expires',
-                       'grants']
-        for cmd in commands_list:
-            self.parameters = {}
-            add_cmd_params(self.parameters, cmd)
-            for param in self.parameters.keys():
-                self.assertIn(param, params_list)
-
 
 class S3SubCommandTest(unittest.TestCase):
     """