Repository: aws/aws-cli

Issue:
Text output is inconsistent based on order
https://forums.aws.amazon.com/thread.jspa?messageID=534743&#534743



Discussion:
I can confirm that this is a problem

Side note, the original AWS forum link is: https://forums.aws.amazon.com/thread.jspa?messageID=534743&#534743


Resolving patch:
diff --git a/awscli/text.py b/awscli/text.py
--- a/awscli/text.py
+++ b/awscli/text.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -19,36 +19,9 @@ def format_text(data, stream):
 
 def _format_text(item, stream, identifier=None, scalar_keys=None):
     if isinstance(item, dict):
-        scalars, non_scalars = _partition_dict(item, scalar_keys=scalar_keys)
-        if scalars:
-            if identifier is not None:
-                scalars.insert(0, identifier.upper())
-            stream.write('\t'.join(scalars))
-            stream.write('\n')
-        for new_identifier, non_scalar in non_scalars:
-            _format_text(item=non_scalar, stream=stream,
-                            identifier=new_identifier)
+        _format_dict(scalar_keys, item, identifier, stream)
     elif isinstance(item, list):
-        if item:
-            if isinstance(item[0], dict):
-                all_keys = _all_scalar_keys(item)
-                for element in item:
-                    _format_text(element,
-                                    stream=stream,
-                                    identifier=identifier,
-                                    scalar_keys=all_keys)
-            elif isinstance(item[0], list):
-                for list_element in item:
-                    _format_text(list_element, stream=stream,
-                                 identifier=identifier)
-            elif identifier is not None:
-                for list_element in item:
-                    stream.write('%s\t%s\n' % (identifier.upper(),
-                                               list_element))
-            else:
-                # For a bare list, just print the contents.
-                stream.write('\t'.join([six.text_type(el) for el in item]))
-                stream.write('\n')
+        _format_list(item, identifier, stream)
     else:
         # If it's not a list or a dict, we just write the scalar
         # value out directly.
@@ -56,6 +29,59 @@ def _format_text(item, stream, identifier=None, scalar_keys=None):
         stream.write('\n')
 
 
+def _format_list(item, identifier, stream):
+    if not item:
+        return
+    if any(isinstance(el, dict) for el in item):
+        all_keys = _all_scalar_keys(item)
+        for element in item:
+            _format_text(element, stream=stream, identifier=identifier,
+                         scalar_keys=all_keys)
+    elif any(isinstance(el, list) for el in item):
+        scalar_elements, non_scalars = _partition_list(item)
+        if scalar_elements:
+            _format_scalar_list(scalar_elements, identifier, stream)
+        for non_scalar in non_scalars:
+            _format_text(non_scalar, stream=stream,
+                         identifier=identifier)
+    else:
+        _format_scalar_list(item, identifier, stream)
+
+
+def _partition_list(item):
+    scalars = []
+    non_scalars = []
+    for element in item:
+        if isinstance(element, (list, dict)):
+            non_scalars.append(element)
+        else:
+            scalars.append(element)
+    return scalars, non_scalars
+
+
+def _format_scalar_list(elements, identifier, stream):
+    if identifier is not None:
+        for item in elements:
+            stream.write('%s\t%s\n' % (identifier.upper(),
+                                       item))
+    else:
+        # For a bare list, just print the contents.
+        stream.write('\t'.join([six.text_type(item) for item in elements]))
+        stream.write('\n')
+
+
+def _format_dict(scalar_keys, item, identifier, stream):
+    scalars, non_scalars = _partition_dict(item, scalar_keys=scalar_keys)
+    if scalars:
+        if identifier is not None:
+            scalars.insert(0, identifier.upper())
+        stream.write('\t'.join(scalars))
+        stream.write('\n')
+    for new_identifier, non_scalar in non_scalars:
+        _format_text(item=non_scalar, stream=stream,
+                     identifier=new_identifier)
+
+
 def _all_scalar_keys(list_of_dicts):
     keys_seen = set()
     for item_dict in list_of_dicts:
@@ -74,6 +100,9 @@ def _partition_dict(item_dict, scalar_keys):
     scalar = []
     non_scalar = []
     if scalar_keys is None:
+        # scalar_keys can have more than just the keys in the item_dict,
+        # but if user does not provide scalar_keys, we'll grab the keys
+        # from the current item_dict
         for key, value in sorted(item_dict.items()):
             if isinstance(value, (dict, list)):
                 non_scalar.append((key, value))
@@ -84,6 +113,5 @@ def _partition_dict(item_dict, scalar_keys):
             scalar.append(six.text_type(item_dict.get(key, '')))
         remaining_keys = sorted(set(item_dict.keys()) - set(scalar_keys))
         for remaining_key in remaining_keys:
-            if remaining_key in item_dict:
-                non_scalar.append((remaining_key, item_dict[remaining_key]))
+            non_scalar.append((remaining_key, item_dict[remaining_key]))
     return scalar, non_scalar


Test patch:
diff --git a/tests/unit/test_text.py b/tests/unit/test_text.py
--- a/tests/unit/test_text.py
+++ b/tests/unit/test_text.py
@@ -37,6 +37,7 @@ def format_text(self, data, stream=None):
 
     def assert_text_renders_to(self, data, expected_rendering):
         rendered = self.format_text(data)
+        print(rendered)
         self.assertEqual(rendered, expected_rendering)
 
     def test_dict_format(self):
@@ -81,6 +82,17 @@ def test_different_keys_in_nested_sublists(self):
             'FOO\t8\t9\t\n'
         )
 
+    def test_different_keys_in_deeply_nested_sublists(self):
+        self.assert_text_renders_to({'bar':[
+            {'foo': [[[dict(a=1, b=2, c=3), dict(a=4, c=5)]]]},
+            {'foo': [[[dict(b=6, d=7), dict(b=8, c=9)]]]},
+        ]},
+            'FOO\t1\t2\t3\n'
+            'FOO\t4\t\t5\n'
+            'FOO\t6\t\t7\n'
+            'FOO\t8\t9\t\n'
+        )
+
     def test_scalars_and_complex_types(self):
         self.assert_text_renders_to(
             {'foo': [dict(a=1, b=dict(y='y', z='z'), c=3),
@@ -111,7 +123,7 @@ def test_deeply_nested_lists(self):
 
     def test_unicode_text(self):
         self.assert_text_renders_to([['1', '2', u'\u2713']],
-                                    u'1\t2\t\u2713\n')
+                                     u'1\t2\t\u2713\n')
 
     def test_single_scalar_value(self):
         self.assert_text_renders_to('foobarbaz', 'foobarbaz\n')
@@ -122,6 +134,12 @@ def test_empty_list(self):
     def test_empty_inner_list(self):
         self.assert_text_renders_to([[]], '')
 
+    def test_deeploy_nested_empty_list(self):
+        self.assert_text_renders_to([[[[]]]], '')
+
+    def test_deeploy_nested_single_scalar(self):
+        self.assert_text_renders_to([[[['a']]]], 'a\n')
+
     def test_empty_list_mock_calls(self):
         # We also need this test as well as test_empty_list
         # because we want to ensure that write() is never called with
@@ -133,10 +151,66 @@ def test_empty_list_mock_calls(self):
 
     def test_list_of_strings_in_dict(self):
         self.assert_text_renders_to(
-            {"KeyName": ['a', 'b', 'c']},
-            'KEYNAME\ta\n'
-            'KEYNAME\tb\n'
-            'KEYNAME\tc\n')
+            {'KeyName': ['a', 'b', 'c']},
+             'KEYNAME\ta\n'
+             'KEYNAME\tb\n'
+             'KEYNAME\tc\n')
+
+    def test_inconsistent_sublists(self):
+        self.assert_text_renders_to(
+            [
+                [['1', '2'], ['3', '4', '5', '6']],
+                [['7', '8', '9'], ['0']]
+            ],
+            '1\t2\n'
+            '3\t4\t5\t6\n'
+            '7\t8\t9\n'
+            '0\n'
+        )
+
+    def test_lists_mixed_with_scalars(self):
+        self.assert_text_renders_to(
+            [
+                ['a', 'b', ['c', 'd']],
+                ['e', 'f', ['g', 'h']]
+            ],
+            'a\tb\n'
+            'c\td\n'
+            'e\tf\n'
+            'g\th\n'
+        )
+
+    def test_deeply_nested_with_scalars(self):
+        self.assert_text_renders_to(
+            [
+                ['a', 'b', ['c', 'd', ['e', 'f', ['g', 'h']]]],
+                ['i', 'j', ['k', 'l', ['m', 'n', ['o', 'p']]]],
+            ],
+            'a\tb\n'
+            'c\td\n'
+            'e\tf\n'
+            'g\th\n'
+            'i\tj\n'
+            'k\tl\n'
+            'm\tn\n'
+            'o\tp\n'
+        )
+
+    def test_deeply_nested_with_identifier(self):
+        self.assert_text_renders_to(
+            {'foo': [
+                ['a', 'b', ['c', 'd']],
+                ['e', 'f', ['g', 'h']]
+            ]},
+            'FOO\ta\n'
+            'FOO\tb\n'
+            'FOO\tc\n'
+            'FOO\td\n'
+            'FOO\te\n'
+            'FOO\tf\n'
+            'FOO\tg\n'
+            'FOO\th\n'
+        )
 
 
 if __name__ == '__main__':