Repository: ansible-collections/amazon.aws

Issue:
Add descend in to list option for 'scrub_none_parameters'
##### SUMMARY
During the implementation of the `scrub_none_parameters` helper function we [talked about](https://github.com/ansible-collections/amazon.aws/pull/184) whether a "descend in to lists" option was necessary.  For the sake of getting it merged, the option was omitted.

Now I have sat down to start to try to use the helper function it has become clear that we definitely need the option and so this is why this issue is being raised.

##### ISSUE TYPE
- Feature Idea

##### COMPONENT NAME
scrub_none_parameters helper function

##### ADDITIONAL INFORMATION
As `scrub_none_parameters` has not yet been implemented anywhere I am going to make the change to descend in to lists y default.



Discussion:
Files identified in the description:
None

If these files are inaccurate, please update the `component name` section of the description or use the `!component` bot command.

[click here for bot help](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md)
<!--- boilerplate: components_banner --->

Resolving patch:
diff --git a/plugins/__init__.py b/plugins/__init__.py
new file mode 100644
diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py
--- a/plugins/module_utils/core.py
+++ b/plugins/module_utils/core.py
@@ -359,13 +359,14 @@ def get_boto3_client_method_parameters(client, method_name, required=False):
     return parameters
 
 
-def scrub_none_parameters(parameters):
+def scrub_none_parameters(parameters, descend_into_lists=False):
     """
     Iterate over a dictionary removing any keys that have a None value
 
     Reference: https://github.com/ansible-collections/community.aws/issues/251
     Credit: https://medium.com/better-programming/how-to-remove-null-none-values-from-a-dictionary-in-python-1bedf1aab5e4
 
+    :param descend_into_lists: whether or not to descend in to lists to continue to remove None values
     :param parameters: parameter dict
     :return: parameter dict with all keys = None removed
     """
@@ -374,7 +375,9 @@ def scrub_none_parameters(parameters):
 
     for k, v in parameters.items():
         if isinstance(v, dict):
-            clean_parameters[k] = scrub_none_parameters(v)
+            clean_parameters[k] = scrub_none_parameters(v, descend_into_lists=descend_into_lists)
+        elif descend_into_lists and isinstance(v, list):
+            clean_parameters[k] = [scrub_none_parameters(vv, descend_into_lists=descend_into_lists) if isinstance(vv, dict) else vv for vv in v]
         elif v is not None:
             clean_parameters[k] = v
 


Test patch:
diff --git a/tests/unit/module_utils/core/test_scrub_none_parameters.py b/tests/unit/module_utils/core/test_scrub_none_parameters.py
--- a/tests/unit/module_utils/core/test_scrub_none_parameters.py
+++ b/tests/unit/module_utils/core/test_scrub_none_parameters.py
@@ -1,56 +1,88 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
 import pytest
 
+from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
+
 scrub_none_test_data = [
-    (dict(),
-     dict()
+    (dict(),  # Input
+     dict(),  # Output with descend_into_lists=False
+     dict(),  # Output with descend_into_lists=True
+     ),
+    (dict(param1=None, param2=None),
+     dict(),
+     dict(),
      ),
     (dict(param1='something'),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1=False),
-     dict(param1=False)
+     dict(param1=False),
+     dict(param1=False),
+     ),
+    (dict(param1=None, param2=[]),
+     dict(param2=[]),
+     dict(param2=[]),
+     ),
+    (dict(param1=None, param2=["list_value"]),
+     dict(param2=["list_value"]),
+     dict(param2=["list_value"]),
      ),
     (dict(param1='something', param2='something_else'),
-     dict(param1='something', param2='something_else')
+     dict(param1='something', param2='something_else'),
+     dict(param1='something', param2='something_else'),
      ),
     (dict(param1='something', param2=dict()),
-     dict(param1='something', param2=dict())
+     dict(param1='something', param2=dict()),
+     dict(param1='something', param2=dict()),
      ),
     (dict(param1='something', param2=None),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1='something', param2=None, param3=None),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1='something', param2=None, param3=None, param4='something_else'),
-     dict(param1='something', param4='something_else')
-     ),
-    (dict(param1=dict(sub_param1='something', sub_param2=None), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something'), param4='something_else')
+     dict(param1='something', param4='something_else'),
+     dict(param1='something', param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict()), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=False), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else'),
      ),
-    (dict(param1=None, param2=None),
-     dict()
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1')], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1=[dict()])], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=None),
+     dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)]),
+     dict(param1=[dict(sub_param1=[dict()])]),
      ),
-    (dict(param1=None, param2=[]),
-     dict(param2=[])
-     )
 ]
 
 
-@pytest.mark.parametrize("input_params, output_params", scrub_none_test_data)
-def test_scrub_none_parameters(input_params, output_params):
-
-    assert scrub_none_parameters(input_params) == output_params
+@pytest.mark.parametrize("input_params, output_params_no_descend, output_params_descend", scrub_none_test_data)
+def test_scrub_none_parameters(input_params, output_params_no_descend, output_params_descend):
+    assert scrub_none_parameters(input_params) == output_params_no_descend
+    assert scrub_none_parameters(input_params, descend_into_lists=False) == output_params_no_descend
+    assert scrub_none_parameters(input_params, descend_into_lists=True) == output_params_descend