Repository: ansible/ansible

Issue:
iam-group returns state 'changed' for large groups even when no change is applied
##### ISSUE TYPE
 - Bug Report

##### COMPONENT NAME
iam-group module

##### ANSIBLE VERSION
<!--- Paste verbatim output from "ansible --version" between quotes below -->
```
ansible 2.4.0.0
  config file =<path>/ansible.cfg
  configured module search path = [u'<path>/.ansible/plugins/modules', u'/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/local/lib/python2.7/site-packages/ansible
  executable location = /usr/local/bin/ansible
  python version = 2.7.3 (default, Jun  6 2017, 13:15:29) [GCC 4.2.1 Compatible Apple LLVM 8.1.0 (clang-802.0.42)]
```

##### CONFIGURATION

```
ANSIBLE_SSH_ARGS(<path>/ansible.cfg) = -o ControlMaster=auto -o ControlPersis
ANSIBLE_SSH_CONTROL_PATH(<path>/ansible.cfg) = /tmp/ansible-ssh-%%h-%%p-%%r
DEFAULT_GATHERING(<path>/ansible.cfg) = explicit
DEFAULT_REMOTE_TMP(<path>/ansible.cfg) = /tmp/ansible/$USER
```

##### OS / ENVIRONMENT

OS X 10.12.6

##### SUMMARY
The iam-group module returns a state of 'changed' for two of our groups every time. These are large groups with 100+ users, and I think that's what's causing the issue. I've verified that the module is not actually applying any changes in AWS (we have a job that dumps our AWS configs and checks for drift against what's checked into code), so I'm confident that nothing is actually changing.

One interesting thing I found when running the playbook in verbose mode is that for the large groups, the return JSON has `"is_truncated": true`, whereas it's always `false` for the other groups that don't have this issue.

##### STEPS TO REPRODUCE
Have a large IAM group? Not sure how many users is too many users for this issue to trigger.

##### EXPECTED RESULTS
Module should return `ok` and not `changed`

##### ACTUAL RESULTS
Module returns `changed`

```
changed: [localhost] => {
    "changed": true,
    "failed": false,
    "iam_group": {
        "group": {
            "arn": "arn:aws:iam::<redacted>:group/console-users",
            "create_date": "2015-08-26T20:08:38+00:00",
            "group_id": "<redacted>",
            "group_name": "console-users",
            "path": "/"
        },
        "is_truncated": true,
        "marker": "<redacted>",
        "response_metadata": {
            "http_headers": {
                "content-length": "34869",
                "content-type": "text/xml",
                "date": "Tue, 10 Oct 2017 18:15:56 GMT",
                "vary": "Accept-Encoding",
                "x-amzn-requestid": "<redacted>"
            },
            "http_status_code": 200,
            "request_id": "<redacted>",
            "retry_attempts": 0
        },
        "users": [
            {
                "arn": "arn:aws:iam::<redacted>:user/<redacted>@<redacted>.com",
                "create_date": "2016-04-14T18:06:22+00:00",
                "password_last_used": "2017-09-27T13:23:50+00:00",
                "path": "/",
                "user_id": "<redacted>",
                "user_name": "<redacted>@<redacted>.com"
            },
            ...
        ]
    },
    "invocation": {
        "module_args": {
            "aws_access_key": null,
            "aws_secret_key": null,
            "ec2_url": null,
            "managed_policy": [
                "arn:aws:iam::<redacted>:policy/<redacted>"
            ],
            "name": "console-users",
            "profile": null,
            "purge_policy": false,
            "purge_users": false,
            "region": null,
            "security_token": null,
            "state": "present",
            "users": [
                "<redacted>@<redacted>.com",
                "<redacted>@<redacted>.com",
            ],
            "validate_certs": true
        }
    }
}
```



Discussion:
cc @infectsoldier @naslanidis @willthames
[click here for bot help](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md)
<!--- boilerplate: notify --->

Resolving patch:
diff --git a/lib/ansible/modules/cloud/amazon/iam_group.py b/lib/ansible/modules/cloud/amazon/iam_group.py
--- a/lib/ansible/modules/cloud/amazon/iam_group.py
+++ b/lib/ansible/modules/cloud/amazon/iam_group.py
@@ -160,7 +160,7 @@
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, ec2_argument_spec, get_aws_connection_info, boto3_conn
-from ansible.module_utils.ec2 import HAS_BOTO3
+from ansible.module_utils.ec2 import HAS_BOTO3, AWSRetry
 
 import traceback
 
@@ -227,7 +227,11 @@ def create_or_update_group(connection, module):
     changed = False
 
     # Get group
-    group = get_group(connection, module, params['GroupName'])
+    try:
+        group = get_group(connection, module, params['GroupName'])
+    except ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(),
+                         **camel_dict_to_snake_dict(e.response))
 
     # If group is None, create it
     if group is None:
@@ -273,7 +277,12 @@ def create_or_update_group(connection, module):
         changed = True
 
     # Manage group memberships
-    current_group_members = get_group(connection, module, params['GroupName'])['Users']
+    try:
+        current_group_members = get_group(connection, module, params['GroupName'])['Users']
+    except ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(),
+                         **camel_dict_to_snake_dict(e.response))
+
     current_group_members_list = []
     for member in current_group_members:
         current_group_members_list.append(member['UserName'])
@@ -303,7 +312,11 @@ def create_or_update_group(connection, module):
         changed = True
 
     # Get the group again
-    group = get_group(connection, module, params['GroupName'])
+    try:
+        group = get_group(connection, module, params['GroupName'])
+    except ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(),
+                         **camel_dict_to_snake_dict(e.response))
 
     module.exit_json(changed=changed, iam_group=camel_dict_to_snake_dict(group))
 
@@ -313,7 +326,12 @@ def destroy_group(connection, module):
     params = dict()
     params['GroupName'] = module.params.get('name')
 
-    if get_group(connection, module, params['GroupName']):
+    try:
+        group = get_group(connection, module, params['GroupName'])
+    except ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(),
+                         **camel_dict_to_snake_dict(e.response))
+    if group:
 
         # Remove any attached policies otherwise deletion fails
         try:
@@ -327,7 +345,11 @@ def destroy_group(connection, module):
 
         # Remove any users in the group otherwise deletion fails
         current_group_members_list = []
-        current_group_members = get_group(connection, module, params['GroupName'])['Users']
+        try:
+            current_group_members = get_group(connection, module, params['GroupName'])['Users']
+        except ClientError as e:
+            module.fail_json(msg=e.message, exception=traceback.format_exc(),
+                             **camel_dict_to_snake_dict(e.response))
         for member in current_group_members:
             current_group_members_list.append(member['UserName'])
         for user in current_group_members_list:
@@ -353,29 +375,29 @@ def destroy_group(connection, module):
     module.exit_json(changed=True)
 
 
+@AWSRetry.exponential_backoff()
 def get_group(connection, module, name):
-
-    params = dict()
-    params['GroupName'] = name
-
     try:
-        return connection.get_group(**params)
+        paginator = connection.get_paginator('get_group')
+        return paginator.paginate(GroupName=name).build_full_result()
     except ClientError as e:
         if e.response['Error']['Code'] == 'NoSuchEntity':
             return None
         else:
-            module.fail_json(msg=e.message, **camel_dict_to_snake_dict(e.response))
+            raise
 
 
+@AWSRetry.exponential_backoff()
 def get_attached_policy_list(connection, module, name):
 
     try:
-        return connection.list_attached_group_policies(GroupName=name)['AttachedPolicies']
+        paginator = connection.get_paginator('list_attached_group_policies')
+        return paginator.paginate(GroupName=name).build_full_result()['AttachedPolicies']
     except ClientError as e:
         if e.response['Error']['Code'] == 'NoSuchEntity':
             return None
         else:
-            module.fail_json(msg=e.message, **camel_dict_to_snake_dict(e.response))
+            raise
 
 
 def main():


Test patch: