Repository: ansible-collections/amazon.aws

Issue:
ec2_group tests regularly failing
### Summary

The ec2_group tests seem to be consistently failing

### Issue Type

Bug Report

### Component Name

ec2_group

### Ansible Version

(CI)

### Collection Versions

(CI)

### AWS SDK versions

(CI)

### Configuration

(CI)

### OS / Environment

CI

### Steps to Reproduce

Trigger integration tests

### Expected Results

Success

### Actual Results

https://dashboard.zuul.ansible.com/t/ansible/build/b31992de56524ef4a4d325a63c3bb46c/console

```
TASK [ec2_group : add a rule that auto creates another security group (CHECK MODE)] ***
task path: /home/zuul/.ansible/collections/ansible_collections/amazon/aws/tests/integration/targets/ec2_group/tasks/main.yml:446
<testhost> ESTABLISH LOCAL CONNECTION FOR USER: zuul
<testhost> EXEC /bin/sh -c 'echo ~zuul && sleep 0'
<testhost> EXEC /bin/sh -c '( umask 77 && mkdir -p "` echo /home/zuul/.ansible/tmp `"&& mkdir "` echo /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135 `" && echo ansible-tmp-1627748612.2796433-5619-201358591405135="` echo /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135 `" ) && sleep 0'
Using module file /home/zuul/.ansible/collections/ansible_collections/amazon/aws/plugins/modules/ec2_group.py
<testhost> PUT /home/zuul/.ansible/tmp/ansible-local-3520gvm4u706/tmpzwjhczcj TO /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py
<testhost> EXEC /bin/sh -c 'chmod u+x /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/ /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py && sleep 0'
<testhost> EXEC /bin/sh -c 'ANSIBLE_DEBUG_BOTOCORE_LOGS=True /home/zuul/venv/bin/python /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py && sleep 0'
<testhost> EXEC /bin/sh -c 'rm -f -r /home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/ > /dev/null 2>&1 && sleep 0'
The full traceback is:
Traceback (most recent call last):
  File "/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py", line 114, in <module>
    _ansiballz_main()
  File "/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py", line 106, in _ansiballz_main
    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)
  File "/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py", line 55, in invoke_module
    run_name='__main__', alter_sys=True)
  File "/usr/lib64/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib64/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib64/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py", line 1379, in <module>
  File "/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py", line 1369, in main
  File "/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py", line 1114, in get_diff_final_resource
  File "/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py", line 1087, in get_final_rules
  File "/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible/module_utils/common/dict_transformations.py", line 44, in camel_dict_to_snake_dict
AttributeError: 'NoneType' object has no attribute 'items'
fatal: [testhost]: FAILED! => {
    "changed": false,
    "module_stderr": "Traceback (most recent call last):\n  File \"/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py\", line 114, in <module>\n    _ansiballz_main()\n  File \"/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py\", line 106, in _ansiballz_main\n    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)\n  File \"/home/zuul/.ansible/tmp/ansible-tmp-1627748612.2796433-5619-201358591405135/AnsiballZ_ec2_group.py\", line 55, in invoke_module\n    run_name='__main__', alter_sys=True)\n  File \"/usr/lib64/python3.6/runpy.py\", line 205, in run_module\n    return _run_module_code(code, init_globals, run_name, mod_spec)\n  File \"/usr/lib64/python3.6/runpy.py\", line 96, in _run_module_code\n    mod_name, mod_spec, pkg_name, script_name)\n  File \"/usr/lib64/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py\", line 1379, in <module>\n  File \"/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py\", line 1369, in main\n  File \"/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py\", line 1114, in get_diff_final_resource\n  File \"/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible_collections/amazon/aws/plugins/modules/ec2_group.py\", line 1087, in get_final_rules\n  File \"/tmp/ansible_ec2_group_payload_zwf74pzk/ansible_ec2_group_payload.zip/ansible/module_utils/common/dict_transformations.py\", line 44, in camel_dict_to_snake_dict\nAttributeError: 'NoneType' object has no attribute 'items'\n",
    "module_stdout": "",
    "msg": "MODULE FAILURE\nSee stdout/stderr for the exact error",
    "rc": 1
}
```


### Code of Conduct

- [X] I agree to follow the Ansible Code of Conduct


Discussion:
Files identified in the description:
* [`plugins/modules/ec2_group.py`](https://github.com/['ansible-collections/amazon.aws', 'ansible-collections/community.aws', 'ansible-collections/community.vmware']/blob/main/plugins/modules/ec2_group.py)

If these files are inaccurate, please update the `component name` section of the description or use the `!component` bot command.

[click here for bot help](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md)
<!--- boilerplate: components_banner --->
cc @adq @jillr @s-hertel @wimnat
[click here for bot help](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md)
<!--- boilerplate: notify --->

Resolving patch:
diff --git a/plugins/modules/ec2_group.py b/plugins/modules/ec2_group.py
--- a/plugins/modules/ec2_group.py
+++ b/plugins/modules/ec2_group.py
@@ -137,6 +137,7 @@
     required: false
     type: list
     elements: dict
+    aliases: ['egress_rules']
     suboptions:
         cidr_ip:
             type: str
@@ -232,7 +233,7 @@
       - Purge existing rules_egress on security group that are not found in rules_egress.
     required: false
     default: 'true'
-    aliases: []
+    aliases: ['purge_egress_rules']
     type: bool
 
 extends_documentation_fragment:
@@ -1163,16 +1164,28 @@ def get_final_rules(client, module, security_group_rules, specified_rules, purge
                             rule[source_type] = [rule[source_type]]
                         format_rule[rule_key] = [{source_type: target} for target in rule[source_type]]
             if rule.get('group_id') or rule.get('group_name'):
-                rule_sg = camel_dict_to_snake_dict(group_exists(client, module, module.params['vpc_id'], rule.get('group_id'), rule.get('group_name'))[0])
-                format_rule['user_id_group_pairs'] = [{
-                    'description': rule_sg.get('description', rule_sg.get('group_desc')),
-                    'group_id': rule_sg.get('group_id', rule.get('group_id')),
-                    'group_name': rule_sg.get('group_name', rule.get('group_name')),
-                    'peering_status': rule_sg.get('peering_status'),
-                    'user_id': rule_sg.get('user_id', get_account_id(security_group, module)),
-                    'vpc_id': rule_sg.get('vpc_id', module.params['vpc_id']),
-                    'vpc_peering_connection_id': rule_sg.get('vpc_peering_connection_id')
-                }]
+                rule_sg = group_exists(client, module, module.params['vpc_id'], rule.get('group_id'), rule.get('group_name'))[0]
+                if rule_sg is None:
+                    # --diff during --check
+                    format_rule['user_id_group_pairs'] = [{
+                        'group_id': rule.get('group_id'),
+                        'group_name': rule.get('group_name'),
+                        'peering_status': None,
+                        'user_id': get_account_id(security_group, module),
+                        'vpc_id': module.params['vpc_id'],
+                        'vpc_peering_connection_id': None
+                    }]
+                else:
+                    rule_sg = camel_dict_to_snake_dict(rule_sg)
+                    format_rule['user_id_group_pairs'] = [{
+                        'description': rule_sg.get('description', rule_sg.get('group_desc')),
+                        'group_id': rule_sg.get('group_id', rule.get('group_id')),
+                        'group_name': rule_sg.get('group_name', rule.get('group_name')),
+                        'peering_status': rule_sg.get('peering_status'),
+                        'user_id': rule_sg.get('user_id', get_account_id(security_group, module)),
+                        'vpc_id': rule_sg.get('vpc_id', module.params['vpc_id']),
+                        'vpc_peering_connection_id': rule_sg.get('vpc_peering_connection_id')
+                    }]
                 for k, v in list(format_rule['user_id_group_pairs'][0].items()):
                     if v is None:
                         format_rule['user_id_group_pairs'][0].pop(k)
@@ -1243,7 +1256,7 @@ def get_ip_permissions_sort_key(rule):
         return rule.get('prefix_list_ids')[0]['prefix_list_id']
     elif rule.get('user_id_group_pairs'):
         rule.get('user_id_group_pairs').sort(key=get_rule_sort_key)
-        return rule.get('user_id_group_pairs')[0]['group_id']
+        return rule.get('user_id_group_pairs')[0].get('group_id', '')
     return None
 
 
@@ -1254,10 +1267,10 @@ def main():
         description=dict(),
         vpc_id=dict(),
         rules=dict(type='list', elements='dict'),
-        rules_egress=dict(type='list', elements='dict'),
+        rules_egress=dict(type='list', elements='dict', aliases=['egress_rules']),
         state=dict(default='present', type='str', choices=['present', 'absent']),
         purge_rules=dict(default=True, required=False, type='bool'),
-        purge_rules_egress=dict(default=True, required=False, type='bool'),
+        purge_rules_egress=dict(default=True, required=False, type='bool', aliases=['purge_egress_rules']),
         tags=dict(required=False, type='dict', aliases=['resource_tags']),
         purge_tags=dict(default=True, required=False, type='bool')
     )


Test patch:
diff --git a/tests/integration/targets/ec2_group/aliases b/tests/integration/targets/ec2_group/aliases
--- a/tests/integration/targets/ec2_group/aliases
+++ b/tests/integration/targets/ec2_group/aliases
@@ -1,9 +1,6 @@
-# reason: broken
-# Tests frequently failing
-# https://github.com/ansible-collections/amazon.aws/issues/440
-disabled
-
+# duration: 15
 slow
 
 cloud/aws
+
 ec2_group_info
diff --git a/tests/integration/targets/ec2_group/tasks/ec2_classic.yml b/tests/integration/targets/ec2_group/tasks/ec2_classic.yml
deleted file mode 100644
--- a/tests/integration/targets/ec2_group/tasks/ec2_classic.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-- module_defaults:
-    group/aws:
-      aws_access_key: "{{ aws_access_key }}"
-      aws_secret_key: "{{ aws_secret_key }}"
-      security_token: "{{ security_token | default(omit) }}"
-      region: "{{ aws_region }}"
-  block:
-  - name: Get available AZs
-    aws_az_info:
-      filters:
-        region-name: "{{ aws_region }}"
-    register: az_facts
-
-  - name: Create a classic ELB with classic networking
-    ec2_elb_lb:
-      name: "{{ resource_prefix }}-elb"
-      state: present
-      zones:
-        - "{{ az_facts['availability_zones'][0]['zone_name'] }}"
-        - "{{ az_facts['availability_zones'][1]['zone_name'] }}"
-      listeners:
-        - protocol: http # options are http, https, ssl, tcp
-          load_balancer_port: 80
-          instance_port: 80
-          proxy_protocol: True
-    register: classic_elb
-
-  - name: Assert the elb was created
-    assert:
-      that:
-        - classic_elb.changed
-
-  - name: Create a security group with a classic elb-sg rule
-    ec2_group:
-      name: "{{ resource_prefix }}-sg-a"
-      description: "EC2 classic test security group"
-      rules:
-      - proto: tcp
-        ports: 80
-        group_id: amazon-elb/amazon-elb-sg
-      state: present
-    register: classic_sg
-
-  - name: Assert the SG was created
-    assert:
-      that:
-        - classic_sg.changed
-        - "{{ classic_sg.ip_permissions | length }} == 1"
-
-  - set_fact:
-      elb_sg_id: "{{ classic_sg.ip_permissions[0].user_id_group_pairs[0].user_id }}/{{ classic_sg.ip_permissions[0].user_id_group_pairs[0].group_id }}/{{ classic_sg.ip_permissions[0].user_id_group_pairs[0].group_name }}"
-
-  - name: Update the security group
-    ec2_group:
-      name: "{{ resource_prefix }}-sg-a"
-      description: "EC2 classic test security group"
-      rules:
-      - proto: tcp
-        ports: 8080
-        group_id: "{{ elb_sg_id }}"
-      - proto: tcp
-        ports:
-          - 80
-        cidr_ip: 0.0.0.0/0
-      state: present
-    register: updated_classic_sg
-
-
-  - name: Assert the SG was updated
-    assert:
-      that:
-        - updated_classic_sg.changed
-        - "{{ updated_classic_sg.ip_permissions | length }} == 2"
-        - "{{ classic_sg.ip_permissions[0]}} not in {{ updated_classic_sg.ip_permissions }}"
-
-  # ===========================================
-  always:
-    - name: Terminate classic ELB
-      ec2_elb_lb:
-        name: "{{ resource_prefix }}-classic-elb"
-        state: absent
-
-    - name: Delete security group
-      ec2_group:
-        name: "{{ resource_prefix }}-sg-a"
-        state: absent
diff --git a/tests/integration/targets/ec2_group/tasks/icmp_verbs.yml b/tests/integration/targets/ec2_group/tasks/icmp_verbs.yml
--- a/tests/integration/targets/ec2_group/tasks/icmp_verbs.yml
+++ b/tests/integration/targets/ec2_group/tasks/icmp_verbs.yml
@@ -178,9 +178,39 @@
           - result is failed
 
   always:
+    - name: tidy up egress rule test security group rules
+      ec2_group:
+        name: '{{ec2_group_name}}-auto-create-2'
+        description: 'sg-group-referencing'
+        vpc_id: '{{ vpc_result.vpc.id }}'
+        rules: []
+        rules_egress: []
+      ignore_errors: yes
+
+    - name: tidy up egress rule test security group rules
+      ec2_group:
+        name: '{{ec2_group_name}}-icmp-{{ item }}'
+        description: '{{ec2_group_description}}'
+        vpc_id: '{{ vpc_result.vpc.id }}'
+        rules: []
+        rules_egress: []
+      ignore_errors: yes
+      with_items:
+        - 1
+        - 2
+        - 3
+        - 4
+
+    - name: tidy up egress rule test security group rules
+      ec2_group:
+        name: '{{ec2_group_name}}-auto-create-2'
+        state: absent
+        vpc_id: '{{ vpc_result.vpc.id }}'
+      ignore_errors: yes
+
     - name: tidy up egress rule test security group
       ec2_group:
-        name: '{{ec2_group_name}}-auto-create-{{ item }}'
+        name: '{{ec2_group_name}}-icmp-{{ item }}'
         state: absent
         vpc_id: '{{ vpc_result.vpc.id }}'
       ignore_errors: yes
diff --git a/tests/integration/targets/ec2_group/tasks/main.yml b/tests/integration/targets/ec2_group/tasks/main.yml
--- a/tests/integration/targets/ec2_group/tasks/main.yml
+++ b/tests/integration/targets/ec2_group/tasks/main.yml
@@ -1,35 +1,15 @@
 ---
 - set_fact:
-    aws_security_token: '{{ security_token | default("") }}'
+    # lookup plugins don't have access to module_defaults
+    connection_args:
+        region: "{{ aws_region }}"
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        aws_security_token: "{{ security_token | default(omit) }}"
   no_log: True
 
 # ============================================================
-# EC2 Classic tests can only be run on a pre-2013 AWS account with supported-platforms=EC2
-# Ansible CI does NOT have classic EC2 support; these tests are provided as-is for the
-# community and can be run if you have access to a classic account.  To check if your account
-# has support for EC2 Classic you can use the `amazon.aws.aws_account_attribute` plugin.
-
-- name: determine if this is an EC2 Classic account
-  set_fact:
-    has_ec2_classic: "{{ lookup('amazon.aws.aws_account_attribute',
-                                attribute='has-ec2-classic',
-                                region=aws_region,
-                                aws_access_key=aws_access_key,
-                                aws_secret_key=aws_secret_key,
-                                aws_security_token=aws_security_token,
-                                wantlist=True) }}"
-
-# ============================================================
-- name: Run EC2 Classic accounts if account type is EC2
-  include: ./ec2_classic.yml
-  when: has_ec2_classic
-
-# ============================================================
-# Other tests depend on attribute='default-vpc', ie no vpc_id is set. This is
-# incompatible with EC2 classic accounts, so these tests can only be run in a
-# VPC-type account. See "Q. I really want a default VPC for my existing EC2
-# account. Is that possible?" in https://aws.amazon.com/vpc/faqs/#Default_VPCs
-- name: Run all other tests if account type is VPC
+- name: Run all tests
   module_defaults:
       group/aws:
         aws_access_key: "{{ aws_access_key }}"
@@ -39,12 +19,7 @@
   block:
     - name: determine if there is a default VPC
       set_fact:
-        defaultvpc: "{{ lookup('amazon.aws.aws_account_attribute',
-                              attribute='default-vpc',
-                              region=aws_region,
-                              aws_access_key=aws_access_key,
-                              aws_secret_key=aws_secret_key,
-                              aws_security_token=aws_security_token) }}"
+        defaultvpc: "{{ lookup('amazon.aws.aws_account_attribute', attribute='default-vpc', **connection_args) }}"
       register: default_vpc
 
     - name: create a VPC
@@ -1340,34 +1315,51 @@
         that:
            - 'result.changed'
            - 'not result.group_id'
-  when: not has_ec2_classic
-
   always:
+
     # ============================================================
-    - name: tidy up security group
-      ec2_group:
-        name: '{{ec2_group_name}}'
-        state: absent
-      ignore_errors: yes
+    # Describe state of remaining resources
+
+    - name: Retrieve security group info based on SG VPC
+      ec2_group_info:
+        filters:
+          vpc-id: '{{ vpc_result.vpc.id }}'
+      register: remaining_groups
+
+    - name: Retrieve subnet info based on SG VPC
+      ec2_vpc_subnet_info:
+        filters:
+          vpc-id: '{{ vpc_result.vpc.id }}'
+      register: remaining_subnets
+
+    - name: Retrieve VPC info based on SG VPC
+      ec2_vpc_net_info:
+        vpc_ids:
+          - '{{ vpc_result.vpc.id }}'
+      register: remaining_vpc
 
-    - name: tidy up security group for IPv6 EC2-Classic tests
-      ec2_group:
-        name: '{{ ec2_group_name }}-2'
-        state: absent
-      ignore_errors: yes
+    # ============================================================
+    # Delete all remaining SGs
 
-    - name: tidy up default VPC security group
+    - name: Delete rules from remaining SGs
       ec2_group:
-        name: '{{ec2_group_name}}-default-vpc'
-        state: absent
+        name: '{{ item.group_name }}'
+        group_id: '{{ item.group_id }}'
+        description: '{{ item.description }}'
+        rules: []
+        rules_egress: []
+      loop: '{{ remaining_groups.security_groups }}'
       ignore_errors: yes
 
-    - name: tidy up automatically created SG
+    - name: Delete remaining SGs
       ec2_group:
-        name: "{{ resource_prefix }} - Another security group"
         state: absent
+        group_id: '{{ item.group_id }}'
+      loop: '{{ remaining_groups.security_groups }}'
       ignore_errors: yes
 
+    # ============================================================
+
     - name: tidy up VPC
       ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
diff --git a/tests/integration/targets/ec2_group/tasks/rule_group_create.yml b/tests/integration/targets/ec2_group/tasks/rule_group_create.yml
--- a/tests/integration/targets/ec2_group/tasks/rule_group_create.yml
+++ b/tests/integration/targets/ec2_group/tasks/rule_group_create.yml
@@ -110,11 +110,12 @@
           - result.warning is not defined
 
   always:
-    - name: tidy up egress rule test security group
+    - name: tidy up egress rule test security group rules
       ec2_group:
         name: '{{ec2_group_name}}-auto-create-{{ item }}'
-        state: absent
-        vpc_id: '{{ vpc_result.vpc.id }}'
+        description: '{{ec2_group_description}}'
+        rules: []
+        rules_egress: []
       ignore_errors: yes
       with_items: [5, 4, 3, 2, 1]
     - name: tidy up egress rule test security group