Repository: mantl/mantl

Issue:
Terraform inventory plugin doesn't work when region is blank.
When using terraform automation and then the inventory plugin; terraform.py fails when the region in the terraform manifest is set to `region = ""`.

This is necessary for openstack environments which do not have region support enabled.



Discussion:
@matjohn2 I'll fix that - could you send me your .tfstate or a stacktrace so I can reproduce?

Got everything you needed from our chat or want me to paste still?


Resolving patch:
diff --git a/plugins/inventory/terraform.py b/plugins/inventory/terraform.py
--- a/plugins/inventory/terraform.py
+++ b/plugins/inventory/terraform.py
@@ -1,4 +1,18 @@
 #!/usr/bin/env python
+#
+# Copyright 2015 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 """\
 Dynamic inventory for Terraform - finds all `.tfstate` files below the working
 directory and generates an inventory based on them.
@@ -11,6 +25,8 @@
 import os
 import re
 
+VERSION = '0.3.0pre'
+
 
 def tfstates(root=None):
     root = root or os.getcwd()
@@ -25,8 +41,9 @@ def iterresources(filenames):
         with open(filename, 'r') as json_file:
             state = json.load(json_file)
             for module in state['modules']:
+                name = module['path'][-1]
                 for key, resource in module['resources'].items():
-                    yield key, resource
+                    yield name, key, resource
 
 ## READ RESOURCES
 PARSERS = {}
@@ -40,14 +57,14 @@ def _clean_dc(dcname):
 
 def iterhosts(resources):
     '''yield host tuples of (name, attributes, groups)'''
-    for key, resource in resources:
+    for module_name, key, resource in resources:
         resource_type, name = key.split('.', 1)
         try:
             parser = PARSERS[resource_type]
         except KeyError:
             continue
 
-        yield parser(resource)
+        yield parser(resource, module_name)
 
 
 def parses(prefix):
@@ -129,7 +146,7 @@ def parse_bool(string_form):
 
 @parses('openstack_compute_instance_v2')
 @calculate_mi_vars
-def openstack_host(resource, tfvars=None):
+def openstack_host(resource, module_name):
     raw_attrs = resource['primary']['attributes']
     name = raw_attrs['name']
     groups = []
@@ -145,7 +162,7 @@ def openstack_host(resource, tfvars=None):
         'key_pair': raw_attrs['key_pair'],
         'metadata': parse_dict(raw_attrs, 'metadata'),
         'network': parse_attr_list(raw_attrs, 'network'),
-        'region': raw_attrs['region'],
+        'region': raw_attrs.get('region', ''),
         'security_groups': parse_list(raw_attrs, 'security_groups'),
         #ansible
         'ansible_ssh_port': 22,
@@ -166,8 +183,8 @@ def openstack_host(resource, tfvars=None):
 
     # attrs specific to microservices-infrastructure
     attrs.update({
-        'consul_dc': _clean_dc(attrs['metadata'].get('dc', attrs['region'])),
-        'role': attrs['metadata'].get('role', 'none')
+        'consul_dc': _clean_dc(attrs['metadata'].get('dc', module_name)),
+        'role': attrs['metadata'].get('role', 'none'),
     })
 
     # add groups based on attrs
@@ -186,7 +203,7 @@ def openstack_host(resource, tfvars=None):
 
 @parses('aws_instance')
 @calculate_mi_vars
-def aws_host(resource, tfvars=None):
+def aws_host(resource, module_name):
     name = resource['primary']['attributes']['tags.Name']
     raw_attrs = resource['primary']['attributes']
 
@@ -221,7 +238,7 @@ def aws_host(resource, tfvars=None):
 
     # attrs specific to microservices-infrastructure
     attrs.update({
-        'consul_dc': attrs['tags'].get('dc'),
+        'consul_dc': _clean_dc(attrs['tags'].get('dc', module_name)),
         'role': attrs['tags'].get('role', 'none')
     })
 
@@ -245,7 +262,7 @@ def aws_host(resource, tfvars=None):
 
 @parses('google_compute_instance')
 @calculate_mi_vars
-def gce_host(resource, tfvars=None):
+def gce_host(resource, module_name):
     name = resource['primary']['id']
     raw_attrs = resource['primary']['attributes']
     groups = []
@@ -278,8 +295,8 @@ def gce_host(resource, tfvars=None):
 
     # attrs specific to microservices-infrastructure
     attrs.update({
-        'consul_dc': _clean_dc(attrs['metadata'].get('dc', attrs['zone'])),
-        'role': attrs['metadata'].get('role', 'none')
+        'consul_dc': _clean_dc(attrs['metadata'].get('dc', module_name)),
+        'role': attrs['metadata'].get('role', 'none'),
     })
 
     try:
@@ -347,6 +364,9 @@ def main():
                        action='store_true',
                        help='list all variables')
     modes.add_argument('--host', help='list variables for a single host')
+    modes.add_argument('--version',
+                       action='store_true',
+                       help='print version and exit')
     parser.add_argument('--pretty',
                         action='store_true',
                         help='pretty-print output JSON')
@@ -361,6 +381,10 @@ def main():
 
     args = parser.parse_args()
 
+    if args.version:
+        print('%s %s' % (__file__, VERSION))
+        parser.exit()
+
     hosts = iterhosts(iterresources(tfstates(args.root)))
     if args.list:
         output = query_list(hosts)


Test patch:
diff --git a/tests/inventory/terraform/conftest.py b/tests/inventory/terraform/conftest.py
deleted file mode 100644
--- a/tests/inventory/terraform/conftest.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# -*- coding: utf-8 -*-
-import pytest
-import sys
-import os
-
-
-@pytest.yield_fixture(autouse=True)
-def pathify():
-    path = os.path.abspath(os.path.join(
-        os.path.dirname(__file__),  # here
-        '..', '..', '..',  # root
-        'plugins', 'inventory',  # location of terraform.py
-    ))
-
-    sys.path.append(path)
-    yield
-    sys.path = [p for p in sys.path if p != path]
diff --git a/tests/inventory/terraform/test_aws.py b/tests/inventory/terraform/test_aws.py
deleted file mode 100644
--- a/tests/inventory/terraform/test_aws.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# -*- coding: utf-8 -*-
-import pytest
-
-
-@pytest.fixture
-def aws_host():
-    from terraform import aws_host
-    return aws_host
-
-
-@pytest.fixture
-def aws_resource():
-    return {
-        "type": "aws_instance",
-        "depends_on": ["aws_key_pair.deployer", "aws_security_group.control",
-                       "aws_subnet.main", "aws_vpc.main"],
-        "primary": {
-            "id": "i-c99f6f60",
-            "attributes": {
-                "ami": "ami-fe100a96",
-                "associate_public_ip_address": "true",
-                "availability_zone": "us-east-1e",
-                "ebs_block_device.#": "0",
-                "ebs_optimized": "false",
-                "ephemeral_block_device.#": "0",
-                "id": "i-c99f6f60",
-                "instance_type": "m1.medium",
-                "key_name": "key-mi",
-                "private_dns": "ip-10-0-152-191.ec2.internal",
-                "private_ip": "10.0.152.191",
-                "public_dns": "ec2-52-7-74-115.compute-1.amazonaws.com",
-                "public_ip": "52.7.74.115",
-                "root_block_device.#": "1",
-                "root_block_device.0.delete_on_termination": "true",
-                "root_block_device.0.iops": "18",
-                "root_block_device.0.volume_size": "6",
-                "root_block_device.0.volume_type": "gp2",
-                "security_groups.#": "0",
-                "subnet_id": "subnet-1155c03a",
-                "tags.#": "4",
-                "tags.Name": "mi-control-01",
-                "tags.dc": "aws",
-                "tags.role": "control",
-                "tags.sshUser": "ec2-user",
-                "tenancy": "default",
-                "vpc_security_group_ids.#": "2",
-                "vpc_security_group_ids.1636704399": "sg-9c360cf8",
-                "vpc_security_group_ids.3543019159": "sg-9d360cf9"
-            },
-            "meta": {"schema_version": "1"}
-        }
-    }
-
-
-def test_name(aws_resource, aws_host):
-    name, _, _ = aws_host(aws_resource)
-    assert name == 'mi-control-01'
-
-
-@pytest.mark.parametrize('attr,should', {
-    'ami': 'ami-fe100a96',
-    'availability_zone': 'us-east-1e',
-    'ebs_block_device': [],
-    'ebs_optimized': False,
-    'ephemeral_block_device': [],
-    'id': 'i-c99f6f60',
-    'key_name': 'key-mi',
-    'private': {'ip': '10.0.152.191',
-                'dns': 'ip-10-0-152-191.ec2.internal'},
-    'public': {
-        'ip': '52.7.74.115',
-        'dns': 'ec2-52-7-74-115.compute-1.amazonaws.com'
-    },
-    'role': 'control',
-    'root_block_device': [{
-        'volume_size': '6',
-        'iops': '18',
-        'delete_on_termination': 'true',
-        'volume_type': 'gp2'
-    }],
-    'security_groups': [],
-    'subnet': {'id': 'subnet-1155c03a'},
-    'tags': {
-        'sshUser': 'ec2-user',
-        'role': 'control',
-        'dc': 'aws',
-        'Name': 'mi-control-01'
-    },
-    'tenancy': 'default',
-    'vpc_security_group_ids': ['sg-9c360cf8', 'sg-9d360cf9'],
-    # ansible
-    'ansible_ssh_host': '52.7.74.115',
-    'ansible_ssh_port': 22,
-    'ansible_ssh_user': 'ec2-user',
-    # mi
-    'consul_dc': 'aws',
-}.items())
-def test_attrs(aws_resource, aws_host, attr, should):
-    _, attrs, _ = aws_host(aws_resource)
-    assert attr in attrs
-    assert attrs[attr] == should
-
-
-@pytest.mark.parametrize(
-    'group',
-    ['aws_ami=ami-fe100a96', 'aws_az=us-east-1e', 'aws_key_name=key-mi',
-     'aws_tenancy=default', 'aws_tag_sshUser=ec2-user', 'aws_tag_role=control',
-     'aws_tag_dc=aws', 'aws_tag_Name=mi-control-01',
-     'aws_vpc_security_group=sg-9c360cf8',
-     'aws_vpc_security_group=sg-9d360cf9', 'aws_subnet_id=subnet-1155c03a'])
-def test_groups(aws_resource, aws_host, group):
-    _, _, groups = aws_host(aws_resource)
-    assert group in groups
diff --git a/tests/inventory/terraform/test_common.py b/tests/inventory/terraform/test_common.py
deleted file mode 100644
--- a/tests/inventory/terraform/test_common.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# -*- coding: utf-8 -*-
-import pytest
-
-
-@pytest.fixture
-def calculate_mi_vars():
-    from terraform import calculate_mi_vars
-    return calculate_mi_vars
-
-
-def mirror(item):
-    def inner(*args, **kwargs):
-        return item
-
-    return inner
-
-
-@pytest.mark.parametrize('role,serverstate',
-                         [('control', True), ('worker', False),
-                          ('none', False)])
-def test_attrs(calculate_mi_vars, role, serverstate):
-    mirrorer = mirror(('', {'role': role}, []))
-    func = calculate_mi_vars(mirrorer)
-    _, attrs, _ = func()
-    assert 'consul_is_server' in attrs
-    assert attrs['consul_is_server'] == serverstate
-
-
-@pytest.mark.parametrize('routable', [True, False])
-def test_publicly_routable(calculate_mi_vars, routable):
-    attrs = {}
-    if routable:
-        attrs['publicly_routable'] = True
-
-    mirrorer = mirror(('', attrs, []))
-    func = calculate_mi_vars(mirrorer)
-    _, _, groups = func()
-    if routable:
-        assert 'publicly_routable' in groups
-    else:
-        assert 'publicly_routable' not in groups
diff --git a/tests/inventory/terraform/test_gce.py b/tests/inventory/terraform/test_gce.py
deleted file mode 100644
--- a/tests/inventory/terraform/test_gce.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# -*- coding: utf-8 -*-
-import pytest
-
-
-@pytest.fixture
-def gce_host():
-    from terraform import gce_host
-    return gce_host
-
-
-@pytest.fixture
-def gce_resource():
-    return {
-        "type": "google_compute_instance",
-        "depends_on": ["google_compute_network.mi-network"],
-        "primary": {
-            "id": "mi-control-01",
-            "attributes": {
-                "can_ip_forward": "false",
-                "description": "microservices-infrastructure control node #01",
-                "disk.#": "1",
-                "disk.0.auto_delete": "true",
-                "disk.0.device_name": "",
-                "disk.0.disk": "",
-                "disk.0.image": "centos-7-v20150423",
-                "disk.0.scratch": "false",
-                "disk.0.size": "0",
-                "disk.0.type": "",
-                "id": "mi-control-01",
-                "machine_type": "n1-standard-1",
-                "metadata.#": "3",
-                "metadata.dc": "gce-dc",
-                "metadata.role": "control",
-                "metadata.sshKeys": "fake ssh key",
-                "metadata_fingerprint": "RRQXhOpeylY=",
-                "name": "mi-control-01",
-                "network.#": "0",
-                "network_interface.#": "1",
-                "network_interface.0.access_config.#": "1",
-                "network_interface.0.access_config.0.nat_ip": "104.197.63.156",
-                "network_interface.0.address": "10.0.237.130",
-                "network_interface.0.name": "nic0",
-                "network_interface.0.network": "microservices-infrastructure",
-                "self_link":
-                "https://www.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a/instances/mi-control-01",
-                "service_account.#": "0",
-                "tags.#": "2",
-                "tags.2783239913": "mi",
-                "tags.3990563915": "control",
-                "tags_fingerprint": "hq1EeKIUnfk=",
-                "zone": "us-central1-a"
-            },
-            "meta": {"schema_version": "2"}
-        }
-    }
-
-
-def test_name(gce_resource, gce_host):
-    name, _, _ = gce_host(gce_resource)
-    assert name == 'mi-control-01'
-
-
-@pytest.mark.parametrize('attr,should', {
-    'can_ip_forward': False,
-    'disks': [{
-        'auto_delete': 'true',
-        'device_name': '',
-        'disk': '',
-        'image': 'centos-7-v20150423',
-        'scratch': 'false',
-        'size': '0',
-        'type': '',
-    }],
-    'machine_type': 'n1-standard-1',
-    'metadata':
-    {'dc': 'gce-dc',
-     'role': 'control',
-     'sshKeys': 'fake ssh key', },
-    'network': [],
-    'network_interface': [{
-        'network': 'microservices-infrastructure',
-        'access_config': [{'nat_ip': '104.197.63.156'}],
-        'address': '10.0.237.130',
-        'name': 'nic0'
-    }],
-    'self_link':
-    'https://www.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a/instances/mi-control-01',
-    'service_account': [],
-    'tags': ['mi', 'control'],
-    'zone': 'us-central1-a',
-    # ansible
-    'ansible_ssh_user': 'deploy',
-    'ansible_ssh_host': '104.197.63.156',
-    'ansible_ssh_port': 22,
-    # mi
-    'consul_dc': 'gce-dc',
-    'role': 'control',
-    'publicly_routable': True,
-}.items())
-def test_attrs(gce_resource, gce_host, attr, should):
-    _, attrs, _ = gce_host(gce_resource)
-    assert attr in attrs
-    assert attrs[attr] == should
-
-
-@pytest.mark.parametrize('group', [
-    'gce_image=centos-7-v20150423',
-    'gce_machine_type=n1-standard-1',
-    'gce_metadata_dc=gce-dc',
-    'gce_metadata_role=control',
-    'gce_tag=mi',
-    'gce_tag=control',
-    'gce_publicly_routable',
-    'role=control',
-    'dc=gce-dc',
-])
-def test_groups(gce_resource, gce_host, group):
-    _, _, groups = gce_host(gce_resource)
-    assert group in groups
diff --git a/tests/inventory/terraform/test_openstack.py b/tests/inventory/terraform/test_openstack.py
deleted file mode 100644
--- a/tests/inventory/terraform/test_openstack.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-import pytest
-
-
-@pytest.fixture
-def openstack_host():
-    from terraform import openstack_host
-    return openstack_host
-
-
-@pytest.fixture
-def openstack_resource():
-    return {
-        "type": "openstack_compute_instance_v2",
-        "primary": {
-            "id": "81467bb5-e214-4764-b501-8b59892d88e2",
-            "attributes": {
-                "access_ip_v4": "173.39.243.27",
-                "access_ip_v6": "",
-                "flavor_id": "c6580dce-3cf4-488a-8eb0-8d3766c5e6f7",
-                "flavor_name": "CO2-2XLarge",
-                "id": "81467bb5-e214-4764-b501-8b59892d88e2",
-                "image_id": "1bbd1f1f-d32c-43ca-a727-3b2b075c3e28",
-                "image_name": "centos-7_x86_64-2015-01-27-v6",
-                "key_pair": "ansible_pubkey_sborrelli",
-                "metadata.#": "1",
-                "metadata.role": "control",
-                "name": "mi-control-01",
-                "network.#": "1",
-                "network.0.fixed_ip_v4": "173.39.243.27",
-                "network.0.fixed_ip_v6": "",
-                "network.0.mac": "fa:16:3e:12:13:d7",
-                "network.0.name": "public-direct-600",
-                "network.0.port": "",
-                "network.0.uuid": "e7b1be4f-e0d2-4024-948f-b9e6c4911123",
-                "region": "eu-amsterdam-1",
-                "security_groups.#": "1",
-                "security_groups.0": "default"
-            }
-        }
-    }
-
-
-def test_name(openstack_resource, openstack_host):
-    name, _, _ = openstack_host(openstack_resource)
-    assert name == 'mi-control-01'
-
-
-@pytest.mark.parametrize('attr,should', {
-    'access_ip_v4': '173.39.243.27',
-    'access_ip_v6': '',
-    'flavor': {
-        'id': 'c6580dce-3cf4-488a-8eb0-8d3766c5e6f7',
-        'name': 'CO2-2XLarge',
-    },
-    'id': '81467bb5-e214-4764-b501-8b59892d88e2',
-    'image': {
-        'id': '1bbd1f1f-d32c-43ca-a727-3b2b075c3e28',
-        'name': 'centos-7_x86_64-2015-01-27-v6',
-    },
-    'key_pair': 'ansible_pubkey_sborrelli',
-    'metadata': {'role': 'control', },
-    'network': [{
-        'fixed_ip_v4': '173.39.243.27',
-        'fixed_ip_v6': '',
-        'mac': 'fa:16:3e:12:13:d7',
-        'name': 'public-direct-600',
-        'port': '',
-        'uuid': 'e7b1be4f-e0d2-4024-948f-b9e6c4911123'
-    }],
-    'region': 'eu-amsterdam-1',
-    'security_groups': ['default'],
-    # ansible
-    'ansible_ssh_user': 22,
-    'ansible_ssh_user': 'centos',
-    'ansible_ssh_host': '173.39.243.27',
-    'publicly_routable': True,
-    # mi
-    'consul_dc': 'eu-amsterdam-1',
-    'role': 'control',
-    # and the bugfix
-    'use_host_domain': True,
-    'host_domain': 'novalocal',
-}.items())
-def test_attrs(openstack_resource, openstack_host, attr, should):
-    _, attrs, _ = openstack_host(openstack_resource)
-    assert attr in attrs
-    assert attrs[attr] == should
-
-@pytest.mark.parametrize('group', [
-    'os_image=centos-7_x86_64-2015-01-27-v6',
-    'os_flavor=CO2-2XLarge',
-    'os_metadata_role=control',
-    'os_region=eu-amsterdam-1',
-    'role=control',
-    'dc=eu-amsterdam-1',
-])
-def test_groups(openstack_resource, openstack_host, group):
-    _, _, groups = openstack_host(openstack_resource)
-    assert group in groups