Repository: cloud-custodian/cloud-custodian

Issue:
azure - add HDInsight resource



Discussion:
Duplicate of #3311 ?
### ACCEPTANCE CRITERIA

1. Follow steps to add a new resource [see [here]](https://cloudcustodian.io/docs/azure/advanced/contribute.html#adding-new-azure-resources)

FYI @erwelch 

Resolving patch:
diff --git a/tools/c7n_azure/c7n_azure/entry.py b/tools/c7n_azure/c7n_azure/entry.py
--- a/tools/c7n_azure/c7n_azure/entry.py
+++ b/tools/c7n_azure/c7n_azure/entry.py
@@ -67,7 +67,8 @@
 import c7n_azure.resources.event_hub
 import c7n_azure.resources.record_set
 import c7n_azure.resources.postgresql_server
-import c7n_azure.resources.postgresql_database  # noqa: F401
+import c7n_azure.resources.postgresql_database
+import c7n_azure.resources.hdinsight  # noqa: F401
 
 
 def initialize_azure():
diff --git a/tools/c7n_azure/c7n_azure/resources/hdinsight.py b/tools/c7n_azure/c7n_azure/resources/hdinsight.py
new file mode 100644
--- /dev/null
+++ b/tools/c7n_azure/c7n_azure/resources/hdinsight.py
@@ -0,0 +1,68 @@
+# Copyright 2019 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from c7n_azure.provider import resources
+from c7n_azure.resources.arm import ArmResourceManager
+
+
+@resources.register('hdinsight')
+class Hdinsight(ArmResourceManager):
+    """HDInsight Resource
+
+    :example:
+
+    Finds all Hadoop HDInsight Clusters
+
+    .. code-block:: yaml
+
+        policies:
+          - name: hdinsight-policy
+            resource: azure.hdinsight
+            filters:
+              - type: value
+                key: properties.clusterDefinition.kind
+                value_type: normalize
+                value: hadoop
+
+    :example:
+
+    Finds all HDInsight Clusters with 3 worker nodes
+
+    .. code-block:: yaml
+
+        policies:
+          - name: hdinsight-policy
+            resource: azure.hdinsight
+            filters:
+              - type: value
+                key: properties.computeProfile.roles[?name=='workernode'].targetInstanceCount | [0]
+                op: eq
+                value_type: integer
+                value: 3
+
+    """
+
+    class resource_type(ArmResourceManager.resource_type):
+        doc_groups = ['Analytics']
+
+        service = 'azure.mgmt.hdinsight'
+        client = 'HDInsightManagementClient'
+        enum_spec = ('clusters', 'list', None)
+        default_report_fields = (
+            'name',
+            'location',
+            'resourceGroup',
+            'properties.clusterDefinition.kind'
+        )
+        resource_type = 'Microsoft.HDInsight/clusters'
diff --git a/tools/c7n_azure/setup.py b/tools/c7n_azure/setup.py
--- a/tools/c7n_azure/setup.py
+++ b/tools/c7n_azure/setup.py
@@ -64,6 +64,7 @@
                       "azure-mgmt-dns",
                       "azure-mgmt-eventgrid",
                       "azure-mgmt-eventhub",
+                      "azure-mgmt-hdinsight",
                       "azure-mgmt-iothub",
                       "azure-mgmt-keyvault",
                       "azure-mgmt-managementgroups",


Test patch:
diff --git a/tools/c7n_azure/tests/cassettes/HdinsightTest.test_find_hdinsight_cluster_by_name.json b/tools/c7n_azure/tests/cassettes/HdinsightTest.test_find_hdinsight_cluster_by_name.json
new file mode 100644
--- /dev/null
+++ b/tools/c7n_azure/tests/cassettes/HdinsightTest.test_find_hdinsight_cluster_by_name.json
@@ -0,0 +1,140 @@
+{
+    "version": 1,
+    "interactions": [
+        {
+            "request": {
+                "method": "GET",
+                "uri": "https://management.azure.com/subscriptions/ea42f556-5106-4743-99b0-c129bfa71a47/providers/Microsoft.HDInsight/clusters?api-version=2018-06-01-preview",
+                "body": null,
+                "headers": {}
+            },
+            "response": {
+                "status": {
+                    "code": 200,
+                    "message": "OK"
+                },
+                "headers": {
+                    "content-type": [
+                        "application/json; charset=utf-8"
+                    ],
+                    "cache-control": [
+                        "no-cache"
+                    ],
+                    "x-ms-hdi-served-by": [
+                        "global"
+                    ],
+                    "date": [
+                        "Thu, 05 Sep 2019 00:53:38 GMT"
+                    ],
+                    "content-length": [
+                        "1797"
+                    ]
+                },
+                "body": {
+                    "data": {
+                        "value": [
+                            {
+                                "id": "/subscriptions/ea42f556-5106-4743-99b0-c129bfa71a47/resourceGroups/test_hdinsight/providers/Microsoft.HDInsight/clusters/cctesthdinsighthbheaulh37fm4",
+                                "name": "cctesthdinsighthbheaulh37fm4",
+                                "type": "Microsoft.HDInsight/clusters",
+                                "location": "South Central US",
+                                "etag": "ae079228-3287-484a-9446-d6761cdd1c98",
+                                "tags": null,
+                                "properties": {
+                                    "clusterVersion": "3.6.1000.67",
+                                    "clusterHdpVersion": "2.6.5.3009-43",
+                                    "osType": "Linux",
+                                    "clusterDefinition": {
+                                        "blueprint": "https://blueprints.azurehdinsight.net/hadoop-3.6.1000.67.1908082122.json",
+                                        "kind": "HADOOP",
+                                        "componentVersion": {
+                                            "Hadoop": "2.7"
+                                        }
+                                    },
+                                    "computeProfile": {
+                                        "roles": [
+                                            {
+                                                "name": "headnode",
+                                                "targetInstanceCount": 2,
+                                                "hardwareProfile": {
+                                                    "vmSize": "standard_a2m_v2"
+                                                },
+                                                "osProfile": {
+                                                    "linuxOperatingSystemProfile": {
+                                                        "username": "sshuser"
+                                                    }
+                                                },
+                                                "encryptDataDisks": false
+                                            },
+                                            {
+                                                "name": "workernode",
+                                                "targetInstanceCount": 1,
+                                                "hardwareProfile": {
+                                                    "vmSize": "standard_a2m_v2"
+                                                },
+                                                "osProfile": {
+                                                    "linuxOperatingSystemProfile": {
+                                                        "username": "sshuser"
+                                                    }
+                                                },
+                                                "encryptDataDisks": false
+                                            },
+                                            {
+                                                "name": "zookeepernode",
+                                                "targetInstanceCount": 3,
+                                                "hardwareProfile": {
+                                                    "vmSize": "standard_a2_v2"
+                                                },
+                                                "osProfile": {
+                                                    "linuxOperatingSystemProfile": {
+                                                        "username": "sshuser"
+                                                    }
+                                                },
+                                                "encryptDataDisks": false
+                                            }
+                                        ]
+                                    },
+                                    "provisioningState": "Succeeded",
+                                    "clusterState": "Running",
+                                    "createdDate": "2019-09-04T23:20:47.837",
+                                    "quotaInfo": {
+                                        "coresUsed": 6
+                                    },
+                                    "connectivityEndpoints": [
+                                        {
+                                            "name": "SSH",
+                                            "protocol": "TCP",
+                                            "location": "cctesthdinsighthbheaulh37fm4-ssh.azurehdinsight.net",
+                                            "port": 22
+                                        },
+                                        {
+                                            "name": "HTTPS",
+                                            "protocol": "TCP",
+                                            "location": "cctesthdinsighthbheaulh37fm4.azurehdinsight.net",
+                                            "port": 443
+                                        }
+                                    ],
+                                    "tier": "standard",
+                                    "storageProfile": {
+                                        "storageaccounts": [
+                                            {
+                                                "name": "ccstoragehbheaulh37fm4.blob.core.windows.net",
+                                                "resourceId": null,
+                                                "msiResourceId": null,
+                                                "key": null,
+                                                "fileSystem": null,
+                                                "container": "cctesthdinsighthbheaulh37fm4",
+                                                "saskey": null,
+                                                "isDefault": true
+                                            }
+                                        ]
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tools/c7n_azure/tests/templates/hdinsight.json b/tools/c7n_azure/tests/templates/hdinsight.json
new file mode 100644
--- /dev/null
+++ b/tools/c7n_azure/tests/templates/hdinsight.json
@@ -0,0 +1,127 @@
+{
+    "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "clusterName": {
+            "type": "string",
+            "defaultValue": "[concat('cctesthdinsight', uniqueString(resourceGroup().id))]"
+        },
+        "location": {
+            "type": "string",
+            "defaultValue": "[resourceGroup().location]"
+        },
+        "sshUserName": {
+            "type": "string",
+            "defaultValue": "sshuser",
+            "metadata": {
+                "description": "These credentials can be used to remotely access the cluster."
+            }
+        },
+        "sshPassword": {
+            "type": "securestring",
+            "defaultValue": "[concat('sshPassw0rd!', uniquestring(resourceGroup().id))]",
+            "metadata": {
+                "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter."
+            }
+        }
+    },
+    "variables": {
+        "defaultStorageAccount": {
+          "name": "[concat('ccstorage', uniqueString(resourceGroup().id))]",
+          "type": "Standard_LRS"
+        }
+      },
+    "resources": [
+        {
+            "type": "Microsoft.Storage/storageAccounts",
+            "name": "[variables('defaultStorageAccount').name]",
+            "location": "[parameters('location')]",
+            "apiVersion": "2016-01-01",
+            "sku": {
+              "name": "[variables('defaultStorageAccount').type]"
+            },
+            "kind": "Storage",
+            "properties": {}
+        },
+        {
+            "type": "Microsoft.HDInsight/clusters",
+            "apiVersion": "2015-03-01-preview",
+            "name": "[parameters('clusterName')]",
+            "location": "[parameters('location')]",
+            "dependsOn": [],
+            "properties": {
+                "clusterVersion": "3.6",
+                "osType": "Linux",
+                "tier": "Standard",
+                "clusterDefinition": {
+                    "kind": "HADOOP",
+                    "componentVersion": {
+                        "Hadoop": "2.7"
+                    },
+                    "configurations": {
+                        "gateway": {
+                            "restAuthCredential.isEnabled": true,
+                            "restAuthCredential.username": "ccusername",
+                            "restAuthCredential.password": "[concat('cctestPassw0rd!', uniquestring(resourceGroup().id))]"
+                        }
+                    }
+                },
+                "storageProfile": {
+                    "storageaccounts": [
+                        {
+                            "name": "[replace(replace(concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('defaultStorageAccount').name), '2016-01-01').primaryEndpoints.blob),'https:',''),'/','')]",
+                            "isDefault": true,
+                            "container": "[parameters('clusterName')]",
+                            "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageAccount').name), '2016-01-01').keys[0].value]"
+                        }
+                    ]
+                },
+                "computeProfile": {
+                    "roles": [
+                        {
+                            "name": "headnode",
+                            "minInstanceCount": 1,
+                            "targetInstanceCount": 2,
+                            "hardwareProfile": {
+                                "vmSize": "Standard_A2m_v2"
+                            },
+                            "osProfile": {
+                                "linuxOperatingSystemProfile": {
+                                    "username": "[parameters('sshUserName')]",
+                                    "password": "[parameters('sshPassword')]"
+                                }
+                            }
+                        },
+                        {
+                            "name": "workernode",
+                            "targetInstanceCount": 1,
+                            "hardwareProfile": {
+                                "vmSize": "Standard_A2m_v2"
+                            },
+                            "osProfile": {
+                                "linuxOperatingSystemProfile": {
+                                    "username": "[parameters('sshUserName')]",
+                                    "password": "[parameters('sshPassword')]"
+                                }
+                            }
+                        },
+                        {
+                            "name": "zookeepernode",
+                            "minInstanceCount": 1,
+                            "targetInstanceCount": 3,
+                            "hardwareProfile": {
+                                "vmSize": "Small"
+                            },
+                            "osProfile": {
+                                "linuxOperatingSystemProfile": {
+                                    "username": "[parameters('sshUserName')]",
+                                    "password": "[parameters('sshPassword')]"
+                                }
+                            }
+                        }
+                    ]
+                }
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tools/c7n_azure/tests/test_hdinsight.py b/tools/c7n_azure/tests/test_hdinsight.py
new file mode 100644
--- /dev/null
+++ b/tools/c7n_azure/tests/test_hdinsight.py
@@ -0,0 +1,44 @@
+# Copyright 2019 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+from azure_common import BaseTest, arm_template
+
+
+class HdinsightTest(BaseTest):
+
+    def test_hdinsight_schema_validate(self):
+        p = self.load_policy({
+            'name': 'test-hdinsight-schema-validate',
+            'resource': 'azure.hdinsight'
+        }, validate=True)
+        self.assertTrue(p)
+
+    @arm_template('hdinsight.json')
+    def test_find_hdinsight_cluster_by_name(self):
+        p = self.load_policy({
+            'name': 'test-azure-hdinsight-by-name',
+            'resource': 'azure.hdinsight',
+            'filters': [
+                {
+                    'type': 'value',
+                    'key': 'name',
+                    'op': 'glob',
+                    'value_type': 'normalize',
+                    'value': 'cctesthdinsight*'
+                }
+            ],
+        })
+        resources = p.run()
+        self.assertEqual(len(resources), 1)