Repository: localstack/localstack

Issue:
--elasticsearch-cluster-config is (at least partially) ignored
<!-- Love localstack? Please consider supporting our collective:
👉  https://opencollective.com/localstack/donate -->

# Type of request: This is a ...

[x] bug report
[ ] feature request

# Detailed description

I'm using `localstack:latest` as a docker image. But when I try to create a new Elasticsearch domain with a specific configuration, the `InstanceType` and `InstanceCount` are ignored.

## Expected behavior

`InstanceCount` and `InstanceType` are recognised.

## Actual behavior

InstanceCount and InstanceType are ignored.

# Steps to reproduce

Setup `localstack:latest` as a docker image and try to setup an elasticsearch domain with `--elasticsearch-cluster-config` included.

## Command used to start LocalStack

Since I'm using docker (or more specific docker-compose) I'm using `docker-compose up -d`

## Client code (AWS SDK code snippet, or sequence of "awslocal" commands)

I've tried 2 different approaches, once with shorthand syntax and one with json syntax, both got the same unexpected Output:

```sh
aws --endpoint-url=http://localhost:4566 es create-elasticsearch-domain --domain-name mylogs-2 --elasticsearch-version 7.10 --elasticsearch-cluster-config '{ "InstanceType": "m3.xlarge.elasticsearch", "InstanceCount": 4, "DedicatedMasterEnabled": true, "ZoneAwarenessEnabled": true, "DedicatedMasterType": "m3.xlarge.elasticsearch", "DedicatedMasterCount": 3}'
```

I'm expecting to get `m3.xlarge.elasticsearch` as `InstanceType` and 4 as `InstanceCount` but the output looks like this:

```json
{
    "DomainStatus": {
        "DomainId": "000000000000/mylogs-2",
        "DomainName": "mylogs-2",
        "ARN": "arn:aws:es:us-east-1:000000000000:domain/mylogs-2",
        "Created": true,
        "Deleted": false,
        "Endpoint": "http://localhost:4571",
        "Processing": false,
        "ElasticsearchVersion": "7.10",
        "ElasticsearchClusterConfig": {
            "InstanceType": "m3.medium.elasticsearch",
            "InstanceCount": 1,
            "DedicatedMasterEnabled": true,
            "ZoneAwarenessEnabled": false,
            "DedicatedMasterType": "m3.medium.elasticsearch",
            "DedicatedMasterCount": 1
        },
        "EBSOptions": {
            "EBSEnabled": true,
            "VolumeType": "gp2",
            "VolumeSize": 10,
            "Iops": 0
        },
        "CognitoOptions": {
            "Enabled": false
        }
    }
}
```

What I get is `m3.medium.elasticsearch` as `InstanceType` instead of `m3.xlarge.elasticsearch`  and 1 as `InstanceCount` instead of 4.

I've used the json syntax as well:
```sh
aws es create-elasticsearch-domain --domain-name mylogs-1 --elasticsearch-version 7.10 --elasticsearch-cluster-config InstanceType=r6g.large.elasticsearch,InstanceCount=6 --endpoint-url=http://localhost:4566
```

but got the same result. :/


Discussion:
Hello @Dgame, the config is hardcoded for elasticsearch, I will add this to our pipeline and get back to you with a fix. Cheers.

Resolving patch:
diff --git a/localstack/services/es/es_api.py b/localstack/services/es/es_api.py
--- a/localstack/services/es/es_api.py
+++ b/localstack/services/es/es_api.py
@@ -22,6 +22,15 @@
 
 ES_DOMAINS = {}
 
+DEFAULT_ES_CLUSTER_CONFIG = {
+    'InstanceType': 'm3.medium.elasticsearch',
+    'InstanceCount': 1,
+    'DedicatedMasterEnabled': True,
+    'ZoneAwarenessEnabled': False,
+    'DedicatedMasterType': 'm3.medium.elasticsearch',
+    'DedicatedMasterCount': 1
+}
+
 TAGS = TaggingService()
 
 app = Flask(APP_NAME)
@@ -50,6 +59,9 @@ def get_domain_config_status():
 
 
 def get_domain_config(domain_name):
+    status = ES_DOMAINS.get(domain_name) or {}
+    cluster_cfg = status.get('ElasticsearchClusterConfig') or {}
+    default_cfg = DEFAULT_ES_CLUSTER_CONFIG
     config_status = get_domain_config_status()
     return {
         'DomainConfig': {
@@ -76,12 +88,18 @@ def get_domain_config(domain_name):
             },
             'ElasticsearchClusterConfig': {
                 'Options': {
-                    'DedicatedMasterCount': 1,
-                    'DedicatedMasterEnabled': True,
-                    'DedicatedMasterType': 'm3.medium.elasticsearch',
-                    'InstanceCount': 1,
-                    'InstanceType': 'm3.medium.elasticsearch',
-                    'ZoneAwarenessEnabled': False
+                    'DedicatedMasterCount': cluster_cfg.get(
+                        'DedicatedMasterCount', default_cfg['DedicatedMasterCount']),
+                    'DedicatedMasterEnabled': cluster_cfg.get(
+                        'DedicatedMasterEnabled', default_cfg['DedicatedMasterEnabled']),
+                    'DedicatedMasterType': cluster_cfg.get(
+                        'DedicatedMasterType', default_cfg['DedicatedMasterType']),
+                    'InstanceCount': cluster_cfg.get(
+                        'InstanceCount', default_cfg['InstanceCount']),
+                    'InstanceType': cluster_cfg.get(
+                        'InstanceType', default_cfg['InstanceType']),
+                    'ZoneAwarenessEnabled': cluster_cfg.get(
+                        'ZoneAwarenessEnabled', default_cfg['ZoneAwarenessEnabled']),
                 },
                 'Status': config_status
             },
@@ -139,6 +157,8 @@ def get_domain_config(domain_name):
 
 def get_domain_status(domain_name, deleted=False):
     status = ES_DOMAINS.get(domain_name) or {}
+    cluster_cfg = status.get('ElasticsearchClusterConfig') or {}
+    default_cfg = DEFAULT_ES_CLUSTER_CONFIG
     endpoint = '%s://%s:%s' % (get_service_protocol(), config.HOSTNAME_EXTERNAL, config.PORT_ELASTICSEARCH)
     return {
         'DomainStatus': {
@@ -148,12 +168,18 @@ def get_domain_status(domain_name, deleted=False):
             'DomainId': '%s/%s' % (TEST_AWS_ACCOUNT_ID, domain_name),
             'DomainName': domain_name,
             'ElasticsearchClusterConfig': {
-                'DedicatedMasterCount': 1,
-                'DedicatedMasterEnabled': True,
-                'DedicatedMasterType': 'm3.medium.elasticsearch',
-                'InstanceCount': 1,
-                'InstanceType': 'm3.medium.elasticsearch',
-                'ZoneAwarenessEnabled': False
+                'DedicatedMasterCount': cluster_cfg.get(
+                    'DedicatedMasterCount', default_cfg['DedicatedMasterCount']),
+                'DedicatedMasterEnabled': cluster_cfg.get(
+                    'DedicatedMasterEnabled', default_cfg['DedicatedMasterEnabled']),
+                'DedicatedMasterType': cluster_cfg.get(
+                    'DedicatedMasterType', default_cfg['DedicatedMasterType']),
+                'InstanceCount': cluster_cfg.get(
+                    'InstanceCount', default_cfg['InstanceCount']),
+                'InstanceType': cluster_cfg.get(
+                    'InstanceType', default_cfg['InstanceType']),
+                'ZoneAwarenessEnabled': cluster_cfg.get(
+                    'ZoneAwarenessEnabled', default_cfg['ZoneAwarenessEnabled']),
             },
             'ElasticsearchVersion': status.get('ElasticsearchVersion') or DEFAULT_ES_VERSION,
             'Endpoint': endpoint,


Test patch:
diff --git a/tests/integration/test_elasticsearch.py b/tests/integration/test_elasticsearch.py
--- a/tests/integration/test_elasticsearch.py
+++ b/tests/integration/test_elasticsearch.py
@@ -15,6 +15,14 @@
     'Accept-encoding': 'identity'
 }
 TEST_DOMAIN_NAME = 'test_es_domain_1'
+ES_CLUSTER_CONFIG = {
+    'InstanceType': 'm3.xlarge.elasticsearch',
+    'InstanceCount': 4,
+    'DedicatedMasterEnabled': True,
+    'ZoneAwarenessEnabled': True,
+    'DedicatedMasterType': 'm3.xlarge.elasticsearch',
+    'DedicatedMasterCount': 3
+}
 
 
 class ElasticsearchTest(unittest.TestCase):
@@ -49,9 +57,10 @@ def test_domain_es_version(self):
         self.assertEqual(status['ElasticsearchVersion'], DEFAULT_ES_VERSION)
 
         domain_name = 'es-%s' % short_uid()
-        self._create_domain(name=domain_name, version='6.8')
+        self._create_domain(name=domain_name, version='6.8', es_cluster_config=ES_CLUSTER_CONFIG)
         status = es_client.describe_elasticsearch_domain(DomainName=domain_name)['DomainStatus']
         self.assertEqual(status['ElasticsearchVersion'], '6.8')
+        self.assertEqual(status['ElasticsearchClusterConfig'], ES_CLUSTER_CONFIG)
 
     def test_create_indexes_and_domains(self):
         indexes = ['index1', 'index2']
@@ -142,12 +151,14 @@ def _delete_document(self, id):
         return resp
 
     @classmethod
-    def _create_domain(cls, name=None, version=None):
+    def _create_domain(cls, name=None, version=None, es_cluster_config=None):
         es_client = aws_stack.connect_to_service('es')
         name = name or TEST_DOMAIN_NAME
         kwargs = {}
         if version:
             kwargs['ElasticsearchVersion'] = version
+        if es_cluster_config:
+            kwargs['ElasticsearchClusterConfig'] = es_cluster_config
         es_client.create_elasticsearch_domain(DomainName=name, **kwargs)
         assert_in(name, [d['DomainName'] for d in es_client.list_domain_names()['DomainNames']])