Repository: elastic/elasticsearch-dsl-py

Issue:
Error with DateHistogramFacet, bucket['key'] is None instead of int()
I am using Doctype with Date() field. When I want to use DateHistogramFacet I get this error: 
`TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'`

Error is raised in file faceted_search.py in DateHistogramFacet.get_value() function on line 144: `return datetime.utcfromtimestamp(int(bucket['key']) / 1000)`

`bucket` variable shows this values during error:

```
bucket = {Bucket} {'key_as_string': '1970-01-01T00:00:00.000Z', 'key': None, '...}
 doc_count = {int} 84
 key = {NoneType} None
 key_as_string = {str} '1970-01-01T00:00:00.000Z'
```
Before calling get_value() function `data` variable looks like this:
```
[{'key_as_string': '1970-01-01T00:00:00.000Z', 'key': None, 'doc_count': 4},
{'key_as_string': '1970-02-01T00:00:00.000Z', 'key': 2678400000, 'doc_count': 3},
{'key_as_string': '1970-03-01T00:00:00.000Z', 'key': 5097600000, 'doc_count': 0}, 
{'key_as_string': '1970-04-01T00:00:00.000Z', 'key': 7776000000, 'doc_count': 5}, 
{'key_as_string': '1970-05-01T00:00:00.000Z', 'key': 10368000000, 'doc_count': 17}, 
{'key_as_string': '1970-06-01T00:00:00.000Z', 'key': 13046400000, 'doc_count': 4}, 
{'key_as_string': '1970-07-01T00:00:00.000Z', 'key': 15638400000, 'doc_count': 8}, 
{'key_as_string': '1970-08-01T00:00:00.000Z', 'key': 18316800000, 'doc_count': 2}, 
{'key_as_string': '1970-09-01T00:00:00.000Z', 'key': 20995200000, 'doc_count': 1}, 
{'key_as_string': '1970-10-01T00:00:00.000Z', 'key': 23587200000, 'doc_count': 0}, 
{'key_as_string': '1970-11-01T00:00:00.000Z', 'key': 26265600000, 'doc_count': 6}, 
{'key_as_string': '1970-12-01T00:00:00.000Z', 'key': 28857600000, 'doc_count': 34}, 
.....]
```



Discussion:

Patch:
diff --git a/.gitignore b/.gitignore
index 4d5d0eb..d08fec9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ test_elasticsearch_dsl/htmlcov
 docs/_build
 .cache
 venv
+.idea
diff --git a/Changelog.rst b/Changelog.rst
index 8aa4e7b..d35153a 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -3,6 +3,12 @@
 Changelog
 =========
 
+5.4.0 (dev)
+-----------
+ * fix ``ip_range`` aggregation and rename the class to ``IPRange``.
+   ``Iprange`` is kept for bw compatibility
+ * fix bug in loading an aggregation with meta data from dict
+
 5.3.0 (2017-05-18)
 ------------------
  * fix constant score query definition
diff --git a/README b/README
index 0422153..c6ac2fd 100644
--- a/README
+++ b/README
@@ -64,7 +64,7 @@ Let's have a typical search request written directly as a ``dict``:
           "query": {
             "bool": {
               "must": [{"match": {"title": "python"}}],
-              "must_not": [{"match": {"description": "beta"}}]
+              "must_not": [{"match": {"description": "beta"}}],
               "filter": [{"term": {"category": "search"}}]
             }
           },
diff --git a/docs/faceted_search.rst b/docs/faceted_search.rst
index a08e649..169534c 100644
--- a/docs/faceted_search.rst
+++ b/docs/faceted_search.rst
@@ -54,9 +54,9 @@ There are several different facets available:
 ``HistogramFacet``
   similar to ``DateHistogramFacet`` but for numerical values: ``HistogramFacet(field="rating", interval=2)``
 
-``Rangefacet``
+``RangeFacet``
   allows you to define your own ranges for a numerical fields:
-  ``Rangefacet(field="comment_count", ranges=[("few", (None, 2)), ("lots", (2, None))])``
+  ``RangeFacet(field="comment_count", ranges=[("few", (None, 2)), ("lots", (2, None))])``
 
 Advanced
 ~~~~~~~~
diff --git a/elasticsearch_dsl/__init__.py b/elasticsearch_dsl/__init__.py
index 0a11f2c..74c7e2a 100644
--- a/elasticsearch_dsl/__init__.py
+++ b/elasticsearch_dsl/__init__.py
@@ -9,6 +9,6 @@ from .index import Index
 from .analysis import analyzer, token_filter, char_filter, tokenizer
 from .faceted_search import *
 
-VERSION = (5, 3, 0)
+VERSION = (5, 4, 0, 'dev')
 __version__ = VERSION
 __versionstr__ = '.'.join(map(str, VERSION))
diff --git a/elasticsearch_dsl/aggs.py b/elasticsearch_dsl/aggs.py
index fba5980..5a56172 100644
--- a/elasticsearch_dsl/aggs.py
+++ b/elasticsearch_dsl/aggs.py
@@ -17,14 +17,19 @@ def A(name_or_agg, filter=None, **params):
         agg = name_or_agg.copy()
         # pop out nested aggs
         aggs = agg.pop('aggs', None)
+        # pop out meta data
+        meta = agg.pop('meta', None)
         # should be {"terms": {"field": "tags"}}
         if len(agg) != 1:
             raise ValueError('A() can only accept dict with an aggregation ({"terms": {...}}). '
-                 'Instead it got (%r)' % name_or_agg)
+                             'Instead it got (%r)' % name_or_agg)
         agg_type, params = agg.popitem()
         if aggs:
             params = params.copy()
             params['aggs'] = aggs
+        if meta:
+            params = params.copy()
+            params['meta'] = meta
         return Agg.get_dsl_class(agg_type)(_expand__to_dot=False, **params)
 
     # Terms(...) just return the nested agg
@@ -62,7 +67,7 @@ class AggBase(object):
         return key in self._params.get('aggs', {})
 
     def __getitem__(self, agg_name):
-        agg = self._params.setdefault('aggs', {})[agg_name] # propagate KeyError
+        agg = self._params.setdefault('aggs', {})[agg_name]  # propagate KeyError
 
         # make sure we're not mutating a shared state - whenever accessing a
         # bucket, return a shallow copy of it to be safe
@@ -171,8 +176,12 @@ class Histogram(Bucket):
     def result(self, search, data):
         return FieldBucketData(self, search, data)
 
+# TODO: remove in 6.0
 class Iprange(Bucket):
-    name = 'iprange'
+    name = 'ip_range'
+
+class IPRange(Bucket):
+    name = 'ip_range'
 
 class Missing(Bucket):
     name = 'missing'
diff --git a/elasticsearch_dsl/analysis.py b/elasticsearch_dsl/analysis.py
index b735365..8424283 100644
--- a/elasticsearch_dsl/analysis.py
+++ b/elasticsearch_dsl/analysis.py
@@ -1,7 +1,7 @@
 from .utils import DslBase
 
 __all__ = [
-    'tokenizer', 'analyzer', 'char_filter', 'token_filter'
+    'tokenizer', 'analyzer', 'char_filter', 'token_filter', 'normalizer'
 ]
 
 class AnalysisBase(object):
@@ -34,6 +34,26 @@ class CustomAnalysis(object):
         d['type'] = self._builtin_type
         return d
 
+class CustomAnalysisDefinition(CustomAnalysis):
+    def get_analysis_definition(self):
+        out = {self._type_name: {self._name: self.get_definition()}}
+
+        t = getattr(self, 'tokenizer', None)
+        if 'tokenizer' in self._param_defs and hasattr(t, 'get_definition'):
+            out['tokenizer'] = {t._name: t.get_definition()}
+
+        filters = dict((f._name, f.get_definition())
+                       for f in self.filter if hasattr(f, 'get_definition'))
+        if filters:
+            out['filter'] = filters
+
+        char_filters = dict((f._name, f.get_definition())
+                            for f in self.char_filter if hasattr(f, 'get_definition'))
+        if char_filters:
+            out['char_filter'] = char_filters
+
+        return out
+
 class BuiltinAnalysis(object):
     name = 'builtin'
     def __init__(self, name):
@@ -52,33 +72,26 @@ class BuiltinAnalyzer(BuiltinAnalysis, Analyzer):
     def get_analysis_definition(self):
         return {}
 
-class CustomAnalyzer(CustomAnalysis, Analyzer):
+class CustomAnalyzer(CustomAnalysisDefinition, Analyzer):
     _param_defs = {
         'filter': {'type': 'token_filter', 'multi': True},
         'char_filter': {'type': 'char_filter', 'multi': True},
         'tokenizer': {'type': 'tokenizer'},
     }
 
-    def get_analysis_definition(self):
-        out = {'analyzer': {self._name: self.get_definition()}}
-
-        t = getattr(self, 'tokenizer', None)
-        if hasattr(t, 'get_definition'):
-            out['tokenizer'] = {t._name: t.get_definition()}
-
-        filters = dict((f._name, f.get_definition())
-                for f in self.filter if hasattr(f, 'get_definition'))
-        if filters:
-            out['filter'] = filters
-
-
-        char_filters = dict((f._name, f.get_definition())
-                for f in self.char_filter if hasattr(f, 'get_definition'))
-        if char_filters:
-            out['char_filter'] = char_filters
+class Normalizer(AnalysisBase, DslBase):
+    _type_name = 'normalizer'
+    name = None
 
-        return out
+class BuiltinNormalizer(BuiltinAnalysis, Normalizer):
+    def get_analysis_definition(self):
+        return {}
 
+class CustomNormalizer(CustomAnalysisDefinition, Normalizer):
+    _param_defs = {
+        'filter': {'type': 'token_filter', 'multi': True},
+        'char_filter': {'type': 'char_filter', 'multi': True}
+    }
 
 class Tokenizer(AnalysisBase, DslBase):
     _type_name = 'tokenizer'
@@ -113,9 +126,9 @@ class CustomCharFilter(CustomAnalysis, CharFilter):
     pass
 
 
-
 # shortcuts for direct use
 analyzer = Analyzer._type_shortcut
 tokenizer = Tokenizer._type_shortcut
 token_filter = TokenFilter._type_shortcut
 char_filter = CharFilter._type_shortcut
+normalizer = Normalizer._type_shortcut
diff --git a/elasticsearch_dsl/document.py b/elasticsearch_dsl/document.py
index 1c4e5f9..a0a103f 100644
--- a/elasticsearch_dsl/document.py
+++ b/elasticsearch_dsl/document.py
@@ -25,16 +25,19 @@ META_FIELDS = frozenset((
     'index', 'using', 'score',
 )).union(DOC_META_FIELDS)
 
+
 class MetaField(object):
     def __init__(self, *args, **kwargs):
         self.args, self.kwargs = args, kwargs
 
+
 class DocTypeMeta(type):
     def __new__(cls, name, bases, attrs):
         # DocTypeMeta filters attrs in place
         attrs['_doc_type'] = DocTypeOptions(name, bases, attrs)
         return super(DocTypeMeta, cls).__new__(cls, name, bases, attrs)
 
+
 class DocTypeOptions(object):
     def __init__(self, name, bases, attrs):
         meta = attrs.pop('Meta', None)
@@ -337,7 +340,7 @@ class DocType(ObjectBase):
         meta['_source'] = d
         return meta
 
-    def update(self, using=None, index=None, **fields):
+    def update(self, using=None, index=None,  detect_noop=True, doc_as_upsert=False, **fields):
         """
         Partial update of the document, specify fields you wish to update and
         both the instance and the document in elasticsearch will be updated::
@@ -356,21 +359,37 @@ class DocType(ObjectBase):
         if not fields:
             raise IllegalOperation('You cannot call update() without updating individual fields. '
                                    'If you wish to update the entire object use save().')
+
         es = self._get_connection(using)
 
-        # update the data locally
+        # update given fields locally
         merge(self._d_, fields)
 
+        # prepare data for ES
+        values = self.to_dict()
+
+        # if fields were given: partial update
+        doc = dict(
+            (k, values.get(k))
+            for k in fields.keys()
+        )
+
         # extract parent, routing etc from meta
         doc_meta = dict(
             (k, self.meta[k])
             for k in DOC_META_FIELDS
             if k in self.meta
         )
+        body = {
+            'doc': doc,
+            'doc_as_upsert': doc_as_upsert,
+            'detect_noop': detect_noop,
+        }
+
         meta = es.update(
             index=self._get_index(index),
             doc_type=self._doc_type.name,
-            body={'doc': fields},
+            body=body,
             **doc_meta
         )
         # update meta information from ES
diff --git a/elasticsearch_dsl/faceted_search.py b/elasticsearch_dsl/faceted_search.py
index d896625..75c5fb0 100644
--- a/elasticsearch_dsl/faceted_search.py
+++ b/elasticsearch_dsl/faceted_search.py
@@ -141,6 +141,10 @@ class DateHistogramFacet(Facet):
 
     def get_value(self, bucket):
         if not isinstance(bucket['key'], datetime):
+            # Elasticsearch returns key=None instead of 0 for date 1970-01-01,
+            # so we need to set key to 0 to avoid TypeError exception
+            if bucket['key'] is None:
+                bucket['key'] = 0
             return datetime.utcfromtimestamp(int(bucket['key']) / 1000)
         else:
             return bucket['key']
diff --git a/elasticsearch_dsl/field.py b/elasticsearch_dsl/field.py
index 3fbc80d..129b53c 100644
--- a/elasticsearch_dsl/field.py
+++ b/elasticsearch_dsl/field.py
@@ -247,6 +247,7 @@ class Keyword(Field):
     _param_defs = {
         'fields': {'type': 'field', 'hash': True},
         'search_analyzer': {'type': 'analyzer'},
+        'normalizer': {'type': 'normalizer'}
     }
     name = 'keyword'
 
diff --git a/elasticsearch_dsl/index.py b/elasticsearch_dsl/index.py
index 7fcb168..634c00f 100644
--- a/elasticsearch_dsl/index.py
+++ b/elasticsearch_dsl/index.py
@@ -133,7 +133,7 @@ class Index(object):
 
     def search(self):
         """
-        Rteurn a :class:`~elasticsearch_dsl.Search` object searching over this
+        Return a :class:`~elasticsearch_dsl.Search` object searching over this
         index and its ``DocType``\s.
         """
         return Search(
diff --git a/elasticsearch_dsl/mapping.py b/elasticsearch_dsl/mapping.py
index c738ed2..385416d 100644
--- a/elasticsearch_dsl/mapping.py
+++ b/elasticsearch_dsl/mapping.py
@@ -57,7 +57,7 @@ class Mapping(object):
             fields.append(Text(**self._meta['_all']))
 
         for f in chain(fields, self.properties._collect_fields()):
-            for analyzer_name in ('analyzer', 'search_analyzer', 'search_quote_analyzer'):
+            for analyzer_name in ('analyzer', 'normalizer', 'search_analyzer', 'search_quote_analyzer'):
                 if not hasattr(f, analyzer_name):
                     continue
                 analyzer = getattr(f, analyzer_name)
diff --git a/elasticsearch_dsl/search.py b/elasticsearch_dsl/search.py
index bb2706d..fbb2d24 100644
--- a/elasticsearch_dsl/search.py
+++ b/elasticsearch_dsl/search.py
@@ -135,13 +135,24 @@ class Request(object):
 
             s = Search()
             s = s.index('twitter-2015.01.01', 'twitter-2015.01.02')
+            s = s.index(['twitter-2015.01.01', 'twitter-2015.01.02'])
         """
         # .index() resets
         s = self._clone()
         if not index:
             s._index = None
         else:
-            s._index = (self._index or []) + list(index)
+            indexes = []
+            for i in index:
+                if isinstance(i, str):
+                    indexes.append(i)
+                elif isinstance(i, list):
+                    indexes += i
+                elif isinstance(i, tuple):
+                    indexes += list(i)
+
+            s._index = (self._index or []) + indexes
+
         return s
 
     def _add_doc_type(self, doc_type):
@@ -566,7 +577,7 @@ class Search(Request):
 
             d.update(self._extra)
 
-            if not self._source in (None, {}):
+            if self._source not in (None, {}):
                 d['_source'] = self._source
 
             if self._highlight:
@@ -654,7 +665,7 @@ class Search(Request):
                 index=self._index,
                 doc_type=self._doc_type,
                 **self._params
-            ):
+        ):
             callback = self._doc_type_map.get(hit['_type'], Hit)
             callback = getattr(callback, 'from_es', callback)
             yield callback(hit)
@@ -676,7 +687,6 @@ class Search(Request):
         )
 
 
-
 class MultiSearch(Request):
     """
     Combine multiple :class:`~elasticsearch_dsl.Search` objects into a single
diff --git a/setup.py b/setup.py
index d45101d..bfd7f45 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ import sys
 from os.path import join, dirname
 from setuptools import setup, find_packages
 
-VERSION = (5, 3, 0)
+VERSION = (5, 4, 0, 'dev')
 __version__ = VERSION
 __versionstr__ = '.'.join(map(str, VERSION))
 


Test patch:
diff --git a/test_elasticsearch_dsl/test_aggs.py b/test_elasticsearch_dsl/test_aggs.py
index 4f7dd10..a35de40 100644
--- a/test_elasticsearch_dsl/test_aggs.py
+++ b/test_elasticsearch_dsl/test_aggs.py
@@ -18,6 +18,12 @@ def test_meta():
         'meta': {'some': 'metadata'}
     } == a.to_dict()
 
+def test_meta_from_dict():
+    max_score = aggs.Max(field='score')
+    a = aggs.A('terms', field='tags', aggs={'max_score': max_score}, meta={'some': 'metadata'})
+
+    assert aggs.A(a.to_dict()) == a
+
 def test_A_creates_proper_agg():
     a = aggs.A('terms', field='tags')
 
diff --git a/test_elasticsearch_dsl/test_analysis.py b/test_elasticsearch_dsl/test_analysis.py
index 5e9ad94..014c43d 100644
--- a/test_elasticsearch_dsl/test_analysis.py
+++ b/test_elasticsearch_dsl/test_analysis.py
@@ -19,6 +19,24 @@ def test_analyzer_has_definition():
         'filter': ["lowercase"],
     } == a.get_definition()
 
+def test_normalizer_serializes_as_name():
+    n = analysis.normalizer('my_normalizer')
+
+    assert 'my_normalizer' == n.to_dict()
+
+def test_normalizer_has_definition():
+    n = analysis.CustomNormalizer(
+        'my_normalizer',
+        filter=['lowercase', 'asciifolding'],
+        char_filter=['quote']
+    )
+
+    assert {
+        'type': 'custom',
+        'filter': ['lowercase', 'asciifolding'],
+        'char_filter': ['quote']
+    } == n.get_definition()
+
 def test_tokenizer():
     t = analysis.tokenizer('trigram', 'nGram', min_gram=3, max_gram=3)
 
diff --git a/test_elasticsearch_dsl/test_faceted_search.py b/test_elasticsearch_dsl/test_faceted_search.py
index a06d10d..c887629 100644
--- a/test_elasticsearch_dsl/test_faceted_search.py
+++ b/test_elasticsearch_dsl/test_faceted_search.py
@@ -1,4 +1,8 @@
-from elasticsearch_dsl.faceted_search import FacetedSearch, TermsFacet
+from datetime import datetime
+
+from elasticsearch_dsl.faceted_search import (FacetedSearch, TermsFacet,
+                                              DateHistogramFacet)
+
 
 class BlogSearch(FacetedSearch):
     doc_types = ['user', 'post']
@@ -131,3 +135,8 @@ def test_filters_are_applied_to_search_ant_relevant_facets():
         'highlight': {'fields': {'body': {}, 'title': {}}}
     } == d
 
+
+def test_date_histogram_facet_with_1970_01_01_date():
+    dhf = DateHistogramFacet()
+    assert dhf.get_value({'key': None}) == datetime(1970, 1, 1, 0, 0)
+    assert dhf.get_value({'key': 0}) == datetime(1970, 1, 1, 0, 0)
diff --git a/test_elasticsearch_dsl/test_mapping.py b/test_elasticsearch_dsl/test_mapping.py
index f42d866..367005f 100644
--- a/test_elasticsearch_dsl/test_mapping.py
+++ b/test_elasticsearch_dsl/test_mapping.py
@@ -62,7 +62,7 @@ def test_properties_can_iterate_over_all_the_fields():
 
     assert set(('f1', 'f2', 'f3', 'f4')) == set(f.test_attr for f in m.properties._collect_fields())
 
-def test_mapping_can_collect_all_analyzers():
+def test_mapping_can_collect_all_analyzers_and_normalizers():
     a1 = analysis.analyzer('my_analyzer1',
         tokenizer='keyword',
         filter=['lowercase', analysis.token_filter('my_filter1', 'stop', stopwords=['a', 'b'])],
@@ -74,6 +74,13 @@ def test_mapping_can_collect_all_analyzers():
         filter=[analysis.token_filter('my_filter2', 'stop', stopwords=['c', 'd'])],
     )
     a5 = analysis.analyzer('my_analyzer3', tokenizer='keyword')
+    n1 = analysis.normalizer('my_normalizer1',
+        filter=['lowercase']
+    )
+    n2 = analysis.normalizer('my_normalizer2',
+        filter=['my_filter1', 'my_filter2', analysis.token_filter('my_filter3', 'stop', stopwords=['e', 'f'])]
+    )
+    n3 = analysis.normalizer('unknown_custom')
 
     m = mapping.Mapping('article')
     m.field('title', 'text', analyzer=a1,
@@ -85,6 +92,9 @@ def test_mapping_can_collect_all_analyzers():
     m.field('comments', Nested(properties={
         'author': Text(analyzer=a4)
     }))
+    m.field('normalized_title', 'keyword', normalizer=n1)
+    m.field('normalized_comment', 'keyword', normalizer=n2)
+    m.field('unknown', 'keyword', normalizer=n3)
     m.meta('_all', analyzer=a5)
 
     assert {
@@ -93,9 +103,14 @@ def test_mapping_can_collect_all_analyzers():
             'my_analyzer2': {'filter': ['my_filter2'], 'tokenizer': 'trigram', 'type': 'custom'},
             'my_analyzer3': {'tokenizer': 'keyword', 'type': 'custom'},
         },
+        'normalizer': {
+            'my_normalizer1': {'filter': ['lowercase'], 'type': 'custom'},
+            'my_normalizer2': {'filter': ['my_filter1', 'my_filter2', 'my_filter3'], 'type': 'custom'},
+        },
         'filter': {
             'my_filter1': {'stopwords': ['a', 'b'], 'type': 'stop'},
             'my_filter2': {'stopwords': ['c', 'd'], 'type': 'stop'},
+            'my_filter3': {'stopwords': ['e', 'f'], 'type': 'stop'},
         },
         'tokenizer': {
             'trigram': {'max_gram': 3, 'min_gram': 3, 'type': 'nGram'},
diff --git a/test_elasticsearch_dsl/test_search.py b/test_elasticsearch_dsl/test_search.py
index 24cd37d..bf7dd55 100644
--- a/test_elasticsearch_dsl/test_search.py
+++ b/test_elasticsearch_dsl/test_search.py
@@ -162,6 +162,16 @@ def test_search_index():
     s2 = s.index('i3')
     assert s._index == ['i', 'i2']
     assert s2._index == ['i', 'i2', 'i3']
+    s = search.Search()
+    s = s.index(['i', 'i2'], 'i3')
+    assert s._index == ['i', 'i2', 'i3']
+    s2 = s.index('i4')
+    assert s._index == ['i', 'i2', 'i3']
+    assert s2._index == ['i', 'i2', 'i3', 'i4']
+    s2 = s.index(['i4'])
+    assert s2._index == ['i', 'i2', 'i3', 'i4']
+    s2 = s.index(('i4', 'i5'))
+    assert s2._index == ['i', 'i2', 'i3', 'i4', 'i5']
 
 def test_search_doc_type():
     s = search.Search(doc_type='i')