Issue 1:
If you manually move a shard using the Reroute API, it appears that the Cluster node stats API breaks for the duration of the shard reassignment (500 Internal Server error, message at bottom of the gist). It begins working again once the shard has been fully relocated and intialized.

Here is a recreation. The index in question is "test". The cluster has three data nodes (S1, S2, S3) and one non-data client node (C1)

Edit: this is with ES Version 0.20.2

curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
"commands":[
{
"move":{
"index":"test",
"shard":2,
"from_node":"kH152vsLTL-y20mcLFs9GQ",
"to_node":"J47gdOIwQMq2GTmzzmzJBA"
}
}
]
}'

{
"ok":true,
"state":{
"master_node":"J47gdOIwQMq2GTmzzmzJBA",
"blocks":{

},
"nodes":{
"ez-rzcnfSESrVYP6zTWksA":{
"name":"S1",
"transport_address":"inet[/144.76.3.102:9300]",
"attributes":{

}
},
"-VNWEiiaSG2IRlwM6kzcxg":{
"name":"C1",
"transport_address":"inet[/144.76.8.228:9300]",
"attributes":{
"data":"false"
}
},
"kH152vsLTL-y20mcLFs9GQ":{
"name":"S3",
"transport_address":"inet[/144.76.2.205:9300]",
"attributes":{

}
},
"J47gdOIwQMq2GTmzzmzJBA":{
"name":"S2",
"transport_address":"inet[/144.76.3.103:9300]",
"attributes":{

}
}
},
"routing_table":{
"indices":{
"test":{
"shards":{
"0":[
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":0,
"index":"test"
}
],
"1":[
{
"state":"STARTED",
"primary":true,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":1,
"index":"test"
}
],
"2":[
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":2,
"index":"test"
},
{
"state":"RELOCATING",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":"J47gdOIwQMq2GTmzzmzJBA",
"shard":2,
"index":"test"
}
]
}
},
"test123":{
"shards":{
"0":[
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":0,
"index":"test123"
},
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test123"
}
]
}
}
}
},
"routing_nodes":{
"unassigned":[

],
"nodes":{
"ez-rzcnfSESrVYP6zTWksA":[
{
"state":"STARTED",
"primary":true,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":2,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":0,
"index":"test123"
}
],
"kH152vsLTL-y20mcLFs9GQ":[
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"RELOCATING",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":"J47gdOIwQMq2GTmzzmzJBA",
"shard":2,
"index":"test"
},
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test123"
}
],
"J47gdOIwQMq2GTmzzmzJBA":[
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"INITIALIZING",
"primary":true,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":"kH152vsLTL-y20mcLFs9GQ",
"shard":2,
"index":"test"
}
]
}
},
"allocations":[

]
}
}

##For the duration of the move, this API returns a 500 error
$ curl -XGET localhost:9200/_cluster/nodes/stats?all=true
{
"error": "ArithmeticException[Value cannot fit in an int: -2562047788015]",
"status": 500
}

Issue 2:
To reproduce, start a single node and as soon as node starts run the following script:

# Create and delete index a few times
for i in {1..4}
do
curl -XDELETE localhost:9200/test
curl -XPUT localhost:9200/test -d '{
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 0
}
}'
curl -XPUT localhost:9200/test/doc/1 -d '{"foo": "bar"}'
curl -XPUT localhost:9200/test/doc/2 -d '{"foo": "bar"}'
curl -XPUT localhost:9200/test/doc/3 -d '{"foo": "bar"}'
curl -XPOST localhost:9200/test/_refresh
# Do search to fill filter cache
curl "localhost:9200/test/doc/_search"
done
# Sleep for indices.cache.filter.clean_interval
# Reduce it to make this repro run faster
echo
echo "Waiting 1 min for cache to be cleaned"
sleep 60
curl -s -XGET 'http://localhost:9200/_nodes/stats?pretty=true'

The result of this script is

...
"cache" : {
"filter_count" : -2,
"filter_evictions" : 3,
"filter_size" : "-48b",
"filter_size_in_bytes" : -48,
"id_cache_size" : "0b",
"id_cache_size_in_bytes" : 0
}
...

Analysis:

IndicesFilterCache is using index name to dispatch removal notifications.

When index closes, it doesn't clean the cache immediately. So, it's possible that the cache for the old index will be cleaned when a new index with the same name is already created.

As a result, it's possible for WeightedFilterCache to receive notifications targeted for the previous incarnations of the index with the same name.

WeightedFilterCache decrements stats counters without checking if removal notification was sent for this index or for one of its previous incarnations.