Issue 1:
We take snpashots every hour to AWS S3 directly using cloud-aws plugin

snapshot was stuck because of a reboot on master node during the snapshot, ES version 1.4.0, ArchLinux

Cluster was later upgraded to 1.4.2 at which point we noticed that there is a snapshot stuck

while trying to create a new snapshot

{"error":"ConcurrentSnapshotExecutionException[[my_s3_repository:2015.04.13.09.16.42] a snapshot is already running]","status":503}

curl localhost:9200/_snapshot/my_s3_repository/_all?pretty doesn't have the stuck snapshot in the list.

Delete snapshot command never completes, it hangs forever.

rolling restart (restarting one node after the other) didn't help.

Clean up script doesn't work for me imotov/elasticsearch-snapshot-cleanup#2

In S3, there is no metadata file for the stuck snapshot, there are snapshot- files for some of the indices

How do I atleast forget/discard old snapshots..create a new repository and start fresh. I can not stop the whole cluster as it is a production cluster.

curl localhost:9200/_snapshot/_status?pretty

{
"snapshots" : [ {
"snapshot" : "2015.03.11.06.04.39",
"repository" : "my_s3_repository",
"state" : "ABORTED",
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 17,
"failed" : 3,
"total" : 20
},
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
},
"indices" : {
"playground_video" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 5,
"failed" : 0,
"total" : 5
},
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
},
"shards" : {
"4" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"3" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"2" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"1" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"0" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
}
}
},
"playground_feed" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 4,
"failed" : 1,
"total" : 5
},
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
},
"shards" : {
"4" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"2" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"3" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"0" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"1" : {
"stage" : "FAILURE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
}
}
},
"playground_profile" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 4,
"failed" : 1,
"total" : 5
},
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
},
"shards" : {
"2" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"1" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"4" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"3" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"0" : {
"stage" : "FAILURE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
}
}
},
"playground_admin" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 4,
"failed" : 1,
"total" : 5
},
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
},
"shards" : {
"2" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"1" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"0" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"4" : {
"stage" : "FAILURE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
},
"3" : {
"stage" : "DONE",
"stats" : {
"number_of_files" : 0,
"processed_files" : 0,
"total_size_in_bytes" : 0,
"processed_size_in_bytes" : 0,
"start_time_in_millis" : 0,
"time_in_millis" : 0
}
}
}
}
}
} ]
}

Issue 2:
The issue was originally reported in #7980 (comment) If a current master node that contains all primary shards is restarted in the middle of snapshot operation, it might leave the snapshot hanging in ABORTED state.