elasticsearch/elasticsearch — Issue #1500: Convenient rolling index method
Author: karussell
Created: 2011-11-25 21:24:58+00:00
State: open

Here is some code where a rolling index pattern is implemented. Imagine you have a logical index named 'tweets', now you want to create every day a new index to keep the indices small (Its a better scalable 'sharding', but only if you have time dependent data). Now, in the proposed code you will have to call rollIndex(maximumIndices) once a day.

Then the new indices are all 'tagged' with a 'tweets_roll' alias (for later retrieval -> imrovable?), there is a group of indices for searching (tweets_search) and feeding (tweets_feed). Per default it creates a search alias on all indices and a feed alias only for the very latest. It separates the search and the roll alias as it could be the case that one wants to keep some older indices but do not want to search on it.

What do you think? Its rather simple but it works - a simple test below the code.

private static final String simpleDateString = "yyyy-MM-dd-HH-mm-ss";
public void rollIndex(int maxRollIndices) {
rollIndex(getIndexName(), maxRollIndices, maxRollIndices, 1);
}

public void rollIndex(String indexName, int maxRollIndices, int maxSearchIndices, int maxFeedIndices) {
String rollAlias = indexName + "_roll";
SimpleDateFormat formatter = new SimpleDateFormat(simpleDateString);

if (maxRollIndices < 1 || maxSearchIndices < 1 || maxFeedIndices < 1)
throw new RuntimeException("remaining indices, search indices and feeding indices must be at least 1");

// get old aliases
Map allRollingAliases = getAliases(rollAlias);

// always create new index and append aliases
String searchAlias = getSearchIndexName();
String feedAlias = getFeedIndexName();
String newIndexName = indexName + "_" + formatter.format(new Date());

createIndex(newIndexName);
addAlias(newIndexName, feedAlias);
addAlias(newIndexName, searchAlias);
addAlias(newIndexName, rollAlias);

if (allRollingAliases.isEmpty()) {
// original indexName is not in the form of index_date so do not tag it
// addAlias(indexName, feedAlias);
// addAlias(indexName, searchAlias);
// addAlias(indexName, rollAlias);
} else {
TreeMap sortedIndices = new TreeMap(reverseSorter);
String[] concreteIndices = getConcreteIndices(allRollingAliases.keySet());
//logger.info("aliases:" + allRollingAliases + ", indices:" + Arrays.toString(concreteIndices));
for (String index : concreteIndices) {
int pos = index.indexOf("_");
if (pos < 0)
throw new IllegalStateException("index " + index + " is not in the format " + simpleDateString);

String indexDateStr = index.substring(pos + 1);
Long timeLong;
try {
timeLong = formatter.parse(indexDateStr).getTime();
} catch (Exception ex) {
throw new IllegalStateException("index " + index + " is not in the format " + simpleDateString + " error:" + ex.getMessage());
}
String old = sortedIndices.put(timeLong, index);
if (old != null)
throw new IllegalStateException("indices with the identical date are not supported " + old + " vs. " + index);
}
int counter = 1;
Iterator indexIter = sortedIndices.values().iterator();
while (indexIter.hasNext()) {
String currentIndexName = indexIter.next();
if (counter >= maxRollIndices) {
deleteIndex(currentIndexName);
// delete all the older indices
continue;
}

if (counter >= maxFeedIndices)
removeAlias(currentIndexName, feedAlias);

if (counter >= maxSearchIndices)
removeAlias(currentIndexName, searchAlias);

counter++;
}
}
}

public String getSearchIndexName() {
return getIndexName() + "_search";
}

public String getFeedIndexName() {
return getIndexName() + "_feed";
}

public void createIndex(String indexName) {
client.admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
}

public void deleteIndex(String indexName) {
client.admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet();
}

public void addAlias(String indexName, String alias) {
client.admin().indices().aliases(new IndicesAliasesRequest().addAlias(indexName, alias)).actionGet();
}

public void removeAlias(String indexName, String alias) {
client.admin().indices().aliases(new IndicesAliasesRequest().removeAlias(indexName, alias)).actionGet();
}

public Map getAliases(String index) {
Map md = client.admin().cluster().state(new ClusterStateRequest()).
actionGet().getState().getMetaData().aliases().get(index);
if (md == null)
return Collections.emptyMap();

return md;
}
private static Comparator reverseSorter = new Comparator() {

@Override
public int compare(Long o1, Long o2) {
return -o1.compareTo(o2);
}
};

public String[] getConcreteIndices(Set set) {
return client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().
getMetaData().concreteIndices(set.toArray(new String[set.size()]));
}

TEST

@Test public void rollingIndex() throws Exception {
search.setClient(createTestClient());
search.setIndexName("tweets");
String rollIndexTag = search.getIndexName() + "_roll";
String searchIndex = search.getIndexName() + "_search";
String feedIndex = search.getIndexName() + "_feed";
search.rollIndex(4);
assertEquals(1, search.getAliases(rollIndexTag).size());
assertEquals(1, search.getAliases(searchIndex).size());
assertEquals(1, search.getAliases(feedIndex).size());

Thread.sleep(2000);
search.rollIndex(4);
assertEquals(2, search.getAliases(rollIndexTag).size());
assertEquals(2, search.getAliases(searchIndex).size());
assertEquals(1, search.getAliases(feedIndex).size());

Thread.sleep(2000);
search.rollIndex(4);
assertEquals(3, search.getAliases(rollIndexTag).size());
assertEquals(3, search.getAliases(searchIndex).size());
assertEquals(1, search.getAliases(feedIndex).size());

Thread.sleep(2000);
search.rollIndex(4);
assertEquals(4, search.getAliases(rollIndexTag).size());
assertEquals(4, search.getAliases(searchIndex).size());
assertEquals(1, search.getAliases(feedIndex).size());

Thread.sleep(2000);
search.rollIndex(4);
assertEquals(4, search.getAliases(rollIndexTag).size());
assertEquals(4, search.getAliases(searchIndex).size());
assertEquals(1, search.getAliases(feedIndex).size());

Thread.sleep(2000);
search.rollIndex(search.getIndexName(), 4, 3, 2);
assertEquals(4, search.getAliases(rollIndexTag).size());
assertEquals(3, search.getAliases(searchIndex).size());
assertEquals(2, search.getAliases(feedIndex).size());
}