piskvorky/gensim — pull request #119 by florentchandelier (2013-07-21 22:52:43+00:00)
Title: Fixing merge_with method in Dictionary with validation code (see extended summary)

# Code used for Validation

## global represents the dict derived from the complete corpus analyzed at once
dict_global = corpora.dictionary.Dictionary.load('global.dict')

## dict_f/g represent the 2 dict derived from analyzing part of the complete corpus where f+g = complete corpus
dict_f = corpora.dictionary.Dictionary.load('f.dict')
dict_g = corpora.dictionary.Dictionary.load('g.dict'

merge_with(dict_f, dict_g)
dict_f.filter_extremes()
dict_f.compactify()

## CONSIDERING AFTER FILTERING/COMPACTIFY THE MERGED_DICT SHALL BE EQUIVALENT TO THE GLOBAL DICT (where filering/compactify was applied), NO ERROR MESSAGE SHOULD APPEAR ON SCREEN.
for ids,token in dict_f.iteritems():
if dict_f.dfs[ids] != dict_global.dfs[dict_global.token2id[token]]:
ErrMsg = "Error at ID " + str(ids)
paste(ErrMsg)

count = 0;
for ids, token in dict_global.iteritems():
try:
dict_f.token2id[token]
except:
count = count + 1;
ErrMsg = "Error " +str(count) + " -- Unpresent token : " + token
print(ErrMsg)