#!/usr/bin/env python
# -*- coding: utf-8 -*-

# https://radimrehurek.com/gensim/tut2.html

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

import os
from gensim import corpora, models, similarities
if (os.path.exists("/tmp/deerwester.dict")):
   dictionary = corpora.Dictionary.load('/tmp/deerwester.dict')
   corpus = corpora.MmCorpus('/tmp/deerwester.mm')
   print("Used files generated from first tutorial")
else:
   print("Please run first tutorial to generate data set")


tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model

doc_bow = [(0, 1), (1, 1)]
print(tfidf[doc_bow]) # step 2 -- use the model to transform vectors

corpus_tfidf = tfidf[corpus]
for doc in corpus_tfidf:
    print(doc)

lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize an LSI transformation
corpus_lsi = lsi[corpus_tfidf] # create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi
lsi.print_topics(2)

for doc in corpus_lsi: # both bow->tfidf and tfidf->lsi transformations are actually executed here, on the fly
    print(doc)

lsi.save('/tmp/model.lsi') # same for tfidf, lda, ...
lsi = models.LsiModel.load('/tmp/model.lsi')

tfidf_model = models.TfidfModel(corpus, normalize=True)
lsi_model = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=300)
rp_model = models.RpModel(corpus_tfidf, num_topics=500)
lda_model = models.LdaModel(corpus, id2word=dictionary, num_topics=100)
hdp_model = models.HdpModel(corpus, id2word=dictionary)

print(tfidf_model)
print(lsi_model)
print(rp_model)
print(lda_model)
print(hdp_model)
