xls-r-300m-sv-robust / kenlm /lm /fragment_main.cc
marinone94's picture
Training in progress, epoch 0
1ce325b
raw
history blame
1.07 kB
#include "binary_format.hh"
#include "model.hh"
#include "left.hh"
#include "../util/tokenize_piece.hh"
template <class Model> void Query(const char *name) {
Model model(name);
std::string line;
lm::ngram::ChartState ignored;
while (getline(std::cin, line)) {
lm::ngram::RuleScore<Model> scorer(model, ignored);
for (util::TokenIter<util::SingleCharacter, true> i(line, ' '); i; ++i) {
scorer.Terminal(model.GetVocabulary().Index(*i));
}
std::cout << scorer.Finish() << '\n';
}
}
int main(int argc, char *argv[]) {
if (argc != 2) {
std::cerr << "Expected model file name." << std::endl;
return 1;
}
const char *name = argv[1];
lm::ngram::ModelType model_type = lm::ngram::PROBING;
lm::ngram::RecognizeBinary(name, model_type);
switch (model_type) {
case lm::ngram::PROBING:
Query<lm::ngram::ProbingModel>(name);
break;
case lm::ngram::REST_PROBING:
Query<lm::ngram::RestProbingModel>(name);
break;
default:
std::cerr << "Model type not supported yet." << std::endl;
}
}