|
#ifndef LM_INTERPOLATE_TUNE_INSTANCE_H |
|
#define LM_INTERPOLATE_TUNE_INSTANCE_H |
|
|
|
#include "lm/interpolate/tune_matrix.hh" |
|
#include "lm/word_index.hh" |
|
#include "util/scoped.hh" |
|
#include "util/stream/config.hh" |
|
#include "util/string_piece.hh" |
|
|
|
#include <boost/optional.hpp> |
|
|
|
#include <vector> |
|
|
|
namespace util { namespace stream { |
|
class Chain; |
|
class FileBuffer; |
|
}} |
|
|
|
namespace lm { namespace interpolate { |
|
|
|
typedef uint32_t InstanceIndex; |
|
typedef uint32_t ModelIndex; |
|
|
|
struct Extension { |
|
|
|
InstanceIndex instance; |
|
WordIndex word; |
|
ModelIndex model; |
|
|
|
float ln_prob; |
|
|
|
bool operator<(const Extension &other) const; |
|
}; |
|
|
|
class ExtensionsFirstIteration; |
|
|
|
struct InstancesConfig { |
|
|
|
std::size_t model_read_chain_mem; |
|
|
|
std::size_t extension_write_chain_mem; |
|
std::size_t lazy_memory; |
|
util::stream::SortConfig sort; |
|
}; |
|
|
|
class Instances { |
|
private: |
|
typedef Eigen::Matrix<Accum, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> BackoffMatrix; |
|
|
|
public: |
|
Instances(int tune_file, const std::vector<StringPiece> &model_names, const InstancesConfig &config); |
|
|
|
|
|
~Instances(); |
|
|
|
|
|
typedef BackoffMatrix::ConstRowXpr FullBackoffs; |
|
FullBackoffs LNBackoffs(InstanceIndex instance) const { |
|
return ln_backoffs_.row(instance); |
|
} |
|
|
|
InstanceIndex NumInstances() const { return ln_backoffs_.rows(); } |
|
|
|
const Vector &CorrectGradientTerm() const { return neg_ln_correct_sum_; } |
|
|
|
const Matrix &LNUnigrams() const { return ln_unigrams_; } |
|
|
|
|
|
std::size_t ReadExtensionsEntrySize() const; |
|
void ReadExtensions(util::stream::Chain &chain); |
|
|
|
|
|
WordIndex BOS() const { return bos_; } |
|
|
|
private: |
|
|
|
friend class MockInstances; |
|
Instances(); |
|
|
|
|
|
BackoffMatrix ln_backoffs_; |
|
|
|
|
|
|
|
Vector neg_ln_correct_sum_; |
|
|
|
|
|
Matrix ln_unigrams_; |
|
|
|
|
|
util::scoped_ptr<ExtensionsFirstIteration> extensions_first_; |
|
|
|
|
|
util::scoped_ptr<util::stream::FileBuffer> extensions_subsequent_; |
|
|
|
WordIndex bos_; |
|
|
|
std::string temp_prefix_; |
|
}; |
|
|
|
}} |
|
#endif |
|
|