tools/mosesdecoder/moses/PP/SpanLengthPhraseProperty.cpp

#include "SpanLengthPhraseProperty.h"
#include "moses/Util.h"
#include "util/exception.hh"

using namespace std;

namespace Moses
{
SpanLengthPhraseProperty::SpanLengthPhraseProperty()
{
}

void SpanLengthPhraseProperty::ProcessValue(const std::string &value)
{
  vector<string> toks;
  Tokenize(toks, value);

  set< vector<string> > indices;

  for (size_t i = 0; i < toks.size(); ++i) {
    const string &span = toks[i];

    // is it a ntIndex,sourceSpan,targetSpan  or count ?
    vector<string> toks;
    Tokenize<string>(toks, span, ",");
    UTIL_THROW_IF2(toks.size() != 1 && toks.size() != 3, "Incorrect format for SpanLength: " << span);

    if (toks.size() == 1) {
      float count = Scan<float>(toks[0]);
      Populate(indices, count);

      indices.clear();
    } else {
      indices.insert(toks);
    }
  }

  // totals
  CalcTotals(m_source);
  CalcTotals(m_target);
}

void SpanLengthPhraseProperty::Populate(const set< vector<string> > &indices, float count)
{
  set< vector<string> >::const_iterator iter;
  for (iter = indices.begin(); iter != indices.end(); ++iter) {
    const vector<string> &toksStr = *iter;
    vector<size_t> toks = Scan<size_t>(toksStr);
    UTIL_THROW_IF2(toks.size() != 3, "Incorrect format for SpanLength. Size is " << toks.size());

    Populate(toks, count);
  }
}

void SpanLengthPhraseProperty::Populate(const std::vector<size_t> &toks, float count)
{
  size_t ntInd = toks[0];
  size_t sourceLength = toks[1];
  size_t targetLength = toks[2];
  if (ntInd >=  m_source.size() ) {
    m_source.resize(ntInd + 1);
    m_target.resize(ntInd + 1);
  }

  Map &sourceMap = m_source[ntInd].first;
  Map &targetMap = m_target[ntInd].first;
  Populate(sourceMap, sourceLength, count);
  Populate(targetMap, targetLength, count);
}

void SpanLengthPhraseProperty::Populate(Map &map, size_t span, float count)
{
  Map::iterator iter;
  iter = map.find(span);
  if (iter != map.end()) {
    float &value = iter->second;
    value += count;
  } else {
    map[span] = count;
  }
}

void SpanLengthPhraseProperty::CalcTotals(Vec &vec)
{
  for (size_t i = 0; i < vec.size(); ++i) {
    float total = 0;

    const Map &map = vec[i].first;
    Map::const_iterator iter;
    for (iter = map.begin(); iter != map.end(); ++iter) {
      float count = iter->second;
      total += count;
    }

    vec[i].second = total;
  }
}

float SpanLengthPhraseProperty::GetProb(size_t ntInd, size_t sourceWidth, float smoothing) const
{
  float count;

  const std::pair<Map, float> &data = m_source[ntInd];
  const Map &map = data.first;

  if (map.size() == 0) {
    // should this ever be reached? there shouldn't be any span length proprty so FF shouldn't call this
    return 1.0f;
  }

  Map::const_iterator iter = map.find(sourceWidth);
  if (iter == map.end()) {
    count = 0;
  } else {
    count = iter->second;
  }
  count += smoothing;

  float total = data.second + smoothing * (float) map.size();
  float ret = count / total;
  return ret;
}

}