sakharamg's picture
Uploading all files
158b61b
#pragma once
#include "util/string_piece.hh"
#include "util/murmur_hash.hh"
#include "util/string_piece.hh" //Tokenization and work with StringPiece
#include "util/tokenize_piece.hh"
#include <vector>
namespace probingpt
{
//Gets the MurmurmurHash for give string
uint64_t getHash(StringPiece text);
std::vector<uint64_t> getVocabIDs(const StringPiece &textin);
}