Repository: jerinphilip/slimt (language: python)

Issue:
Integrate `gemmology` as an 8-bit multiply backend
Integrate [gemmology](https://github.com/mozilla/gemmology) as a possible alternate backend. 
Gemmology requires [xsimd](https://github.com/xtensor-stack/xsimd) which currently is not in the list of the 3rd-party libraries.


Discussion / hints:

Fix patch:
diff --git a/.clang-tidy b/.clang-tidy
index 6f4d3060..b132e7a3 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -75,7 +75,9 @@ Checks: >
   -bugprone-narrowing-conversions,
   -bugprone-easily-swappable-parameters,
   -bugprone-implicit-widening-of-multiplication-result,
-  -bugprone-unchecked-optional-access
+  -bugprone-exception-escape,
+  -bugprone-unchecked-optional-access,
+  -portability-simd-intrinsics
 
 # Turn all the warnings from the checks above into errors.
 WarningsAsErrors: "*"
@@ -87,14 +89,14 @@ CheckOptions:
   - { key: readability-identifier-naming.ClassCase,              value: CamelCase  }
   - { key: readability-identifier-naming.StructCase,             value: CamelCase  }
   - { key: readability-identifier-naming.TemplateParameterCase,  value: CamelCase  }
-  - { key: readability-identifier-naming.FunctionCase,           value: aNy_CasE  }
+  - { key: readability-identifier-naming.FunctionCase,           value: lower_case }
   - { key: readability-identifier-naming.VariableCase,           value: lower_case }
   - { key: readability-identifier-naming.ClassMemberCase,        value: lower_case }
+  - { key: readability-identifier-naming.ClassMethodCase,        value: lower_case }
   - { key: readability-identifier-naming.ClassMemberSuffix,      value: _          }
   - { key: readability-identifier-naming.PrivateMemberSuffix,    value: _          }
   - { key: readability-identifier-naming.ProtectedMemberSuffix,  value: _          }
   - { key: readability-identifier-naming.EnumConstantCase,         value: CamelCase }
-  - { key: readability-identifier-naming.EnumConstantPrefix,       value: k         }
   - { key: readability-identifier-naming.ConstexprVariableCase,    value: CamelCase }
   - { key: readability-identifier-naming.ConstexprVariablePrefix,  value: k         }
   - { key: readability-identifier-naming.GlobalConstantCase,       value: CamelCase }
@@ -106,3 +108,4 @@ CheckOptions:
   - { key: readability-implicit-bool-conversion.AllowIntegerConditions,  value: 1   }
   - { key: readability-implicit-bool-conversion.AllowPointerConditions,  value: 1   }
   - { key: readability-function-cognitive-complexity.IgnoreMacros,  value: 1   }
+  - { key: readability-identifier-naming.TemplateParameterIgnoredRegexp, value: 'expr-type'}
diff --git a/.gitmodules b/.gitmodules
index 7152576a..f6cde281 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -7,6 +7,9 @@
 [submodule "3rd-party/ruy"]
 	path = 3rd-party/ruy
 	url = https://github.com/google/ruy
+[submodule "3rd-party/ssplit-cpp"]
+	path = 3rd-party/ssplit-cpp
+	url = https://github.com/browsermt/ssplit-cpp
 [submodule "3rd-party/gemmology"]
 	path = 3rd-party/gemmology
 	url = https://github.com/mozilla/gemmology.git
diff --git a/.vscode/launch.json b/.vscode/launch.json
index da01d22f..4d7f5b36 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -40,8 +40,14 @@
             "request": "launch",
             "program": "${workspaceFolder}/build/bin/slimt",
             "args": [
-                "/home/jerin/.local/share/bergamot/models/browsermt/ende.student.tiny11/model.intgemm.alphas.bin",
-                "/home/jerin/.local/share/bergamot/models/browsermt/ende.student.tiny11/vocab.deen.spm",
+                "--root",
+                "/home/jerin/.local/share/bergamot/models/browsermt/ende.student.tiny11/",
+                "--model",
+                "model.intgemm.alphas.bin",
+                "--vocabulary",
+                "vocab.deen.spm",
+                "--shortlist",
+                "lex.s2t.bin",
                 "<",
                 "${workspaceFolder}/data/sample.txt"
             ],
diff --git a/3rd-party/CMakeLists.txt b/3rd-party/CMakeLists.txt
index 08ffc546..2b7ce2dd 100644
--- a/3rd-party/CMakeLists.txt
+++ b/3rd-party/CMakeLists.txt
@@ -20,3 +20,13 @@ add_subdirectory(intgemm EXCLUDE_FROM_ALL)
 
 # ruy
 add_subdirectory(ruy EXCLUDE_FROM_ALL)
+
+add_subdirectory(ssplit-cpp EXCLUDE_FROM_ALL)
+# To enable "#include "ssplit.h"
+# TODO(any): Fix later.
+get_property(
+  INCLUDE_DIRECTORIES
+  DIRECTORY ssplit-cpp/src
+  PROPERTY INCLUDE_DIRECTORIES)
+target_include_directories(ssplit PUBLIC ${INCLUDE_DIRECTORIES})
+add_library(Ssplit::Ssplit ALIAS ssplit)
diff --git a/3rd-party/ssplit-cpp b/3rd-party/ssplit-cpp
new file mode 160000
index 00000000..a311f986
--- /dev/null
+++ b/3rd-party/ssplit-cpp
@@ -0,0 +1,1 @@
+Subproject commit a311f9865ade34db1e8e080e6cc146f55dafb067
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e6bccbe8..95fc74aa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,6 +11,8 @@ option(WITH_TESTS "Enable building tests (requires a few extra libraries)" OFF)
 add_subdirectory(3rd-party)
 
 set(SLIMT_COMPILE_OPTIONS -Wall -Wpedantic -Wextra -Wno-unknown-pragmas -Werror)
+set(SLIMT_COMPILE_OPTIONS ${SLIMT_COMPILE_OPTIONS}
+                          -Wno-missing-field-initializers -Wno-unused-function)
 set(SLIMT_COMPILE_DEFINITIONS "")
 
 option(WITH_INTGEMM "Use intgemm" ON)
@@ -18,7 +20,7 @@ option(WITH_RUY "Use ruy" OFF)
 option(WITH_GEMMOLOGY "Use gemmology" OFF)
 option(WITH_BLAS "Use BLAS. Otherwise moves to ruy" ON)
 
-set(SLIMT_EXTERNAL_LIBS Sentencepiece::Sentencepiece)
+set(SLIMT_EXTERNAL_LIBS Sentencepiece::Sentencepiece Ssplit::Ssplit)
 
 if(WITH_BLAS)
   find_package(BLAS)
diff --git a/app/main.cc b/app/main.cc
index 218711ca..42079417 100644
--- a/app/main.cc
+++ b/app/main.cc
@@ -5,6 +5,14 @@
 #include "3rd-party/CLI11.hpp"
 #include "slimt/slimt.hh"
 
+inline std::string read_from_stdin() {
+  // Read a large input text blob from stdin
+  std::ostringstream stream;
+  stream << std::cin.rdbuf();
+  std::string input = stream.str();
+  return input;
+}
+
 template <class Field>
 struct Record {
   Field model;
@@ -15,7 +23,7 @@ struct Record {
 struct Options {
   Record<std::string> translator;
   std::string root;
-  size_t max_tokens_per_batch = 1024;
+  size_t max_tokens_per_batch = 1024;  // NOLINT
 
   template <class App>
   void setup_onto(App &app) {
@@ -56,77 +64,20 @@ void run(const Options &options) {
       .shortlist = io::MmapFile(adjusted.shortlist),    //
   };
 
-  // Tokenize into numeric-ids using sentencepiece.
-  Vocabulary vocabulary(mmap.vocabulary.data(), mmap.vocabulary.size());
-  ShortlistGenerator shortlist_generator(            //
-      mmap.shortlist.data(), mmap.shortlist.size(),  //
-      vocabulary, vocabulary                         //
-  );
-
-  // Load model
-  auto items = io::loadItems(mmap.model.data());
-  Model model(Tag::tiny11, vocabulary, std::move(items),
-              std::move(shortlist_generator));
-
-  using Sentences = std::vector<Words>;
-
-  AverageMeter<float> wps;
-
-  auto batch_and_translate = [&vocabulary, &model, &wps](    //
-                                 Sentences &sentences,       //
-                                 size_t max_sequence_length  //
-                             ) {
-    Timer timer;
-    uint64_t batch_size = sentences.size();
-    Batch batch(batch_size, max_sequence_length, vocabulary.pad_id());
-    for (auto &sentence : sentences) {
-      batch.add(sentence);
-    }
-
-    auto translated = model.translate(batch);
-    for (auto &sentence : translated) {
-      auto [result, views] = vocabulary.decode(sentence);
-      std::cout << result << "\n";
-    }
-    size_t words_processed = max_sequence_length * sentences.size();
-    float batch_wps = words_processed / timer.elapsed();
-    wps.record(batch_wps);
+  Record<View> view{
+      .model = {mmap.model.data(), mmap.model.size()},                 //
+      .vocabulary = {mmap.vocabulary.data(), mmap.vocabulary.size()},  //
+      .shortlist = {mmap.shortlist.data(), mmap.shortlist.size()},     //
   };
 
-  std::string line;
-  size_t max_sequence_length = 0;
-  size_t token_count = 0;
-  size_t line_no = 0;
-  Sentences sentences;
-  while (getline(std::cin, line)) {
-    auto [words, views] = vocabulary.encode(line, /*add_eos =*/true);
-    // std::cout << "Adding ";
-    // for (const auto &view : views) {
-    //   std::cout << "[" << view << "]";
-    // }
-    // std::cout << "\n";
-
-    size_t candidate_max_sequence_length =
-        std::max(words.size(), max_sequence_length);
-    ++line_no;
-
-    token_count = candidate_max_sequence_length * (line_no);
-    if (token_count > options.max_tokens_per_batch) {
-      batch_and_translate(sentences, max_sequence_length);
-      sentences.clear();
-      max_sequence_length = 0;
-    }
-    sentences.push_back(std::move(words));
-    max_sequence_length = candidate_max_sequence_length;
-  }
-
-  // Overhang.
-  if (!sentences.empty()) {
-    batch_and_translate(sentences, max_sequence_length);
-    sentences.clear();
-  }
+  Config config;
+  Translator translator(config, view.model, view.shortlist, view.vocabulary);
+  std::string source = read_from_stdin();
+  slimt::Options opts;
+  Response response = translator.translate(source, opts);
+  fprintf(stdout, "%s\n", response.target.text.c_str());
 
-  fprintf(stdout, "wps: %f\n", wps.value());
+  // fprintf(stdout, "wps: %f\n", wps.value());
 }
 
 int main(int argc, char *argv[]) {
diff --git a/scripts/ci/android/02-build.sh b/scripts/ci/android/02-build.sh
index b56ee031..85ba28ea 100644
--- a/scripts/ci/android/02-build.sh
+++ b/scripts/ci/android/02-build.sh
@@ -13,6 +13,7 @@ function cmake-configure {
     -DWITH_RUY=ON
     -DWITH_INTGEMM=OFF
     -DWITH_BLAS=OFF
+    -DSSPLIT_USE_INTERNAL_PCRE2=ON
   )
 
   OTHER_ANDROID_ARGS=(
diff --git a/scripts/ci/format-check.sh b/scripts/ci/format-check.sh
index 4e19b9e9..92e0ab7a 100644
--- a/scripts/ci/format-check.sh
+++ b/scripts/ci/format-check.sh
@@ -39,3 +39,4 @@ slimt-check-clang-format
 slimt-check-python
 slimt-check-sh
 slimt-check-cmake
+slimt-check-clang-tidy
diff --git a/slimt/Aligned.hh b/slimt/Aligned.hh
index e044c16b..0c884377 100644
--- a/slimt/Aligned.hh
+++ b/slimt/Aligned.hh
@@ -4,6 +4,8 @@
 
 namespace slimt {
 
+constexpr size_t kAlignWidth = 64;
+
 class Aligned {
  public:
   Aligned() = default;
diff --git a/slimt/Annotation.cc b/slimt/Annotation.cc
new file mode 100644
index 00000000..8113058a
--- /dev/null
+++ b/slimt/Annotation.cc
@@ -0,0 +1,73 @@
+#include "Annotation.hh"
+
+#include <cassert>
+#include <string>
+
+namespace slimt {
+
+AnnotatedText::AnnotatedText(std::string &&t) : text(std::move(t)) {
+  // Treat the entire text as a gap that record_existing_sentence will break.
+  annotation.token_begin_.back() = text.size();
+}
+
+void AnnotatedText::append_sentence(
+    std::string_view prefix, std::vector<std::string_view>::iterator begin,
+    std::vector<std::string_view>::iterator end) {
+  assert(annotation.token_begin_.back() == text.size());
+
+  // prefix is just end of the previous one.
+  append_ending_whitespace(prefix);
+
+  // Appending sentence text.
+  std::size_t offset = text.size();
+  for (auto token = begin; token != end; ++token) {
+    offset += token->size();
+    annotation.token_begin_.push_back(offset);
+  }
+  if (begin != end) {
+    text.append(begin->data(), (end - 1)->data() + (end - 1)->size());
+    assert(offset == text.size());  // Tokens should be contiguous.
+  }
+
+  // Add the gap after the sentence.  This is empty for now, but will be
+  // extended with append_ending_whitespace or another append_sentence.
+  annotation.gap_.push_back(annotation.token_begin_.size() - 1);
+  annotation.token_begin_.push_back(offset);
+}
+
+void AnnotatedText::append_ending_whitespace(std::string_view whitespace) {
+  text.append(whitespace.data(), whitespace.size());
+  annotation.token_begin_.back() = text.size();
+}
+
+void AnnotatedText::record_existing_sentence(
+    std::vector<std::string_view>::iterator begin,
+    std::vector<std::string_view>::iterator end, const char *sentence_begin) {
+  assert(sentence_begin >= text.data());
+  assert(sentence_begin <= text.data() + text.size());
+  assert(begin == end || sentence_begin == begin->data());
+  assert(!annotation.token_begin_.empty());
+  assert(annotation.token_begin_.back() == text.size());
+  // Clip off size token ending.
+  annotation.token_begin_.pop_back();
+  for (auto i = begin; i != end; ++i) {
+    assert(i->data() >= text.data());                            // In range.
+    assert(i->data() + i->size() <= text.data() + text.size());  // In range
+    assert(i + 1 == end ||
+           i->data() + i->size() == (i + 1)->data());  // Contiguous
+    annotation.token_begin_.push_back(i->data() - text.data());
+  }
+  // Gap token after sentence.
+  annotation.gap_.push_back(annotation.token_begin_.size());
+  if (begin != end) {
+    annotation.token_begin_.push_back((end - 1)->data() + (end - 1)->size() -
+                                      text.data());
+  } else {
+    // empty sentence.
+    annotation.token_begin_.push_back(sentence_begin - text.data());
+  }
+  // Add back size token ending.
+  annotation.token_begin_.push_back(text.size());
+}
+
+}  // namespace slimt
diff --git a/slimt/Annotation.hh b/slimt/Annotation.hh
new file mode 100644
index 00000000..d336a4c0
--- /dev/null
+++ b/slimt/Annotation.hh
@@ -0,0 +1,253 @@
+#pragma once
+
+#include <algorithm>
+#include <cassert>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "slimt/Types.hh"
+
+namespace slimt {
+
+/// Annotation expresses sentence and token boundary information as ranges of
+/// bytes in a string, but does not itself own the string.
+///
+/// See also AnnotatedText, which owns Annotation and the string. AnnotatedText
+/// wraps these Range functions to provide a std::string_view interface.
+///
+/// Text is divided into gaps (whitespace between sentences) and sentences like
+/// so:
+///   gap sentence gap sentence gap
+/// Because gaps appear at the beginning and end of the text, there's always
+/// one more gap than there are sentences.
+///
+/// The entire text is a unbroken sequence of tokens (i.e. the end of a token
+/// is the beginning of the next token).  A gap is exactly one token containing
+/// whatever whitespace is between the sentences.  A sentence is a sequence of
+/// tokens.
+///
+/// Since we are using SentencePiece, a token can include whitespace.  The term
+/// "word" is used, somewhat incorrectly, as a synonym of token.
+///
+/// A gap can be empty (for example there may not have been whitespace at the
+/// beginning).  A sentence can also be empty (typically the translation system
+/// produced empty output).  That's fine, these are just empty ranges as you
+/// would expect.
+class Annotation {
+ public:
+  /// Initially an empty string.  Populated by AnnotatedText.
+  Annotation() {
+    token_begin_.push_back(0);
+    token_begin_.push_back(0);
+    gap_.push_back(0);
+  }
+
+  size_t sentence_count() const { return gap_.size() - 1; }
+
+  /// Returns number of words in the sentence identified by `sentence_id`.
+  size_t word_count(size_t sentence_id) const {
+    return gap_[sentence_id + 1] - gap_[sentence_id] - 1 /* minus the gap */;
+  }
+
+  /// Returns a Range representing `word_id` in sentence indexed by
+  /// `sentence_id`. `word_id` follows 0-based indexing, and should be less
+  /// than
+  /// `.word_count()` for `sentence_id` for defined behaviour.
+  Range word(size_t sentence_id, size_t word_id) const {
+    size_t token_idx = gap_[sentence_id] + 1 + word_id;
+    return Range{token_begin_[token_idx], token_begin_[token_idx + 1]};
+  }
+
+  /// Returns a Range representing sentence corresponding to `sentence_id`.
+  /// `sentence_id` follows 0-based indexing, and behaviour is defined only
+  /// when less than `.sentence_count()`.
+  Range sentence(size_t sentence_id) const {
+    return Range{
+        token_begin_[gap_[sentence_id] + 1], /*end of whitespace before */
+        token_begin_[gap_[sentence_id + 1]]  /*beginning of whitespace after */
+    };
+  }
+
+  Range gap(size_t gap_idx) const {
+    size_t token_idx = gap_[gap_idx];
+    return Range{token_begin_[token_idx], token_begin_[token_idx + 1]};
+  }
+
+ private:
+  friend class AnnotatedText;
+  /// Map from token index to byte offset at which it begins.  Token i is:
+  ///   [token_begin_[i], token_begin_[i+1])
+  /// The vector is padded so that these indices are always valid, even at the
+  /// end.  So tokens_begin_.size() is the number of tokens plus 1.
+  std::vector<size_t> token_begin_;
+
+  /// Indices of tokens that correspond to gaps between sentences.  These are
+  /// indices into token_begin_.
+  /// Gap g is byte range:
+  ///   [token_begin_[gap_[w]], token_begin_[gap_[w]+1])
+  /// Sentence s is byte range:
+  ///   [token_begin_[gap_[s]+1], token_begin_[gap_[s+1]])
+  /// A sentence does not include whitespace at the beginning or end.
+  ///
+  /// gap_.size() == sentence_count() + 1.
+  ///
+  /// Example: empty text "" -> just an empty gap.
+  /// token_begin_ = {0, 0};
+  /// gap_ = {0};
+  ///
+  /// Example: only space " " -> just a gap containing the space.
+  /// token_begin_ = {0, 1};
+  /// gap_ = {0};
+  ///
+  /// Example: one token "hi" -> empty gap, sentence with one token, empty gap
+  /// token_begin_ = {0, 0, 2, 2};
+  /// gap_ = {0, 2};
+  std::vector<size_t> gap_;
+};
+
+/// AnnotatedText is effectively std::string text + Annotation, providing the
+/// following additional desiderata.
+///
+/// 1. Access to processed std::string_views for convenience rather than
+/// Ranges (which only provides index information).
+///
+/// 2. Transparently convert std::string_views into Ranges for the
+/// Annotation referring to the text bound by this structure.
+///
+/// 3. Bind the text and annotations together, to move around as a meaningful
+/// unit.
+class AnnotatedText {
+ public:
+  std::string text;       ///< Blob of string elements in annotation refers to.
+  Annotation annotation;  ///< sentence and (sub-) word annotations.
+
+  /// Construct an empty AnnotatedText. This is useful when the target string or
+  /// Ranges are not known yet, but the public members can be used to
+  /// populate it. One use-case, when translated-text is created decoding from
+  /// histories and the Ranges only known after the string has been
+  /// constructed.
+  AnnotatedText() = default;
+
+  /// Construct moving in a string (for efficiency purposes, copying string
+  /// constructor is disallowed).
+  explicit AnnotatedText(std::string &&text);
+
+  /// Appends a sentence to the existing text and transparently rebases
+  /// std::string_views.  Since this tracks only prefix, remember
+  /// append_ending_whitespace.
+  /// The std::string_views must not already be in text.
+  void append_sentence(std::string_view prefix,
+                       std::vector<std::string_view>::iterator tokens_begin,
+                       std::vector<std::string_view>::iterator tokens_end);
+
+  /// Append the whitespace at the end of input. std::string_view must not be in
+  /// text.
+  void append_ending_whitespace(std::string_view whitespace);
+
+  /// Record the existence of a sentence that is already in text.  The
+  /// iterators are over std::string_views for each token that must be in text
+  /// already.  This function must be called to record sentences in order.
+  /// Normally the beginning of the sentence can be inferred from
+  /// tokens_begin->data() but the tokens could be empty, so sentence_begin is
+  /// required to know where the sentence is.
+  void record_existing_sentence(
+      std::vector<std::string_view>::iterator tokens_begin,
+      std::vector<std::string_view>::iterator tokens_end,
+      const char *sentence_begin);
+
+  /// Returns the number of sentences in the annotation structure.
+  size_t sentence_count() const { return annotation.sentence_count(); }
+
+  /// Returns number of words in the sentece identified by sentence_id.
+  size_t word_count(size_t sentence_id) const {
+    return annotation.word_count(sentence_id);
+  }
+
+  /// Returns a std::string_view representing word_id in sentence_id
+  std::string_view word(size_t sentence_id, size_t word_id) const {
+    return as_view(annotation.word(sentence_id, word_id));
+  }
+
+  /// Returns a std::string_view representing sentence corresponding to
+  /// sentence_id.
+  std::string_view sentence(size_t sentence_id) const {
+    return as_view(annotation.sentence(sentence_id));
+  }
+
+  /// Returns the std::string_view of the gap between two sentences in the
+  /// container.
+  ///
+  /// More precisely where `i = sentence_id, N = sentence_count()` for brevity:
+  ///
+  /// * For `i = 0`: The gap between the start of text and the 0th sentence.
+  /// * For `i = 1...N-1`, returns the text comprising of the gap
+  ///   between the `i`-th and `i+1`-th sentence.
+  /// * For `i = N`, the gap between the last (N-1th) sentence and end of
+  ///   text.
+  /// @param sentence_id: Can be between `[0, sentence_count()]`.
+  std::string_view gap(size_t sentence_id) const {
+    return as_view(annotation.gap(sentence_id));
+  }
+
+  /// Returns a Range representing word_id in sentence_id
+  Range word_as_range(size_t sentence_id, size_t word_id) const {
+    return annotation.word(sentence_id, word_id);
+  }
+
+  /// Returns a Range representing sentence corresponding to sentence_id.
+  Range sentence_as_range(size_t sentence_id) const {
+    return annotation.sentence(sentence_id);
+  }
+
+  /// Utility function to call `fun` on each word (subword token effectively) in
+  /// an `AnnotatedText`. `fun` is called with the `Range`, the
+  /// `std::string_view` with the word, and a `bool` to indicate whether it is
+  /// the last word in the `AnnotatedText`, which is also the ending whitespace
+  /// slot of AnnotatedText.
+  template <typename Fun>
+  AnnotatedText apply(Fun fun) const {
+    AnnotatedText out;
+
+    for (size_t sentence_id = 0; sentence_id < sentence_count();
+         ++sentence_id) {
+      std::string sentence;
+      std::vector<Range> tokens;
+
+      std::string prefix =
+          fun(annotation.gap(sentence_id), gap(sentence_id), false);
+
+      for (size_t word_id = 0; word_id < word_count(sentence_id); ++word_id) {
+        std::string token = fun(word_as_range(sentence_id, word_id),
+                                word(sentence_id, word_id), false);
+        tokens.push_back(
+            Range{sentence.size(), sentence.size() + token.size()});
+        sentence += token;
+      }
+
+      // Convert our Ranges to std::string_views since that's what
+      // append_sentence expects
+      std::vector<std::string_view> views(tokens.size());
+      std::transform(tokens.begin(), tokens.end(), views.begin(),
+                     [&](Range const &range) {
+                       return std::string_view(sentence.data() + range.begin,
+                                               range.size());
+                     });
+
+      out.append_sentence(prefix, views.begin(), views.end());
+    }
+
+    out.append_ending_whitespace(
+        fun(annotation.gap(sentence_count()), gap(sentence_count()), true));
+
+    return out;
+  }
+
+ private:
+  std::string_view as_view(const Range &range) const {
+    return std::string_view(text.data() + range.begin, range.size());
+  }
+};
+
+}  // namespace slimt
diff --git a/slimt/Batcher.cc b/slimt/Batcher.cc
new file mode 100644
index 00000000..101d7a4b
--- /dev/null
+++ b/slimt/Batcher.cc
@@ -0,0 +1,122 @@
+#include "slimt/Batcher.hh"
+
+#include <cassert>
+
+#include "slimt/Batch.hh"
+#include "slimt/Model.hh"
+
+namespace slimt::rd {
+
+Batcher::Batcher(size_t max_words, size_t wrap_length,
+                 float tgt_length_limit_factor)
+    : max_words_(max_words), running_bucket_max_size_(0) {
+  // For the time being, we add some slack, which only Batcher is aware of.
+  // Since the TextProcessor still wraps at first request in, most of the
+  // Batches generated will be under max-length break.
+  //
+  // In the unlikely event of a few sentences overflowing, this allows the
+  // exceeding words to be put in the slack area. Very few batches are expected
+  // to be generated at a higher length.
+  size_t pivot_slack = wrap_length * tgt_length_limit_factor - wrap_length;
+  bucket_.resize(wrap_length + pivot_slack + 1);
+
+  SLIMT_ABORT_IF(bucket_.size() - 1 > max_words_,
+                 "Fatal: wrap_length > max_words  will lead to sentences "
+                 "longer than what can fit in a batch.");
+}
+
+Batch Batcher::generate() {
+  // For now simply iterates on buckets and converts batches greedily.  This
+  // has to be enhanced with optimizing over priority. The baseline
+  // implementation should at least be as fast as marian's maxi-batch with full
+  // corpus size as maxi-batch size.
+  Batch batch;
+  size_t padded_batch_size = 0;
+
+  for (size_t length = 0; length <= running_bucket_max_size_; length++) {
+    auto p = bucket_[length].begin();
+    while (p != bucket_[length].end()) {
+      padded_batch_size = (batch.size() + 1) * length;
+      if (padded_batch_size <= max_words_) {
+        auto q = p++;
+        batch.add(*q);
+        bucket_[length].erase(q);
+      } else {
+        // Check if elements exist
+        assert(!batch.empty());
+        return batch;
+      }
+    }
+  }
+
+  return batch;
+}
+
+size_t Batcher::enqueue(const Ptr<Request>& request) {
+  size_t to_be_translated = 0;
+  for (size_t i = 0; i < request->segment_count(); i++) {
+    if (!request->is_prefilled_from_cache(i)) {
+      SegmentRef sentence(i, request);
+      size_t bucket_id = sentence.size();
+
+      // Due to a workaround for pivoting, unless we can discipline the
+      // vocabulary to get stronger static requirements, it is difficult to
+      // rework the rest of the components. Instead, we allow dynamic growth
+      // here. We let std::vector take care of the dynamic growth.
+      // https://en.cppreference.com/w/cpp/container/vector/resize#Complexity
+      if (bucket_id >= bucket_.size()) {
+        bucket_.resize(bucket_id + 1);
+      }
+
+      bucket_[bucket_id].insert(sentence);
+      running_bucket_max_size_ =
+          std::max<size_t>(bucket_id, running_bucket_max_size_);
+
+      to_be_translated += 1;
+    }
+  }
+
+  return to_be_translated;
+}
+
+void Batcher::clear() {
+  for (auto& item : bucket_) {
+    item.clear();
+  }
+}
+
+AggregateBatcher::AggregateBatcher() = default;
+
+size_t AggregateBatcher::enqueue(const Ptr<Model>& model,
+                                 const Ptr<Request>& request) {
+  (void)queue_;
+  (void)model;
+  (void)request;
+  return 0;
+  // size_t sentences_enqueued = model->enqueue(request);
+  // queue_.insert(model);
+  // return sentences_enqueued;
+}
+
+Batch AggregateBatcher::generate(Ptr<Model>& model) {
+  (void)queue_;
+  (void)model;
+  // while (!queue_.empty()) {
+  //   auto candidate_iterator = queue_.begin();
+  //   Ptr<Model> candidate = *candidate_iterator;
+  //   Batch batch = candidate->generate();
+  //   if (batch.size() > 0) {
+  //     model = candidate;
+  //     return batch;
+  //   }
+  //   // Try the next model's batching pool.
+  //   queue_.erase(candidate_iterator);
+  // }
+  // // Empty.
+  Batch batch;
+  return batch;
+}
+
+void AggregateBatcher::clear() { queue_.clear(); }
+
+}  // namespace slimt::rd
diff --git a/slimt/Batcher.hh b/slimt/Batcher.hh
new file mode 100644
index 00000000..acf3febc
--- /dev/null
+++ b/slimt/Batcher.hh
@@ -0,0 +1,197 @@
+#pragma once
+
+#include <set>
+#include <vector>
+
+#include "slimt/Batch.hh"
+#include "slimt/Request.hh"
+#include "slimt/Types.hh"
+#include "slimt/Utils.hh"
+
+namespace slimt {
+
+class Model;
+
+namespace rd {
+
+class Batcher {
+ public:
+  explicit Batcher(size_t max_words, size_t wrap_length,
+                   float tgt_length_limit_factor = 3.0);
+
+  // SegmentRef incorporates (tentative) notions of priority with each
+  // sentence. This method inserts the sentence into the internal data-structure
+  // which maintains priority among sentences from multiple concurrent requests.
+  size_t enqueue(const Ptr<Request>& request);
+
+  // Loads sentences with sentences compiled from (tentatively) multiple
+  // requests optimizing for both padding and priority.
+  Batch generate();
+
+  // Removes any pending requests from the pool.
+  void clear();
+
+ private:
+  size_t max_words_;
+  std::vector<std::set<SegmentRef>> bucket_;
+  size_t running_bucket_max_size_;
+};
+
+/// Aggregates request queueing and generation of batches from multiple
+/// Models (Batchers within, specifically), thereby acting as an
+/// intermediary to enable multiple translation model capability in
+/// BlockingService and AsyncService.
+///
+/// A simple queue containing shared owning references to Models are
+/// held here from which batches are generated on demand. Since a queue is
+/// involved, the ordering is first-come first serve on requests except there
+/// are leaks effectively doing priority inversion if an earlier request with
+/// the same Model is pending to be consumed for translation.
+//
+/// Actual storage for the request and batch generation are within the
+/// respective Models, which owns its own Batcher.
+///
+/// Matches API provided by Batcher except arguments additionally
+/// parameterized by Model.
+///
+/// Note: This class is not thread-safe. You may use this class wrapped with
+/// ThreadsafeBatcher for a thread-safe equivalent of this class, if
+/// needed.
+class AggregateBatcher {
+ public:
+  /// Create an AggregateBatcher with (tentatively) global (across all
+  /// Batchers) limits imposed here.
+  AggregateBatcher();
+
+  /// Enqueue an existing request onto model, also keep account of that this
+  /// model and request are now pending.
+  ///
+  /// @param [in] model: Model to use in translation. A shared ownership to this
+  /// model is accepted by this object to keep the model alive until translation
+  /// is complete.
+  /// @param [in] request: A request to be enqueued to model.
+  /// @returns number of sentences added for translation.
+  size_t enqueue(const Ptr<Model>& model, const Ptr<Request>& request);
+
+  /// Generate a batch from pending requests, obtained from available
+  /// Models.
+  ///
+  /// @param [out] model: Model
+  /// @param [out] batch: Batch to write onto, which is consumed at translation
+  /// elsewhere.
+  /// @returns Number of sentences in the generated batch.
+  Batch generate(Ptr<Model>& model);
+
+  /// Clear the aggregate queue. Does not clear the underlying model/request
+  /// pairs but the next call to `generate()` will return 0. (Unless
+  /// `enqueue()` was called in the mean time.)
+  void clear();
+
+ private:
+  std::unordered_set<std::shared_ptr<Model>, HashPtr<Model>> queue_;
+};
+
+/// The following mechanism operates in a multithreaded async-workflow guarding
+/// access to the pushes to the structure keeping sentences bucketed by length
+/// and sorted by priority.
+///
+/// This is a wrap of a producer-consumer queue implemented as a monitor, where
+/// there is a mutex guarding the underlying data structure (BatcherType)
+/// and (worker/consumer) threads waiting on a condition variable and the
+/// queuing thread producing and notifying waiting threads (consumers) through
+/// the same condition variable.
+///
+/// Originally written by for a single model (where items are produce: Request,
+/// consume: Batch), converted to also work for multiple models where items are
+/// produce: (Model, Request), consume: (TranlsationModel, Batch).
+/// This is accomplished by template parameter packs.
+///
+/// Requires BatcherType to implement the following:
+///
+/// * produce: `size_t enqueue(...)` (returns number elements produced)
+/// * consume: `size_t generate(...)` (returns number of elements available
+/// to be consumed)
+
+template <class BatcherType>
+class ThreadsafeBatcher {
+ public:
+  template <class... Args>
+  explicit ThreadsafeBatcher(Args&&... args);
+  ~ThreadsafeBatcher();
+
+  template <class... Args>
+  void enqueue(Args&&... args);
+
+  template <class... Args>
+  size_t generate(Args&&... args);
+
+  // Removes any pending requests from the batching pool.
+  void clear();
+
+  // Signals shut down of batching pool. After this no new requests can be
+  // enqueued, but all enqueued requests will be processed. To prevent this from
+  // happening, call `clear()` before `shutdown()`.
+  void shutdown();
+
+ private:
+  BatcherType backend_;
+
+  // Number of sentences in backend_;
+  size_t enqueued_ = 0;
+
+  // Are we shutting down?
+  bool shutdown_ = false;
+
+  // Lock on this object.
+  std::mutex mutex_;
+
+  // Signaled when there are sentences to translate.
+  std::condition_variable work_;
+};
+
+template <class BatcherType>
+template <class... Args>
+ThreadsafeBatcher<BatcherType>::ThreadsafeBatcher(Args&&... args)
+    : backend_(std::forward<Args>(args)...) {}
+
+template <class BatcherType>
+ThreadsafeBatcher<BatcherType>::~ThreadsafeBatcher() {
+  shutdown();
+}
+
+template <class BatcherType>
+template <class... Args>
+void ThreadsafeBatcher<BatcherType>::enqueue(Args&&... args) {
+  std::unique_lock<std::mutex> lock(mutex_);
+  assert(!shutdown_);
+  enqueued_ += backend_.enqueue(std::forward<Args>(args)...);
+  work_.notify_all();
+}
+
+template <class BatcherType>
+void ThreadsafeBatcher<BatcherType>::clear() {
+  std::unique_lock<std::mutex> lock(mutex_);
+  backend_.clear();
+  enqueued_ = 0;
+}
+
+template <class BatcherType>
+void ThreadsafeBatcher<BatcherType>::shutdown() {
+  std::unique_lock<std::mutex> lock(mutex_);
+  shutdown_ = true;
+  work_.notify_all();
+}
+
+template <class BatcherType>
+template <class... Args>
+size_t ThreadsafeBatcher<BatcherType>::generate(Args&&... args) {
+  std::unique_lock<std::mutex> lock(mutex_);
+  work_.wait(lock, [this]() { return enqueued_ || shutdown_; });
+  size_t sentences_in_batch = backend_.generate(std::forward<Args>(args)...);
+  assert(sentences_in_batch > 0 || shutdown_);
+  enqueued_ -= sentences_in_batch;
+  return sentences_in_batch;
+}
+}  // namespace rd
+
+}  // namespace slimt
diff --git a/slimt/CMakeLists.txt b/slimt/CMakeLists.txt
index 29b48fb5..6aa5c7d3 100644
--- a/slimt/CMakeLists.txt
+++ b/slimt/CMakeLists.txt
@@ -1,14 +1,21 @@
 add_library(
   slimt STATIC
   Aligned.cc
+  Annotation.cc
   Batch.cc
+  Batcher.cc
+  Frontend.cc
   Io.cc
   Model.cc
   Modules.cc
   QMM.cc
+  Request.cc
+  Response.cc
+  ResponseBuilder.cc
   Shortlist.cc
-  TensorOps.cc
+  Splitter.cc
   Tensor.cc
+  TensorOps.cc
   Utils.cc
   Vocabulary.cc)
 
diff --git a/slimt/Cache.hh b/slimt/Cache.hh
new file mode 100644
index 00000000..7d3507e2
--- /dev/null
+++ b/slimt/Cache.hh
@@ -0,0 +1,91 @@
+#pragma once
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "slimt/Macros.hh"
+
+namespace slimt {
+
+template <class Key, class Value, class Hash = std::hash<Key>,
+          class Equals = std::equal_to<Key>>
+class AtomicCache {
+ public:
+  struct Stats {
+    size_t hits{0};
+    size_t misses{0};
+  };
+
+  explicit AtomicCache(size_t size, size_t buckets)
+      : records_(size), mutexBuckets_(buckets) {}
+
+  std::pair<bool, Value> find(const Key &key) const {
+    Value value;
+    bool found = atomicLoad(key, value);
+    return std::make_pair(found, value);
+  }
+
+  void store(const Key &key, Value value) { atomicStore(key, value); }
+
+  Stats stats() const {
+#ifdef ENABLE_CACHE_STATS
+    return Stats{hits_.load(), misses_.load()};
+#else
+    SLIMT_ABORT(
+        "Cache statistics requested without enabling in builds. Please use "
+        "-DENABLE_CACHE_STATS with cmake.");
+    return Stats{0, 0};
+#endif
+  }
+
+ private:
+  using Record = std::pair<Key, Value>;
+
+  bool atomicLoad(const Key &key, Value &value) const {
+    // No probing, direct map onto records_
+    size_t index = hash_(key) % records_.size();
+    size_t mutex_id = index % mutexBuckets_.size();
+
+    std::lock_guard<std::mutex> lock(mutexBuckets_[mutex_id]);
+    const Record &candidate = records_[index];
+    if (equals_(key, candidate.first)) {
+      value = candidate.second;
+#ifdef ENABLE_CACHE_STATS
+      ++hits_;
+#endif
+      return true;
+    }
+
+#ifdef ENABLE_CACHE_STATS
+    ++misses_;
+#endif
+    return false;
+  }
+
+  void atomicStore(const Key &key, Value value) {
+    // No probing, direct map onto records_
+    size_t index = hash_(key) % records_.size();
+    size_t mutex_id = index % mutexBuckets_.size();
+
+    std::lock_guard<std::mutex> lock(mutexBuckets_[mutex_id]);
+    Record &candidate = records_[index];
+
+    candidate.first = key;
+    candidate.second = value;
+  }
+
+  std::vector<Record> records_;
+
+  mutable std::vector<std::mutex> mutexBuckets_;
+
+#ifdef ENABLE_CACHE_STATS
+  mutable std::atomic<size_t> hits_{0};
+  mutable std::atomic<size_t> misses_{0};
+#endif
+
+  Hash hash_;
+  Equals equals_;
+};
+
+}  // namespace slimt
diff --git a/slimt/Frontend.cc b/slimt/Frontend.cc
new file mode 100644
index 00000000..e8d97ab7
--- /dev/null
+++ b/slimt/Frontend.cc
@@ -0,0 +1,152 @@
+
+#include "slimt/Frontend.hh"
+
+#include "slimt/Batcher.hh"
+#include "slimt/Request.hh"
+#include "slimt/Response.hh"
+#include "slimt/TensorOps.hh"
+
+namespace slimt {
+
+namespace {
+
+Batch convert(rd::Batch &rd_batch) {
+  const auto &segment_refs = rd_batch.segment_refs();
+  Batch batch(rd_batch.size(), rd_batch.max_length(), 0);
+  for (const auto &segment_ref : segment_refs) {
+    Segment segment = segment_ref.get();
+    batch.add(segment);
+  }
+
+  return batch;
+}
+
+}  // namespace
+Translator::Translator(const Config &config, View model, View shortlist,
+                       View vocabulary)
+    : config_(config),
+      vocabulary_(vocabulary.data, vocabulary.size),
+      processor_(config.wrap_length, config.split_mode, vocabulary_,
+                 config.prefix_path),
+      model_(config, model),
+      shortlist_generator_(shortlist.data, shortlist.size, vocabulary_,
+                           vocabulary_) {}
+
+Histories Translator::decode(Tensor &encoder_out, Tensor &mask,
+                             const Words &source) {
+  // Prepare a shortlist for the entire batch.
+  size_t batch_size = encoder_out.dim(-3);
+  size_t source_sequence_length = encoder_out.dim(-2);
+
+  Shortlist shortlist = shortlist_generator_.generate(source);
+  Words indices = shortlist.words();
+  // The following can be used to check if shortlist is going wrong.
+  // std::vector<uint32_t> indices(vocabulary_.size());
+  // std::iota(indices.begin(), indices.end(), 0);
+
+  std::vector<bool> complete(batch_size, false);
+  uint32_t eos = vocabulary_.eos_id();
+  auto record = [eos, &complete](Words &step, Sentences &sentences) {
+    size_t finished = 0;
+    for (size_t i = 0; i < step.size(); i++) {
+      if (not complete[i]) {
+        complete[i] = (step[i] == eos);
+        sentences[i].push_back(step[i]);
+      }
+      finished += static_cast<int>(complete[i]);
+    }
+    return sentences.size() - finished;
+  };
+
+  // Initialize a first step.
+  Sentences sentences(batch_size);
+
+  Decoder &decoder = model_.decoder();
+  Words previous_slice = {};
+  std::vector<Tensor> states = decoder.start_states(batch_size);
+  Tensor logits =
+      decoder.step(encoder_out, mask, states, previous_slice, indices);
+
+  previous_slice = greedy_sample(logits, indices, batch_size);
+  record(previous_slice, sentences);
+
+  size_t remaining = sentences.size();
+  size_t max_seq_length =
+      config_.tgt_length_limit_factor * source_sequence_length;
+  for (size_t i = 1; i < max_seq_length && remaining > 0; i++) {
+    Tensor logits =
+        decoder.step(encoder_out, mask, states, previous_slice, indices);
+
+    previous_slice = greedy_sample(logits, indices, batch_size);
+    remaining = record(previous_slice, sentences);
+  }
+
+  Histories histories;
+  Alignments alignments(sentences.size());
+  for (size_t i = 0; i < sentences.size(); i++) {
+    Hypothesis hypothesis{
+        .target = std::move(sentences[i]),     //
+        .alignment = std::move(alignments[i])  //
+    };
+    auto history = std::make_shared<Hypothesis>(std::move(hypothesis));
+    histories.push_back(std::move(history));
+  }
+
+  return histories;
+}
+
+Histories Translator::forward(Batch &batch) {
+  Tensor &indices = batch.indices();
+  Tensor &mask = batch.mask();
+
+  // uint64_t batch_size = indices.dim(-2);
+  // uint64_t sequence_length = indices.dim(-1);
+  // uint64_t embed_dim = embedding_.dim(-1);
+
+  Tensor word_embedding =
+      index_select(model_.embedding(), indices, "word_embedding");
+  transform_embedding(word_embedding);
+
+  // https://github.com/browsermt/marian-dev/blob/14c9d9b0e732f42674e41ee138571d5a7bf7ad94/src/models/transformer.h#L570
+  // https://github.com/browsermt/marian-dev/blob/14c9d9b0e732f42674e41ee138571d5a7bf7ad94/src/models/transformer.h#L133
+  modify_mask_for_pad_tokens_in_attention(mask.data<float>(), mask.size());
+  Tensor encoder_out = model_.encoder().forward(word_embedding, mask);
+  Histories histories = decode(encoder_out, mask, batch.words());
+  return histories;
+}
+
+Response Translator::translate(std::string source, const Options &options) {
+  // Create a request
+  auto [annotated_source, segments] = processor_.process(std::move(source));
+
+  std::promise<Response> promise;
+  auto future = promise.get_future();
+
+  ResponseBuilder response_builder(options, std::move(annotated_source),
+                                   vocabulary_, std::move(promise));
+
+  auto request = std::make_shared<rd::Request>(  //
+      id_, model_id_,                            //
+      std::move(segments),                       //
+      std::move(response_builder),               //
+      cache_                                     //
+  );
+
+  rd::Batcher batcher(config_.max_words, config_.wrap_length,
+                      config_.tgt_length_limit_factor);
+  batcher.enqueue(request);
+
+  rd::Batch rd_batch = batcher.generate();
+  while (!rd_batch.empty()) {
+    // convert between batches.
+    Batch batch = convert(rd_batch);
+    Histories histories = forward(batch);
+    rd_batch.complete(histories);
+    rd_batch = batcher.generate();
+  }
+
+  future.wait();
+  return future.get();
+}
+
+}  // namespace slimt
diff --git a/slimt/Frontend.hh b/slimt/Frontend.hh
new file mode 100644
index 00000000..a2b74a94
--- /dev/null
+++ b/slimt/Frontend.hh
@@ -0,0 +1,37 @@
+
+#pragma once
+#include <optional>
+
+#include "slimt/Model.hh"
+#include "slimt/Response.hh"
+#include "slimt/Splitter.hh"
+#include "slimt/Types.hh"
+
+namespace slimt {
+
+struct Data {
+  Aligned model;
+  Aligned shortlist;
+  Aligned vocabulary;
+};
+
+class Translator {
+ public:
+  Translator(const Config &config, View model, View shortlist, View vocabulary);
+  Histories decode(Tensor &encoder_out, Tensor &mask, const Words &source);
+  Histories forward(Batch &batch);
+  Response translate(std::string source, const Options &options);
+
+ private:
+  Config config_;
+  Vocabulary vocabulary_;
+  TextProcessor processor_;
+  Model model_;
+  ShortlistGenerator shortlist_generator_;
+  std::optional<TranslationCache> cache_;
+
+  size_t id_ = 0;
+  size_t model_id_ = 0;
+};
+
+}  // namespace slimt
diff --git a/slimt/Io.cc b/slimt/Io.cc
index ccd16be6..0a8c7b85 100644
--- a/slimt/Io.cc
+++ b/slimt/Io.cc
@@ -184,7 +184,7 @@ std::vector<io::Item> loadItems(void* current) {
 
         // Allocate aligned storage to write out unquantized embeddings.
         size_t size_as_float = num_elements * sizeof(float);
-        Aligned aligned(/*alignment=*/64, size_as_float);
+        Aligned aligned(kAlignWidth, size_as_float);
 
         auto* quantized_weights = reinterpret_cast<int8_t*>(ptr);
         auto* weights = reinterpret_cast<float*>(aligned.data());
@@ -204,10 +204,10 @@ std::vector<io::Item> loadItems(void* current) {
         size_t prepared_size =
             embedding_processed.shape.elements() * sizeof(int8_t) +
             sizeof(float);
-        Aligned embedding_aligned(/*alignment=*/64, prepared_size);
+        Aligned embedding_aligned(kAlignWidth, prepared_size);
         auto* prepared = reinterpret_cast<int8_t*>(embedding_aligned.data());
-        qmm::PrepareBTransposed(weights, prepared, quantization_multiplier,
-                                cols, rows);
+        qmm::prepare_weight_transposed(weights, prepared,
+                                       quantization_multiplier, cols, rows);
 
         // Save quantization multiplier.
         auto* embedding_quantization_multiplier_addr =
@@ -222,10 +222,10 @@ std::vector<io::Item> loadItems(void* current) {
         size_t cols = item.shape.dim(-1);
         auto* input = reinterpret_cast<int8_t*>(ptr);
 
-        Aligned aligned(/*alignment=*/64, rows * cols + sizeof(float));
+        Aligned aligned(kAlignWidth, rows * cols + sizeof(float));
 
         auto* output = reinterpret_cast<int8_t*>(aligned.data());
-        qmm::PrepareBQuantizedTransposed(input, output, rows, cols);
+        qmm::prepare_weight_quantized_transposed(input, output, rows, cols);
 
         // Set b_quant at end.
         auto* output_end = reinterpret_cast<float*>(output + rows * cols);
diff --git a/slimt/Io.hh b/slimt/Io.hh
index 8c487576..f332d289 100644
--- a/slimt/Io.hh
+++ b/slimt/Io.hh
@@ -33,6 +33,8 @@ struct Item {
   Type type{Type::f32};
 };
 
+using Items = std::vector<Item>;
+
 void set_item(Item& item, Aligned&& aligned);
 
 std::vector<io::Item> loadItems(void* current);
diff --git a/slimt/Model.cc b/slimt/Model.cc
index eb4b7d73..43f3fa19 100644
--- a/slimt/Model.cc
+++ b/slimt/Model.cc
@@ -12,7 +12,7 @@
 
 namespace slimt {
 
-void transform_embedding(Tensor &word_embedding, size_t start = 0) {
+void transform_embedding(Tensor &word_embedding, size_t start /* = 0*/) {
   // This is a pain, why does marian-transpose here, I do not get yet.
 
   uint64_t embed_dim = word_embedding.dim(-1);
@@ -39,21 +39,14 @@ void transform_embedding(Tensor &word_embedding, size_t start = 0) {
                            word_embedding_ptr);
 }
 
-Sentences Model::translate(Batch &batch) {
-  Tensor &indices = batch.indices();
-  Tensor &mask = batch.mask();
-
-  // uint64_t batch_size = indices.dim(-2);
-  // uint64_t sequence_length = indices.dim(-1);
-  // uint64_t embed_dim = embedding_.dim(-1);
-
-  Tensor word_embedding = index_select(embedding_, indices, "word_embedding");
-  transform_embedding(word_embedding);
-
-  // https://github.com/browsermt/marian-dev/blob/14c9d9b0e732f42674e41ee138571d5a7bf7ad94/src/models/transformer.h#L570
-  // https://github.com/browsermt/marian-dev/blob/14c9d9b0e732f42674e41ee138571d5a7bf7ad94/src/models/transformer.h#L133
-  modify_mask_for_pad_tokens_in_attention(mask.data<float>(), mask.size());
+Encoder::Encoder(const Config &config) {
+  for (size_t i = 0; i < config.encoder_layers; i++) {
+    encoder_.emplace_back(i + 1, config.feed_forward_depth,
+                          config.attention_num_heads);
+  }
+}
 
+Tensor Encoder::forward(Tensor &word_embedding, Tensor &mask) {
   auto [x, attn] = encoder_[0].forward(word_embedding, mask);
 
   for (size_t i = 1; i < encoder_.size(); i++) {
@@ -63,82 +56,14 @@ Sentences Model::translate(Batch &batch) {
     // Overwriting x so that x is destroyed and we need lesser working memory.
     x = std::move(y);
   }
-
-  Tensor &encoder_out = x;
-  return decoder_.decode(encoder_out, mask, batch.words());
-}
-
-Words Decoder::greedy_sample(Tensor &logits, const Words &words,
-                             size_t batch_size) {
-  Words sampled_words;
-  for (size_t i = 0; i < batch_size; i++) {
-    auto *data = logits.data<float>();
-    size_t max_index = 0;
-    float max_value = data[0];
-    size_t stride = words.size();
-    for (size_t cls = 1; cls < stride; cls++) {
-      float value = data[i * stride + cls];
-      if (value > max_value) {
-        max_index = cls;
-        max_value = value;
-      }
-    }
-
-    sampled_words.push_back(words[max_index]);
-  }
-  return sampled_words;
+  return std::move(x);
 }
 
-Sentences Decoder::decode(Tensor &encoder_out, Tensor &mask,
-                          const Words &source) {
-  // Prepare a shortlist for the entire batch.
-  size_t batch_size = encoder_out.dim(-3);
-  size_t source_sequence_length = encoder_out.dim(-2);
-
-  Shortlist shortlist = shortlist_generator_.generate(source);
-  const auto &indices = shortlist.words();
-  // The following can be used to check if shortlist is going wrong.
-  // std::vector<uint32_t> indices(vocabulary_.size());
-  // std::iota(indices.begin(), indices.end(), 0);
-
-  std::vector<bool> complete(batch_size, false);
-  uint32_t eos = vocabulary_.eos_id();
-  auto record = [eos, &complete](Words &step, Sentences &sentences) {
-    size_t finished = 0;
-    for (size_t i = 0; i < step.size(); i++) {
-      if (not complete[i]) {
-        complete[i] = (step[i] == eos);
-        sentences[i].push_back(step[i]);
-      }
-      finished += static_cast<int>(complete[i]);
-    }
-    return sentences.size() - finished;
-  };
-
-  // Initialize a first step.
-  Sentences sentences(batch_size);
-
-  Words previous_slice = {};
-  std::vector<Tensor> states = start_states(batch_size);
-  Tensor decoder_out = step(encoder_out, mask, states, previous_slice);
-
-  Tensor logits = affine_with_select(output_, decoder_out, indices, "logits");
-
-  previous_slice = greedy_sample(logits, indices, batch_size);
-  record(previous_slice, sentences);
-
-  size_t remaining = sentences.size();
-  size_t max_seq_length = max_target_length_factor_ * source_sequence_length;
-  for (size_t i = 1; i < max_seq_length && remaining > 0; i++) {
-    Tensor decoder_out = step(encoder_out, mask, states, previous_slice);
-
-    Tensor logits = affine_with_select(output_, decoder_out, indices, "logits");
-
-    previous_slice = greedy_sample(logits, indices, batch_size);
-    remaining = record(previous_slice, sentences);
+void Encoder::register_parameters(const std::string &prefix,
+                                  ParameterMap &parameters) {
+  for (EncoderLayer &layer : encoder_) {
+    layer.register_parameters(prefix, parameters);
   }
-
-  return sentences;
 }
 
 std::vector<Tensor> Decoder::start_states(size_t batch_size) {
@@ -150,32 +75,18 @@ std::vector<Tensor> Decoder::start_states(size_t batch_size) {
   return states;
 }
 
-Model::Model(Tag tag, Vocabulary &vocabulary, std::vector<io::Item> &&items,
-             ShortlistGenerator &&shortlist_generator)
-    : tag_(tag),
-      items_(std::move(items)),
-      decoder_(                                //
-          Config::tiny11::decoder_layers,      //
-          Config::tiny11::feed_forward_depth,  //
-          vocabulary,                          //
-          embedding_,                          //
-          std::move(shortlist_generator)       //
-      ) {
-  for (size_t i = 0; i < Config::tiny11::encoder_layers; i++) {
-    encoder_.emplace_back(i + 1, Config::tiny11::feed_forward_depth);
-  }
-
-  load_parameters_from_items();
-  (void)tag_;  // Apparently tag not used. This should fix.
+Model::Model(const Config &config, View model)
+    : items_(io::loadItems(model.data)),
+      encoder_(config),  //
+      decoder_(config, embedding_) {
+  load_parameters();
 }
 
-Decoder::Decoder(size_t decoders, size_t ffn_count, Vocabulary &vocabulary,
-                 Tensor &embedding, ShortlistGenerator &&shortlist_generator)
-    : vocabulary_(vocabulary),
-      embedding_(embedding),
-      shortlist_generator_(std::move(shortlist_generator)) {
-  for (size_t i = 0; i < decoders; i++) {
-    decoder_.emplace_back(i + 1, ffn_count);
+Decoder::Decoder(const Config &config, Tensor &embedding)
+    : embedding_(embedding) {
+  for (size_t i = 0; i < config.decoder_layers; i++) {
+    decoder_.emplace_back(i + 1, config.feed_forward_depth,
+                          config.attention_num_heads);
   }
 }
 
@@ -193,7 +104,8 @@ void Decoder::register_parameters(const std::string &prefix,
 }
 
 Tensor Decoder::step(Tensor &encoder_out, Tensor &mask,
-                     std::vector<Tensor> &states, Words &previous_step) {
+                     std::vector<Tensor> &states, Words &previous_step,
+                     Words &shortlist) {
   // Infer batch-size from encoder_out.
   size_t encoder_feature_dim = encoder_out.dim(-1);
   size_t source_sequence_length = encoder_out.dim(-2);
@@ -240,10 +152,11 @@ Tensor Decoder::step(Tensor &encoder_out, Tensor &mask,
     x = std::move(y);
   }
 
-  return std::move(x);
+  Tensor logits = affine_with_select(output_, x, shortlist, "logits");
+  return logits;
 }
 
-void Model::load_parameters_from_items() {
+void Model::load_parameters() {
   // Get the parameterss from strings to target tensors to load.
   ParameterMap parameters;
   std::string prefix;
@@ -277,10 +190,10 @@ void Model::load_parameters_from_items() {
   }
 
   for (std::string &entry : missed) {
-    std::cerr << "Failed to ingest expected load of " << entry << "\n";
+    std::cerr << "[warn] Failed to ingest expected load of " << entry << "\n";
   }
   for (auto &parameter : parameters) {
-    std::cerr << "Failed to complete expected load of ";
+    std::cerr << "[warn] Failed to complete expected load of ";
     std::cerr << parameter.first << "\n";
   }
 }
@@ -288,10 +201,28 @@ void Model::load_parameters_from_items() {
 void Model::register_parameters(const std::string &prefix,
                                 ParameterMap &parameters) {
   parameters.emplace("Wemb", &embedding_);
-  for (EncoderLayer &layer : encoder_) {
-    layer.register_parameters(prefix, parameters);
-  }
+  encoder_.register_parameters(prefix, parameters);
   decoder_.register_parameters(prefix, parameters);
 }
 
+Words greedy_sample(Tensor &logits, const Words &words, size_t batch_size) {
+  Words sampled_words;
+  for (size_t i = 0; i < batch_size; i++) {
+    auto *data = logits.data<float>();
+    size_t max_index = 0;
+    float max_value = data[0];
+    size_t stride = words.size();
+    for (size_t cls = 1; cls < stride; cls++) {
+      float value = data[i * stride + cls];
+      if (value > max_value) {
+        max_index = cls;
+        max_value = value;
+      }
+    }
+
+    sampled_words.push_back(words[max_index]);
+  }
+  return sampled_words;
+}
+
 }  // namespace slimt
diff --git a/slimt/Model.hh b/slimt/Model.hh
index 6fbc40c1..23c6992c 100644
--- a/slimt/Model.hh
+++ b/slimt/Model.hh
@@ -1,3 +1,4 @@
+#pragma once
 #include <unordered_map>
 
 #include "slimt/Batch.hh"
@@ -9,68 +10,69 @@
 
 namespace slimt {
 
-class Decoder {
- public:
-  Decoder(size_t decoders, size_t ffn_count, Vocabulary &vocabulary,
-          Tensor &embedding, ShortlistGenerator &&shortlist_generator);
+struct Config {
+  // NOLINTBEGIN
+  size_t encoder_layers = 6;
+  size_t decoder_layers = 2;
+  size_t feed_forward_depth = 2;
+  size_t tgt_length_limit_factor = 2;
+  size_t attention_num_heads = 8;
 
-  void register_parameters(const std::string &prefix, ParameterMap &parameters);
+  size_t max_words = 1024;
+  size_t wrap_length = 128;
 
-  Sentences decode(Tensor &encoder_out, Tensor &mask, const Words &source);
+  std::string prefix_path;
+  std::string split_mode = "sentence";
+
+  // NOLINTEND
+};
+
+class Encoder {
+ public:
+  explicit Encoder(const Config &config);
+  Tensor forward(Tensor &embedding, Tensor &mask);
+  void register_parameters(const std::string &prefix, ParameterMap &parameters);
 
  private:
-  Tensor step(Tensor &encoder_out, Tensor &mask, std::vector<Tensor> &states,
-              Words &previous_step);
+  std::vector<EncoderLayer> encoder_;
+};
+
+class Decoder {
+ public:
+  Decoder(const Config &config, Tensor &embedding);
 
-  static Words greedy_sample(Tensor &logits, const Words &words,
-                             size_t batch_size);
+  void register_parameters(const std::string &prefix, ParameterMap &parameters);
 
   std::vector<Tensor> start_states(size_t batch_size);
+  Tensor step(Tensor &encoder_out, Tensor &mask, std::vector<Tensor> &states,
+              Words &previous_step, Words &shortlist);
 
-  Vocabulary &vocabulary_;
-
+ private:
   Tensor &embedding_;
   std::vector<DecoderLayer> decoder_;
   Affine output_;
-
-  ShortlistGenerator shortlist_generator_;
-
-  float max_target_length_factor_ = 1.5;  // FIXME(-1): HARDCODE
 };
 
-// Restrict the models that can be created by a few kinds.
-enum class Tag {
-  // NOLINTBEGIN (readability-identifier-naming)
-  tiny11  //
-  // NOLINTEND
-};
-
-struct Config {
-  // NOLINTBEGIN (readability-identifier-naming)
-  struct tiny11 {
-    static constexpr size_t encoder_layers = 6;
-    static constexpr size_t decoder_layers = 2;
-    static constexpr size_t feed_forward_depth = 2;
-  };
-  // NOLINTEND
-};
+Words greedy_sample(Tensor &logits, const Words &words, size_t batch_size);
+void transform_embedding(Tensor &word_embedding, size_t start = 0);
 
 class Model {
  public:
-  explicit Model(Tag tag, Vocabulary &vocabulary, std::vector<io::Item> &&items,
-                 ShortlistGenerator &&shortlist_generator);
+  explicit Model(const Config &config, View model);
 
-  Sentences translate(Batch &batch);
+  Config &config() { return config_; }
+  Tensor &embedding() { return embedding_; }
+  Encoder &encoder() { return encoder_; }
+  Decoder &decoder() { return decoder_; }
 
  private:
-  void load_parameters_from_items();
   void register_parameters(const std::string &prefix, ParameterMap &parameters);
+  void load_parameters();
 
-  Tag tag_;
-
+  Config config_;
   std::vector<io::Item> items_;
   Tensor embedding_;
-  std::vector<EncoderLayer> encoder_;
+  Encoder encoder_;
   Decoder decoder_;
 };
 
diff --git a/slimt/Modules.cc b/slimt/Modules.cc
index cb11184f..42ff92c5 100644
--- a/slimt/Modules.cc
+++ b/slimt/Modules.cc
@@ -258,15 +258,15 @@ std::tuple<Tensor, Tensor> DecoderLayer::forward(Tensor &encoder_out,
   return std::make_tuple(std::move(normalized_ffn_out), std::move(attn));
 }
 
-EncoderLayer::EncoderLayer(size_t depth, size_t ffn_count)
-    : depth_(depth), attention_("self") {
+EncoderLayer::EncoderLayer(size_t depth, size_t ffn_count, size_t num_heads)
+    : depth_(depth), attention_("self", num_heads) {
   for (size_t i = 0; i < ffn_count; i++) {
     ffn_.emplace_back(i + 1);
   }
 }
 
-DecoderLayer::DecoderLayer(size_t depth, size_t ffn_count)
-    : depth_(depth), attention_("context") {
+DecoderLayer::DecoderLayer(size_t depth, size_t ffn_count, size_t num_heads)
+    : depth_(depth), attention_("context", num_heads) {
   for (size_t i = 0; i < ffn_count; i++) {
     ffn_.emplace_back(i + 1);
   }
@@ -369,7 +369,8 @@ void DecoderLayer::register_parameters(const std::string &prefix,
   ffn_ffn_.register_parameters(decoder_prefix + "_ffn_ffn", parameters);
 }
 
-Attention::Attention(std::string name) : name_(std::move(name)) {}
+Attention::Attention(std::string name, size_t num_heads)
+    : name_(std::move(name)), num_heads_(num_heads) {}
 
 void Attention::register_parameters(const std::string &prefix,
                                     ParameterMap &parameters) {
diff --git a/slimt/Modules.hh b/slimt/Modules.hh
index 62ea5cc8..ae62cb4a 100644
--- a/slimt/Modules.hh
+++ b/slimt/Modules.hh
@@ -29,7 +29,7 @@ class LayerNorm {
 
 class Attention {
  public:
-  explicit Attention(std::string name);
+  explicit Attention(std::string name, size_t num_heads);
   void register_parameters(const std::string &prefix, ParameterMap &parameters);
   std::tuple<Tensor, Tensor> forward(Tensor &q, Tensor &k, Tensor &v,
                                      Tensor &mask);
@@ -38,7 +38,7 @@ class Attention {
   std::string name_;
   Affine Q_, K_, V_, O_;
   LayerNorm ln_;
-  size_t num_heads_ = 8;  // FIXME(-1): HARDCODE
+  size_t num_heads_;
 };
 
 class SSRU {
@@ -67,7 +67,7 @@ class FFN {
 
 class EncoderLayer {
  public:
-  EncoderLayer(size_t depth, size_t ffn_count);
+  EncoderLayer(size_t depth, size_t ffn_count, size_t num_heads);
   void register_parameters(const std::string &prefix, ParameterMap &parameters);
   std::tuple<Tensor, Tensor> forward(Tensor &x, Tensor &mask);
 
@@ -80,7 +80,7 @@ class EncoderLayer {
 
 class DecoderLayer {
  public:
-  explicit DecoderLayer(size_t depth, size_t ffn_count);
+  explicit DecoderLayer(size_t depth, size_t ffn_count, size_t num_heads);
   void register_parameters(const std::string &prefix, ParameterMap &parameters);
   std::tuple<Tensor, Tensor> forward(Tensor &encoder_out, Tensor &mask,
                                      Tensor &state, Tensor &x);
diff --git a/slimt/QMM.cc b/slimt/QMM.cc
index dc3aad73..5e17e34c 100644
--- a/slimt/QMM.cc
+++ b/slimt/QMM.cc
@@ -27,7 +27,7 @@ namespace detail {
 
 #ifdef SLIMT_HAS_INTGEMM
 template <>
-Tensor affine_with_select<Provider::kIntgemm>(
+Tensor affine_with_select<Provider::Intgemm>(
     Tensor& x, Tensor& W, Tensor& b, float a_quant, float b_quant,
     const std::vector<uint32_t>& indices, const std::string& name) {
   // Naming is to simplify thinking with the intgemm API below.
@@ -56,7 +56,7 @@ Tensor affine_with_select<Provider::kIntgemm>(
 
   // Prepare bias
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  constexpr float kMax8bit = 127.0F;
+  constexpr float kMax8bit = kInt8Maxf;
   float a_alpha = kMax8bit / a_quant;
   float b_alpha = kMax8bit / b_quant;
 
@@ -111,9 +111,8 @@ Tensor affine_with_select<Provider::kIntgemm>(
 }
 
 template <>
-Tensor affine<Provider::kIntgemm>(Tensor& x, Tensor& W, Tensor& b,
-                                  float a_quant, float b_quant,
-                                  const std::string& name) {
+Tensor affine<Provider::Intgemm>(Tensor& x, Tensor& W, Tensor& b, float a_quant,
+                                 float b_quant, const std::string& name) {
   // Naming is to simplify thinking with the intgemm API below.
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
@@ -141,10 +140,10 @@ Tensor affine<Provider::kIntgemm>(Tensor& x, Tensor& W, Tensor& b,
 
   // Prepare bias
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  float a_alpha = 127.0F / a_quant;
-  float b_alpha = 127.0F / b_quant;
+  float a_alpha = kInt8Maxf / a_quant;
+  float b_alpha = kInt8Maxf / b_quant;
 
-  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / 127.0F;
+  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / kInt8Maxf;
   auto prepare_bias_callback = intgemm::callbacks::UnquantizeAndAddBiasAndWrite(
       bias_unquant_multiplier, bias.data<float>(),  //
       prepared_bias.data<float>()                   //
@@ -177,8 +176,8 @@ Tensor affine<Provider::kIntgemm>(Tensor& x, Tensor& W, Tensor& b,
 }
 
 template <>
-Tensor dot<Provider::kIntgemm>(Tensor& x, Tensor& W, float a_quant,
-                               float b_quant, const std::string& name) {
+Tensor dot<Provider::Intgemm>(Tensor& x, Tensor& W, float a_quant,
+                              float b_quant, const std::string& name) {
   // Naming is to simplify thinking with the intgemm API below.
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
@@ -210,10 +209,10 @@ Tensor dot<Provider::kIntgemm>(Tensor& x, Tensor& W, float a_quant,
   bias.fill_in_place(0.0F);
 
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  float a_alpha = 127.0F / a_quant;
-  float b_alpha = 127.0F / b_quant;
+  float a_alpha = kInt8Maxf / a_quant;
+  float b_alpha = kInt8Maxf / b_quant;
 
-  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / 127.0F;
+  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / kInt8Maxf;
   auto prepare_bias_callback = intgemm::callbacks::UnquantizeAndAddBiasAndWrite(
       bias_unquant_multiplier, bias.data<float>(),  //
       prepared_bias.data<float>()                   //
@@ -247,18 +246,19 @@ Tensor dot<Provider::kIntgemm>(Tensor& x, Tensor& W, float a_quant,
 }
 
 template <>
-void PrepareBTransposed<Provider::kIntgemm>(const float* weights,
-                                            int8_t* prepared,
-                                            float quantization_multiplier,
-                                            size_t cols, size_t rows) {
+void prepare_weight_transposed<Provider::Intgemm>(const float* weights,
+                                                  int8_t* prepared,
+                                                  float quantization_multiplier,
+                                                  size_t cols, size_t rows) {
   intgemm::Int8::PrepareBTransposed(weights, prepared, quantization_multiplier,
                                     cols, rows);
 }
 
 template <>
-void PrepareBQuantizedTransposed<Provider::kIntgemm>(const int8_t* input,
-                                                     int8_t* output,
-                                                     size_t rows, size_t cols) {
+void prepare_weight_quantized_transposed<Provider::Intgemm>(const int8_t* input,
+                                                            int8_t* output,
+                                                            size_t rows,
+                                                            size_t cols) {
   intgemm::Int8::PrepareBQuantizedTransposed(input, output, rows, cols);
 }
 
@@ -278,8 +278,8 @@ void quantize(const float* input, float scale, Index rows, Index width,
 
     // Since float can store bigger values, we threshold anything that's gone
     // higher and can't fit in int8.
-    value = std::max<float>(-127.0F, value);
-    value = std::min<float>(127.0F, value);
+    value = std::max<float>(-kInt8Maxf, value);
+    value = std::min<float>(kInt8Maxf, value);
 
     // Finally a static cast.
     output[i] = static_cast<int8_t>(value);
@@ -320,8 +320,8 @@ void unquantizeAddBias(const int32_t* input, const float* input_bias_prepared,
 
 // Ruy.
 template <>
-Tensor affine<Provider::kRuy>(Tensor& x, Tensor& W, Tensor& b, float a_quant,
-                              float b_quant, const std::string& name) {
+Tensor affine<Provider::Ruy>(Tensor& x, Tensor& W, Tensor& b, float a_quant,
+                             float b_quant, const std::string& name) {
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
   Tensor& bias = b;
@@ -380,10 +380,10 @@ Tensor affine<Provider::kRuy>(Tensor& x, Tensor& W, Tensor& b, float a_quant,
 }
 
 template <>
-Tensor affine_with_select<Provider::kRuy>(Tensor& x, Tensor& W, Tensor& b,
-                                          float a_quant, float b_quant,
-                                          const std::vector<uint32_t>& indices,
-                                          const std::string& name) {
+Tensor affine_with_select<Provider::Ruy>(Tensor& x, Tensor& W, Tensor& b,
+                                         float a_quant, float b_quant,
+                                         const std::vector<uint32_t>& indices,
+                                         const std::string& name) {
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
   Tensor& bias = b;
@@ -464,8 +464,8 @@ Tensor affine_with_select<Provider::kRuy>(Tensor& x, Tensor& W, Tensor& b,
 }
 
 template <>
-Tensor dot<Provider::kRuy>(Tensor& x, Tensor& W, float a_quant, float b_quant,
-                           const std::string& name) {
+Tensor dot<Provider::Ruy>(Tensor& x, Tensor& W, float a_quant, float b_quant,
+                          const std::string& name) {
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
 
@@ -520,16 +520,18 @@ Tensor dot<Provider::kRuy>(Tensor& x, Tensor& W, float a_quant, float b_quant,
 }
 
 template <>
-void PrepareBTransposed<Provider::kRuy>(const float* weights, int8_t* prepared,
-                                        float quantization_multiplier,
-                                        size_t cols, size_t rows) {
+void prepare_weight_transposed<Provider::Ruy>(const float* weights,
+                                              int8_t* prepared,
+                                              float quantization_multiplier,
+                                              size_t cols, size_t rows) {
   detail::quantize(weights, quantization_multiplier, cols, rows, prepared);
 }
 
 template <>
-void PrepareBQuantizedTransposed<Provider::kRuy>(const int8_t* input,
-                                                 int8_t* output, size_t rows,
-                                                 size_t cols) {
+void prepare_weight_quantized_transposed<Provider::Ruy>(const int8_t* input,
+                                                        int8_t* output,
+                                                        size_t rows,
+                                                        size_t cols) {
   std::memcpy(output, input,
               /*count=*/sizeof(int8_t) * (rows * cols));
 }
@@ -537,7 +539,7 @@ void PrepareBQuantizedTransposed<Provider::kRuy>(const int8_t* input,
 
 #ifdef SLIMT_HAS_GEMMOLOGY
 template <>
-Tensor affine_with_select<Provider::kGemmology>(
+Tensor affine_with_select<Provider::Gemmology>(
     Tensor& x, Tensor& W, Tensor& b, float a_quant, float b_quant,
     const std::vector<uint32_t>& indices, const std::string& name) {
   // Naming is to simplify thinking with the gemmology API below.
@@ -566,7 +568,7 @@ Tensor affine_with_select<Provider::kGemmology>(
 
   // Prepare bias
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  constexpr float kMax8bit = 127.0F;
+  constexpr float kMax8bit = kInt8Maxf;
   float a_alpha = kMax8bit / a_quant;
   float b_alpha = kMax8bit / b_quant;
 
@@ -622,9 +624,9 @@ Tensor affine_with_select<Provider::kGemmology>(
 }
 
 template <>
-Tensor affine<Provider::kGemmology>(Tensor& x, Tensor& W, Tensor& b,
-                                    float a_quant, float b_quant,
-                                    const std::string& name) {
+Tensor affine<Provider::Gemmology>(Tensor& x, Tensor& W, Tensor& b,
+                                   float a_quant, float b_quant,
+                                   const std::string& name) {
   // Naming is to simplify thinking with the gemmology API below.
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
@@ -652,10 +654,10 @@ Tensor affine<Provider::kGemmology>(Tensor& x, Tensor& W, Tensor& b,
 
   // Prepare bias
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  float a_alpha = 127.0F / a_quant;
-  float b_alpha = 127.0F / b_quant;
+  float a_alpha = kInt8Maxf / a_quant;
+  float b_alpha = kInt8Maxf / b_quant;
 
-  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / 127.0F;
+  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / kInt8Maxf;
   auto prepare_bias_callback =
       gemmology::callbacks::UnquantizeAndAddBiasAndWrite(
           bias_unquant_multiplier, bias.data<float>(),  //
@@ -689,8 +691,8 @@ Tensor affine<Provider::kGemmology>(Tensor& x, Tensor& W, Tensor& b,
 }
 
 template <>
-Tensor dot<Provider::kGemmology>(Tensor& x, Tensor& W, float a_quant,
-                                 float b_quant, const std::string& name) {
+Tensor dot<Provider::Gemmology>(Tensor& x, Tensor& W, float a_quant,
+                                float b_quant, const std::string& name) {
   // Naming is to simplify thinking with the gemmology API below.
   Tensor& A = x;  // NOLINT
   Tensor& B = W;  // NOLINT
@@ -722,10 +724,10 @@ Tensor dot<Provider::kGemmology>(Tensor& x, Tensor& W, float a_quant,
   bias.fill_in_place(0.0F);
 
   Tensor prepared_bias(Type::f32, bias.shape(), "prepared_bias");
-  float a_alpha = 127.0F / a_quant;
-  float b_alpha = 127.0F / b_quant;
+  float a_alpha = kInt8Maxf / a_quant;
+  float b_alpha = kInt8Maxf / b_quant;
 
-  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / 127.0F;
+  float bias_unquant_multiplier = (-1.0F * (a_alpha * b_alpha)) / kInt8Maxf;
   auto prepare_bias_callback =
       gemmology::callbacks::UnquantizeAndAddBiasAndWrite(
           bias_unquant_multiplier, bias.data<float>(),  //
@@ -760,19 +762,16 @@ Tensor dot<Provider::kGemmology>(Tensor& x, Tensor& W, float a_quant,
 }
 
 template <>
-void PrepareBTransposed<Provider::kGemmology>(const float* weights,
-                                              int8_t* prepared,
-                                              float quantization_multiplier,
-                                              size_t cols, size_t rows) {
+void prepare_weight_transposed<Provider::Gemmology>(
+    const float* weights, int8_t* prepared, float quantization_multiplier,
+    size_t cols, size_t rows) {
   gemmology::PrepareBTransposed(weights, prepared, quantization_multiplier,
                                 cols, rows);
 }
 
 template <>
-void PrepareBQuantizedTransposed<Provider::kGemmology>(const int8_t* input,
-                                                       int8_t* output,
-                                                       size_t rows,
-                                                       size_t cols) {
+void prepare_weight_quantized_transposed<Provider::Gemmology>(
+    const int8_t* input, int8_t* output, size_t rows, size_t cols) {
   gemmology::PrepareBQuantizedTransposed(input, output, rows, cols);
 }
 
@@ -803,19 +802,19 @@ Tensor dot(Tensor& x, Tensor& W, float a_quant, float b_quant,
   return dot<kAutoProvider>(x, W, a_quant, b_quant, name);
 }
 
-void PrepareBTransposed(const float* weights, int8_t* prepared,
-                        float quantization_multiplier, size_t cols,
-                        size_t rows) {
+void prepare_weight_transposed(const float* weights, int8_t* prepared,
+                               float quantization_multiplier, size_t cols,
+                               size_t rows) {
   using detail::kAutoProvider;
-  using detail::PrepareBTransposed;
-  PrepareBTransposed<kAutoProvider>(weights, prepared, quantization_multiplier,
-                                    cols, rows);
+  using detail::prepare_weight_transposed;
+  prepare_weight_transposed<kAutoProvider>(weights, prepared,
+                                           quantization_multiplier, cols, rows);
 }
-void PrepareBQuantizedTransposed(const int8_t* input, int8_t* output,
-                                 size_t rows, size_t cols) {
+void prepare_weight_quantized_transposed(const int8_t* input, int8_t* output,
+                                         size_t rows, size_t cols) {
   using detail::kAutoProvider;
-  using detail::PrepareBQuantizedTransposed;
-  PrepareBQuantizedTransposed<kAutoProvider>(input, output, rows, cols);
+  using detail::prepare_weight_quantized_transposed;
+  prepare_weight_quantized_transposed<kAutoProvider>(input, output, rows, cols);
 }
 
 }  // namespace slimt::qmm
diff --git a/slimt/QMM.hh b/slimt/QMM.hh
index a6f02262..ab8707e3 100644
--- a/slimt/QMM.hh
+++ b/slimt/QMM.hh
@@ -7,13 +7,15 @@
 
 namespace slimt::qmm {
 
+constexpr float kInt8Maxf = 127.0F;
+
 namespace detail {
 
 enum class Provider {
-  kNone,      //
-  kIntgemm,   //
-  kRuy,       //
-  kGemmology  //
+  None,      //
+  Intgemm,   //
+  Ruy,       //
+  Gemmology  //
 };
 
 template <enum Provider>
@@ -30,23 +32,23 @@ Tensor dot(Tensor& x, Tensor& W, float a_quant, float b_quant,
            const std::string& name = "");
 
 template <enum Provider>
-void PrepareBTransposed(const float* weights, int8_t* prepared,
-                        float quantization_multiplier, size_t cols,
-                        size_t rows);
+void prepare_weight_transposed(const float* weights, int8_t* prepared,
+                               float quantization_multiplier, size_t cols,
+                               size_t rows);
 template <enum Provider>
-void PrepareBQuantizedTransposed(const int8_t* input, int8_t* output,
-                                 size_t rows, size_t cols);
+void prepare_weight_quantized_transposed(const int8_t* input, int8_t* output,
+                                         size_t rows, size_t cols);
 
 #ifdef SLIMT_HAS_INTGEMM
-constexpr Provider kAutoProvider = Provider::kIntgemm;
+constexpr Provider kAutoProvider = Provider::Intgemm;
 #endif
 
 #ifdef SLIMT_HAS_RUY
-constexpr Provider kAutoProvider = Provider::kRuy;
+constexpr Provider kAutoProvider = Provider::Ruy;
 #endif
 
 #ifdef SLIMT_HAS_GEMMOLOGY
-constexpr Provider kAutoProvider = Provider::kGemmology;
+constexpr Provider kAutoProvider = Provider::Gemmology;
 #endif
 }  // namespace detail
 
@@ -60,10 +62,10 @@ Tensor affine_with_select(Tensor& x, Tensor& W, Tensor& b, float a_quant,
 Tensor dot(Tensor& x, Tensor& W, float a_quant, float b_quant,
            const std::string& name = "");
 
-void PrepareBTransposed(const float* weights, int8_t* prepared,
-                        float quantization_multiplier, size_t cols,
-                        size_t rows);
-void PrepareBQuantizedTransposed(const int8_t* input, int8_t* output,
-                                 size_t rows, size_t cols);
+void prepare_weight_transposed(const float* weights, int8_t* prepared,
+                               float quantization_multiplier, size_t cols,
+                               size_t rows);
+void prepare_weight_quantized_transposed(const int8_t* input, int8_t* output,
+                                         size_t rows, size_t cols);
 
 }  // namespace slimt::qmm
diff --git a/slimt/Request.cc b/slimt/Request.cc
new file mode 100644
index 00000000..0053e064
--- /dev/null
+++ b/slimt/Request.cc
@@ -0,0 +1,152 @@
+#include "slimt/Request.hh"
+
+#include <string>
+
+#include "slimt/Annotation.hh"
+#include "slimt/Cache.hh"
+#include "slimt/Types.hh"
+#include "slimt/Utils.hh"
+
+namespace slimt {
+
+size_t cache_key(size_t model_id, const Words &words) {
+  auto seed = model_id;
+  for (size_t word : words) {
+    hash_combine<size_t>(seed, word);
+  }
+  return seed;
+}
+
+namespace rd {
+
+// -----------------------------------------------------------------
+Request::Request(size_t Id, size_t model_id, Segments &&segments,
+                 ResponseBuilder &&responseBuilder,
+                 std::optional<TranslationCache> &cache)
+    : id_(Id),
+      model_id_(model_id),
+      segments_(std::move(segments)),
+      responseBuilder_(std::move(responseBuilder)),
+      cache_(cache) {
+  counter_ = segments_.size();
+  histories_.resize(segments_.size(), nullptr);
+
+  // 1. If there are no segments_, we are never able to trigger the
+  // responseBuilder calls from a different thread. This happens when the use
+  // provides empty input, or the unit and subword preprocessing deems no
+  // translatable units present. However, in this case we want an empty valid
+  // response. There's no need to do any additional processing here.
+  if (segments_.empty()) {
+    responseBuilder_(std::move(histories_));
+  } else {
+    counter_ = segments_.size();
+    histories_.resize(segments_.size());
+
+    if (cache_) {
+      // Iterate through segments, see if any can be prefilled from cache. If
+      // prefilled, mark the particular segments as complete (non-empty
+      // ProcessedSegmentRef). Also update accounting used elsewhere
+      // (counter_) to reflect one less segment to translate.
+      for (size_t idx = 0; idx < segments_.size(); idx++) {
+        size_t key = cache_key(model_id_, segment(idx));
+        auto [found, history] = cache_->find(key);
+        if (found) {
+          histories_[idx] = history;
+          --counter_;
+        }
+      }
+      // 2. Also, if cache somehow manages to decrease all counter prefilling
+      // histories, then we'd have to trigger ResponseBuilder as well. No
+      // segments go into batching and therefore no complete triggers.
+      if (counter_.load() == 0) {
+        responseBuilder_(std::move(histories_));
+      }
+    }
+  }
+}
+
+size_t Request::segment_count() const { return segments_.size(); }
+
+size_t Request::word_count(size_t index) const {
+  return (segments_[index].size());
+}
+
+Segment Request::segment(size_t index) const { return segments_[index]; }
+
+void Request::complete(size_t index, History history) {
+  // Concurrently called by multiple workers as a history from translation is
+  // ready. The container storing histories is set with the value obtained.
+
+  // Fill in placeholder from History obtained by freshly translating. Since
+  // this was a cache-miss to have got through, update cache if available to
+  // store the result.
+  histories_[index] = std::move(history);
+  if (cache_) {
+    size_t key = cache_key(model_id_, segment(index));
+    cache_->store(key, histories_[index]);
+  }
+
+  // In case this is last request in, completeRequest is called, which sets the
+  // value of the promise.
+  if (--counter_ == 0) {
+    responseBuilder_(std::move(histories_));
+  }
+}
+
+// ------------------------------------------------------------------
+
+SegmentRef::SegmentRef(size_t index, Ptr<Request> request)
+    : index_(index), request_(std::move(request)) {}
+
+size_t SegmentRef::size() const { return (request_->word_count(index_)); }
+
+void SegmentRef::complete(History history) {
+  // Relays complete into request's complete, using index
+  // information.
+  request_->complete(index_, std::move(history));
+}
+
+Segment SegmentRef::get() const { return request_->segment(index_); }
+
+bool operator<(const Request &a, const Request &b) {
+  // Among Requests, only sequence id is used for obtaining priority.
+  return a.id_ < b.id_;
+}
+
+bool operator<(const SegmentRef &a, const SegmentRef &b) {
+  // Operator overload for usage in priority-queue / set.
+  if (a.request_ == b.request_) {
+    return a.index_ < b.index_;
+  }
+  return a.request_ < b.request_;
+}
+
+// ----------------------------------------------------------------------
+
+void Batch::log() {
+  (void)token_count_;
+  LOG(info, "Batch(tokens={}, max-length={}, segment_refs_={})", token_count_,
+      max_length_, segment_refs_.size());
+}
+
+void Batch::add(const SegmentRef &segment_ref) {
+  segment_refs_.push_back(segment_ref);
+  token_count_ += segment_ref.size();
+  max_length_ = std::max(max_length_, static_cast<size_t>(segment_ref.size()));
+}
+
+void Batch::complete(const Histories &histories) {
+  for (size_t i = 0; i < segment_refs_.size(); i++) {
+    segment_refs_[i].complete(histories[i]);
+  }
+}
+
+void Batch::clear() {
+  segment_refs_.clear();
+  token_count_ = 0;
+  max_length_ = 0;
+}
+
+}  // namespace rd
+
+}  // namespace slimt
diff --git a/slimt/Request.hh b/slimt/Request.hh
new file mode 100644
index 00000000..e1f37d86
--- /dev/null
+++ b/slimt/Request.hh
@@ -0,0 +1,167 @@
+#pragma once
+
+#include <cassert>
+#include <future>
+#include <optional>
+#include <vector>
+
+#include "slimt/Annotation.hh"
+#include "slimt/ResponseBuilder.hh"
+#include "slimt/Types.hh"
+
+namespace slimt::rd {
+
+using TranslationCache = slimt::TranslationCache;
+
+/// A Request is an internal representation used to represent a request after
+/// processed by TextProcessor into segments constituted by marian::Words.
+///
+/// The batching mechanism (BatchingPool) draws from multiple Requests and
+/// compiles segments into a batch. When a batch completes translation (at
+/// BatchTranslator, intended in a different thread), backward propogation
+/// happens through:
+///
+/// ```cpp
+///   Batch::complete(...)
+///       -> SegmentRef::complete(..)
+///          -> Request::complete(...)
+/// ```
+///
+/// When all segments in a Request are completed, responseBuilder is
+/// triggered with the compiled Histories, to construct the Response
+/// corresponding to the Request and set value of the promise which triggers
+/// the future at client.
+class Request {
+ public:
+  /// Constructs an internal representation of the Request identified by Id,
+  /// processed Segments and accepts a callback (ResponseBuilder) which builds
+  /// the Response upon completion of the Request.
+  ///
+  ///
+  /// @param [in] Id: Identifier assigned to Request by Service.
+  /// @param [in] model: Model for identifying a unique translation
+  /// segment key (model, words in a segment) for cache.
+  /// @param [in] segments: Each segment is a segment to be translated.
+  /// @param [in] responseBuilder: Callback function (of ResponseBuilder type)
+  /// to be triggered upon the completion of translation of all segments in a
+  /// Request.
+  /// @param [in] cache: Cache supplied externally to attempt to fetch
+  /// translations or store them after completion for reuse later.
+  Request(size_t Id, size_t model_id, Segments &&segments,
+          ResponseBuilder &&responseBuilder,
+          std::optional<TranslationCache> &cache);
+
+  /// Obtain the count of tokens in the segment correponding to index. Used to
+  /// insert segment from multiple requests into the corresponding size
+  /// bucket.
+  size_t word_count(size_t index) const;
+
+  /// Obtain number of segments in a request.
+  size_t segment_count() const;
+
+  /// Obtains segment corresponding to index  to create a batch of segments
+  /// among several requests.
+  Segment segment(size_t index) const;
+
+  /// For notions of priority among requests, used to enable std::set in
+  /// BatchingPool.
+  friend bool operator<(const Request &a, const Request &b);
+
+  /// Processes a history obtained after translating in a heterogenous batch
+  /// compiled from requests.
+  void complete(size_t index, History history);
+
+  bool is_prefilled_from_cache(size_t index) const {
+    return histories_[index] != nullptr;
+  }
+
+ private:
+  size_t id_;
+
+  /// Model associated with this request
+  size_t model_id_;
+
+  /// Multiple translation-workers can concurrently access the same Request.
+  /// The following atomic atomically operates on the variable holding
+  /// segments remaining to be translated.
+  std::atomic<int> counter_;
+
+  /// segments_ hold the segments processed into Words which generated from
+  /// input string.
+  Segments segments_;
+
+  /// histories_ is a buffer which eventually stores the translations of each
+  /// segment in the corresponding index.
+  Histories histories_;
+
+  /// Constructing Response requires the vocabs_ used to generate Request.
+  /// std::vector<Ptr<Vocab const>> *vocabs_;
+  ResponseBuilder responseBuilder_;
+
+  /// Cache used to hold segment translations. If nullopt, means no-caching.
+  std::optional<TranslationCache> &cache_;
+};
+
+/// A SegmentRef provides a view to a segment within a Request. Existence
+/// of this class allows the segments and associated information to be kept
+/// within Request, while batching mechanism (BatchingPool) compiles Batch
+/// from SegmentRef-s coming from different Requests.
+class SegmentRef {
+ public:
+  SegmentRef(size_t, Ptr<Request>);
+
+  /// Number of tokens in the segment this SegmentRef represents. Used to
+  /// order by length in batching.
+  size_t size() const;
+
+  /// Accessor to the segment represented by the SegmentRef.
+  Segment get() const;
+
+  /// Forwards history to Request to set history corresponding to this
+  /// SegmentRef.
+  void complete(History history);
+
+  friend bool operator<(const SegmentRef &a, const SegmentRef &b);
+
+ private:
+  size_t index_;
+  Ptr<Request> request_;
+};
+
+using SegmentRefs = std::vector<SegmentRef>;
+
+// An empty batch is poison.
+class Batch {
+ public:
+  Batch() = default;
+  void clear();
+
+  size_t size() const { return segment_refs_.size(); }
+  bool empty() const { return segment_refs_.empty(); }
+  size_t max_length() const { return max_length_; }
+
+  void add(const SegmentRef &segment_ref);
+
+  // Accessors to read from a Batch. For use in BatchTranslator (consumer on a
+  // PCQueue holding batches).
+  //
+  // segment_refs() are used to access segment_refs to construct marian internal
+  // batch.
+  const SegmentRefs &segment_refs() { return segment_refs_; }
+
+  // On obtaining Histories after translating a batch, complete can be
+  // called with Histories , which forwards the call to Request through
+  // SegmentRef and triggers completion, by setting the promised value to
+  // the future given to client.
+  void complete(const Histories &histories);
+
+  // Convenience function to log batch-statistics. size, max-length.
+  void log();
+
+ private:
+  SegmentRefs segment_refs_;
+  size_t token_count_ = 0;
+  size_t max_length_ = 0;
+};
+
+}  // namespace slimt::rd
diff --git a/slimt/Response.cc b/slimt/Response.cc
new file mode 100644
index 00000000..28beedcf
--- /dev/null
+++ b/slimt/Response.cc
@@ -0,0 +1,184 @@
+#include "slimt/Response.hh"
+
+#include "slimt/Annotation.hh"
+#include "slimt/Types.hh"
+
+namespace slimt {
+
+// We're marginalizing q out of p(s | q) x p( q | t). However, we have different
+// representations of q on source side to intermediate - p(s_i | q_j) and
+// intermediate to target side - p(q'_j' | t_k).
+//
+// The matrix p(q'_j' | t_k) is rewritten into p(q_j | t_k) by means of
+// spreading the probability in the former over bytes and collecting it at the
+// ranges specified by latter, using a two pointer accumulation strategy.
+Alignment transfer_through_characters(
+    const std::vector<Range> &source_side_pivots,
+    const std::vector<Range> &target_side_pivots,
+    const Alignment &pivot_given_targets) {
+  // Initialize an empty alignment matrix.
+  Alignment remapped(pivot_given_targets.size(),
+                     std::vector<float>(source_side_pivots.size(), 0.0F));
+
+  size_t sq;
+  size_t qt;
+  for (sq = 0, qt = 0; sq < source_side_pivots.size() && qt < target_side_pivots.size();
+       /*each branch inside increments either sq or qt or both, therefore the loop terminates */) {
+    const auto &source_side_pivot = source_side_pivots[sq];
+    const auto &target_side_pivot = target_side_pivots[qt];
+    if (source_side_pivot.begin == target_side_pivot.begin &&
+        source_side_pivot.end == target_side_pivot.end) {
+      for (size_t t = 0; t < pivot_given_targets.size(); t++) {
+        remapped[t][sq] += pivot_given_targets[t][qt];
+      }
+
+      // Perfect match, move pointer from both.
+      sq++, qt++;
+    } else {
+      // Do we have overlap?
+      size_t left = std::max(target_side_pivot.begin, source_side_pivot.begin);
+      size_t right = std::min(target_side_pivot.end, source_side_pivot.end);
+
+      assert(left < right);  // there should be overlap.
+
+      size_t character_count = right - left;
+      size_t probability_spread = target_side_pivot.size();
+      for (size_t t = 0; t < pivot_given_targets.size(); t++) {
+        remapped[t][sq] += character_count * pivot_given_targets[t][qt] /
+                           static_cast<float>(probability_spread);
+      }
+
+      // Which one is ahead? sq or qt or both end at same point?
+      if (source_side_pivot.end == target_side_pivot.end) {
+        sq++;
+        qt++;
+      } else if (source_side_pivot.end > target_side_pivot.end) {
+        qt++;
+      } else {  // source_side_pivot.end < target_side_pivot.end
+        sq++;
+      }
+    }
+  }
+
+  // The following is left in here for future debugging. Every token in source
+  // is expected to have been processed in the above pipeline. We advance the
+  // pivot-token index based on overlap with source-token. @jerinphilip is
+  // worried about EOS not existing when people try weird 4-model things in the
+  // future and would like to keep this check here.
+  assert(sq == source_side_pivots.size());
+
+  while (qt < target_side_pivots.size()) {
+    // There is a case of EOS not being predicted. In this case the two pointer
+    // algorithm will fail. The just author will redistribute the surplus among
+    // subjects.
+
+    // assert in DEBUG, that this is only EOS - occuring at the end and with
+    // zero-surface.
+    assert(qt == target_side_pivots.size() - 1 &&
+           target_side_pivots[qt].size() == 0);
+    for (size_t t = 0; t < pivot_given_targets.size(); t++) {
+      float gift = pivot_given_targets[t][qt] / source_side_pivots.size();
+      for (size_t sq = 0; sq < source_side_pivots.size(); sq++) {
+        remapped[t][sq] += gift;
+      }
+    }
+
+    qt++;
+  }
+
+#ifdef DEBUG
+  // The following sanity check ensures when DEBUG is enabled that we have
+  // successfully transferred all probabily mass available over pivot tokens
+  // given a target token in our original input to the new remapped
+  // representation.
+  //
+  // It's been discovered that floating point arithmetic before we get the
+  // Alignment matrix can have values such that the distribution does not sum
+  // upto 1.
+  const float EPS = 1e-6;
+  for (size_t t = 0; t < pivot_given_targets.size(); t++) {
+    float sum = 0.0f, expectedSum = 0.0f;
+    for (size_t qt = 0; qt < target_side_pivots.size(); qt++) {
+      expectedSum += pivot_given_targets[t][qt];
+    }
+    for (size_t sq = 0; sq < source_side_pivots.size(); sq++) {
+      sum += remapped[t][sq];
+    }
+    std::cerr << fmt::format(
+                     "Sum @ token {} = {} to be compared with expected {}.", t,
+                     sum, expectedSum)
+              << std::endl;
+    ABORT_IF(std::abs(sum - expectedSum) > EPS,
+             "Haven't accumulated probabilities, re-examine");
+  }
+#endif  // DEBUG
+
+  return remapped;
+}
+
+std::vector<Alignment> remap_alignments(const Response &first,
+                                        const Response &second) {
+  std::vector<Alignment> alignments;
+  for (size_t sentence_id = 0; sentence_id < first.source.sentence_count();
+       sentence_id++) {
+    const Alignment &source_given_pivots = first.alignments[sentence_id];
+    const Alignment &pivot_given_targets = second.alignments[sentence_id];
+
+    // TODO(any): Allow range iterators and change algorithm, directly tapping
+    // into AnnotatedText Extracts Ranges corresponding to a words
+    // constituting a sentence from an annotation.
+    auto extract_word_ranges = [](const AnnotatedText &annotatedText,
+                                  size_t sentence_id) -> std::vector<Range> {
+      size_t num_words = annotatedText.word_count(sentence_id);
+      std::vector<Range> output;
+
+      for (size_t i = 0; i < num_words; i++) {
+        output.push_back(annotatedText.word_as_range(sentence_id, i));
+      }
+      return output;
+    };
+
+    auto source_side_pivots = extract_word_ranges(first.target, sentence_id);
+    auto target_side_pivots = extract_word_ranges(second.source, sentence_id);
+
+    // Reintrepret probability p(q'_j' | t_k) as p(q_j | t_k)
+    Alignment remapped_pivot_given_targets = transfer_through_characters(
+        source_side_pivots, target_side_pivots, pivot_given_targets);
+
+    // Marginalize out q_j.
+    // p(s_i | t_k) = \sum_{j} p(s_i | q_j) x p(q_j | t_k)
+    size_t source_token_count = first.source.word_count(sentence_id);
+    size_t target_token_count = second.target.word_count(sentence_id);
+    Alignment output(target_token_count,
+                     std::vector<float>(source_token_count, 0.0F));
+    for (size_t idt = 0; idt < target_token_count; idt++) {
+      for (size_t idq = 0; idq < source_side_pivots.size(); idq++) {
+        for (size_t ids = 0; ids < source_token_count; ids++) {
+          // Matrices are of form p(s | t) = P[t][s], hence idq appears on the
+          // extremes.
+          output[idt][ids] += source_given_pivots[idq][ids] *
+                              remapped_pivot_given_targets[idt][idq];
+        }
+      }
+    }
+
+    alignments.push_back(output);
+  }
+  return alignments;
+}
+
+Response combine(Response &&first, Response &&second) {
+  Response combined;
+
+  // Compute alignment first using internal matrices and mappings.
+  if (!first.alignments.empty()) {
+    combined.alignments = remap_alignments(first, second);
+  }
+
+  combined.source = std::move(first.source);
+  combined.target = std::move(second.target);
+
+  return combined;
+}
+
+}  // namespace slimt
diff --git a/slimt/Response.hh b/slimt/Response.hh
new file mode 100644
index 00000000..e43fea77
--- /dev/null
+++ b/slimt/Response.hh
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include "slimt/Annotation.hh"
+#include "slimt/Types.hh"
+
+namespace slimt {
+
+/// Response holds AnnotatedText(s) of source-text and translated text,
+/// alignment information between source and target sub-words and sentences.
+///
+/// AnnotatedText provides an API to access markings of (sub)-word and
+/// sentences boundaries, which are required to interpret Quality and
+/// Alignment (s) at the moment.
+struct Response {
+  /// source text and annotations of (sub-)words and sentences.
+  AnnotatedText source;
+
+  /// translated text and annotations of (sub-)words and sentences.
+  AnnotatedText target;
+
+  /// Alignments between source and target. This is a collection of dense
+  /// matrices providing
+  ///    P[t][s] = p(source-token s  | target token t)
+  /// with an alignment matrix for each sentence.
+  std::vector<std::vector<std::vector<float>>> alignments;
+
+  /// Convenience function to obtain number of units translated. Same as
+  /// `.source.sentence_count()` and `.target.sentence_count().` The processing
+  /// of a text of into sentences are handled internally, and this information
+  /// can be used to iterate through meaningful units of translation for which
+  /// alignment and quality information are available.
+  size_t size() const { return source.sentence_count(); }
+};
+
+/// Options dictate how to construct a Response for an input string of
+/// text to be translated.
+struct Options {
+  bool alignment{false};  ///< Include alignments or not.
+  bool HTML{false};       ///< Remove HTML tags from text and insert in output.
+};
+
+std::vector<Alignment> remap_alignments(const Response &first,
+                                        const Response &second);
+
+// Combines two responses with first.target == second.source mapping alignments
+// etc accordingly. There are several constraints which are matched by only the
+// pivoting workflow in <>Service source, therefore this function is not for
+// external use and in a hidden namespace.
+Response combine(Response &&first, Response &&second);
+
+using Responses = std::vector<Response>;
+
+}  // namespace slimt
diff --git a/slimt/ResponseBuilder.cc b/slimt/ResponseBuilder.cc
new file mode 100644
index 00000000..2689d879
--- /dev/null
+++ b/slimt/ResponseBuilder.cc
@@ -0,0 +1,43 @@
+#include "slimt/ResponseBuilder.hh"
+
+#include "slimt/Response.hh"
+
+namespace slimt {
+
+void ResponseBuilder::operator()(Histories &&histories) {
+  // TODO(jerinphilip) load Options into options and turn build
+  // functions on or off.
+  // options_ is unused, but we can try something here.
+  SLIMT_ABORT_IF(source_.sentence_count() != histories.size(),
+                 "Mismatch in source and translated sentences");
+  Response response;
+
+  // Move source_ into response.
+  response.source = std::move(source_);
+  // Reserving length at least as much as source_ seems like a reasonable
+  // thing to do to avoid reallocations.
+  response.target.text.reserve(response.source.text.size());
+
+  for (size_t sentence_id = 0; sentence_id < histories.size(); sentence_id++) {
+    Words words = histories[sentence_id]->target;
+    std::string decoded;
+    auto views = vocabulary_.decode(words, decoded, /*ignore_eos=*/false);
+
+    // For each sentence, prepend the filler text between the corresponding
+    // source-sentence and the source-sentence before.
+    std::string_view pre = response.source.gap(sentence_id);
+    response.target.append_sentence(pre, views.begin(), views.end());
+
+    // If this is the last history to be decoded and translated-text
+    // constructed, append the text till the end, which could be spaces or
+    // empty.
+    if (sentence_id + 1 == histories.size()) {
+      response.target.append_ending_whitespace(
+          response.source.gap(sentence_id + 1));
+    }
+  }
+
+  promise_.set_value(std::move(response));
+}
+
+}  // namespace slimt
diff --git a/slimt/ResponseBuilder.hh b/slimt/ResponseBuilder.hh
new file mode 100644
index 00000000..00305e93
--- /dev/null
+++ b/slimt/ResponseBuilder.hh
@@ -0,0 +1,55 @@
+#pragma once
+
+#include <functional>
+#include <future>
+#include <optional>
+
+#include "slimt/Annotation.hh"
+#include "slimt/Macros.hh"
+#include "slimt/Response.hh"
+#include "slimt/Types.hh"
+#include "slimt/Vocabulary.hh"
+
+// For now we will work with this, to avoid complaints another structure is hard
+// to operate with.
+
+namespace slimt {
+
+/// ResponseBuilder is a callback functor. It is expected to be bound to a
+/// Request after giving it the context of options, vocabulary and promise to
+/// set. It constructs the Response and it's members based on options
+/// (quality=on|off, alignments=on|off, mappings=on|off, splitmode=sentence |
+/// paragraph).
+
+class ResponseBuilder {
+ public:
+  /// @param [in] options: Options, indicating what to include
+  /// or not in the response and any additional configurable parameters.
+  /// @param [in] vocabulary: marian vocab object (used in decoding)
+  /// @param [in] callback: callback with operates on the constructed Response.
+  /// @param [in] qualityEstimator: the QualityEstimator model that can be used
+  /// to provide translation quality probability.
+  ResponseBuilder(Options options, AnnotatedText &&source,
+                  const Vocabulary &vocabulary,
+                  std::promise<Response> &&promise)
+      : options_(options),
+        vocabulary_(vocabulary),
+        source_(std::move(source)),
+        promise_(std::move(promise)) {}
+
+  /// Constructs and sets the promise of a Response object from obtained
+  /// histories after translating.
+  /// @param [in] histories: Histories obtained after translating the Request
+  /// from which this functor is called.
+  void operator()(Histories &&histories);
+
+ private:
+  // Data members are context/curried args for the functor.
+  Options options_;
+  const Vocabulary &vocabulary_;  // vocabulary are required for decoding
+                                  // and any source validation checks.
+  AnnotatedText source_;
+
+  std::promise<Response> promise_;
+};
+}  // namespace slimt
diff --git a/slimt/Shortlist.cc b/slimt/Shortlist.cc
index 2ce52251..47b4b590 100644
--- a/slimt/Shortlist.cc
+++ b/slimt/Shortlist.cc
@@ -141,8 +141,8 @@ Shortlist ShortlistGenerator::generate(const Words& words) const {
   // Ensure that the generated vocabulary items from a shortlist are a
   // multiple-of-eight This is necessary until intgemm supports
   // non-multiple-of-eight matrices.
-  for (size_t i = frequent_; i < target_size && target_table_ones % 8 != 0;
-       i++) {
+  for (size_t i = frequent_;
+       i < target_size && target_table_ones % kVExtAlignment != 0; i++) {
     if (!target_table[i]) {
       target_table[i] = true;
       target_table_ones++;
diff --git a/slimt/Shortlist.hh b/slimt/Shortlist.hh
index 673f9d2d..9688d760 100644
--- a/slimt/Shortlist.hh
+++ b/slimt/Shortlist.hh
@@ -37,6 +37,7 @@ class ShortlistGenerator {
   static constexpr uint64_t kMagic = 0xF11A48D5013417F5;
   static constexpr uint64_t kFrequent = 100;
   static constexpr uint64_t kBest = 100;
+  static constexpr size_t kVExtAlignment = 8;
 
   // construct directly from buffer
   ShortlistGenerator(
diff --git a/slimt/Splitter.cc b/slimt/Splitter.cc
new file mode 100644
index 00000000..7130ee59
--- /dev/null
+++ b/slimt/Splitter.cc
@@ -0,0 +1,198 @@
+#include "slimt/Splitter.hh"
+
+#include <string>
+#include <vector>
+
+#include "slimt/Annotation.hh"
+#include "slimt/Macros.hh"
+
+namespace slimt {
+
+namespace {
+ug::ssplit::SentenceStream::splitmode string2splitmode(const std::string &m) {
+  using splitmode = ug::ssplit::SentenceStream::splitmode;
+  if (m == "sentence") {
+    return splitmode::one_sentence_per_line;
+  }
+  if (m == "paragraph") {
+    return splitmode::one_paragraph_per_line;
+  }
+  if (m == "wrapped_text") {
+    return splitmode::wrapped_text;
+  }
+  SLIMT_ABORT(
+      "Unknown ssplitmode {}, Please choose one of "
+      "{sentence,paragraph,wrapped_text}");
+}
+
+ug::ssplit::SentenceSplitter load_splitter(const std::string &prefix_path) {
+  // Temporarily supports empty, will be removed when mozilla passes
+  // prefix_path
+  ug::ssplit::SentenceSplitter splitter;
+  if (!prefix_path.empty()) {
+    LOG(info, "Loading protected prefixes for sentence splitting from {}",
+        prefix_path);
+    splitter.load(prefix_path);
+  } else {
+    LOG(warn,
+        "Missing list of protected prefixes for sentence splitting. "
+        "Set with --ssplit-prefix-file.");
+  }
+  return splitter;
+}
+
+ug::ssplit::SentenceSplitter load_splitter(const Aligned &memory) {
+  // Temporarily supports empty, will be removed when mozilla passes memory
+  ug::ssplit::SentenceSplitter splitter;
+  if (!memory.empty()) {
+    std::string_view serialized(memory.begin(), memory.size());
+    splitter.loadFromSerialized(serialized);
+  }
+  return splitter;
+}
+
+}  // namespace
+
+Segment TextProcessor::tokenize(
+    const std::string_view &segment,
+    std::vector<std::string_view> &word_ranges) const {
+  auto [words, views] = vocabulary_.encode(segment, /*add_eos=*/false);
+  word_ranges.reserve(word_ranges.size() +
+                      distance(views.begin(), views.end()));
+  word_ranges.insert(word_ranges.end(), views.begin(), views.end());
+  return words;
+}
+
+TextProcessor::TextProcessor(size_t wrap_length, const std::string &mode,
+                             const Vocabulary &vocabulary,
+                             const std::string &prefix_path)
+    : wrap_length_(wrap_length),
+      ssplit_mode_(string2splitmode(mode)),
+      vocabulary_(vocabulary),
+      ssplit_(load_splitter(prefix_path)) {}
+
+TextProcessor::TextProcessor(size_t wrap_length, const std::string &mode,
+                             const Vocabulary &vocabulary,
+                             const Aligned &memory)
+    : wrap_length_(wrap_length),
+      ssplit_mode_(string2splitmode(mode)),
+      vocabulary_(vocabulary) {
+  // This is not the best of the solutions at the moment, but is consistent
+  // with // what happens among other structures like model, vocabulary or
+  // shortlist.  First, we check if the bytearray is empty. If not, we load
+  // from ByteArray.  In case empty, the string based loader which reads from
+  // file is called.  However, ssplit allows for not supplying
+  // ssplit-prefix-file where-in the purely regular expression based splitter
+  // is activated.
+  //
+  // For now, we allow not supplying an ssplit-prefix-file.
+
+  SLIMT_ABORT_IF(memory.empty(),
+                 "ssplit: Empty blob supplied for initialization.");
+  ssplit_ = load_splitter(memory);
+}
+
+std::tuple<AnnotatedText, Segments> TextProcessor::process(
+    std::string &&input) const {
+  AnnotatedText source(std::move(input));
+  Segments segments;
+  std::string_view input_converted(source.text.data(), source.text.size());
+  auto sentence_stream =
+      ug::ssplit::SentenceStream(input_converted, ssplit_, ssplit_mode_);
+
+  std::string_view sentence_string_piece;
+
+  while (sentence_stream >> sentence_string_piece) {
+    std::string_view sentence(sentence_string_piece.data(),
+                              sentence_string_piece.size());
+
+    std::vector<std::string_view> word_ranges;
+    Segment segment = tokenize(sentence, word_ranges);
+
+    // There are some cases where SentencePiece or vocab returns no words
+    // after normalization. 0 prevents any empty entries from being added.
+    if (!segment.empty()) {
+      // Wrap segment into sentences of at most wrap_length_ tokens and
+      // tell source about them.
+      wrap(segment, word_ranges, segments, source);
+    }
+  }
+  return {std::move(source), std::move(segments)};
+}
+
+void TextProcessor::wrap(Segment &segment,
+                         std::vector<std::string_view> &word_ranges,
+                         Segments &segments, AnnotatedText &source) const {
+  // There's an EOS token added to the words, manually.
+  // SentencePiece/marian-vocab is set to not append EOS. Marian requires EOS to
+  // be at the end as a marker to start translating. So while we're supplied
+  // wrap_length_ from outside, we need to ensure there's space for EOS in
+  // each wrapped segment.
+  Word eos_id = vocabulary_.eos_id();
+  size_t wrap_step_size = wrap_length_ - 1;
+
+  for (size_t offset = 0; offset < segment.size(); offset += wrap_step_size) {
+    auto start = segment.begin() + offset;
+
+    // Restrict the range within bounds.
+    size_t left = segment.size() - offset;
+    size_t diff = std::min(wrap_step_size, left);
+
+    segments.emplace_back(start, start + diff);
+    segments.back().push_back(eos_id);
+
+    auto astart = word_ranges.begin() + offset;
+
+    // Construct a part vector of std::string_view representing wrapped segment,
+    // use the last std::string_view to create an EOS std::string_view manually.
+    std::vector<std::string_view> partial_word_ranges(astart, astart + diff);
+    std::string_view &last = partial_word_ranges.back();
+    const char *end = last.data() + last.size();
+    partial_word_ranges.emplace_back(end, 0);
+    // diff > 0
+    source.record_existing_sentence(partial_word_ranges.begin(),
+                                    partial_word_ranges.end(), astart->data());
+  }
+}
+
+void TextProcessor::processFromAnnotation(AnnotatedText &source,
+                                          Segments &segments) const {
+  std::string text = source.text;
+  AnnotatedText replacement(std::move(text));
+
+  for (size_t s = 0; s < source.sentence_count(); s++) {
+    // This is our sentence_stream
+    Range sentence_range = source.sentence_as_range(s);
+
+    // Fool tokenization using Ranges into looking at replacement. They're
+    // same, so okay.
+    std::string_view sentence{&replacement.text[sentence_range.begin],
+                              sentence_range.size()};
+
+    std::vector<std::string_view> word_ranges;
+    Segment segment = tokenize(sentence, word_ranges);
+
+    // Manually add EoS
+    Word eos_id = vocabulary_.eos_id();
+    segment.push_back(eos_id);
+
+    if (!word_ranges.empty()) {
+      std::string_view &last =
+          word_ranges.back();  // this is a possible segfault if
+                               // word_ranges is empty. So guard.
+      const char *end = last.data() + last.size();
+      word_ranges.emplace_back(end, 0);
+    } else {
+      const char *end = sentence.data() + sentence.size();
+      word_ranges.emplace_back(end, 0);
+    }
+
+    segments.push_back(std::move(segment));
+    replacement.record_existing_sentence(word_ranges.begin(), word_ranges.end(),
+                                         word_ranges.begin()->data());
+  }
+
+  source = replacement;
+}
+
+}  // namespace slimt
diff --git a/slimt/Splitter.hh b/slimt/Splitter.hh
new file mode 100644
index 00000000..e625d2fe
--- /dev/null
+++ b/slimt/Splitter.hh
@@ -0,0 +1,80 @@
+#pragma once
+
+#include <vector>
+
+#include "slimt/Aligned.hh"
+#include "slimt/Annotation.hh"
+#include "slimt/Types.hh"
+#include "slimt/Vocabulary.hh"
+#include "ssplit.h"
+
+namespace slimt {
+
+class TextProcessor {
+  /// TextProcessor handles loading the sentencepiece vocabulary and also
+  /// contains an instance of sentence-splitter based on ssplit.
+  ///
+  /// Used in Service to convert an incoming blob of text to a vector of
+  /// sentences (vector of words). In addition, the Ranges of the
+  /// source-tokens in unnormalized text are provided as string_views.
+ public:
+  // There are two ways to construct text-processor, different in a file-system
+  // based prefix file load and a memory based prefix file store. @jerinphilip
+  // is not doing magic inference inside to determine file-based or memory
+  // based on one being empty or not.
+
+  /// Construct TextProcessor from options, vocabs and prefix-file.
+  /// @param [in] options: expected to contain `max-length-break`,
+  /// `ssplit-mode`.
+  /// @param [in] vocabs: Vocabularies used to process text into sentences to
+  /// marian::Words and corresponding Range information in AnnotatedText.
+  /// @param [in] ssplit_prefix_file: Path to ssplit-prefix file compatible with
+  /// moses-tokenizer.
+  TextProcessor(size_t wrap_length, const std::string &mode,
+                const Vocabulary &vocabulary, const std::string &prefix_path);
+
+  /// Construct TextProcessor from options, vocabs and prefix-file supplied as a
+  /// bytearray. For other parameters, see the path based constructor. Note:
+  /// This falls back to string based loads if memory is null, this behaviour
+  /// will be deprecated in the future.
+  ///
+  /// @param [in] memory: ssplit-prefix-file contents in memory, passed as a
+  /// bytearray.
+  TextProcessor(size_t wrap_length, const std::string &mode,
+                const Vocabulary &vocabulary, const Aligned &memory);
+
+  /// Wrap into sentences of at most maxLengthBreak_ tokens and add to source.
+  /// @param [in] blob: Input blob, will be bound to source and annotations on
+  /// it stored.
+  /// @param [out] source: AnnotatedText instance holding input and annotations
+  /// of sentences and pieces
+  /// @param [out] segments: marian::Word equivalents of the sentences processed
+  /// and stored in AnnotatedText for consumption of marian translation
+  /// pipeline.
+
+  std::tuple<AnnotatedText, Segments> process(std::string &&input) const;
+
+  void processFromAnnotation(AnnotatedText &source, Segments &segments) const;
+
+ private:
+  /// Tokenizes an input string, returns Words corresponding. Loads the
+  /// corresponding byte-ranges into word_ranges.
+  Segment tokenize(const std::string_view &segment,
+                   std::vector<std::string_view> &word_ranges) const;
+
+  /// Wrap into sentences of at most maxLengthBreak_ tokens and add to source.
+  void wrap(Segment &segment, std::vector<std::string_view> &word_ranges,
+            Segments &segments, AnnotatedText &source) const;
+
+  size_t wrap_length_;  ///< Parameter used to wrap sentences to a maximum
+                        ///< number of tokens
+  /// Mode of splitting, can be line ('\n') based, paragraph based, also
+  /// supports a wrapped mode.
+  ug::ssplit::SentenceStream::splitmode ssplit_mode_;
+
+  const Vocabulary &vocabulary_;  ///< Vocabularies used to tokenize a sentence
+  /// SentenceSplitter compatible with moses sentence-splitter
+  ug::ssplit::SentenceSplitter ssplit_;
+};
+
+}  // namespace slimt
diff --git a/slimt/Tensor.hh b/slimt/Tensor.hh
index e968d14e..28fbe5af 100644
--- a/slimt/Tensor.hh
+++ b/slimt/Tensor.hh
@@ -4,6 +4,7 @@
 #include <vector>
 
 #include "slimt/Aligned.hh"
+#include "slimt/Types.hh"
 
 namespace slimt {
 
@@ -83,7 +84,8 @@ class Tensor {
   Tensor() = default;
   void load(View view, Type type, Shape shape, std::string name);
 
-  static Aligned allocate(Type type, const Shape &shape, size_t alignment = 64);
+  static Aligned allocate(Type type, const Shape &shape,
+                          size_t alignment = kAlignWidth);
 
   template <class Scalar>
   Scalar *data() {
diff --git a/slimt/TensorOps.cc b/slimt/TensorOps.cc
index a3f9c034..8b5e3713 100644
--- a/slimt/TensorOps.cc
+++ b/slimt/TensorOps.cc
@@ -38,11 +38,11 @@ Tensor index_select(Tensor& x, Tensor& indices,
 void modify_mask_for_pad_tokens_in_attention(float* mask, size_t size) {
   // Adopted from:
   // https://github.com/browsermt/marian-dev/blob/14c9d9b0e732f42674e41ee138571d5a7bf7ad94/src/models/transformer.h#L132
-  float f16_lowest = std::numeric_limits<float>::lowest() / 2.0F;
-  float minus_inf = std::max(f16_lowest, -99999999.0F);
+  float f16_lowest = std::numeric_limits<float>::lowest() / 2.0F;  // NOLINT
+  float minus_inf = std::max(f16_lowest, -99999999.0F);            // NOLINT
   for (size_t i = 0; i < size; i++) {
-    float& x = mask[i];
-    x = (1.0F - x) * minus_inf;
+    float* x = mask + i;
+    *x = (1.0F - *x) * minus_inf;
   }
 }
 
@@ -239,7 +239,8 @@ void sinusoidal_signal(int start, size_t sequence_length, size_t embed_dim,
   // Imported from:
   // https://github.com/jerinphilip/marian/blob/8c4170fa08c46df1cf4c987e493b7a3772c380b3/src/graph/node_initializers.cpp#L216
   float num_timescales = static_cast<float>(embed_dim) / 2;
-  float log_timescale_increment = std::log(10000.0F) / (num_timescales - 1.0F);
+  float log_timescale_increment =
+      std::log(10000.0F) / (num_timescales - 1.0F);  // NOLINT
 
   for (size_t p = start; p < sequence_length + start; ++p) {
     for (int i = 0; i < num_timescales; ++i) {
@@ -272,13 +273,13 @@ void add_positional_embedding(const float* word_embedding,
 
 void softmax(float* logits, size_t batch_size, size_t num_classes, float* out) {
 #ifdef VEXT_W8_AVAILABLE
-  if (num_classes % 8 == 0) {
+  if (num_classes % VDatum<VExt::w8>::kWidth == 0) {
     vext::softmax<VExt::w8>(logits, batch_size, num_classes, out);
     return;
   }
 #endif
 #ifdef VEXT_W4_AVAILABLE
-  if (num_classes % 4 == 0) {
+  if (num_classes % VDatum<VExt::w4>::kWidth == 0) {
     vext::softmax<VExt::w4>(logits, batch_size, num_classes, out);
     return;
   }
diff --git a/slimt/Types.hh b/slimt/Types.hh
index 5e6cdd28..8be5561a 100644
--- a/slimt/Types.hh
+++ b/slimt/Types.hh
@@ -6,17 +6,19 @@
 #include <string_view>
 #include <vector>
 
+#include "slimt/Cache.hh"
+
 namespace slimt {
 
-/// ByteRange stores indices for half-interval [begin, end) in a string. Can be
+/// Range stores indices for half-interval [begin, end) in a string. Can be
 /// used to represent a sentence, word.
-struct ByteRange {
+struct Range {
   size_t begin;
   size_t end;
   size_t size() const { return end - begin; }
 };
 
-inline bool operator==(ByteRange &a, ByteRange b) {
+inline bool operator==(Range &a, Range b) {
   return a.begin == b.begin && a.end == b.end;
 }
 
@@ -28,16 +30,19 @@ using Segment = Words;
 using Segments = std::vector<Segment>;
 using Sentences = std::vector<Words>;
 
-struct Hypothesis {
-  std::string source;
-  std::string target;
-  std::string alignments;
-};
-
 template <class T>
 using Ptr = std::shared_ptr<T>;
 
+using Alignment = std::vector<std::vector<float>>;
+using Alignments = std::vector<Alignment>;
+
+struct Hypothesis {
+  Segment target;
+  Alignment alignment;
+};
+
 using History = Ptr<Hypothesis>;
 using Histories = std::vector<History>;
+using TranslationCache = AtomicCache<size_t, History>;
 
 }  // namespace slimt
diff --git a/slimt/Vocabulary.cc b/slimt/Vocabulary.cc
index 71954187..e73e2289 100644
--- a/slimt/Vocabulary.cc
+++ b/slimt/Vocabulary.cc
@@ -54,11 +54,11 @@ std::tuple<Words, Views> Vocabulary::encode(const std::string_view &line,
     std_views.emplace_back(view.data(), view.size());
   }
 
-  return {std::move(words), std::move(std_views)};
+  return std::make_tuple(words, std_views);
 }
 
-std::tuple<std::string, Views> Vocabulary::decode(const Words &words,
-                                                  bool ignore_eos) const {
+Views Vocabulary::decode(const Words &words, std::string &decoded,
+                         bool ignore_eos) const {
   sentencepiece::SentencePieceText sentencepiece_text;
   std::vector<std::string_view> views;
 
@@ -72,11 +72,10 @@ std::tuple<std::string, Views> Vocabulary::decode(const Words &words,
   processor_.Decode(sentence, &sentencepiece_text);
 
   // Creates copy of string.
-  auto decoded = sentencepiece_text.text();
-  std::string_view decoded_view(decoded);
+  decoded = std::move(sentencepiece_text.text());
   for (const auto &piece : sentencepiece_text.pieces()) {
     size_t size = piece.end() - piece.begin();
-    std::string_view view = decoded_view.substr(piece.begin(), size);
+    std::string_view view(decoded.data() + piece.begin(), size);
     views.push_back(view);
   }
 
@@ -84,7 +83,7 @@ std::tuple<std::string, Views> Vocabulary::decode(const Words &words,
     views.pop_back();
   }
 
-  return {std::move(decoded), std::move(views)};
+  return views;
 }
 
 }  // namespace slimt
diff --git a/slimt/Vocabulary.hh b/slimt/Vocabulary.hh
index cc3efe15..abbfecf2 100644
--- a/slimt/Vocabulary.hh
+++ b/slimt/Vocabulary.hh
@@ -20,8 +20,8 @@ class Vocabulary {
   Vocabulary(void *data, size_t size);
   std::tuple<Words, Views> encode(const std::string_view &line,
                                   bool add_eos = false) const;
-  std::tuple<std::string, Views> decode(const Words &words,
-                                        bool ignore_eos = true) const;
+  Views decode(const Words &words, std::string &decoded,
+               bool ignore_eos = true) const;
 
   Word pad_id() const { return std::max(0, processor_.pad_id()); }
   Word eos_id() const { return processor_.eos_id(); }
diff --git a/slimt/slimt.hh b/slimt/slimt.hh
index 24b6450d..9f5b7f19 100644
--- a/slimt/slimt.hh
+++ b/slimt/slimt.hh
@@ -1,5 +1,6 @@
 #pragma once
 #include "slimt/Batch.hh"
+#include "slimt/Frontend.hh"
 #include "slimt/Io.hh"
 #include "slimt/Macros.hh"
 #include "slimt/Model.hh"


Test patch:
diff --git a/app/test.cc b/app/test.cc
index eff9db7d..0618bf27 100644
--- a/app/test.cc
+++ b/app/test.cc
@@ -1,3 +1,4 @@
+// NOLINTBEGIN
 #include <cmath>
 #include <cstdint>
 #include <iostream>
@@ -637,7 +638,7 @@ namespace slimt {
 template <class Field>
 struct Record {
   Field model;
-  Field vocab;
+  Field vocabulary;
   Field shortlist;
 };
 
@@ -655,57 +656,28 @@ void integration() {
 
   Record<std::string> path{
       .model = prefix_browsermt("model.intgemm.alphas.bin"),  //
-      .vocab = prefix_browsermt("vocab.deen.spm"),            //
+      .vocabulary = prefix_browsermt("vocab.deen.spm"),       //
       .shortlist = prefix_browsermt("lex.s2t.bin")            //
   };
 
   Record<io::MmapFile> mmap{
-      .model = io::MmapFile(path.model),          //
-      .vocab = io::MmapFile(path.vocab),          //
-      .shortlist = io::MmapFile(path.shortlist),  //
+      .model = io::MmapFile(path.model),            //
+      .vocabulary = io::MmapFile(path.vocabulary),  //
+      .shortlist = io::MmapFile(path.shortlist),    //
   };
 
-  // Tokenize into numeric-ids using sentencepiece.
-  Vocabulary vocab(mmap.vocab.data(), mmap.vocab.size());
-
-  uint64_t batch_size = 2;
-  uint64_t sequence_length = 4;
-  Batch batch(batch_size, sequence_length, vocab.pad_id());
-  std::vector<std::string> lines = {"1 2", "1 2 3"};
-  for (const auto &line : lines) {
-    bool add_eos = true;
-    auto [words, views] = vocab.encode(line, add_eos);
-    // std::cout << "Adding ";
-    for (size_t i = 0; i < views.size(); i++) {
-      // std::cout << "[" << views[i] << ": " << words[i] << "]";
-    }
-    // std::cout << "\n";
-    batch.add(words);
-  }
-
-  // Load model
-  auto items = io::loadItems(mmap.model.data());
-  ShortlistGenerator shortlist_generator(            //
-      mmap.shortlist.data(), mmap.shortlist.size(),  //
-      vocab, vocab                                   //
-  );
-
-  size_t encoder_layers = 6;
-  size_t decoder_layers = 2;
-  size_t ffn_depth = 2;
-
-  Model model(                        //
-      Tag::tiny11,                    //
-      vocab,                          //
-      std::move(items),               //
-      std::move(shortlist_generator)  //
-  );
+  Record<View> view{
+      .model = {mmap.model.data(), mmap.model.size()},                 //
+      .vocabulary = {mmap.vocabulary.data(), mmap.vocabulary.size()},  //
+      .shortlist = {mmap.shortlist.data(), mmap.shortlist.size()},     //
+  };
 
-  auto sentences = model.translate(batch);
-  for (auto &sentence : sentences) {
-    auto [result, views] = vocab.decode(sentence);
-    std::cout << result << "\n";
-  }
+  Config config;
+  Translator translator(config, view.model, view.shortlist, view.vocabulary);
+  std::string source = "1 2\n1 2 3\n";
+  slimt::Options opts;
+  Response response = translator.translate(source, opts);
+  fprintf(stdout, "%s\n", response.target.text.c_str());
 }
 
 void ShortlistGen() {
@@ -738,7 +710,8 @@ void ShortlistGen() {
   Shortlist shortlist = shortlist_generator.generate(words);
 
   const auto &likely_target_words = shortlist.words();
-  auto [decoded, dviews] = vocab.decode(likely_target_words);
+  std::string decoded;
+  auto dviews = vocab.decode(likely_target_words, decoded);
   for (size_t i = 0; i < likely_target_words.size(); i++) {
     std::cout << "[" << dviews[i] << ": " << likely_target_words[i] << "] ";
   }
@@ -808,3 +781,5 @@ int main(int argc, char **argv) {
   }
   return 0;
 }
+
+// NOLINTEND