Repository: deepset-ai/haystack

Issue:
Implementing data augmentation from TinyBERT
**Is your feature request related to a problem? Please describe.**
A basic version of model distillation was implemented with #1758. However, there is still room for improvement. The TinyBERT paper (https://arxiv.org/pdf/1909.10351.pdf) details an approach for finetuning an already pretrained small language model. 

**Describe the solution you'd like**
Adding the functionality to generate more data samples by using approach outlined in TinyBERT paper. This could be implemented as an additional DataSilo.

**Describe alternatives you've considered**
https://arxiv.org/pdf/1910.08381.pdf: Seems to depend too heavily on expensive retraining and seems to be too task specific.
https://arxiv.org/pdf/2002.10957.pdf, https://arxiv.org/pdf/1910.01108.pdf: Seem only to focus on pretraining

**Additional context**
This is the second of two issues for implementing finetuning as described in the TinyBERT paper. This issue focusses on data augmentation. The first issue focussed on the loss functions. 


Discussion:

Resolving patch:
diff --git a/haystack/utils/augment_squad.py b/haystack/utils/augment_squad.py
new file mode 100644
--- /dev/null
+++ b/haystack/utils/augment_squad.py
@@ -0,0 +1,236 @@
+"""
+Script to perform data augmentation on a SQuAD like dataset to increase training data.
+It follows the approach oultined in the TinyBERT paper: https://arxiv.org/pdf/1909.10351.pdf.
+It takes a SQuAD like dataset as input and writes the augmented dataset to a new file in the same format.
+There are no answer labels in the augmented dataset and every question is marked impossible.
+This isn't a problem for distillation. However, this means that the augmented dataset is not suitable for training
+a model on its own.
+Usage:
+    python augment_squad.py --squad_path <squad_path> --output_path <output_path> \
+        --multiplication_factor <multiplication_factor> --word_possibilities <word_possibilities> \
+        --replace_probability <replace_probability> --glove_path <glove_path> \
+        --batch_size <batch_size> --device <device> --tokenizer <tokenizer> --model <model>
+Arguments:
+    squad_path: Path to the input dataset. Must have the same structure as the official squad json.
+    output_path: Path to the output dataset.
+    multiplication_factor: Number of times to augment the dataset.
+    word_possibilities: Number of possible words to replace a word with.
+    replace_probability: Probability of replacing a word with a different word.
+    glove_path: Path to the GloVe vectors. If it does not exist, it will be downloaded.
+    batch_size: Batch size for MLM model.
+    device: Device to use for MLM model. Usually either "cpu:0" or "cuda:0".
+    tokenizer: Huggingface tokenizer identifier.
+    model: Huggingface MLM model identifier.
+"""
+
+
+import torch
+from torch.nn import functional as F
+from transformers import AutoModelForMaskedLM, AutoTokenizer, PreTrainedModel, PreTrainedTokenizerBase
+from copy import copy, deepcopy
+from pathlib import Path
+import requests
+from zipfile import ZipFile
+import numpy as np
+import random
+import argparse
+import json
+import logging
+from tqdm import tqdm
+from typing import Tuple, List
+
+logger = logging.getLogger(__name__)
+
+def load_glove(glove_path: Path = Path("glove.txt"), vocab_size: int = 100_000, device: str = "cpu:0") -> Tuple[dict, dict, torch.Tensor]:
+    """Loads the GloVe vectors and returns a mapping from words to their GloVe vector indices and the other way around."""
+
+    if not glove_path.exists(): # download and extract glove if necessary
+        logger.info("Provided glove file not found. Downloading it instead.")
+        glove_path.parent.mkdir(parents=True, exist_ok=True)
+        zip_path = glove_path.parent / (glove_path.name + ".zip")
+        request = requests.get("https://nlp.stanford.edu/data/glove.42B.300d.zip", allow_redirects=True)
+        with zip_path.open("wb") as downloaded_file:
+            downloaded_file.write(request.content)
+        with ZipFile(zip_path, "r") as zip_file:
+            glove_file = zip_file.namelist()[0]
+            with glove_path.open("wb") as g:
+                g.write(zip_file.read(glove_file))
+
+    word_id_mapping = {}
+    id_word_mapping = {}
+    vector_list = []
+    with open(glove_path, "r") as f:
+        for i, line in enumerate(f):
+            if i == vocab_size: # limit vocab size
+                break
+            split = line.split()
+            word_id_mapping[split[0]] = i
+            id_word_mapping[i] = split[0]
+            vector_list.append(torch.tensor([float(x) for x in split[1:]]))
+    vectors = torch.stack(vector_list)
+    with torch.no_grad():
+        vectors = vectors.to(device)
+        vectors = F.normalize(vectors, dim=1)
+    return word_id_mapping, id_word_mapping, vectors
+
+def tokenize_and_extract_words(text: str, tokenizer: PreTrainedTokenizerBase) -> Tuple[torch.Tensor, List[str], dict]:
+    # tokenizes text and returns, in addition to the tokens, indices and mapping of the words that were not split into subwords
+    # this is important as MLM is not used for subwords
+    words = tokenizer.basic_tokenizer.tokenize(text)
+
+    subwords = [tokenizer.wordpiece_tokenizer.tokenize(word) for word in words]
+
+    word_subword_mapping = {}
+
+    j = 0
+    for i, subwords_ in enumerate(subwords):
+        j += len(subwords_)
+        if j >= 510: # sequence length may not be longer than 512 (1 cls + 510 tokens + 1 sep)
+            break
+        if len(subwords_) == 1:
+                word_subword_mapping[i] = j
+        
+    
+    subwords = [subword for subwords_ in subwords for subword in subwords_] # flatten list of lists
+
+    input_ids = tokenizer.convert_tokens_to_ids(subwords[:510])
+    input_ids.insert(0, tokenizer.cls_token_id)
+    input_ids.append(tokenizer.sep_token_id)
+
+    return input_ids, words, word_subword_mapping
+
+def get_replacements(glove_word_id_mapping: dict, glove_id_word_mapping: dict, glove_vectors: np.ndarray,
+    model: PreTrainedModel, tokenizer: PreTrainedTokenizerBase, text: str, word_possibilities: int = 20,
+    batch_size: int = 16, device: str = "cpu:0") -> List[List[str]]:
+    """Returns a list of possible replacements for each word in the text."""
+    input_ids, words, word_subword_mapping = tokenize_and_extract_words(text, tokenizer)
+
+    # masks words which were not split into subwords by the tokenizer
+    inputs = []
+    for word_index in word_subword_mapping:
+        subword_index = word_subword_mapping[word_index]
+        input_ids_ = copy(input_ids)
+        input_ids_[subword_index] = tokenizer.mask_token_id
+        inputs.append((input_ids_, subword_index))
+        
+    # doing batched forward pass
+    with torch.no_grad():
+        prediction_list = []
+        while len(inputs) != 0:
+            batch_list, token_indices = tuple(zip(*inputs[:batch_size]))
+            batch = torch.tensor(batch_list)
+            batch = batch.to(device)
+            logits = model(input_ids=batch)["logits"]
+            relevant_logits = logits[torch.arange(batch.shape[0]), token_indices]
+            ranking = torch.topk(relevant_logits, word_possibilities, dim=1)
+            prediction_list.append(ranking.indices.cpu())
+            inputs = inputs[batch_size:]
+        predictions = torch.cat(prediction_list, dim=0)
+
+    # creating list of possible replacements for each word
+    possible_words = []
+
+    batch_index = 0
+    for i, word in enumerate(words):
+        if i in word_subword_mapping: # word was not split into subwords so we can use MLM output
+            subword_index = word_subword_mapping[i]
+            ranking = predictions[batch_index]
+            possible_words_ = [word]
+            for token in ranking:
+                word = tokenizer.convert_ids_to_tokens([token])[0]
+                if not word.startswith("##"):
+                    possible_words_.append(word)
+
+            possible_words.append(possible_words_)
+
+            batch_index += 1
+        elif word in glove_word_id_mapping: # word was split into subwords so we use glove instead
+            word_id = glove_word_id_mapping[word]
+            glove_vector = glove_vectors[word_id]
+            with torch.no_grad():
+                word_similarities = torch.mm(glove_vectors, glove_vector.unsqueeze(1)).squeeze(1)
+                ranking = torch.argsort(word_similarities, descending=True)[:word_possibilities + 1]
+                possible_words.append([glove_id_word_mapping[int(id_)] for id_ in ranking.cpu()])
+        else: # word was not in glove either so we can't find any replacements
+            possible_words.append([word])
+
+    return possible_words
+
+def augment(word_id_mapping: dict, id_word_mapping: dict, vectors: np.ndarray, model: PreTrainedModel,
+    tokenizer: PreTrainedTokenizerBase, text: str, multiplication_factor: int = 20, word_possibilities: int = 20,
+    replace_probability: float = 0.4, batch_size: int = 16, device: str = "cpu:0") -> List[str]:
+    # returns a list of different augmented versions of the text
+    replacements = get_replacements(glove_word_id_mapping=word_id_mapping, glove_id_word_mapping=id_word_mapping,
+        glove_vectors=vectors, model=model, tokenizer=tokenizer, text=text, word_possibilities=word_possibilities,
+        batch_size=batch_size, device=device)
+    new_texts = []
+    for i in range(multiplication_factor):
+        new_text = []
+        for possible_words in replacements:
+            if len(possible_words) == 1:
+                new_text.append(possible_words[0])
+                continue
+            if random.random() < replace_probability:
+                new_text.append(random.choice(possible_words[1:]))
+            else:
+                new_text.append(possible_words[0])
+        new_texts.append(" ".join(new_text))
+    return new_texts
+
+def augment_squad(model: str, tokenizer: str, squad_path: Path, output_path: Path,
+        glove_path: Path = Path("glove.txt"), multiplication_factor: int = 20, word_possibilities: int = 20,
+        replace_probability: float = 0.4, device: str = "cpu:0", batch_size: int = 16):
+    """Loads a squad dataset, augments the contexts, and saves the result in SQuAD format."""
+    # loading model and tokenizer
+    transformers_model = AutoModelForMaskedLM.from_pretrained(model)
+    transformers_model.to(device)
+    transformers_tokenizer = AutoTokenizer.from_pretrained(tokenizer, use_fast=False)
+    # load glove for words that do not have one distinct token, but are split into subwords
+    word_id_mapping, id_word_mapping, vectors = load_glove(glove_path=glove_path, device=device)
+
+    # load squad dataset
+    with open(squad_path, "r") as f:
+        squad = json.load(f)
+        
+    topics = []
+
+    for topic in tqdm(squad["data"]):
+        paragraphs = []
+        for paragraph in topic["paragraphs"]:
+            # make every question unanswerable as answer strings will probably not match and aren't relevant for distillation
+            for question in paragraph["qas"]:
+                question["answers"] = []
+            context = paragraph["context"]
+            contexts = augment(word_id_mapping=word_id_mapping, id_word_mapping=id_word_mapping, vectors=vectors,
+                model=transformers_model, tokenizer=transformers_tokenizer, text=context, multiplication_factor=multiplication_factor,
+                word_possibilities=word_possibilities, replace_probability=replace_probability, device=device, batch_size=batch_size)
+            paragraphs_ = []
+            for context in contexts:
+                new_paragraph = deepcopy(paragraph)
+                new_paragraph["context"] = context
+                paragraphs_.append(new_paragraph)
+            paragraphs += paragraphs_
+        topic["paragraphs"] = paragraphs
+        topics.append(topic)
+    squad["topics"] = topics
+
+    # save new dataset
+    with open(output_path, "w") as f:
+        json.dump(squad, f)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--squad_path", type=Path, required=True, help="Path to the squad json file")
+    parser.add_argument("--output_path", type=Path, required=True, help="Path to save augmented dataaset")
+    parser.add_argument("--multiplication_factor", type=int, default=20, help="Factor by which dataset size is multiplied")
+    parser.add_argument("--word_possibilities", type=int, default=5, help="Number of possible words to choose from when replacing a word")
+    parser.add_argument("--replace_probability", type=float, default=0.4, help="Probability of replacing a word")
+    parser.add_argument("--glove_path", type=Path, default="glove.txt", help="Path to the glove file")
+    parser.add_argument("--batch_size", type=int, default=16, help="Batch size for forward pass")
+    parser.add_argument("--device", type=str, default="cuda:0", help="Device to use")
+    parser.add_argument("--model", type=str, default="bert-base-uncased", help="Huggingface model identifier for MLM model")
+    parser.add_argument("--tokenizer", type=str, default="bert-base-uncased", help="Huggingface tokenizer identifier for MLM model")
+
+    args = parser.parse_args()
+
+    augment_squad(**vars(args))
\ No newline at end of file


Test patch:
diff --git a/test/samples/glove/tiny.txt b/test/samples/glove/tiny.txt
new file mode 100644
--- /dev/null
+++ b/test/samples/glove/tiny.txt
@@ -0,0 +1,10 @@
+, 0.18378 -0.12123 -0.11987 0.015227 -0.19121 -0.066074 -2.9876 0.80795 0.067338 -0.13184 -0.5274 0.44521 0.12982 -0.21823 -0.4508 -0.22478 -0.30766 -0.11137 -0.162 -0.21294 -0.46022 -0.086593 -0.24902 0.46729 -0.6023 -0.44972 0.43946 0.014738 0.27498 -0.078421 0.36009 0.12172 0.4298 -0.055345 0.4495 -0.74444 -0.26702 0.16431 -0.19335 0.13468 0.2887 0.23924 -0.23579 -0.28972 0.20149 0.048135 -0.18322 -0.15492 -0.19255 0.40271 0.16051 0.17721 0.32557 0.011625 -0.42572 0.34205 -0.45865 -0.2486 0.034128 0.03306 -0.057065 0.18136 -0.43638 0.0005709 -0.11935 -0.2195 0.16429 -0.18119 -0.19145 -0.081672 -0.2962 0.25803 0.073848 0.54213 -0.15405 -0.49256 0.091719 0.13329 -0.05253 -0.20518 0.34576 -1.0449 0.072779 -0.0003453 -0.16926 0.051019 -0.14753 0.23848 -0.40749 -0.58278 -0.48695 0.25863 -0.20531 -0.4775 0.40645 -0.038512 -2.403 -0.12421 0.63149 0.089419 0.08557 -0.20757 -0.1617 -0.29506 -0.13948 0.14202 -0.30138 -0.15806 0.52984 0.24229 0.075169 0.13792 0.90416 -0.23647 0.027788 0.099915 0.45422 0.60176 0.25044 0.29142 0.040712 -0.08121 -0.43786 -0.3015 -0.17991 -0.52149 0.029446 -0.23051 0.073955 0.34751 0.07806 0.19801 -0.32246 -0.13827 0.10076 0.56601 0.31925 0.09426 -0.045898 0.78329 0.19997 0.1619 0.41579 -0.31467 -0.036655 -0.11687 -0.17942 0.16246 0.42221 0.19588 -0.025058 -0.018717 -0.17965 0.35635 0.25853 0.13139 0.026784 0.017271 -0.14781 0.30598 -0.033228 0.15521 -0.50574 0.1295 0.14602 -0.35552 -0.43194 -0.1029 0.04736 -0.57903 -0.42488 0.67163 -0.11182 0.29306 -0.0033312 0.13091 -0.086655 0.22618 0.29357 -0.3088 -0.42705 0.3268 0.39254 0.17474 -0.19659 0.35665 0.38025 0.24257 -0.17021 0.097295 0.45248 -0.40589 0.27886 -0.33315 0.37076 0.16742 -0.28582 -0.051604 -0.090346 0.095385 0.26395 -0.30008 -0.63244 0.076666 0.14102 0.88613 -0.053817 0.26223 -0.016005 -0.040608 0.082136 -0.08159 -0.068912 -0.62239 -0.014757 -0.033402 0.25847 -0.28878 -0.27143 -0.23709 -0.11285 0.24828 0.14512 0.3373 -4.1005 -0.075261 0.32638 0.21444 0.37972 0.029263 0.24594 0.42935 0.68689 -0.58112 0.22939 -0.38889 0.41684 0.066217 0.47901 0.27427 0.41645 -0.35492 -0.14413 -0.010046 -0.42024 -0.19382 0.36156 -0.13364 -0.29853 0.47537 -0.26989 -0.083662 -0.0741 0.21815 -0.30678 -0.83499 -0.11287 -0.32612 0.12375 0.35341 -0.32607 0.32853 0.060266 -0.21991 0.35671 0.29546 -0.48159 -0.22347 0.31036 0.22132 -0.20994 -0.085675 -0.26173 -0.10764 -0.14802 0.17573 -0.17804 -0.21765 0.3073 -0.4459 0.03913 -0.22065 0.2214 0.32727 -0.40378 0.33021 -0.13942 -0.41003 -0.17526 0.21852 0.13615 0.10999 -0.33474 -0.046109 0.1078 -0.035657 -0.012921 -0.039038 0.18274 0.14654
+the -0.20838 -0.14932 -0.017528 -0.028432 -0.060104 -0.2646 -4.1445 0.62932 0.33672 -0.43395 0.39899 -0.19573 0.13977 -0.021519 0.37823 -0.5525 -0.1123 -0.0081443 0.29059 0.066817 0.10465 -0.086943 -0.048983 -0.26757 -0.47038 0.27469 0.069245 -0.027967 -0.19719 0.016749 -0.29681 0.17838 0.058374 -0.24806 0.085846 0.35043 0.049157 -0.16431 0.50012 -0.18053 0.31422 0.10671 0.031852 0.074278 0.27956 0.080317 0.05478 -0.30349 -0.43215 0.32417 0.40856 0.36192 0.13445 -0.12933 0.11331 -0.15755 0.35755 0.30463 -0.098488 0.012032 0.45581 0.37101 0.1427 -0.43329 -0.10869 0.49849 0.54455 0.44352 0.31804 0.022171 -0.41186 -0.025428 0.21062 -0.3583 0.22028 -0.55391 -0.035364 -0.053998 0.32172 -0.51928 -0.27427 -0.45214 -0.329 -0.48519 0.52966 0.0041434 -0.1718 -0.18748 -0.24365 -0.060786 0.050733 -0.21335 0.27627 0.42745 0.011461 -0.29794 -3.2881 -0.39842 0.16796 -0.12894 0.0020005 0.45613 0.15215 0.15364 -0.21281 -0.25339 0.28955 -0.57817 -0.6074 0.10301 0.28324 0.21506 -0.042325 0.40479 -0.20579 -0.011674 0.26092 -0.15402 0.057961 -0.058576 -0.41974 0.52015 0.15074 -0.088039 -0.14446 -0.17074 0.083752 -0.25708 0.16362 0.14795 -0.059821 0.034473 -0.14534 -0.17965 0.076303 0.33354 -0.14434 0.17618 0.45345 0.15262 -0.0751 0.27592 0.081456 0.30738 -0.072327 0.10706 -0.35581 -0.02669 0.61236 0.70829 -0.28945 -0.024637 0.01189 -0.091899 -0.27272 -0.10157 0.44713 0.092418 -0.10711 -0.015552 0.12822 0.22256 -0.069059 0.29927 -0.10913 0.1618 0.14796 0.1136 0.26634 0.010832 0.071946 0.16973 -0.22769 0.322 -0.083748 0.65269 0.068244 -0.32687 0.31782 0.17035 0.79803 -0.19194 -0.16485 -0.32437 0.079105 -0.35672 -0.26786 -0.24786 0.70512 -0.11909 0.16256 -0.43259 -0.050078 0.050232 -0.1145 -0.041885 0.47866 0.012767 0.19642 0.26196 -0.29425 0.089615 -0.17736 -0.22448 0.22624 0.16749 0.05577 0.14399 0.2158 0.33819 0.23459 0.15826 -0.2856 0.24199 0.11018 0.38164 -0.2984 -0.20169 0.2695 0.11186 -0.21006 -0.04207 0.016507 -0.22866 -3.3882 0.29204 -0.088358 -0.014966 -0.25225 -0.11503 0.036337 -0.14817 0.04622 -0.073466 -0.13866 0.23612 0.033882 0.29495 -0.61234 0.20289 -0.42091 0.37767 0.03626 0.21708 0.12561 -0.21682 -0.0037997 -0.17791 -0.26431 0.31678 -0.051229 0.049269 -0.12622 -0.10117 0.017246 -0.02195 -0.1982 0.03725 -0.16791 -0.055459 0.5767 0.059123 0.22931 0.064201 0.27424 -0.37129 -0.091375 -0.071342 -0.037218 -0.012668 -0.017976 -0.42622 -0.10095 0.044992 -0.090225 0.22915 0.1861 0.36366 -0.20676 -0.33037 0.47302 0.2338 0.079306 0.21083 0.21013 0.15275 0.080873 -0.33013 -0.17181 -0.07017 -0.041244 -0.46182 0.027903 0.54657 -0.25894 0.39515 0.26144 -0.54066 0.21199 -0.0094357
+. 0.10876 0.0022438 0.22213 -0.12102 -0.048959 0.018135 -3.8174 -0.032631 -0.62594 -0.51898 -0.35893 0.78871 -0.29777 0.005186 -0.41814 0.24691 -0.13584 -0.19046 -0.29987 -0.63696 -0.065406 0.11436 -0.021438 0.17501 -0.19494 -0.089508 -0.10631 -0.24107 0.12856 0.33577 0.051281 0.13849 0.24562 -0.40412 -0.16927 0.22167 0.24016 -0.032447 0.11127 -0.01178 -0.19622 -0.59113 -0.13265 -0.21192 0.17192 -0.18961 -0.34916 -0.098324 -0.42555 0.026523 -0.41361 -0.21346 -0.19775 0.0093893 -0.25976 -0.48221 -0.10778 0.029514 0.29254 -0.14844 0.73861 -0.10725 0.13759 0.54075 -0.1347 0.12982 0.036717 0.32338 -0.077501 -0.04642 -0.23924 -0.25071 -0.059339 -0.40909 0.022929 -0.36144 0.016797 -0.0070788 -0.15447 -0.36141 -0.22258 -0.44854 0.070369 0.19246 0.38875 0.10956 0.20907 0.60839 0.060436 0.042908 -0.13592 -0.1407 -0.56612 0.048278 -0.23504 -0.27897 -2.2884 0.11096 -0.19271 0.14455 0.28472 -0.20574 -0.036538 0.35877 0.16868 -0.063331 -0.18492 0.28085 0.85193 0.28055 -0.20985 -0.19593 0.81482 0.25215 -0.62068 0.41435 0.10557 -0.3093 -0.096312 0.13028 0.15084 -0.50123 0.15623 -0.20892 -0.28956 0.012043 -0.097626 -0.26242 0.096474 0.25771 -0.10903 -0.10082 -0.14434 -0.32984 -0.046379 0.031062 0.014825 0.091645 -0.067844 0.046894 0.15272 -0.088693 0.19054 0.67088 -0.059463 -0.21187 0.16674 -0.23207 0.33023 -0.23966 -0.48016 -0.18978 0.51637 0.13811 -0.26918 0.089576 -0.54118 0.53692 -0.18972 0.046074 -0.27932 -0.10529 -0.73563 -0.087796 0.39539 -0.44176 -0.62707 -0.21666 0.36325 -0.38248 0.49247 0.066497 0.016996 0.17387 -0.30563 -0.036247 0.094577 -0.40638 -0.11703 -0.044589 -0.41872 0.55478 -0.050797 -0.51106 0.081849 -0.23704 0.12522 0.56392 0.22459 0.5077 0.12011 -0.18115 -0.11125 -0.060846 -0.582 -0.32602 -0.061308 0.32683 0.099323 0.2721 0.32524 0.021033 -0.047185 0.51143 0.3579 0.35536 0.005939 0.49009 0.056522 -0.1218 0.021951 -0.21458 0.41146 -0.086966 0.42673 -0.18731 -0.29233 -0.019558 -0.5649 -0.25493 0.12111 0.10869 -0.16795 -0.010423 -3.6964 -0.22244 -0.16492 0.12317 -0.010062 -0.53517 0.30557 0.03702 0.4443 0.015176 -0.26279 -0.065307 0.33099 0.35048 -0.3176 -0.2243 -0.3004 0.24022 -0.18502 -0.66931 0.14986 -0.15948 0.141 -0.047219 -0.083048 0.16829 0.044241 0.35777 -0.22207 0.067496 0.24111 -0.20181 0.64122 0.05656 -0.32635 0.28111 -0.15638 -0.045039 0.14662 -0.41362 0.29122 0.48036 -0.65526 0.041017 -0.40086 0.11574 0.02623 0.028116 -0.4088 0.17148 -0.50448 0.10087 0.3922 0.14905 0.20962 -0.11509 0.32092 -0.32045 0.035085 0.67025 -0.61808 -0.13169 -0.032892 0.21476 -0.0033323 0.52406 0.063131 -0.2123 -0.30088 -0.45161 0.2648 0.075971 -0.40688 -0.29696 0.15939 -0.14902
+and -0.09611 -0.25788 -0.3586 -0.32887 0.5795 -0.51774 -4.1582 -0.11371 -0.10848 -0.48885 0.19931 -0.1054 -0.43825 -0.34483 -0.45052 -0.34864 -0.458 -0.81554 0.22006 0.20254 -0.10954 0.1252 -0.54117 0.34731 -0.099998 -0.018998 -0.14277 -0.42481 -0.0094091 -0.43155 -0.038769 0.12147 0.51988 -0.4984 -0.24625 -0.52067 -0.05821 -0.30712 0.25512 0.048033 -0.22313 -0.0069182 0.039824 -0.50088 -0.11972 -0.079045 0.01688 -0.34052 -0.2066 0.081265 0.12352 -0.49007 0.34946 -0.29241 0.14893 0.1366 -0.09783 -0.068472 -0.010913 0.0028454 -0.12656 0.3427 0.1058 -0.46151 0.070133 -0.061343 -0.015021 0.17659 0.17941 -0.51377 -0.31381 -0.1372 0.045186 -0.082259 0.21515 -0.21955 0.10313 -0.20704 0.14041 -0.35151 0.62316 -0.5799 -0.056115 -0.0021746 0.18958 0.22398 0.12246 -0.26178 0.010779 -0.31268 -0.21447 0.35344 -0.026041 0.018232 0.35751 -0.070188 -3.0872 -0.13131 0.017387 0.23244 -0.060585 0.20679 0.57579 0.36338 -0.41574 0.030607 0.23619 -0.11284 -0.36043 0.21635 -0.02752 0.17502 0.43491 -0.088247 0.40754 -0.48551 0.13539 -0.090759 0.14423 0.34118 -0.3794 -0.27344 0.02593 0.073217 -0.10176 0.16551 -0.23278 -0.18563 0.021372 -0.093111 0.15179 0.15057 0.55148 -0.20088 -0.079495 0.22599 0.26243 0.25123 0.60266 -0.20423 0.36972 -0.10694 0.0072887 0.018359 0.22368 -0.14065 0.1112 0.087667 0.8466 0.31545 -0.15348 0.020311 0.020878 0.38651 0.047422 -0.24854 -0.19053 0.49173 0.038161 -0.021038 0.14496 0.11591 -0.15105 -0.18942 0.18703 0.026752 0.0046523 0.39814 -0.018617 -0.73177 0.072832 0.41535 -0.48818 0.00304 -0.22729 0.88248 -0.61612 -0.18901 -0.33491 -0.28672 -0.013143 -0.37545 -0.18443 -0.55218 0.70186 -0.073107 0.6393 0.13098 0.071586 0.0053641 0.24636 -0.70744 -0.45036 0.00060187 -0.39093 -0.02716 0.25589 -0.17313 0.29883 -0.00090947 0.08314 -0.4099 -0.013024 -0.049533 0.3041 0.64302 0.23045 -0.18757 0.037584 -0.26082 0.1753 -0.062815 -0.22569 -0.1213 0.15524 -0.14407 0.088732 0.34674 -0.43494 0.38688 -0.15733 -0.12721 0.30194 0.32034 -3.3264 0.069427 0.13848 -0.058216 -0.027088 0.11028 0.3404 0.18654 0.11522 -0.40381 0.044776 0.15535 0.16247 -0.24051 0.04729 0.03498 -0.075942 0.15598 -0.059873 0.0046743 0.15595 -0.27613 0.13562 0.13485 -0.073724 0.31421 0.031234 -0.23516 0.31005 -0.10375 -0.30783 -0.55327 0.28304 0.081429 0.37778 0.15725 0.011757 0.043006 -0.43423 -0.22718 -0.043292 -0.63617 -0.8939 -0.17406 0.41111 -0.14404 -0.1678 -0.44438 -0.73051 0.10957 0.13122 0.085623 0.12504 -0.40337 0.041765 -0.27574 0.062513 0.051093 0.39926 0.11149 -0.056462 0.26809 -0.39569 0.31033 -0.04975 -0.33139 0.47781 -0.021213 -0.21236 0.42374 0.14083 0.067498 -0.12675 -0.3703 -0.092774 0.39058
+to -0.24837 -0.45461 0.039227 -0.28422 -0.031852 0.26355 -4.6323 0.01389 -0.53928 -0.084454 0.061556 -0.41552 -0.14599 -0.59321 -0.028738 -0.034991 -0.29698 -0.07985 0.27312 0.2204 -0.089859 0.00088265 -0.41991 -0.12536 -0.054629 0.03055 0.1934 -0.063945 0.027405 0.051193 -0.38656 -0.11085 0.17259 0.29804 -0.35183 0.1315 -0.54006 -0.76677 -0.00055168 0.13076 0.025101 0.62106 -0.24797 -0.3979 -0.36116 -0.51967 0.030138 -0.052436 0.069281 0.035252 -0.21402 0.24836 -0.15693 0.12829 0.35425 -0.1608 -0.005072 -0.30656 -0.29514 -0.13554 -0.14385 -0.40552 0.57233 -0.2767 0.30519 0.15586 0.016086 -0.22009 0.48589 -0.41384 0.20546 0.40491 0.041558 -0.13542 0.22544 -0.23629 0.15193 -0.010859 -0.082662 -0.55484 -0.061584 -0.11112 -0.11982 -0.37064 0.16501 0.44063 -0.33883 -0.57676 0.50847 -0.035707 -0.059233 0.030748 -0.27689 -0.070433 0.027786 -0.59336 -2.822 -0.10052 0.67168 -0.17046 -0.25902 0.27938 0.39992 0.03748 -0.26409 -0.26378 0.20645 0.17564 -0.080807 -0.38376 0.26602 0.36214 -0.095112 0.35199 -0.86994 -0.15747 -0.2255 -0.064948 -0.24845 0.15038 -0.32951 -0.22285 -0.025509 -0.29725 -0.37715 0.089296 -0.034399 0.3364 0.35534 0.38253 0.17646 0.13305 -0.32743 -0.47115 0.24673 -0.15964 0.18212 -0.41241 0.098565 0.38118 0.33043 0.051987 -0.21824 0.22214 -0.05945 -0.063743 0.43723 0.11068 0.47444 0.56891 0.31123 -0.20272 0.080078 -0.43905 -0.12246 -0.025057 -0.057162 0.1425 0.094468 0.12991 0.10444 -0.39447 -0.29337 -0.20466 0.20815 -0.1601 -0.14665 0.54511 0.2974 -0.22959 -0.1705 -0.062371 -0.50399 -0.38 -0.39528 0.57552 -0.46892 -0.43308 0.15018 -0.041179 0.62157 0.019874 -0.11969 -0.25611 0.26602 -0.37383 0.12936 -0.050006 -0.11554 -0.17163 -0.4243 0.19844 0.50611 -0.11093 -0.13939 -0.59377 0.67338 0.38497 0.62604 -0.20128 0.30058 -0.13946 -0.16186 0.12168 -0.01841 0.61356 -0.19887 0.1925 0.0084372 -0.50757 0.35858 -0.49729 -0.44725 0.021423 -0.20769 0.083729 0.22032 0.14404 0.001259 -0.44309 -0.17242 -0.353 -0.29477 0.32898 -3.191 0.3891 0.35654 0.052134 0.20576 -0.088649 0.16398 0.11203 0.2859 0.2894 -0.44349 0.91036 -0.30902 -0.13985 -0.39499 -0.027299 -0.15201 0.084418 -0.37196 0.049827 0.14128 -0.15126 -0.16107 0.0040226 0.16799 -0.25429 -0.15074 -0.57409 -0.15611 0.068407 0.24832 0.16828 0.072764 -0.086728 0.0021982 0.13593 0.70224 -0.45976 -0.24506 -0.33874 -0.10952 0.24698 -0.55919 -0.38866 -0.13372 0.091943 -0.10543 -0.31319 -0.29952 -0.20611 0.17976 0.458 -0.072402 0.16118 -0.41649 -0.30103 0.23234 -0.050139 0.10026 0.38974 -0.061342 0.26626 -0.15671 0.075136 -0.42926 -0.12025 0.082736 -0.62469 0.044267 0.60673 -0.12458 -0.15443 -0.16339 0.053097 0.15458 -0.38053
+of -0.036429 -0.28592 0.063387 -0.60122 -0.015309 0.073243 -3.8547 0.52809 -0.077859 -0.40756 0.44713 0.0056261 -0.38887 -0.0092043 0.26061 -0.45271 -0.50423 -0.38904 -0.30564 0.42003 0.011568 -0.25619 0.55622 0.17323 -0.30415 0.06224 -0.29075 -0.51046 -0.45783 -0.49763 -0.18727 -0.24962 -0.73865 0.046438 0.11561 -0.031821 -0.26618 -0.41581 -0.071389 -0.2079 -0.17899 0.040931 0.1425 -0.43102 -0.2068 0.2589 -0.36623 -0.22408 -0.45842 0.82779 0.53323 0.213 0.14102 0.4644 -0.19063 0.13789 0.23394 -0.48319 0.020489 0.035165 0.1129 0.31478 -0.11344 -0.12321 -0.071496 0.12934 0.51611 -0.17039 -0.10672 0.69655 -0.48179 0.037786 -0.2011 0.21367 -0.023501 -0.43007 0.45458 -0.18225 0.045348 -0.73977 -0.018974 -1.1612 0.046104 -0.59842 0.48152 0.077035 0.079665 -0.059979 0.28133 -0.19488 0.33882 -0.022075 0.34987 0.43741 -0.20609 -0.27658 -3.4348 -0.25458 -0.011177 0.2142 0.21506 0.78159 -0.0064574 0.052498 -0.53285 -0.32415 0.032445 -0.46545 -0.86625 0.13234 0.22265 0.63226 -0.23883 0.20949 0.0081047 0.25664 0.12726 0.4862 0.63486 -0.11147 -0.22792 0.46799 0.098441 -0.22575 -0.050439 0.26147 -0.35885 0.34768 0.46181 0.6723 -0.0088817 -0.3072 0.032602 -0.18653 0.014603 0.21552 0.021596 0.13545 0.30614 0.020874 0.19128 -0.31014 -0.091298 -0.03491 0.089525 -0.14151 -0.10139 -0.10558 0.43202 0.38602 0.067725 -0.49517 -0.43656 0.53588 -0.42404 -0.073479 0.31768 -0.1902 -0.10048 -0.41895 -0.085678 -0.31469 0.20415 -0.16602 -0.10681 -0.17592 0.0045544 0.1717 0.15179 0.02964 0.31657 0.11772 -0.060541 0.54928 0.24473 0.60053 -0.27232 -0.010884 -0.32954 0.21139 -0.05083 0.033311 -0.28025 -0.46744 0.73776 -0.5466 0.2121 -0.088026 0.18173 -0.26704 -0.14732 0.010112 0.12324 0.19112 0.2311 -0.13961 -0.14684 0.35858 0.59945 0.25938 -0.33926 0.48621 0.19161 -0.37456 -0.064856 0.19457 0.3635 -0.11403 -0.50548 -0.030436 0.44118 0.11703 -0.15215 0.17065 -0.3151 -0.16841 -0.51375 -0.06952 -0.28814 0.32903 -0.033493 -0.046118 -0.21614 -0.054066 -3.2795 -0.14166 -0.10367 -0.042882 -0.052912 -0.156 0.20207 0.24538 0.0051342 0.067185 0.08875 0.14647 0.24244 -0.25878 -0.22634 -0.07779 -0.41721 0.0017983 -0.078582 -0.1313 0.10876 0.152 -0.41862 -0.18904 -0.33577 -0.022991 -0.19204 0.0017913 -0.22954 0.28383 -0.64747 0.12466 -0.019468 -0.6369 -0.2137 0.51779 0.23749 0.55759 -0.12286 0.20406 0.26386 -0.094897 -0.20743 0.14492 0.24069 0.52458 0.11511 0.044618 -0.57734 0.32207 -0.097548 0.21445 0.16075 0.16482 0.29271 -0.08841 0.67936 0.32115 -0.21867 0.26709 -0.15852 -0.017671 -0.0092939 0.17614 -0.10259 0.033329 -0.021308 -0.66019 0.090383 0.12859 0.22616 0.30235 -0.25703 -0.48926 0.30537 0.27273
+a -0.035835 0.77844 -0.51806 0.080682 -0.13173 -0.28606 -4.2485 0.81827 0.24034 -0.69057 0.21556 -0.12434 -0.54229 -0.15138 -0.13591 -0.5348 0.34595 -0.22926 -0.013789 0.047816 -0.041427 -0.41058 0.10172 -0.097093 -0.00064646 -0.026877 -0.23473 -0.06019 -0.3141 -0.5624 -0.079395 -0.22116 0.098952 0.089909 -0.068699 0.30787 0.35724 0.27748 0.21167 -0.020939 -0.48188 -0.14981 -0.19693 -0.19514 -0.029672 -0.2862 0.059519 -0.15534 -0.041982 0.31305 -0.10834 0.74492 -0.26733 0.090418 0.32783 -0.35072 0.0019352 -0.16018 0.35431 -0.066866 -0.094903 -0.026866 -0.44347 0.13844 0.14952 0.34483 0.12546 0.4631 -0.0058689 -0.41446 0.12612 0.13602 -0.43715 0.096268 -0.18979 -0.75418 0.35777 -0.017479 -0.21907 -0.18382 -0.27002 -0.54582 -0.45421 -0.54994 -0.43079 -0.11863 -0.47369 -0.099825 -0.14261 -0.081525 0.33486 -0.4146 -0.079636 0.30339 -0.42575 -0.34956 -2.7357 -0.74104 0.2163 0.42723 0.035431 -0.05464 0.70669 -0.29739 0.0083191 -0.19281 0.32391 0.0087548 -0.16015 -0.059082 0.17503 0.35089 -0.01354 -0.33457 -0.051774 0.01569 1.029 0.044273 0.44906 -0.29036 -0.78684 -0.060001 0.11784 -0.1194 0.20565 -0.30654 -0.10883 -0.076856 -0.027208 -0.0032139 -0.19201 0.13827 0.15239 -0.085574 0.25771 0.035583 0.25709 -0.13264 0.64585 0.539 0.09338 0.17612 0.45444 -0.30252 -0.51751 -0.0077072 0.52078 -0.22996 0.51313 0.6185 -0.271 0.18719 0.3466 -0.27276 0.34786 -0.31713 0.13038 0.021235 -0.25821 -0.14553 0.65942 -0.19469 -0.17672 -0.17879 -0.29119 0.2739 -0.074949 -0.35214 0.12527 -0.1515 0.16561 -0.35536 0.26169 -0.66952 -0.037627 0.24556 -0.47338 -0.19609 -0.16514 0.18519 0.74366 0.099546 -0.30843 0.082241 -0.14698 -0.53421 -0.0061243 0.57767 0.34576 -0.16245 0.062227 -0.274 -0.10936 0.17974 0.32881 -0.0009029 0.072287 0.11748 0.25142 0.062339 0.44903 -0.00094289 0.18164 -0.29528 0.17274 0.41538 -0.1912 0.14755 0.39594 -0.16013 0.064232 -0.32972 0.2003 0.31493 -0.43744 -0.13939 -0.10015 -0.20848 -0.22608 -0.048169 0.20638 0.51249 -0.341 -0.004226 -3.4638 -0.12878 -0.47151 -0.42011 -0.04058 0.042346 0.11615 -0.041935 0.038201 -0.24746 -0.093521 0.44521 -0.044714 0.32429 -0.13014 -0.3915 -0.461 -0.070122 -0.35223 0.38686 -0.25869 0.47497 0.076472 0.0085433 -0.4047 0.66155 -0.23044 -0.021724 -0.021885 -0.00054206 0.0065346 -0.074982 -0.59198 -0.61204 -0.14 0.19463 -0.17722 -0.51057 0.40299 0.2675 -0.29331 -0.18907 -0.37621 -0.4179 0.54875 0.070255 0.80564 -0.3841 -0.42401 -0.41384 0.43875 -0.29252 -0.091183 0.22039 -0.18372 -0.41012 0.62847 0.21983 -0.079124 0.019266 0.85543 -0.13378 0.60141 0.67718 -0.33309 -0.2561 0.084727 0.10459 -0.25359 -0.12002 -0.38965 -0.2878 0.0036703 0.022321 -0.31591 -0.35608
+in 0.068507 -0.023344 0.28271 -0.40215 0.077815 -0.027003 -3.7645 0.41708 0.058373 -0.067997 0.32045 0.093317 0.35446 0.24415 -0.22822 0.21088 0.092499 -0.0766 -0.0030272 0.65657 0.10773 -0.17828 -0.1082 0.37832 -0.141 0.10373 0.15094 -0.43425 0.00019809 -0.39078 -0.51362 0.09536 -0.42444 0.15611 -0.32383 -0.12708 0.1385 0.21015 -0.3378 -0.4188 0.13787 0.29742 -0.27689 0.04996 -0.54465 -0.20988 0.25044 -0.33203 -0.2728 0.14554 0.60364 0.65656 0.0074453 -0.058784 -0.064545 0.75666 0.087701 -0.1936 -0.34405 -0.1472 0.0067563 0.18505 0.19937 -0.29199 -0.23206 -0.061202 0.21897 0.61196 0.5289 -0.10914 -0.77609 0.44075 -0.63157 0.35396 -0.29959 -0.15911 0.34515 -0.4131 0.41821 0.036273 -0.28904 -1.3799 -0.26149 -0.38145 -0.11302 0.20644 0.19689 -0.16254 0.61176 0.15381 -0.53395 -0.48077 -0.024011 -0.094099 0.22243 -0.14617 -2.9577 0.32627 0.10761 0.25086 0.20389 0.043559 0.37386 0.26832 -0.22787 0.14908 0.14154 -0.10015 0.31386 -0.1703 -0.060314 0.44677 0.64932 -0.3216 -0.55059 0.34674 0.22071 -0.35157 0.57815 0.25799 -0.41677 -0.17724 0.00032002 -0.61324 0.36185 0.055766 -0.136 -0.11536 -0.18133 -0.063293 0.37101 0.41068 -0.45675 0.0019741 0.052907 0.077341 0.047259 0.35521 0.94395 0.44756 -0.23055 -0.29585 -0.24687 0.3778 -0.070669 0.18335 -0.20146 -0.2345 0.60466 0.48813 -0.074877 -0.37743 0.13481 0.10849 -0.016328 -0.56793 0.096551 -0.28974 -0.85841 0.020689 0.55518 -0.20981 0.2382 -0.20757 -0.4081 0.10346 -0.027219 0.23233 0.047661 -0.026813 0.042456 -0.098405 0.40697 0.18637 -0.021596 0.38216 0.033878 -0.2535 -0.29586 -0.019435 -0.73558 -0.2914 -0.50521 -0.47852 -0.22671 -0.26419 -0.11708 0.16524 0.27825 -0.14631 0.051184 -0.009314 0.18186 -0.19979 0.13183 -0.11793 0.27146 0.19013 0.36612 -0.12877 -0.12857 0.094955 0.028475 -0.062946 0.25311 0.15169 0.15963 -0.22283 0.22304 -0.13993 0.23218 0.039904 0.046954 0.3138 -0.26845 -0.26303 0.14821 0.42341 0.34001 0.60787 -0.28699 -0.056269 0.11356 0.029713 -3.479 0.29802 0.03178 -0.035681 0.17751 0.16752 -0.2961 0.044736 -0.036076 0.30142 -0.34389 0.27084 -0.37026 0.2625 -0.47872 -0.0061819 0.055592 -0.061268 0.042886 0.1868 -0.12602 -0.078149 0.33139 -0.26079 -0.25363 0.40634 -0.34571 0.034084 0.1397 -0.1017 -0.11649 -0.19904 -0.027695 -0.31638 -0.038923 0.66143 0.34892 -0.18563 -0.12017 -0.42968 0.56684 -0.46318 0.010384 -0.16257 0.57057 -0.048416 -0.56498 -0.40066 0.17318 0.26049 0.45737 -0.51844 -0.46726 0.012543 -0.3206 0.068685 0.55413 -0.086315 -0.21519 0.66634 0.017061 -0.2999 -0.19917 0.17682 0.15799 0.15006 0.095986 -0.67037 0.59937 -0.14699 0.11697 -0.35628 0.11224 -0.23447 0.4055 -0.28067
+" -0.048032 -0.6914 -0.21957 0.089568 0.28472 -1.0531 -3.4121 0.02696 -0.20037 -0.53091 0.97056 -0.034644 -0.10033 -0.2071 -0.42032 0.16462 0.059077 -0.61837 0.23765 0.22807 -0.083801 -0.11536 0.27572 0.26371 -0.17288 0.0084245 -0.51929 -0.36559 0.14597 -0.81487 -0.51025 0.14477 0.46289 -0.25076 0.70546 0.57385 -0.44765 -0.090041 0.16833 0.38911 0.24072 -0.013165 0.074562 0.34116 0.34232 -0.28445 -0.13614 0.070661 -0.6707 0.19258 -0.55693 0.4442 -0.40277 0.04553 -0.010283 -0.040729 0.042784 0.85237 0.46238 0.31426 0.36427 -0.43114 -0.37496 -0.16093 -0.32705 0.53713 -0.25947 0.096877 0.10915 -0.085802 -0.39206 -0.40433 0.25685 -0.53044 -0.52574 -0.34065 -0.25792 0.30916 0.15165 -0.14527 0.040906 -1.0451 3.0427e-05 -0.10925 -0.22796 0.58295 0.17161 -0.50492 0.046715 -0.2565 -0.8094 0.12377 -0.17735 0.064325 -0.10278 0.20066 -2.6094 -0.13329 0.52934 -0.2907 0.039797 0.411 -0.28274 0.14114 -0.149 0.29174 -0.47222 0.27153 0.33043 -0.8398 -0.21368 0.068345 -0.41806 -0.1394 -0.29861 0.3712 0.30367 -0.16679 0.43494 0.39441 0.23276 -0.56044 0.23202 -0.050038 0.025742 -0.037166 0.31038 -0.37319 -0.071117 -0.16484 -0.034022 0.29113 0.41152 -0.04297 0.19572 0.18262 0.086992 0.38578 -0.80169 1.1235 -0.19665 0.252 0.51106 0.50104 0.30746 0.22131 1.0419 -0.12934 0.48314 -0.1097 -0.085398 0.057882 0.056915 -0.19651 0.13835 0.76558 -0.7776 0.14955 -0.0085004 0.10597 0.098488 -0.28773 -0.37583 -0.0099517 -0.49475 -0.027932 -0.15144 0.12913 0.16105 -0.059968 -0.21213 0.27882 0.75464 0.015144 0.13686 -0.18702 0.60299 -0.045254 -0.20525 0.15743 -0.46815 0.048959 0.3447 -0.37844 0.34106 -0.68152 0.11599 -0.15878 -0.044058 -0.18173 0.31114 -0.93244 0.25892 0.2941 -0.48729 0.11488 0.20706 0.71376 -0.0026516 -0.32355 -0.80349 0.32074 -0.10306 -0.038874 0.20804 0.069746 0.10707 0.13026 0.92798 -1.0789 0.59913 0.054721 -0.20213 0.22438 0.18911 -0.44685 -0.2894 -0.051251 -0.18365 0.15081 -0.59379 -0.029135 -0.20623 0.26559 -3.4438 0.65238 -0.68772 0.41854 -0.33688 0.20371 -0.48257 0.3191 0.050891 -0.13833 -0.40396 -0.25854 -0.082192 0.093764 -0.083204 -0.39219 0.14538 -0.5384 -0.081043 0.40279 -0.36503 0.50993 -0.4367 -0.21211 -0.41628 -0.41527 -0.52174 -0.527 -0.51573 0.046207 0.0099332 -0.37437 0.15547 0.14391 0.37315 0.47392 -0.16113 -0.070403 -0.14696 0.22995 -0.24619 0.14984 1.0486 -0.29904 0.11577 -0.48089 -0.24363 -0.13805 -0.67828 -0.080053 0.2871 0.61649 -0.25851 0.25048 0.0051335 0.040719 0.4885 -0.32601 -0.082974 0.82039 -0.054578 0.25842 0.47645 -0.3331 -0.33535 -0.084167 0.19699 0.26016 -0.59807 0.4776 0.48911 -0.33895 -0.43047 -0.087665 -0.22918 -0.0044677
+is 0.11396 0.34503 -0.055064 -0.20192 0.20848 -0.09699 -4.0236 0.95046 -0.060803 -0.8326 0.39128 0.080882 0.19049 -0.043595 -0.39001 -0.34955 0.11262 -0.14287 -0.032761 0.067908 0.17553 0.31537 0.28348 0.10421 0.35078 -0.032335 -0.36486 -0.34773 -0.20653 -0.0396 -0.23032 -0.077951 0.23275 -0.30656 -0.42728 0.11155 0.56584 0.18156 0.24105 0.26077 0.10482 -0.17409 -0.27386 -0.19085 0.001336 0.15198 -0.085442 -0.45299 -0.25952 -0.10323 0.16033 0.84934 -0.14611 -0.097733 -0.10756 -0.64231 -0.086497 0.19817 -0.145 0.081023 -0.51695 -0.054831 -0.011848 0.14594 0.27602 0.2617 0.47357 0.080774 0.024029 -0.20884 -0.19015 0.13206 -0.3157 -0.38179 -0.099757 -0.47159 0.38781 -0.15529 0.14584 0.061407 -0.34411 -1.2493 -0.55997 -0.51856 0.25404 0.23224 -0.26082 -0.20301 0.091036 -0.15094 0.054542 0.12856 -0.25977 0.57906 0.43496 0.11506 -2.4913 -0.074039 -0.22596 -0.078059 0.37 0.14267 0.29842 0.25159 0.18929 -0.21655 -0.06477 -0.31406 -0.22593 -0.035533 -0.006801 -0.19788 -0.3362 0.11342 -0.26645 0.2006 0.53312 -0.17071 0.29388 -0.1544 -0.32386 -0.23815 -0.074312 -0.0023955 0.31235 0.023316 0.35249 -0.37121 -0.025591 0.11688 -0.1956 -0.037899 -0.059021 0.12694 0.015057 -0.059807 0.11488 0.1346 0.38488 -0.0070541 0.09581 0.26326 0.31792 -0.020803 -0.57065 0.2096 0.34243 -0.13416 0.15929 0.049954 -0.31797 0.092282 -0.36187 -0.15156 0.20105 -0.49372 -0.24939 0.52548 -0.067412 0.0029721 0.48416 -0.29553 -0.1868 0.057142 -0.038733 0.11757 -0.13336 0.2471 0.72767 -0.3194 0.43514 -0.15875 0.20184 0.093158 -0.19095 0.73837 0.3119 -0.18697 0.11706 -0.19537 0.04848 -0.13815 0.083468 0.10713 0.22465 -0.65526 0.51878 0.24839 0.13339 -0.0076668 -0.21111 0.33498 0.42292 -0.19456 0.565 0.45421 0.16063 -0.25718 0.43721 -0.42303 0.097476 -0.015056 0.064036 -0.087359 -0.0071586 0.38121 0.33028 0.29676 0.23904 -0.14327 -0.1011 -0.16298 0.15252 0.066246 0.087008 0.36414 0.23292 -0.54527 -0.17597 -0.57629 -0.27825 -0.21048 0.31815 -0.11145 -3.3707 -0.57578 -0.31548 -0.21493 0.06938 -0.29779 -0.12619 0.40149 -0.1364 -0.60166 0.013699 -0.14001 0.50957 0.034518 -0.082044 -0.15225 0.054835 -0.30277 -0.22071 0.22622 0.26695 -0.032464 0.055506 -0.071883 -0.12037 0.21999 -0.11196 -0.0095203 -0.054642 0.12505 0.38358 0.027555 0.052245 -0.014396 0.13378 0.1317 0.14486 -0.20949 -0.28146 0.063763 0.098775 0.12514 -0.76178 -0.12258 0.21425 0.021548 0.40072 -0.18433 -0.17931 0.079434 -0.62864 0.16945 0.15069 0.25897 -0.31789 -0.29582 0.81134 -0.45501 -0.2029 0.068279 -0.06824 0.29112 0.32883 0.20016 -0.23233 -0.59309 -0.085202 -0.018587 -0.25809 0.21739 0.21592 0.081633 0.12224 -0.37247 0.2879 -0.18029
\ No newline at end of file
diff --git a/test/test_utils.py b/test/test_utils.py
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1,8 +1,10 @@
 import pytest
+from pathlib import Path
 
 from haystack.utils.preprocessing import convert_files_to_dicts, tika_convert_files_to_dicts
 from haystack.utils.cleaning import clean_wiki_text
-
+from haystack.utils.augment_squad import augment_squad
+from haystack.utils.squad_data import SquadData
 
 def test_convert_files_to_dicts():
     documents = convert_files_to_dicts(dir_path="samples", clean_func=clean_wiki_text, split_paragraphs=True)
@@ -14,3 +16,13 @@ def test_tika_convert_files_to_dicts():
     documents = tika_convert_files_to_dicts(dir_path="samples", clean_func=clean_wiki_text, split_paragraphs=True)
     assert documents and len(documents) > 0
 
+def test_squad_augmentation():
+    input_ = Path("samples/squad/tiny.json")
+    output = Path("samples/squad/tiny_augmented.json")
+    glove_path = Path("samples/glove/tiny.txt") # dummy glove file, will not even be use when augmenting tiny.json
+    multiplication_factor = 5
+    augment_squad("distilbert-base-uncased", "distilbert-base-uncased", input_, output,
+                    glove_path, multiplication_factor=multiplication_factor)
+    original_squad = SquadData.from_file(input_)
+    augmented_squad = SquadData.from_file(output)
+    assert original_squad.count(unit="paragraph") == augmented_squad.count(unit="paragraph") * multiplication_factor
\ No newline at end of file