Gut more server things

server/config.py

@@ -1,20 +0,0 @@
-"""
-This file stores the main configuration variables to run a server.
-"""
-from pathlib import Path
-import utils.path_fixes as pf
-import os
-
-ROOT = Path(os.path.abspath(__file__)).parent
-CORPORA = ROOT / "corpora"
-
-# Change this to indicate what data is loaded for searching
-RESOURCE_DIR = CORPORA / "gpt2" / "woz"
-MODEL_VERSION = "gpt2"
-# RESOURCE_DIR = CORPORA / "woz_bert-base-cased" 
-# MODEL_VERSION = "bert-base-cased"
-
-# Below are DEFAULTS. Change only if you changed the way embeddings and contexts are stored and created
-CORPUS = RESOURCE_DIR / "data.hdf5"
-EMBEDDING_FAISS = RESOURCE_DIR / "embedding_faiss"
-CONTEXT_FAISS = RESOURCE_DIR / "context_faiss"

server/data_processing/README.md

@@ -1,35 +0,0 @@
-# Creating an annotated corpus
-This module contains the code necessary for extracting and labeling a corpus with semantic data. 
-
-## Known limitations
-Please note the following:
-
-- There are many cases in which BPE tokenization and spacy's built in tokenization do not align. To remedy this, contractions that would break the BPE tokenization (defined by Spacy's hard coded exceptions in `spacy.lang.en.TOKENIZER_EXCEPTIONS` and `spacy.lang.tokenizer_exceptions.BASE_EXCEPTIONS`) are instead decomposed into the full words the contractions represent.
-- Large corpus files require a LOT of hard drive space to store all the attentions and representations at every layer for every head. When tackling a corpus the size of the Wizard of Oz (207kb), make sure you have at least 9GB of free space. For the validation set of WikiText-2 (1.1MB), you will need 47GB.
-
-## Getting Started
-The raw Wizard of Oz text used to create the annotated corpus can be found [here](http://www.gutenberg.org/ebooks/55). A small version of Wikipedia (WikiText-2) can be found [here](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/).
-
-### Environment 
-Because this module depends on code written in other parts of this repo, we will need to make those files available to the PYTHONPATH. There are several ways to do this, but the easiest way is to do the following:
-
-1. `conda activate exbert` (Assuming you have taken the time to sort out the conda dependencies)
-2. `cd server`
-3. `pip install -e .`
-
-This essentially makes this repository a local pip package, allowing you access to all packages inside of `server/` whenever the conda environment is active. For instance, if writing your own scripts or running a jupyter notebook, the top level `utils/token_processing` module will be available as `import utils.token_processing as tp`.
-
-### Overview
-To create your own dataset from scratch, you will need a large text file whose contents are in English. This repo currently does not support other languages.
-
-1. Run `python create_corpus.py -f <FNAME>.txt -o <OUTDIR>`. This will create, in `<OUTDIR>`, the following files:
-    - `embeddings/` - A folder containing the `<FNAME>.hdf5` file and all the `<layer_**>.faiss` files needed to index into the embeddings. NOTE: These files can be quite large
-    - `headContext/` - A folder containing the `<FNAME>.hdf5` file and all the `<layer_**>.faiss` files needed to index into the head embeddings/context. NOTE: These files can be quite large
-
-If you want to overwrite existing files in the output directory, add the `--force` flag onto the `create_corpus.py` command above.
-
-### Running the individual scripts
-2. Run `python create_hdf5.py -f <FNAME>.txt -o <OUTDIR>`
-3. Run `python create_faiss.py -d <OUTDIR>`. This will assume the creation of the `embeddings` and `headContexts` folders inside of `<OUTDIR>`
-
-You will then need to link these corpora into your application. In the `config.py` file, change the `RESOURCE_DIR` to point at `<OUTDIIR>`. 

server/data_processing/__init__.py

@@ -1,12 +0,0 @@
-from .corpus_data_wrapper import CorpusDataWrapper
-from .convenience_corpus import ConvenienceCorpus, from_model
-from .index_wrapper import Indexes, ContextIndexes
-from .sentence_data_wrapper import TokenH5Data, SentenceH5Data
-
-__all__ = [
-    'CorpusDataWrapper',
-    'Indexes',
-    'ContextIndexes',
-    'TokenH5Data',
-    'SentenceH5Data'
-]

server/data_processing/convenience_corpus.py

@@ -1,71 +0,0 @@
-from pathlib import Path
-from .corpus_data_wrapper import CorpusDataWrapper
-from .index_wrapper import Indexes, ContextIndexes
-from config import CORPORA
-from utils.f import memoize, delegates, GetAttr
-from typing import List
-
-def get_dir_names(path: Path) -> List[str]:
-    available = [g.name for g in filter(lambda g: g.is_dir(), path.glob("*"))]
-    return available
-
-
-@memoize
-def from_model(model_name, corpus_name):
-    """Get the convenience corpus wrapper for a model and a corpus"""
-    model_dir = Path(CORPORA) / model_name
-    available = get_dir_names(model_dir)
-    if not model_dir.exists() or len(available) == 0:
-        raise FileNotFoundError("There are no corpora present for this model")
-
-    base_dir = model_dir / corpus_name
-
-    if not base_dir.exists(): 
-        raise FileNotFoundError(f"Desired corpus '{corpus_name}' not available")
-
-    return ConvenienceCorpus(base_dir)
-
-def files_available(base_dir, glob_pattern="*.faiss"):
-    """Determine whether the base_dir contains indexed files"""
-    if not base_dir.exists() or len(list(base_dir.glob(glob_pattern))) == 0:
-        return False
-
-    return True
-class ConvenienceCorpus(GetAttr):
-    def __init__(self, base_dir):
-        bd = Path(base_dir)
-        self.base_dir = bd
-        self.model_dir = bd.parent
-        self.available_corpora = get_dir_names(self.model_dir)
-
-        self.model_name = self.model_dir.name
-        self.corpus_name = bd.name
-        self.name = f"{self.model_name}_{self.corpus_name}"
-
-        self.corpus_f = bd / 'data.hdf5'
-        self.embedding_dir = bd / 'embedding_faiss'
-        self.context_dir = bd / 'context_faiss'
-
-        # Define whether these different files exist or not
-        if not self.corpus_f.exists(): 
-            raise FileNotFoundError("Main HDF5 file does not exist")
-
-        self.embeddings_available = files_available(self.embedding_dir)
-        self.contexts_available = files_available(self.context_dir)
-
-        self.corpus = CorpusDataWrapper(self.corpus_f, self.name)
-        self.embedding_faiss = Indexes(self.embedding_dir)
-        self.context_faiss = ContextIndexes(self.context_dir)
-
-        self.default = self.corpus # Almost acts like an inherited class, but is rather a composed class
-
-    def search_embeddings(self, layer, query, k):
-        D, I = self.embedding_faiss.search(layer, query, k)
-        return self.find2d(I)[0]
-
-    def search_contexts(self, layer, heads, query, k):
-        D, I = self.context_faiss.search(layer, heads, query, k)
-        return self.find2d(I)[0]
-
-    def __repr__(self):
-        return f"ConvenienceCorpus({self.name})"

server/data_processing/corpus_data_wrapper.py

@@ -1,147 +0,0 @@
-import h5py
-import numpy as np
-from functools import partial
-from utils.gen_utils import map_nlist, vround
-import regex as re
-from spacyface.simple_spacy_token import SimpleSpacyToken
-from data_processing.sentence_data_wrapper import SentenceH5Data, TokenH5Data
-from utils.f import ifnone
-    
-ZERO_BUFFER = 12 # Number of decimal places each index takes
-main_key = r"{:0" + str(ZERO_BUFFER) + r"}"
-
-def to_idx(idx:int):
-    return main_key.format(idx)
-
-def zip_len_check(*iters):
-    """Zip iterables with a check that they are all the same length"""
-    if len(iters) < 2:
-        raise ValueError(f"Expected at least 2 iterables to combine. Got {len(iters)} iterables")
-    n = len(iters[0])
-    for i in iters:
-        n_ = len(i)
-        if n_ != n:
-            raise ValueError(f"Expected all iterations to have len {n} but found {n_}")
-
-    return zip(*iters)
-
-class CorpusDataWrapper:
-    """A wrapper for both the token embeddings and the head context.
-    
-    This class allows access into an HDF5 file designed according to the data/processing module's contents as if it were
-    and in memory dictionary.
-    """
-
-    def __init__(self, fname, name=None):
-        """Open an hdf5 file of the format designed and provide easy access to its contents"""
-                
-        # For iterating through the dataset
-        self.__curr = 0
-        
-        self.__name = ifnone(name, "CorpusData")
-        self.fname = fname
-        self.data = h5py.File(fname, 'r')
-
-        main_keys = self.data.keys()
-        self.__len = len(main_keys)
-
-        assert self.__len > 0, "Cannot process an empty file"
-
-        embeds = self[0].embeddings
-        self.embedding_dim = embeds.shape[-1]
-        self.n_layers = embeds.shape[0] - 1  # 1 was added for the input layer
-        self.refmap, self.total_vectors = self._init_vector_map()
-        
-    def __del__(self):
-        try: self.data.close()
-
-        # If run as a script, won't be able to close because of an import error
-        except ImportError: pass
-
-        except AttributeError:
-            print(f"Never successfully loaded {self.fname}")
-        
-    def __iter__(self):
-        return self
-    
-    def __len__(self):
-        return self.__len
-    
-    def __next__(self):
-        if self.__curr >= self.__len:
-            self.__curr = 0
-            raise StopIteration
-            
-        out = self[self.__curr]
-        self.__curr += 1
-        return out
-    
-    def __getitem__(self, idx):
-        """Index into the embeddings"""
-        if isinstance(idx, slice):
-            
-            start = idx.start or 0
-            step = idx.step or 1
-            stop = idx.stop or (self.__len - 1)
-            stop = min(stop, self.__len)
-            
-            i = start
-            out = []
-            while i < stop:
-                out.append(self[i])
-                i += step
-            
-            return out
-        
-        elif isinstance(idx, int):
-            if idx < 0: i = self.__len + idx
-            else: i = idx
-
-            key = to_idx(i)
-            return SentenceH5Data(self.data[key])
-        
-        else:
-            raise NotImplementedError
-
-    def __repr__(self):
-        return f"{self.__name}: containing {self.__len} items"
-    
-    def _init_vector_map(self):
-        """Create main hashmap for all vectors to get their metadata.
-        
-        TODO Initialization is a little slow... Should this be stored in a separate hdf5 file?
-        
-        This doesn't change. Check for special hdf5 file and see if it exists already. If it does, open it. 
-        If not, create it
-        """
-        refmap = {}
-        print("Initializing reference map for embedding vector...")
-        n_vec = 0
-        for z, sentence in enumerate(self):
-            for i in range(len(sentence)):
-                refs = TokenH5Data(sentence, i)
-                refmap[n_vec] = refs
-                n_vec += 1
-        
-        return refmap, n_vec
-    
-    def extract(self, layer):
-        """Extract embeddings from a particular layer from the dataset
-        
-        For all examples
-        """
-        embeddings = []
-        for i, embeds in enumerate(self):
-            embeddings.append(embeds[layer])
-            
-        out = np.vstack(embeddings)
-        return out
-
-    def find(self, vec_num):
-        """Find a vector's metadata (by id) in the hdf5 file. Needed to find sentence info and other attr"""
-        return self.refmap[vec_num]
-    
-    def find2d(self, idxs):
-        """Find a vector's metadata in the hdf5 file. Needed to find sentence info and other attr"""
-        out = [[self.refmap[i] for i in idx] for idx in idxs]
-        return out

server/data_processing/create_corpus.py

@@ -1,28 +0,0 @@
-import argparse
-from pathlib import Path
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-f', '--file', help="Path to .txt file to analyze and annotate")
-    parser.add_argument("-o", "--outdir", help="Path of output directory inside of which to place <model>/<corpus>/ directory containing hdf5 and faiss files")
-    parser.add_argument("-n", "--name", default=None, help="Name the corpus with a code name. If not given, default to the name of the provided .txt file")
-    parser.add_argument("--force", action="store_true", help="If given, overwrite existing hdf5 and faiss files.")
-    parser.add_argument("-m", "--model", help="Specify the huggingface model to use for attentions")
-    parser.add_argument("--nomask", action='store_false', help="INCLUDE attentions from special tokens like [CLS] and [SEP]. By default, ignore these attentions")
-
-    return parser.parse_args()
-
-if __name__ == "__main__":
-    from utils.f import ifnone
-    import create_hdf5
-    import create_faiss
-
-    args = parse_args()
-
-    f = Path(args.file)
-    corpus_name = ifnone(args.name, f.stem)
-    output_dir = Path(args.outdir) / args.model / corpus_name
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    create_hdf5.main(args.file, output_dir, args.force, args.model, args.nomask)
-    create_faiss.main(output_dir)

server/data_processing/create_faiss.py

@@ -1,78 +0,0 @@
-from pathlib import Path
-import faiss
-import numpy as np
-from data_processing.corpus_data_wrapper import CorpusDataWrapper
-from data_processing.index_wrapper import LAYER_TEMPLATE
-import argparse
-
-# Get model from base_dir
-# Use that information to get the model's configuration
-# From this, get the special tokens associated with that model
-# Have flag to allow model's special tokens to be ignored
-# Test what items match 'bert-base-cased'
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-d", "--directory", help="Path to the directory that contains the 'embeddings' and 'headContext' folders")
-    
-    args = parser.parse_args()
-    return args
-
-def train_indexes(ce:CorpusDataWrapper, stepsize=100, drop_null=True):
-    """
-
-    Parameters:
-    ===========
-    - corpus_embedding: Wrapper around HDF5 file for easy access to data
-    - stepsize: How many sentences to train with at once
-    - drop_null: Don't index the embeddings of special tokens (e.g., [CLS] and [SEP]) whose spacy POS are null
-    """
-    NUM_LAYERS = ce.n_layers # want to account for the input layer, which for attentions + contexts is all value 0
-    
-    embedding_indexes = [faiss.IndexFlatIP(ce.embedding_dim) for i in range(NUM_LAYERS)]
-    context_indexes = [faiss.IndexFlatIP(ce.embedding_dim) for i in range(NUM_LAYERS)]
-
-    for ix in range(0, len(ce), stepsize):
-        cdata = ce[ix:ix+stepsize]
-
-        if drop_null: 
-            embeddings = np.concatenate([c.zero_special_embeddings for c in cdata], axis=1)
-            contexts = np.concatenate([c.zero_special_contexts for c in cdata], axis=1)
-        else:
-            embeddings = np.concatenate([c.embeddings for c in cdata], axis=1)
-            contexts = np.concatenate([c.contexts for c in cdata], axis=1)
-
-        for i in range(NUM_LAYERS):
-            embedding_indexes[i].add(embeddings[i])
-            context_indexes[i].add(contexts[i])
-            
-    return embedding_indexes, context_indexes
-
-def save_indexes(idxs, outdir, base_name=LAYER_TEMPLATE):
-    """Save the faiss index into a file for each index in idxs"""
-
-    base_dir = Path(outdir)
-    if not base_dir.exists(): base_dir.mkdir(exist_ok=True, parents=True)
-
-    out_name = str(base_dir / base_name)
-    for i, idx in enumerate(idxs):
-        name = out_name.format(i)
-        print(f"Saving to {name}")
-        faiss.write_index(idx, name)
-
-def main(basedir):
-    base = Path(basedir)
-    h5_fname = base / 'data.hdf5'
-    corpus = CorpusDataWrapper(h5_fname)
-    embedding_faiss, context_faiss = train_indexes(corpus)
-
-    context_faiss_dir = base / "context_faiss"
-    embedding_faiss_dir = base / "embedding_faiss"
-    save_indexes(embedding_faiss, embedding_faiss_dir)
-    save_indexes(context_faiss, context_faiss_dir)
-
-if __name__ == "__main__":
-    # Creating the indices for both the context and embeddings
-    args = parse_args()
-
-    main(args.directory)

server/data_processing/create_hdf5.py

@@ -1,71 +0,0 @@
-import numpy as np
-import torch
-import h5py
-import pickle
-import argparse
-from pathlib import Path
-
-from data_processing.sentence_extracting import extract_chars, extract_lines
-from data_processing.corpus_data_wrapper import CorpusDataWrapper, to_idx
-from transformer_details import from_pretrained
-
-MIN_SENTENCE_CHARLEN = 24
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-f", "--file", help="Path to .pckl file of unique sentences from a corpus.")
-    parser.add_argument("-o", "--outdir", help="Path of directory in which to store the analyzed sentences as a .hdf5")
-    parser.add_argument("-m", "--model", default="bert-base-cased", help="Which pretrained transformer model to use. See 'transformer_details.py' for supported models")
-    parser.add_argument("--nomask", action='store_false', help="By default, ignore attentions to special tokens like '[CLS]' and '[SEP]'. If given, include these attentions")
-    parser.add_argument("--force", action="store_true", help="If given, overwrite existing hdf5 files.")
-
-    args = parser.parse_args()
-    return args
-
-def main(infile, outdir, force, model_name, mask_attentions):
-    outdir = Path(outdir)
-    outdir.mkdir(parents=True, exist_ok=True)
-    data_outfile = outdir / "data.hdf5"
-    f = h5py.File(data_outfile, 'a')
-    if force: f.clear()
-
-    extractor = from_pretrained(model_name)
-
-    # if "gpt" in model_name:
-    #     mask_attentions = False
-
-    print_every = 50
-    long_strings = extract_chars(infile, 10000)
-    cutoff_sent = ""
-    i = 0
-    for strip in long_strings:
-        sentences = [sent.text for sent in extractor.aligner.spacy_nlp(strip).sents]
-        fixed_sentences = [cutoff_sent + sentences[0]] + sentences[1:-1]
-
-        # This leads to the possibility that there will be an input that is two sentences long. This is ok.
-        cutoff_sent = sentences[-1]
-        for s in fixed_sentences:
-            if len(s) < MIN_SENTENCE_CHARLEN: continue
-            if ((i + 1) % print_every) == 0: print(f"Starting sentence {i+1}: \n", s)
-
-            try:
-                out = extractor.att_from_sentence(s, mask_attentions=mask_attentions)
-
-            except Exception as e:
-                print(f"Error {e} occured at sentence {i}:\n{s}\n\n Skipping, not creating hdf5 grp")
-                continue
-
-            content = out.to_hdf5_content()
-            meta = out.to_hdf5_meta()
-            grp = f.create_group(to_idx(i))
-            for k,v in content.items(): grp.create_dataset(k, data=v)
-            for k, v in meta.items(): grp.attrs[k] = v
-
-            i += 1 # Increment to mark the next sentence
-
-    print("FINISHED CORPUS PROCESSING SUCCESSFULLY")
-
-if __name__ == "__main__":
-    args = parse_args()
-
-    main(args.file, args.outdir, args.force, args.model, args.nomask)

server/data_processing/index_wrapper.py

@@ -1,88 +0,0 @@
-from functools import partial
-import faiss
-import numpy as np
-from pathlib import Path
-from typing import Iterable
-from utils.f import memoize
-from transformers import AutoConfig
-
-@memoize
-def get_config(model_name):
-    return AutoConfig.from_pretrained(model_name)
-
-FAISS_LAYER_PATTERN = 'layer_*.faiss'
-LAYER_TEMPLATE = 'layer_{:02d}.faiss' 
-
-def create_mask(head_size:int , n_heads:int, selected_heads:Iterable[int]):
-    """Create a masked vector of size (head_size * n_heads), where 0 indicates we don't care about the contribution of that head 1 indicates that we do care
-    
-    Parameters:
-    -----------
-        head_size: Hidden dimension of the heads
-        n_heads: Number of heads the model has
-        selected_heads: Which heads we don't want to zero out
-    """
-
-    mask = np.zeros(n_heads)
-    for h in selected_heads:
-        mask[int(h)] = 1
-        
-    return np.repeat(mask, head_size)
-
-class Indexes:
-    """Wrapper around the faiss indices to make searching for a vector simpler and faster.
-    
-    Assumes there are files in the folder matching the pattern input
-    """
-    def __init__(self, folder, pattern=FAISS_LAYER_PATTERN):
-        self.base_dir = Path(folder)
-        self.n_layers = len(list(self.base_dir.glob(pattern))) - 1 # Subtract final output
-        self.indexes = [None] * (self.n_layers + 1) # Initialize empty list, adding 1 for input
-        self.pattern = pattern
-        self.__init_indexes()
-
-        # Extract model name from folder hierarchy
-        self.model_name = self.base_dir.parent.parent.stem
-        self.config = get_config(self.model_name)
-        self.nheads = self.config.num_attention_heads
-        self.hidden_size = self.config.hidden_size
-        assert (self.hidden_size % self.nheads) == 0, "Number of heads does not divide cleanly into the hidden size. Aborting"
-        self.head_size = int(self.config.hidden_size / self.nheads)
-
-        
-    def __getitem__(self, v):
-        """Slices not allowed, but index only"""
-        return self.indexes[v]
-    
-    def __init_indexes(self):
-        for fname in self.base_dir.glob(self.pattern):
-            print(fname)
-            idx = fname.stem.split('_')[-1]
-            self.indexes[int(idx)] = faiss.read_index(str(fname))
-
-    def search(self, layer, query, k):
-        """Search a given layer for the query vector. Return k results"""
-        return self[layer].search(query, k)
-
-
-class ContextIndexes(Indexes):
-    """Special index enabling masking of particular heads before searching"""
-
-    def __init__(self, folder, pattern=FAISS_LAYER_PATTERN):
-        super().__init__(folder, pattern)
-
-        self.head_mask = partial(create_mask, self.head_size, self.nheads)
-
-    # Int -> [Int] -> np.Array -> Int -> (np.Array(),  )
-    def search(self, layer:int, heads:list, query:np.ndarray, k:int):
-        """Search the embeddings for the context layer, masking by selected heads"""
-        assert max(heads) < self.nheads, "max of selected heads must be lest than nheads. Are you indexing by 1 instead of 0?"
-        assert min(heads) >= 0, "What is a negative head?"
-        
-        unique_heads = list(set(heads))
-        mask_vector = self.head_mask(unique_heads)
-        mask_vector = mask_vector.reshape(query.shape)
-
-        new_query = (query * mask_vector).astype(np.float32)
-
-        return self[layer].search(new_query, k)

server/data_processing/sentence_data_wrapper.py

@@ -1,331 +0,0 @@
-import h5py
-import numpy as np
-from functools import partial
-from utils.gen_utils import map_nlist, vround
-import regex as re
-from spacyface.simple_spacy_token import SimpleSpacyToken
-
-ZERO_BUFFER = 12 # Number of decimal places each index takes
-main_key = r"{:0" + str(ZERO_BUFFER) + r"}"
-suppl_attn_key = r"{:0" + str(ZERO_BUFFER) + r"}_attn"
-
-def zip_len_check(*iters):
-    """Zip iterables with a check that they are all the same length"""
-    if len(iters) < 2:
-        raise ValueError(f"Expected at least 2 iterables to combine. Got {len(iters)} iterables")
-    n = len(iters[0])
-    for i in iters:
-        n_ = len(i)
-        if n_ != n:
-            raise ValueError(f"Expected all iterations to have len {n} but found {n_}")
-
-    return zip(*iters)
-
-class SentenceH5Data:
-    def __init__(self, grp):
-        self.grp = grp
-
-    @property
-    def n_layers(self):
-        return self.embeddings.shape[0] - 1 # 1 was added at the input, not a hidden layer
-        
-    @property
-    def sentence(self):
-        return self.grp.attrs['sentence']
-
-    @property
-    def embeddings(self):
-        return self.grp['embeddings'][:]
-
-    @property
-    def zero_special_embeddings(self):
-        out = self.embeddings.copy()
-        out[:, self.mask_is_special] = np.zeros(out[:, self.mask_is_special].shape)
-        return out
-    
-    @property
-    def contexts(self):
-        return self.grp['contexts'][:]
-    
-    @property
-    def zero_special_contexts(self):
-        out = self.contexts.copy()
-        out[:, self.mask_is_special] = np.zeros(out[:, self.mask_is_special].shape)
-        return out
-
-    @property
-    def attentions(self):
-        """Return all attentions, including [CLS] and [SEP]
-        
-        Note that if the hdf5 is created with CLS and SEP attentions, it will have CLS and SEP attentions"""
-        return self.grp['attentions'][:] # Converts to numpy array
-
-    @property
-    def mask_is_special(self):
-        return np.logical_or(self.deps == '', self.poss == '')
-
-    @property
-    def tokens(self):
-        return self.grp.attrs['token']
-
-    @property
-    def poss(self):
-        return self.grp.attrs['pos']
-
-    @property
-    def deps(self):
-        return self.grp.attrs['dep']
-
-    @property
-    def is_ents(self):
-        return self.grp.attrs['is_ent']
-    
-    @property
-    def heads(self):
-        """Not the attention heads, but rather the head word of the orig sentence"""
-        return self.grp.attrs['head']
-    
-    @property
-    def norms(self):
-        return self.grp.attrs['norm']
-    
-    @property
-    def tags(self):
-        return self.grp.attrs['tag']
-    
-    @property
-    def lemmas(self):
-        return self.grp.attrs['lemma']
-
-    def __len__(self):
-        return len(self.tokens)
-
-    def __repr__(self):
-        sent_len = 40
-        if len(self.sentence) > sent_len: s = self.sentence[:(sent_len - 3)] + '...'
-        else: s = self.sentence
-        return f"SentenceH5Data({s})"
-
-class TokenH5Data(SentenceH5Data):
-    """A wrapper around the HDF5 file storage information allowing easy access to information about each 
-    processed sentence.
-
-    Sometimes, and index of -1 is used to represent the entire object in memory
-    """
-    def __init__(self, grp, index):
-        """Represents returned from the refmap of the CorpusEmbedding class"""
-        if type(grp) == SentenceH5Data: super().__init__(grp.grp)
-        elif type(grp) == h5py._hl.group.Group: super().__init__(grp)
-        self.index = index
-
-    @property
-    def embedding(self):
-        return self.embeddings[:, self.index, :]
-    
-    @property
-    def context(self):
-        return self.contexts[:, self.index, :]
-
-    @property
-    def attentions_out(self):
-        """Access all attention OUT of this token"""
-        output = self.attentions[:,:, self.index, :]
-        return output
-
-    @property
-    def attentions_in(self):
-        """Access all attention INTO this token"""
-        new_attention = self.attentions.transpose((0,1,3,2))
-        return new_attention[:,:, self.index, :]
-
-    def _select_from_attention(self, layer, heads):
-        if type(heads) is int:
-            heads = [heads]
-
-        # Select layer and heads
-        modified_attentions = self.attentions[layer, heads].mean(0)
-        attentions_out = modified_attentions
-        attentions_in = modified_attentions.transpose()
-        return attentions_out, attentions_in
-
-    def _calc_offset_single(self, attention):
-        """Get offset to location of max attention"""
-        curr_idx = self.index
-        max_atts = np.argmax(attention)
-        return max_atts - curr_idx
-
-    # Define metadata properties. 
-    # Right now, needs manual curation of fields from SimpleSpacyToken. Ideally, this is automated
-    
-    @property
-    def token(self):
-        return self.tokens[self.index]
-
-    @property
-    def pos(self):
-        return self.poss[self.index]
-
-    @property
-    def dep(self):
-        return self.deps[self.index]
-    
-    @property
-    def is_ent(self):
-        return bool(self.is_ents[self.index])
-
-    @property
-    def norm(self):
-        return self.norms[self.index]
-    
-    @property
-    def head(self):
-        return self.heads[self.index]
-    
-    @property
-    def lemma(self):
-        return self.lemmas[self.index]
-    
-    @property
-    def tag(self):
-        return self.tags[self.index]
-
-    def to_json(self, layer, heads, top_k=5, ndigits=4):
-        """
-        Convert token information and attention to return to frontend
-        
-        Require layer, heads, and top_k to convert the attention into value to return to frontend.
-        
-        Output:
-            {
-                sentence: str
-                index: number
-                match: str
-                is_match: bool
-                is_next_word: bool
-                matched_att: {
-                    in: { att: number[]
-                        , offset_to_max: number
-                        , loc_of_max: float 
-                        }
-                    out: { att: number[]
-                        , offset_to_max: number
-                        , loc_of_max: float 
-                        }
-                },
-                matched_att_plus_1: {
-                    in: { att: number[]
-                        , offset_to_max: number
-                        }
-                    out: { att: number[]
-                        , offset_to_max: number
-                        }
-                }
-                tokens: List[
-                    { token: string
-                    , pos: string
-                    , dep: string
-                    , is_ent: boolean
-                    , inward: number[]
-                    , outward: number[]
-                    }
-                ]
-            }
-        """
-        keys = [
-            "token",
-            "pos",
-            "dep",
-            "is_ent",
-            "inward",
-            "outward",
-        ]
-
-        token_arr = []
-        matched_attentions = {}
-        N = len(self)
-
-        # Iterate through the following
-        tokens = self.tokens.tolist()
-        poss = [p.lower() for p in self.poss.tolist()]
-        deps = [d.lower() for d in self.deps.tolist()]
-        ents = self.is_ents.tolist()
-        attentions_out, attentions_in = self._select_from_attention(layer, heads)
-
-        matched_att_plus_1 = None
-        next_index = None
-
-        for i, tok_info in enumerate(zip_len_check( 
-            tokens
-            , poss
-            , deps
-            , ents
-            , attentions_out.tolist()
-            , attentions_in.tolist())):
-
-            def get_interesting_attentions():
-                return {
-                    "in": {
-                        "att": att_in,
-                        "offset_to_max": self._calc_offset_single(att_in).item(),
-                        # "loc_of_max": np.argmax(att_in), # Broken
-                    },
-                    "out": {
-                        "att": att_out,
-                        "offset_to_max": self._calc_offset_single(att_out).item(),
-                        # "loc_of_max": np.argmax(att_out), # Broken
-                    }
-                }
-
-
-            # Perform rounding of attentions
-            rounder = partial(round, ndigits=ndigits)
-            att_out = map_nlist(rounder, tok_info[-2])
-            att_in = map_nlist(rounder, tok_info[-1])
-
-            obj = {k: v for (k, v) in zip_len_check(keys, tok_info)}
-
-            IS_LAST_TOKEN = i == (N-1)
-
-            if (i == self.index) or ((i - 1) == self.index):
-                interesting_attentions = get_interesting_attentions()
-
-                if i == self.index:
-                    obj['is_match'] = True
-                    matched_attentions = interesting_attentions
-
-                elif (i-1) == self.index:
-                    matched_att_plus_1 = interesting_attentions
-                    obj['is_next_word'] = True
-                    next_index = i
-
-                # Edge case for final iteration through sentence
-
-            else:
-                obj['is_match'] = False
-                obj['is_next_word'] = False
-
-            if (IS_LAST_TOKEN and (matched_att_plus_1 is None)):
-                print("Saving matched_att_plus_1 to: ", interesting_attentions)
-                obj['is_next_word'] = True
-                matched_att_plus_1 = get_interesting_attentions()
-                next_index = i
-            
-            token_arr.append(obj) 
-
-        next_token = self.tokens[next_index]
-
-        obj = {
-            "sentence": self.sentence,
-            "index": self.index,
-            "match": self.token,
-            "next_index": next_index,
-            "match_plus_1": next_token,
-            "matched_att": matched_attentions,
-            "matched_att_plus_1": matched_att_plus_1,
-            "tokens": token_arr,
-        }
-
-        return obj
-    
-    def __repr__(self):
-        return f"{self.token}: [{self.pos}, {self.dep}, {self.is_ent}]"

server/data_processing/sentence_extracting.py

@@ -1,181 +0,0 @@
-"""Extractor functions to retrieve sentences by character chunks from a file
-
-This script contains the logic that allows the user to process and filter
-sentences of the original corpus. By default, this considers a minimum sentence
-length, and removes newlines and multiple consecutive spaces.
-
-Configuration for existing functionality is at the top of the file. Feel free to
-add new processing and/or filter functions. The "process_line" and "filter_line"
-functions contain the pipeline for processing the scripts as needed.
-
-"""
-import regex as re
-import argparse
-from pathlib import Path
-from functools import partial
-from typing import Union
-
-MIN_LINE_LENGTH = 8 # words
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-f", "--file", help="Path to .txt file to analyze and annotate")
-    parser.add_argument("-o", "--outdir", help="Path of directory in which to store the analyzed sentences as a .pckl")
-
-
-    args = parser.parse_args()
-    return args
-
-# ============================================================
-#                  Helper functions
-# ============================================================
-# String -> String
-def replace_newlines(s:str) -> str:
-    return re.sub(r"\n+", r" ", s)
-
-# String -> String
-def replace_multispace(s:str) -> str:
-    return re.sub(r"\s+", r" ", s)
-
-def is_short_sentence(s:str, min_len=8) -> str:
-    """Returns True if the sentence has less than `min_len` number of words"""
-    return len(s.split(' ')) < min_len
-
-def contains_char(char:str, s:str) -> str:
-    return char in s
-
-# ============================================================
-#                  Compilation functions
-# ============================================================
-
-def process_line(line:str) -> str:
-    """"Replaces newlines with spaces and removes multiple consecutive spaces from a chunk of file.
-
-    Args:
-        line: Chunk of text
-
-    Returns:
-        Input that has been stripped of newlines and multiple consecutive spaces.
-    """
-    s = replace_multispace(replace_newlines(line))
-    return s
-
-def filter_line(line:str) -> bool:
-    """Returns True if the sentence passes the MIN_LINE_LENGTH configuration
-
-    Redefine this function with desired helper functions, returning true if you want to keep the line
-    """
-    fails = is_short_sentence(line, MIN_LINE_LENGTH)
-
-    return not fails
-
-# ============================================================
-#                      Main Logic
-# ============================================================
-
-def read_outcomes(chars:str) -> Union[str, None]:
-    """From a chunk of characters, decide whether to return the processed characters or Nothing.
-
-    If the input is the empty string "", raise StopIteration
-
-    Args:
-        chars: Chunk of text to process
-
-    Returns:
-        The processed chunk of text or nothing if the characters do not pass the filtering
-
-    Raises:
-        StopIteration: If the input is the empty string "", raise StopIteration
-    """
-
-    if chars == '': raise StopIteration
-    line = process_line(chars)
-    if filter_line(line): return line
-    return None
-
-def get_chars(n:int, f) -> Union[str, None]:
-    """Extract `n` chars from opened file `f`
-
-    Args:
-        n: Number of characters to read from the opened file
-        f: Opened file from the return of `open(fname)`
-
-    Returns:
-        The processed chunk of text or nothing if the characters do not pass the filtering
-
-    Raises:
-        This function does not raise any errors of its own, but can pass up the StopIteration exception
-          from read_outcomes
-    """
-    chars = f.read(n)
-    return read_outcomes(chars)
-
-def get_line(f):
-    """Given an open file, get the next line and process it. Handles 3 scenarios:
-
-    1. StopIteration indicates the opened file has reached the end
-    2. Return a processed line if it passes the filter
-    3. If line does not pass the filter line, return None
-    """
-    line = f.readline()
-    return read_outcomes(line)
-
-def read_on(reader, f):
-    """Read from an open file `f` according to the function `reader`
-
-    Args:
-        reader: A unary function of signature (f: _io.TextIOWrapper) -> str
-        f: An opened file, as returned by `open(fname)`
-
-    Yields:
-        A generator that returns lines defined by `reader` until the end of the file is reached.
-    """
-    while True:
-        try:
-            line = reader(f)
-        except StopIteration:
-            break
-
-        if line is not None:
-            yield line
-
-
-def extract_chars(infile, n=10000):
-    """Extract `n` characters from a file"""
-    reader = partial(get_chars, n)
-    src = open(infile, 'r')
-    return read_on(reader, src)
-    src.close()
-
-
-def extract_lines(infile):
-    """Given a file, yield the processed lines from that file"""
-    src = open(infile, 'r')
-    return read_on(get_line, src)
-    src.close()
-
-
-def extract_sentences_to_file(infile, outfname:str):
-    """Extract sentences from a file into a new file indicated by `outfname`."""
-    out = open(outfname, 'x')
-
-    linegen = extract_lines(infile)
-
-    for line in linegen:
-        out.write(line + "\n")
-
-    out.close()
-
-def main(infile, outdir):
-    """Main function for creating the outdir and saving the processed sentences to that file"""
-    outfname = Path(infile).stem + '.txt'
-    outdir = Path(outdir)
-    outdir.mkdir(parents=True, exist_ok=True)
-    outfile = outdir / outfname
-    out_path = extract_sentences_to_file(infile, outfile)
-
-    return out_path
-
-if __name__ == "__main__":
-    args = parse_args()
-    main(args.file, args.outdir)

server/main.py

@@ -5,7 +5,6 @@ from flask_cors import CORS
 from flask import render_template, redirect, send_from_directory
 
 import utils.path_fixes as pf
-import config
 from utils.f import ifnone
 
 from data_processing import from_model

server/utils/path_fixes.py

@@ -5,6 +5,7 @@ FAISS_LAYER_PATTERN = 'layer_*.faiss'
 LAYER_TEMPLATE = 'layer_{:02d}.faiss' 
 
 ROOT_DIR = Path(os.path.abspath(__file__)).parent.parent.parent
+CORPORA = ROOT / "corpora"
 DATA_DIR = ROOT_DIR / 'server' / 'data'
 DATASET_DIR = Path.home() / 'Datasets'
 ROOT_DIR = Path(os.path.abspath(__file__)).parent.parent.parent

server/utils/token_processing.py

@@ -5,7 +5,6 @@ If adding more metadata, modify the definitions in `to_spacy_meta` and `meta_to_
 import h5py
 import numpy as np
 import spacy
-import config
 from transformers.tokenization_bert import BertTokenizer
 from .f import flatten_, assoc, memoize, GetAttr