QBGPT V1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,79 @@
+import streamlit as st
+from pages import set_app_title_and_logo, qb_gpt_page, contacts_and_disclaimers
+import json
+import pandas as pd
+import numpy as np
+import os
+from tools import tokenizer
+
+from assets.models import QBGPT
+
+moves_to_pred = 11170
+input_size = 11172
+starts_size = 1954
+scrimmage_size = 100
+positions_id = 29
+
+temp_ids = 52
+off_def_size = 2
+token_type_size = 3
+play_type_size = 9
+
+qbgpt = QBGPT(input_vocab_size = input_size,
+                    positional_vocab_size = temp_ids,
+                    position_vocab_size=positions_id,
+                    start_vocab_size=starts_size,
+                    scrimmage_vocab_size=scrimmage_size,
+                    offdef_vocab_size = off_def_size,
+                    type_vocab_size = token_type_size,
+                    playtype_vocab_size = play_type_size,
+                    embedding_dim = 256,
+                    hidden_dim = 256,
+                    num_heads = 3,
+                    diag_masks = False,
+                    to_pred_size = moves_to_pred)
+
+qbgpt.load_weights("app/assets/model_mediumv2/QBGPT")
+
+
+qb_tok = tokenizer(moves_index="./app/assets/moves_index.parquet",
+                   play_index="./app/assets/plays_index.parquet",
+                   positions_index="./app/assets/positions_index.parquet",
+                   scrimmage_index="./app/assets/scrimmage_index.parquet",
+                   starts_index="./app/assets/starts_index.parquet",
+                   time_index="./app/assets/time_index.parquet",
+                   window_size=20)
+
+print(os.listdir("app"))
+
+with open('./app/assets/ref.json', 'r') as fp:
+    ref_json = json.load(fp)
+    
+def convert_numpy(d):
+    return {k:np.array(v) for k,v in d.items()}
+
+ref_json = {int(k):convert_numpy(v) for k,v in ref_json.items()}
+
+ref_df = pd.read_json("./app/assets/ref_df.json")
+
+
+
+# Define the main function to run the app
+def main():
+    set_app_title_and_logo()
+    
+    # Create a sidebar for navigation
+    st.sidebar.title("Navigation")
+    page = st.sidebar.radio("Go to:", ("QB-GPT", "Contacts and Disclaimers"))
+
+    if page == "QB-GPT":
+        # Page 2: QB-GPT
+        st.title("QB-GPT")
+        qb_gpt_page(ref_df, ref_json, qb_tok, qbgpt)
+        
+    if page == "Contacts and Disclaimers":
+        contacts_and_disclaimers()
+        
+        
+if __name__ == "__main__":
+    main()

assets/index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce0a2888c66b821c990837d1ea76061923d9ab85bde410aa2291f426c63317eb
+size 29240

assets/model_mediumv2/QBGPT.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a725719eb71e137bfc5f126af82012085830f95a2992374e196cf3d7a054fc6
+size 86410198

assets/model_mediumv2/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "QBGPT"
+all_model_checkpoint_paths: "QBGPT"

assets/models.py

@@ -0,0 +1,377 @@
+import tensorflow as tf
+from typing import List, Optional, Union
+import numpy as np
+
+def shape_list(tensor: Union[tf.Tensor, np.ndarray]) -> List[int]:
+    """
+    Deal with dynamic shape in tensorflow cleanly.
+
+    Args:
+        tensor (`tf.Tensor` or `np.ndarray`): The tensor we want the shape of.
+
+    Returns:
+        `List[int]`: The shape of the tensor as a list.
+    """
+    if isinstance(tensor, np.ndarray):
+        return list(tensor.shape)
+
+    dynamic = tf.shape(tensor)
+
+    if tensor.shape == tf.TensorShape(None):
+        return dynamic
+
+    static = tensor.shape.as_list()
+
+    return [dynamic[i] if s is None else s for i, s in enumerate(static)]
+
+class PlayTypeEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(PlayTypeEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["PlayType"])
+    return embed
+
+class PositionEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(PositionEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["position_ids"])
+    return embed
+
+class ScrimmageEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(ScrimmageEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["scrim_ids"])
+    return embed
+
+class StartEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(StartEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["start_ids"])
+    return embed
+
+class OffDefEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(OffDefEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["OffDef"])
+    return embed
+
+class TypeEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(TypeEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["token_type_ids"])
+    return embed
+
+class PositionalEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(PositionalEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["pos_ids"])
+    return embed
+
+class InputEncoder(tf.keras.Model):
+  def __init__(self, vocab_size : int, embedding_dim : int):
+        super(InputEncoder, self).__init__()
+
+        self.Embedding = tf.keras.layers.Embedding(input_dim = vocab_size,
+                                                   output_dim = embedding_dim)
+
+  def call(self, x):
+    embed = self.Embedding(x["input_ids"])
+    return embed
+
+class Embedding(tf.keras.Model):
+  def __init__(self,
+               input_vocab_size : int,
+               positional_vocab_size : int,
+               position_vocab_size : int,
+               scrimmage_vocab_size : int,
+               start_vocab_size: int,
+               offdef_vocab_size : int,
+               type_vocab_size : int,
+               playtype_vocab_size : int,
+               embedding_dim : int):
+        super(Embedding, self).__init__()
+
+        self.InputEmbedding = InputEncoder(vocab_size=input_vocab_size,
+                                           embedding_dim=embedding_dim)
+        self.PositionalEmbedding = PositionalEncoder(vocab_size=positional_vocab_size,
+                                                     embedding_dim=embedding_dim)
+        self.PositionEmbedding = PositionEncoder(vocab_size=position_vocab_size,
+                                                     embedding_dim=embedding_dim)
+        self.ScrimEmbedding = ScrimmageEncoder(vocab_size=scrimmage_vocab_size,
+                                                     embedding_dim=embedding_dim)
+        self.StartEmbedding = StartEncoder(vocab_size=start_vocab_size,
+                                                     embedding_dim=embedding_dim)
+        self.OffDefEmbedding = OffDefEncoder(vocab_size=offdef_vocab_size,
+                                             embedding_dim=embedding_dim)
+        self.TypeEmbedding = TypeEncoder(vocab_size=type_vocab_size,
+                                             embedding_dim=embedding_dim)
+        self.PlayTypeEmbedding = PlayTypeEncoder(vocab_size=playtype_vocab_size,
+                                                 embedding_dim=embedding_dim)
+        self.Add = tf.keras.layers.Add()
+
+  def call(self, x):
+    input_embed = self.InputEmbedding(x)
+    positional_embed = self.PositionalEmbedding(x)
+    position_embed = self.PositionEmbedding(x)
+    scrim_embed = self.ScrimEmbedding(x)
+    start_embed = self.StartEmbedding(x)
+    type_embed = self.TypeEmbedding(x)
+    offdef_embed = self.OffDefEmbedding(x)
+    playtype_embed = self.PlayTypeEmbedding(x)
+
+    embed = self.Add([input_embed,
+                      positional_embed,
+                      position_embed,
+                      scrim_embed,
+                      start_embed,
+                      type_embed,
+                      offdef_embed,
+                      playtype_embed])
+
+    return embed
+
+class Transformers(tf.keras.Model):
+  def __init__(self,
+               num_heads : int,
+               hidden_dim : int,
+               output_dim : int,
+               diag_masks : bool):
+        super(Transformers, self).__init__()
+        
+        self.diag_masks = diag_masks
+        self.num_attention_heads = num_heads
+        self.attention_head_size = hidden_dim
+        self.total_dim = num_heads * hidden_dim
+        self.output_dim = output_dim
+        
+        self.NormIn = tf.keras.layers.LayerNormalization(name = "Norm_in")
+        self.Query = tf.keras.layers.Dense(self.total_dim, name = "Query", use_bias = False)
+        self.Key = tf.keras.layers.Dense(self.total_dim, name = "Key", use_bias = False)
+        self.Value = tf.keras.layers.Dense(self.total_dim, name = "Value", use_bias = False)
+        
+        self.DenseAtt = tf.keras.layers.Dense(hidden_dim, activation = "relu", use_bias = False)
+        
+        self.Add = tf.keras.layers.Add(name = "Add")
+        self.Drop = tf.keras.layers.Dropout(rate = 0.1)
+        
+        self.DenseOut = tf.keras.layers.Dense(output_dim, name = "Dense", activation = "relu")
+        self.NormOut = tf.keras.layers.LayerNormalization(name = "Norm_out")
+
+  def transpose_for_scores(self, tensor: tf.Tensor, batch_size: int) -> tf.Tensor:
+        # Reshape from [batch_size, seq_length, all_head_size] to [batch_size, seq_length, num_attention_heads, attention_head_size]
+        tensor = tf.reshape(tensor=tensor, shape=(batch_size, -1, self.num_attention_heads, self.attention_head_size))
+
+        # Transpose the tensor from [batch_size, seq_length, num_attention_heads, attention_head_size] to [batch_size, num_attention_heads, seq_length, attention_head_size]
+        return tf.transpose(tensor, perm=[0, 2, 1, 3])
+
+  def create_causal_masks(self, temp_ids):
+      # Use broadcasting to create the 2D comparison tensor
+      causal_mask = temp_ids[:, :, tf.newaxis] >= temp_ids[:, tf.newaxis, :]
+      causal_mask = (tf.cast(causal_mask, dtype=tf.float32) - 1) * 1000000
+      reshaped_tensor = tf.expand_dims(causal_mask, axis=1)
+      duplicated_tensor = tf.tile(reshaped_tensor, multiples=[1, self.num_attention_heads, 1, 1])
+      return duplicated_tensor
+    
+  def create_diag_masks(self, hidden_state):
+    dims = shape_list(hidden_state)
+    matrix = tf.linalg.diag(tf.ones((dims[0], dims[1], dims[2]), dtype=tf.float32))
+    return matrix*-1000000
+
+  def create_attention_mask(self, attn_mask):
+    attn_mask = (tf.cast(attn_mask, dtype=tf.float32) -1) * 1000000
+    reshaped_tensor = tf.expand_dims(attn_mask, axis=1)
+    reshaped_tensor = tf.expand_dims(reshaped_tensor, axis=1)
+    duplicated_tensor = tf.tile(reshaped_tensor, multiples=[1, self.num_attention_heads, 1, 1])
+    return duplicated_tensor
+
+  def compute_scaled_attn_scores(self, query, key):
+    attention_scores = tf.matmul(query, key, transpose_b=True)  # Transpose the second sequence
+
+    # If you want scaled dot-product attention, divide by the square root of the embedding dimension
+    embedding_dim = query.shape[-1]
+    scaled_attention_scores = attention_scores / tf.math.sqrt(tf.cast(embedding_dim, dtype=tf.float32))
+
+    return scaled_attention_scores
+
+  def compute_attention_weigths(self, query, key, temp_ids, masks):
+
+    attn_masks = self.create_attention_mask(masks)
+    causal_masks = self.create_causal_masks(temp_ids)
+    scaled_attn_scores = self.compute_scaled_attn_scores(query, key)
+    if self.diag_masks == True:
+      diag_masks = self.create_diag_masks(query)
+      attn_scores = scaled_attn_scores + attn_masks + causal_masks + diag_masks
+    else:
+      attn_scores = scaled_attn_scores + attn_masks + causal_masks
+      
+    return tf.nn.softmax(attn_scores, axis = -1)
+
+  def get_preds_and_attention(self,
+           embeddings,
+           temporal_ids,
+           attention_masks):
+
+    query = self.Query(embeddings)
+    key = self.Key(embeddings)
+    value = self.Value(embeddings)
+
+    attention_weights = self.compute_attention_weigths(query, key, temporal_ids, attention_masks)
+
+    attention_scores = tf.matmul(attention_weights, value)
+    attention_scores = self.Dense(attention_scores)
+
+    output = self.Add([attention_scores, embeddings])
+    output = self.Drop(output)
+    output = self.Norm(output)
+    return output, attention_weights
+
+  def call(self,
+           hidden_states : tf.Tensor,
+           temporal_ids,
+           attention_masks):
+
+    batch_size = shape_list(hidden_states)[0]
+    
+    norm_hidden_states = self.NormIn(hidden_states)
+    
+    query = self.Query(norm_hidden_states)
+    queries = self.transpose_for_scores(query, batch_size)
+
+    key = self.Key(norm_hidden_states)
+    keys = self.transpose_for_scores(key, batch_size)
+
+    value = self.Value(norm_hidden_states)
+    values = self.transpose_for_scores(value, batch_size)
+
+    attention_weights = self.compute_attention_weigths(queries, keys, temporal_ids, attention_masks)
+    attention_scores = tf.matmul(attention_weights, values)
+    attention_scores = tf.transpose(attention_scores, perm=[0, 2, 1, 3])
+    attention_scores = tf.reshape(tensor=attention_scores, shape=(batch_size, -1, self.total_dim))
+    attention_scores = self.DenseAtt(attention_scores)
+    
+    output = self.Add([attention_scores, hidden_states])
+    norm_output = self.NormOut(output)
+    
+    densed_output = self.DenseOut(norm_output)
+    output = self.Add([densed_output, output])
+    output = self.Drop(output)
+    return output
+
+class Encoder(tf.keras.Model):
+  def __init__(self,
+               input_vocab_size : int,
+               positional_vocab_size : int,
+               position_vocab_size : int,
+               scrimmage_vocab_size : int,
+               start_vocab_size: int,
+               offdef_vocab_size : int,
+               type_vocab_size : int,
+               playtype_vocab_size : int,
+               embedding_dim : int,
+               hidden_dim : int,
+               num_heads : int,
+               diag_masks : bool):
+        super(Encoder, self).__init__()
+        
+        self.num_heads = num_heads
+        self.diag_masks = diag_masks
+        self.Embedding = Embedding(input_vocab_size = input_vocab_size,
+                                   positional_vocab_size = positional_vocab_size,
+                                   position_vocab_size = position_vocab_size,
+                                   scrimmage_vocab_size = scrimmage_vocab_size,
+                                   start_vocab_size = start_vocab_size,
+                                   type_vocab_size = type_vocab_size,
+                                   offdef_vocab_size = offdef_vocab_size,
+                                   playtype_vocab_size = playtype_vocab_size,
+                                   embedding_dim = embedding_dim)
+
+        self.Attention1 = Transformers(num_heads = self.num_heads,
+                                       hidden_dim = hidden_dim,
+                                       output_dim = embedding_dim, 
+                                       diag_masks = self.diag_masks)
+
+        self.DenseHead = tf.keras.layers.Dense(embedding_dim, activation = "relu")
+
+  def call(self,
+           x):
+
+    embed = self.Embedding(x)
+    h1 = self.Attention1(embed, x["pos_ids"], x["attention_mask"])
+
+    encoded = self.DenseHead(h1)
+
+    return encoded
+
+
+class QBGPT(tf.keras.Model):
+  def __init__(self,
+               input_vocab_size : int,
+               positional_vocab_size : int,
+               position_vocab_size : int,
+               scrimmage_vocab_size : int,
+               start_vocab_size: int,
+               offdef_vocab_size : int,
+               type_vocab_size : int,
+               playtype_vocab_size : int,
+               embedding_dim : int,
+               hidden_dim : int,
+               num_heads : int,
+               diag_masks : bool,
+               to_pred_size : int):
+        super(QBGPT, self).__init__()
+
+        self.Encoder = Encoder(input_vocab_size = input_vocab_size,
+                               positional_vocab_size = positional_vocab_size,
+                               position_vocab_size = position_vocab_size,
+                               scrimmage_vocab_size = scrimmage_vocab_size,
+                               start_vocab_size = start_vocab_size,
+                               type_vocab_size = type_vocab_size,
+                               offdef_vocab_size = offdef_vocab_size,
+                               playtype_vocab_size = playtype_vocab_size,
+                               embedding_dim = embedding_dim,
+                               hidden_dim = hidden_dim,
+                               num_heads = num_heads,
+                               diag_masks = diag_masks)
+
+        self.Logits = tf.keras.layers.Dense(to_pred_size)
+
+  def call(self, x):
+
+    encoded = self.Encoder(x)
+    logits = self.Logits(encoded)
+
+    return logits

assets/moves_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a42fbda4f9377e74f2d79024f8b8628a9cc462b21a18526b7e7342188a6245e
+size 32456

assets/plays_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9a9697f12ba01faa7a5a395cd6eafe435868b34b5c634322ec67424e24a6fcc
+size 760

assets/positions_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7abdabf96a14a3457287d6147280ab2051d8c65b2875d71a6402f4b880756bfb
+size 1052

assets/ref_df.json

@@ -0,0 +1 @@
+{"gameId":{"0":2019100600,"1":2018122201,"2":2019092600,"3":2019101300,"4":2020092011,"5":2019112801,"6":2018102802,"7":2018120912,"8":2019120109,"9":2019092900,"10":2018122312,"11":2019101311,"12":2019102702,"13":2019120102,"14":2019121507,"15":2019091507,"16":2019112404,"17":2018121605,"18":2019102000,"19":2019122901,"20":2019111002,"21":2018092303,"22":2019120109,"23":2018110404,"24":2018121609,"25":2020101801,"26":2021010302,"27":2020112200,"28":2018102809,"29":2019122909,"30":2019102710,"31":2018110500,"32":2018122314,"33":2020122002,"34":2019122901,"35":2020100401,"36":2019122205,"37":2020102513,"38":2018100706,"39":2018120211,"40":2018112600,"41":2018120200,"42":2019122912,"43":2019102007,"44":2020110500,"45":2019110308,"46":2018112504,"47":2018090903,"48":2018101408,"49":2020092013,"50":2018102110,"51":2019110310,"52":2019111100,"53":2018090900,"54":2019111001,"55":2020121306,"56":2019122102,"57":2021010310,"58":2021010311,"59":2018093009,"60":2020111503,"61":2019111000,"62":2018101404,"63":2019101309,"64":2019120500,"65":2020122012,"66":2018121604,"67":2019092207,"68":2019100600,"69":2019122913,"70":2018112508,"71":2018091601,"72":2020120702,"73":2019100608,"74":2018102808,"75":2019092209,"76":2018112505,"77":2018120909,"78":2018091613,"79":2019110303,"80":2018120204,"81":2019120802,"82":2018102111,"83":2018120902,"84":2018120907,"85":2019100602,"86":2020100501,"87":2020122004,"88":2018101401,"89":2019092211,"90":2018112201,"91":2020121312,"92":2020101200,"93":2020121700,"94":2021010304,"95":2020112204,"96":2018123011,"97":2019111703,"98":2020110900,"99":2020110808,"100":2018102805,"101":2020110802,"102":2019092210,"103":2018120912,"104":2018112510,"105":2019122905,"106":2018100100,"107":2020112202,"108":2020122002,"109":2021010305,"110":2018100706,"111":2019092903,"112":2019122209,"113":2018120300,"114":2019110303,"115":2018101402,"116":2019120106,"117":2020110108,"118":2020101801,"119":2020101812,"120":2018090600,"121":2018123014,"122":2018093004,"123":2021102401,"124":2018122313,"125":2018091607,"126":2018102111,"127":2018121700,"128":2018110405,"129":2018090901,"130":2018121609,"131":2018102900,"132":2018101400,"133":2018110409,"134":2018093009,"135":2018111103,"136":2018111802,"137":2018111809,"138":2021103102,"139":2018121501,"140":2018111101,"141":2021101700,"142":2018102107,"143":2021101707,"144":2021101001,"145":2018123012,"146":2018110404,"147":2018092000,"148":2021091908,"149":2018122309,"150":2018102104,"151":2018110406,"152":2018102109,"153":2021092607,"154":2018111108,"155":2021102100,"156":2018120909,"157":2021091910,"158":2018122313,"159":2018111108,"160":2018091001,"161":2018092000,"162":2018111802,"163":2021103106,"164":2018102801,"165":2018100711,"166":2018091604,"167":2021091203,"168":2018090600,"169":2021092603,"170":2018111111,"171":2018112507,"172":2018101401,"173":2018123013,"174":2021101800,"175":2021101703,"176":2018111111,"177":2021091204,"178":2021102406,"179":2018101100,"180":2021091208,"181":2018101100,"182":2019111001,"183":2021101011,"184":2021103109,"185":2021101009,"186":2021102800,"187":2021101800,"188":2019122210,"189":2021092600,"190":2021101008,"191":2021091209,"192":2018120600,"193":2018112507,"194":2021091910,"195":2021101701,"196":2019122900,"197":2018122313,"198":2021091211,"199":2021091903,"200":2021103104,"201":2021091300,"202":2021091600,"203":2021092700,"204":2021102406,"205":2021091600,"206":2021091300,"207":2021102401,"208":2020111505,"209":2021102401,"210":2021102500,"211":2021091900,"212":2021103100,"213":2021101704,"214":2021102405,"215":2021100308,"216":2021091901,"217":2019111708,"218":2019120108,"219":2021101001,"220":2018123001,"221":2020112601,"222":2021102800,"223":2021102400,"224":2020010400,"225":2019100612,"226":2021092600,"227":2019120200,"228":2021092600,"229":2020010401,"230":2021102405,"231":2021102407,"232":2018111102,"233":2021101703,"234":2021101701,"235":2019111001,"236":2021100400,"237":2021100307,"238":2021092607,"239":2021102100},"playId":{"0":2982,"1":1468,"2":1354,"3":2856,"4":2465,"5":2414,"6":2963,"7":1968,"8":3361,"9":2619,"10":1476,"11":1994,"12":2907,"13":1121,"14":3588,"15":2316,"16":2334,"17":2244,"18":587,"19":4339,"20":4405,"21":2001,"22":3361,"23":1637,"24":2901,"25":3860,"26":2517,"27":752,"28":1593,"29":2913,"30":4343,"31":299,"32":2594,"33":1831,"34":2158,"35":1446,"36":1330,"37":338,"38":3240,"39":3633,"40":2075,"41":3719,"42":2100,"43":366,"44":2860,"45":3363,"46":2578,"47":3690,"48":3602,"49":1482,"50":2823,"51":2104,"52":4387,"53":1226,"54":880,"55":3189,"56":402,"57":812,"58":1148,"59":1087,"60":3389,"61":2781,"62":2709,"63":2091,"64":1389,"65":1290,"66":3032,"67":581,"68":2542,"69":2693,"70":614,"71":618,"72":1864,"73":1899,"74":1114,"75":825,"76":1804,"77":4008,"78":2135,"79":2561,"80":1839,"81":36,"82":1113,"83":1020,"84":3589,"85":441,"86":3336,"87":3817,"88":2558,"89":260,"90":2939,"91":939,"92":4450,"93":2100,"94":3139,"95":2052,"96":3847,"97":1214,"98":1436,"99":1942,"100":4119,"101":770,"102":3615,"103":4107,"104":771,"105":36,"106":3169,"107":960,"108":40,"109":2171,"110":2799,"111":324,"112":3592,"113":1812,"114":36,"115":36,"116":3006,"117":4314,"118":2249,"119":41,"120":1085,"121":555,"122":2577,"123":3528,"124":1547,"125":1021,"126":3193,"127":852,"128":1283,"129":5511,"130":4242,"131":954,"132":2207,"133":1797,"134":3845,"135":1515,"136":4328,"137":325,"138":62,"139":4297,"140":1803,"141":2085,"142":295,"143":2535,"144":2135,"145":1359,"146":3051,"147":3860,"148":3110,"149":400,"150":3904,"151":1592,"152":463,"153":2325,"154":275,"155":1587,"156":566,"157":756,"158":2419,"159":1061,"160":3049,"161":3947,"162":2406,"163":153,"164":3963,"165":3218,"166":508,"167":620,"168":3678,"169":2221,"170":685,"171":2004,"172":3333,"173":2329,"174":3505,"175":3218,"176":1248,"177":1579,"178":1355,"179":2765,"180":3874,"181":2858,"182":1000,"183":54,"184":3119,"185":2668,"186":3007,"187":3998,"188":2421,"189":3942,"190":1905,"191":1295,"192":1238,"193":153,"194":2359,"195":1326,"196":3123,"197":3997,"198":142,"199":1170,"200":3699,"201":148,"202":3392,"203":2565,"204":896,"205":3392,"206":356,"207":893,"208":4156,"209":2098,"210":3030,"211":2784,"212":1624,"213":340,"214":3280,"215":1258,"216":988,"217":3909,"218":3886,"219":3479,"220":3552,"221":3187,"222":3670,"223":3028,"224":122,"225":4026,"226":3544,"227":3214,"228":3639,"229":698,"230":2124,"231":2169,"232":2155,"233":3602,"234":1326,"235":2257,"236":3637,"237":3395,"238":1227,"239":655},"Traj":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0,"8":0,"9":0,"10":0,"11":0,"12":0,"13":0,"14":0,"15":0,"16":0,"17":0,"18":0,"19":0,"20":0,"21":0,"22":0,"23":0,"24":0,"25":0,"26":0,"27":0,"28":0,"29":0,"30":0,"31":0,"32":0,"33":0,"34":0,"35":0,"36":0,"37":0,"38":0,"39":0,"40":0,"41":0,"42":0,"43":0,"44":0,"45":0,"46":0,"47":0,"48":0,"49":0,"50":0,"51":0,"52":0,"53":0,"54":0,"55":0,"56":0,"57":0,"58":0,"59":0,"60":0,"61":0,"62":0,"63":0,"64":0,"65":0,"66":0,"67":0,"68":0,"69":0,"70":0,"71":0,"72":0,"73":0,"74":0,"75":0,"76":0,"77":0,"78":0,"79":0,"80":0,"81":0,"82":0,"83":0,"84":0,"85":0,"86":0,"87":0,"88":0,"89":0,"90":0,"91":0,"92":0,"93":0,"94":0,"95":0,"96":0,"97":0,"98":0,"99":0,"100":0,"101":0,"102":0,"103":0,"104":0,"105":0,"106":0,"107":0,"108":0,"109":0,"110":0,"111":0,"112":0,"113":0,"114":0,"115":0,"116":0,"117":0,"118":0,"119":0,"120":0,"121":0,"122":0,"123":0,"124":0,"125":0,"126":0,"127":0,"128":0,"129":0,"130":0,"131":0,"132":0,"133":0,"134":0,"135":0,"136":0,"137":0,"138":0,"139":0,"140":0,"141":0,"142":0,"143":0,"144":0,"145":0,"146":0,"147":0,"148":0,"149":0,"150":0,"151":0,"152":0,"153":0,"154":0,"155":0,"156":0,"157":0,"158":0,"159":0,"160":0,"161":0,"162":0,"163":0,"164":0,"165":0,"166":0,"167":0,"168":0,"169":0,"170":0,"171":0,"172":0,"173":0,"174":0,"175":0,"176":0,"177":0,"178":0,"179":0,"180":0,"181":0,"182":0,"183":0,"184":0,"185":0,"186":0,"187":0,"188":0,"189":0,"190":0,"191":0,"192":0,"193":0,"194":0,"195":0,"196":0,"197":0,"198":0,"199":0,"200":0,"201":0,"202":0,"203":0,"204":0,"205":0,"206":0,"207":0,"208":0,"209":0,"210":0,"211":0,"212":0,"213":0,"214":0,"215":0,"216":0,"217":0,"218":0,"219":0,"220":0,"221":0,"222":0,"223":0,"224":0,"225":0,"226":0,"227":0,"228":0,"229":0,"230":0,"231":0,"232":0,"233":0,"234":0,"235":0,"236":0,"237":0,"238":0,"239":0},"PlayType":{"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"14":1,"15":1,"16":1,"17":1,"18":1,"19":1,"20":1,"21":1,"22":1,"23":1,"24":1,"25":1,"26":1,"27":1,"28":1,"29":1,"30":1,"31":1,"32":1,"33":1,"34":1,"35":1,"36":1,"37":1,"38":1,"39":1,"40":1,"41":1,"42":1,"43":1,"44":1,"45":1,"46":1,"47":1,"48":1,"49":1,"50":1,"51":1,"52":1,"53":1,"54":1,"55":1,"56":1,"57":1,"58":1,"59":1,"60":2,"61":2,"62":2,"63":2,"64":2,"65":2,"66":2,"67":2,"68":2,"69":2,"70":2,"71":2,"72":2,"73":2,"74":2,"75":2,"76":2,"77":2,"78":2,"79":2,"80":2,"81":2,"82":2,"83":2,"84":2,"85":2,"86":2,"87":2,"88":2,"89":2,"90":2,"91":2,"92":2,"93":2,"94":2,"95":2,"96":2,"97":2,"98":2,"99":2,"100":2,"101":2,"102":2,"103":2,"104":2,"105":2,"106":2,"107":2,"108":2,"109":2,"110":2,"111":2,"112":2,"113":2,"114":2,"115":2,"116":2,"117":2,"118":2,"119":2,"120":4,"121":4,"122":4,"123":4,"124":4,"125":4,"126":4,"127":4,"128":4,"129":4,"130":4,"131":4,"132":4,"133":4,"134":4,"135":4,"136":4,"137":4,"138":4,"139":4,"140":4,"141":4,"142":4,"143":4,"144":4,"145":4,"146":4,"147":4,"148":4,"149":4,"150":4,"151":4,"152":4,"153":4,"154":4,"155":4,"156":4,"157":4,"158":4,"159":4,"160":4,"161":4,"162":4,"163":4,"164":4,"165":4,"166":4,"167":4,"168":4,"169":4,"170":4,"171":4,"172":4,"173":4,"174":4,"175":4,"176":4,"177":4,"178":4,"179":4,"180":7,"181":7,"182":7,"183":7,"184":7,"185":7,"186":7,"187":7,"188":7,"189":7,"190":7,"191":7,"192":7,"193":7,"194":7,"195":7,"196":7,"197":7,"198":7,"199":7,"200":7,"201":7,"202":7,"203":7,"204":7,"205":7,"206":7,"207":7,"208":7,"209":7,"210":7,"211":7,"212":7,"213":7,"214":7,"215":7,"216":7,"217":7,"218":7,"219":7,"220":7,"221":7,"222":7,"223":7,"224":7,"225":7,"226":7,"227":7,"228":7,"229":7,"230":7,"231":7,"232":7,"233":7,"234":7,"235":7,"236":7,"237":7,"238":7,"239":7},"index":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7,"8":8,"9":9,"10":10,"11":11,"12":12,"13":13,"14":14,"15":15,"16":16,"17":17,"18":18,"19":19,"20":20,"21":21,"22":22,"23":23,"24":24,"25":25,"26":26,"27":27,"28":28,"29":29,"30":30,"31":31,"32":32,"33":33,"34":34,"35":35,"36":36,"37":37,"38":38,"39":39,"40":40,"41":41,"42":42,"43":43,"44":44,"45":45,"46":46,"47":47,"48":48,"49":49,"50":50,"51":51,"52":52,"53":53,"54":54,"55":55,"56":56,"57":57,"58":58,"59":59,"60":60,"61":61,"62":62,"63":63,"64":64,"65":65,"66":66,"67":67,"68":68,"69":69,"70":70,"71":71,"72":72,"73":73,"74":74,"75":75,"76":76,"77":77,"78":78,"79":79,"80":80,"81":81,"82":82,"83":83,"84":84,"85":85,"86":86,"87":87,"88":88,"89":89,"90":90,"91":91,"92":92,"93":93,"94":94,"95":95,"96":96,"97":97,"98":98,"99":99,"100":100,"101":101,"102":102,"103":103,"104":104,"105":105,"106":106,"107":107,"108":108,"109":109,"110":110,"111":111,"112":112,"113":113,"114":114,"115":115,"116":116,"117":117,"118":118,"119":119,"120":120,"121":121,"122":122,"123":123,"124":124,"125":125,"126":126,"127":127,"128":128,"129":129,"130":130,"131":131,"132":132,"133":133,"134":134,"135":135,"136":136,"137":137,"138":138,"139":139,"140":140,"141":141,"142":142,"143":143,"144":144,"145":145,"146":146,"147":147,"148":148,"149":149,"150":150,"151":151,"152":152,"153":153,"154":154,"155":155,"156":156,"157":157,"158":158,"159":159,"160":160,"161":161,"162":162,"163":163,"164":164,"165":165,"166":166,"167":167,"168":168,"169":169,"170":170,"171":171,"172":172,"173":173,"174":174,"175":175,"176":176,"177":177,"178":178,"179":179,"180":180,"181":181,"182":182,"183":183,"184":184,"185":185,"186":186,"187":187,"188":188,"189":189,"190":190,"191":191,"192":192,"193":193,"194":194,"195":195,"196":196,"197":197,"198":198,"199":199,"200":200,"201":201,"202":202,"203":203,"204":204,"205":205,"206":206,"207":207,"208":208,"209":209,"210":210,"211":211,"212":212,"213":213,"214":214,"215":215,"216":216,"217":217,"218":218,"219":219,"220":220,"221":221,"222":222,"223":223,"224":224,"225":225,"226":226,"227":227,"228":228,"229":229,"230":230,"231":231,"232":232,"233":233,"234":234,"235":235,"236":236,"237":237,"238":238,"239":239}}

assets/scrimmage_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a46d1fbccfcc76f8adab0a3653ee2eed8c4ebbad5b47348e62f309c0c7235981
+size 1291

assets/starts_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fa4326a5b21d554d21611679dd8eeb69000a8be2258a789617237f49ea447b4
+size 10308

assets/time_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990ef64cc53d1aefcd5916daeb91c090372833f2ac9422f60fef0264fd91a8a7
+size 825

pages.py

@@ -0,0 +1,185 @@
+import streamlit as st
+import plotly.express as px
+import pandas as pd
+import numpy as np
+from PIL import Image
+from tools import generator
+
+def set_app_title_and_logo():
+    st.set_page_config(
+        page_title="QB-GPT",
+        page_icon=":rocket:",
+        layout="wide",
+    )
+
+def qb_gpt_page(ref_df, ref, tokenizer, model):
+    
+    with st.container():
+        cola, colb = st.columns(2)
+        with cola:
+            selected_gameId = st.selectbox("Select Game ID", ref_df['gameId'].unique())
+            filtered_df1 = ref_df[(ref_df['gameId'] == selected_gameId)]
+        with colb:
+            selected_Play= st.selectbox("Select Play ", filtered_df1['playId'].unique())
+        filtered_df = filtered_df1[(filtered_df1['playId'] == selected_Play)].reset_index(drop ="True")
+
+        # Display the filtered DataFrame
+        st.write("Filtered Data:")
+        st.dataframe(filtered_df)
+    
+    
+    with st.container():
+        col1, col2 = st.columns(2)
+        with col1:
+            temperature = st.slider("Temperature", 1.0, 10.0, 1.5, step = 0.5)
+        with col2:
+            n_select = st.slider("N movements to shortlist", 2, 100, 10, step = 1)
+        
+        QB_gen = generator(model=model,
+                        tokenizer=tokenizer,
+                        temp = temperature,
+                        n_select = n_select)
+    
+    
+    selected = filtered_df["index"][0]
+    selection = ref[selected]
+    
+    colc, cold = st.columns(2)
+    
+    with colc:
+        starts = st.slider("Temperature", 1, 21, 1, step = 1)
+    with cold:
+        frames = st.slider("n select", 1, 50, 20, step = 1)
+    
+    if st.button("Generate"):
+        trial_d = QB_gen.tokenizer.truncate_to_time_t(selection, starts)
+        generated = QB_gen.generate_sequence(trial_d, frames)
+        decoded = QB_gen.tokenizer.decode_sequence(generated)
+
+        step1 = QB_gen.prepare_for_plot(decoded)
+        plot = pd.DataFrame(step1)
+
+        decoded_true = QB_gen.tokenizer.decode_sequence(selection)
+        step1_true = QB_gen.prepare_for_plot(decoded_true)
+        plot_true = pd.DataFrame(step1_true)
+        
+        fig_gen = px.line(plot, x="input_ids_x", y="input_ids_y", animation_frame="pos_ids", color="OffDef", symbol="ids",
+                        text="position_ids", title="Player Trajectories Over Time", line_shape="linear",
+                        range_x=[0, 140], range_y=[0, 60], # Set X and Y axis ranges
+                        render_mode="svg")  # Render mode for smoother lines
+
+        # Customize the appearance of the plot
+        fig_gen.update_traces(marker=dict(size=10), selector=dict(mode='lines'))
+        fig_gen.update_layout(width=800, height=600) 
+        st.plotly_chart(fig_gen)
+        
+        fig_true = px.line(plot_true, x="input_ids_x", y="input_ids_y", animation_frame="pos_ids", color="OffDef", symbol="ids",
+                    text="position_ids", title="Player Trajectories Over Time",
+                    range_x=[0, 140], range_y=[0, 60], # Set X and Y axis ranges
+                    line_shape="linear",  # Draw lines connecting points
+                    render_mode="svg")  # Render mode for smoother lines
+
+        # Customize the appearance of the plot
+        fig_true.update_traces(marker=dict(size=10), selector=dict(mode='lines'))
+        fig_true.update_layout(width=800, height=600) 
+        st.plotly_chart(fig_true)
+    
+        
+def contacts_and_disclaimers():
+    
+    
+    st.title("QB-GPT - Your Football Playbook Powerhouse!")
+    
+    qb_gpt_text_intro = """
+    Are you a data scientist, a machine learning enthusiast, or simply a die-hard NFL fan looking to explore the power of Transformers in the world of American football? Look no further!
+    """
+    st.markdown(qb_gpt_text_intro)
+    
+    with st.expander("***What is QB-GPT?***"):
+        
+        qb_gpt_what = """
+        QuarterBack-GPT (QB-GPT) is a companion in the world of football strategy and analytics. It's an innovative application with a model relying on the remarkable capabilities of Transformers to generate football plays that are not only strategic but also incredibly realistic. Imagine having an AI-powered coach in your corner, designing plays that could turn the tide of any game.
+        """
+        st.markdown(qb_gpt_what)
+    
+    with st.expander("***What's inside QB-GPT***"):
+        
+        qb_gpt_transf = """
+        At the heart of QB-GPT lies the cutting-edge Transformer model, a deep learning architecture known for its prowess in understanding sequential data. It doesn't just create plays; it understands the game at a granular level, taking into account player positions, game situations, and historical data. It relies on the same conceptual approach behind the now famous "GPT" model of OpenAI. It's the playbook of the future, driven by the technology of tomorrow.
+        
+        A more detailed blogpost about the model QB-GPT can be found [here](link)
+        """
+        st.markdown(qb_gpt_transf)
+        
+    with st.expander("***QB-GPT in Action***"):
+        
+        qb_gpt_act = """
+        With QB-GPT, you can explore a wide range of football scenarios. Design plays that are tailored to your team's strengths, simulate game situations, and experiment with different strategies—all at your fingertips. Whether you're a coach looking to refine your playbook or an NFL enthusiast seeking the thrill of strategic gameplay, QB-GPT has something for everyone.
+        """
+        st.markdown(qb_gpt_act)
+        
+        
+    
+    with st.expander("***Author***"):
+        col1, col2 = st.columns([4, 1])
+        with col1:
+            author_text = """
+            My name is Samuel Chaineau, I am 26 and live in Paris. This is my second work related to Deep-Learning applied to NFL NGS data. This work is released as an app in order to facilitate the interaction and feedbacks from user. I also hope having constructive academic/scientific feedbacks to improve the model and bring it to the next level. 
+        
+            I have a background in economics, management, statistics and computer sciences. I am currently the CTO of a french healthcare start-up called Nuvocare. Prior to that I worked 2 and a half years as a Data Scientist at Ekimetrics. 
+        
+            ***Contacts***
+            
+            If interested by the project, the app or wishing to discuss any related topics, feel free to contact me on :
+            - My email : s.chaineau@roof.ai
+            
+            - Linkedin : [My profile](https://www.linkedin.com/in/samuel-chaineau-734b13122/)
+            
+            - X (Twitter, nobody says X) : [My profile](https://twitter.com/samboucon)
+            
+            - Or you can follow me on Medium : [My blog](https://medium.com/@sam.chaineau)
+            """
+            st.markdown(author_text)
+        
+        with col2:
+            image = Image.open('app/assets/photo_cv.jpg')
+            st.image(image)
+        
+    with st.expander("***Disclaimers***"):
+        disclaimer_text = """
+        This work is at a very early stage and while I think it shows promising results, I acknowledge that the model may yield disturbing results and potentially wrong.
+        Maintaining and improving QB-GPT will be a long run. 
+
+        I used data only found publicly on the internet (GitHub and Kaggle). I don't hold any relationship with NFL officials or any NFL teams. 
+        I do not intend to have any payments or commercial activities via this app. It is a POC showing abilities, advantages and flaws of current SotA technologies applied to sports analytics.
+        
+        """
+        st.markdown(disclaimer_text)
+        
+    with st.expander("***License***"):
+        license_text = """
+                **License**
+                
+                This application and its associated work are released under the **Creative Commons Attribution-NonCommercial 4.0 International License**.
+                
+                **Creative Commons Attribution-NonCommercial 4.0 International License (CC BY-NC 4.0)**
+
+                You are free to:
+
+                - **Share** - copy and redistribute the material in any medium or format.
+                
+                - **Adapt** - remix, transform, and build upon the material.
+
+                Under the following terms:
+
+                - **Attribution (BY)**: You must give appropriate credits and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+
+                - **Non-Commercial (NC)**: You may not use the material for commercial purposes.
+
+                - **No Derivatives (ND)**: If you remix, transform, or build upon the material, you may not distribute the modified material.
+
+                For a full description of the license terms, please visit the [Creative Commons Attribution-NonCommercial 4.0 International License](https://creativecommons.org/licenses/by-nc/4.0/).
+
+                This license is designed to allow others to use, remix, and build upon your work, but not for commercial purposes. It requires proper attribution and restricts commercial use and the creation of derivative works.
+                """
+        st.markdown(license_text)

tools.py

@@ -0,0 +1,375 @@
+import polars as pl
+import numpy as np
+import tensorflow as tf
+import pandas as pd
+
+class tokenizer:
+    def __init__(self, 
+                 moves_index : str,
+                 play_index : str,
+                 positions_index  : str,
+                 scrimmage_index : str,
+                 starts_index : str,
+                 time_index : str,
+                 window_size : int):
+        self.window = window_size
+        
+        moves_index = pl.read_parquet(moves_index)
+        self.moves_index = self.convert_index_to_dict(moves_index)
+        
+        play_index = pl.read_parquet(play_index)
+        self.play_index= self.convert_index_to_dict(play_index)
+        
+        positions_index = pl.read_parquet(positions_index)
+        self.positions_index = self.convert_index_to_dict(positions_index)
+        
+        scrimmage_index = pl.read_parquet(scrimmage_index)
+        self.scrimmage_index = self.convert_index_to_dict(scrimmage_index)
+        
+        starts_index = pl.read_parquet(starts_index)
+        self.starts_index = self.convert_index_to_dict(starts_index)
+        
+        time_index = pl.read_parquet(time_index)
+        self.time_index = self.convert_index_to_dict(time_index)
+        
+        self.offdef_index = {0 : ["Def"],
+                             1 : ["Off"]}
+        
+        self.index = {"input_ids" : self.moves_index,
+                      "PlayType" : self.play_index,
+                      "position_ids" : self.positions_index,
+                      "scrim_ids" : self.scrimmage_index,
+                      "start_ids" : self.starts_index,
+                      "pos_ids" : self.time_index,
+                      "OffDef" : self.offdef_index}
+    
+    def convert_index_to_dict(self, df : pl.DataFrame):
+    
+        ID_col = [v for v in df.columns if "ID" in v]
+        assert len(ID_col) == 1
+        new_id_name = ["ID"]
+
+        val_cols = [v for v in df.columns if v not in ID_col+["Cat"]]
+        new_val_name = ["Val_"+str(i) for i in range(1, len(val_cols)+1)]
+
+        past_names = ID_col + val_cols
+        new_names = new_id_name+new_val_name
+
+        renaming = {past_names[i]: new_names[i] for i in range(len(new_names))}
+
+        d = (df.
+                drop("Cat").
+                rename(renaming).
+                select(new_names).
+                to_dict(as_series=False))
+
+        final_d = {d["ID"][i] : [d[k][i] for k in new_val_name] for i in range(len(d["ID"]))}
+
+        return final_d
+    
+    def base_decode(self,
+                    pad_element,
+                    inputs : list,
+                    index : dict,
+                    first : bool):
+        if first == True:
+            return [index[v][0] if v in index.keys() else pad_element for v in inputs]
+        else:
+            return [index[v] if v in index.keys() else pad_element for v in inputs]
+        
+    def decode(self,
+               inputs : list,
+               type : str):
+        if type in ["input_ids", "start_ids"]:
+            padding = [-1000, -1000]
+        elif type in ["scrim_ids", "pos_ids"]:
+            padding = -1000
+        else:
+            padding = "[PAD]"
+        
+        if type in ["input_ids", "start_ids"]:
+            return self.base_decode(padding, inputs, index = self.index[type], first=False)
+        else:
+            return self.base_decode(padding, inputs, index = self.index[type], first=True)
+
+    def find_id_by_values(self, 
+                          input_dict : dict, 
+                          target_list : list):
+        
+        for key, values in input_dict.items():
+            if set(target_list) == set(values):
+                return key
+        
+    def base_encode(self,
+                inputs : list,
+                index : dict):
+        return [self.find_id_by_values(index, [v]) for v in inputs]
+    
+    def encode(self,
+               inputs : list,
+               type : str):
+        return self.base_encode(inputs, index = self.index[type])
+    
+    def decode_sequence(self,
+                        input : dict):
+        return {k : self.decode(v, k) if k not in ["side_ids", "token_type_ids", "labels", "attention_mask", "ids"] else v for k,v in input.items()}
+    
+    def encode_sequence(self,
+                        input : dict):
+        return {k : self.encode(v, k) if k not in ["side_ids", "token_type_ids", "labels", "attention_mask", "ids"] else v for k,v in input.items()}
+    
+    def truncate_to_time_t(self,
+                           input : dict,
+                           t : int):
+        to_keep = [i < t for i in input["pos_ids"]]
+        return {k: [v[i] for i in range(len(v)) if to_keep[i] == True] for k,v in input.items()}
+    
+    def resize_window(self,
+                      input : dict,
+                      pos_id):
+        out = input.copy()
+        out["attention_mask"] = [0 if out["pos_ids"][p] <pos_id else 1 for p in range(len(out["pos_ids"]))]
+        return out
+    
+    def prepare_for_call(self, 
+                         input : dict):
+        resize_limit = max([v for v in np.array(input["pos_ids"]).flatten() if v != 51]) - self.window
+        if resize_limit > 0:
+            input = self.resize_window(input, resize_limit)
+        
+        done = {k : tf.constant(v) for k,v in input.items()}
+        if len(done["pos_ids"].shape) == 1:
+            done = {k : tf.expand_dims(v, axis=0) for k,v in input.items()}
+        return done        
+    
+class generator:
+    def __init__(self,
+                 model,
+                 tokenizer,
+                 temp,
+                 n_select):
+        
+        self.QBGPT = model
+        self.tokenizer = tokenizer
+        
+        self.temperature = temp
+        self.n_select = n_select
+        
+    def get_unique_lists(self,
+                         l_of_ls : list):
+        list_of_tuples = [tuple(inner_list) for inner_list in l_of_ls]
+        
+        # Create a set to eliminate duplicate
+        unique_tuples = set(list_of_tuples)
+        
+        # Convert unique tuples back to lists
+        unique_lists = [list(unique_tuple) for unique_tuple in unique_tuples]
+        
+        return unique_lists
+                
+    def cut(self, l, ref):
+        splitted = []
+        cutted = []
+        for i in range(len(l)):
+            if ref[i] == True:
+                cutted.append(l[i])
+            else:
+                splitted.append(cutted)
+                cutted = []
+                cutted.append(l[i])     
+            if i == len(l)-1:
+                splitted.append(cutted) 
+        return splitted
+        
+    def get_last_preds(self,
+                       logits,
+                       input : dict):
+        
+        to_keep = [i == max(input["pos_ids"]) for i in input["pos_ids"]]
+        return np.array([logits[i] for i in range(len(logits)) if to_keep[i] == True])
+        
+    def get_logits(self, 
+                   input : dict):
+        x = self.tokenizer.prepare_for_call(input)
+        return self.QBGPT(x)
+    
+    def convert_to_preds(self,
+                         logits):
+        preds = tf.squeeze(logits, axis=0)
+        return preds
+    
+    def set_temperature(self, 
+                        x):
+        if x < 5:
+            return self.temperature
+        elif x < 10 and x >= 5:
+            return self.temperature/2
+        elif x <20 and x >= 10:
+            return self.temperature/5
+        else: 
+            return 1.0
+        
+    def select_and_temp(self, 
+                        tensor, 
+                        n, 
+                        temp):
+        probas = tf.nn.softmax(tf.sort(tensor/temp, axis = -1)[:,:,-n:], axis = 2)
+        indices = tf.argsort(tensor, axis = -1)[:,:,-n:]
+        return probas, indices
+
+    def draw_random(self,
+                    probas):
+        drawn = np.vstack([np.random.multinomial(1, p.numpy(), 1) for p in probas[0]])
+        drawn = tf.expand_dims(drawn, axis = 0)
+        return tf.cast(drawn, dtype="int32")
+    
+    def get_indices(self, 
+                    drawn, 
+                    ind):
+        return tf.reduce_sum(drawn*ind, axis = 2)
+    
+    def process_logits(self,
+                       logits,
+                       temp,
+                       n):
+        probas, indices = self.select_and_temp(logits, n, temp)
+        drawn = self.draw_random(probas)
+        results = self.get_indices(drawn, indices)
+        return results
+    
+    def generate(self,
+                 input : dict):
+        logits = self.get_logits(input)
+        temperature_parameter = self.set_temperature(max(input["pos_ids"]))
+        processed_logits = self.process_logits(logits, n=self.n_select, temp=temperature_parameter)
+        preds = self.convert_to_preds(processed_logits)
+        return self.get_last_preds(preds, input)
+    
+    def slice_inputs(self,
+                     input : dict):
+        flags = [True] + [input["pos_ids"][i+1] > input["pos_ids"][i] for i in range(len(input["pos_ids"])-1)]
+        cutted_inputs = {k : self.cut(v, flags) for k,v in input.items()}
+        return cutted_inputs
+    
+    def continue_by_token(self, 
+                         arr,
+                         token :str):
+        if token == "input_ids":
+            return arr
+        if token == "pos_ids":
+            insert = max(arr)+1
+            return np.concatenate([arr, np.array([insert])])
+        elif token == "token_type_ids":
+            return  np.concatenate([arr, np.array([1])])
+        else:
+            return  np.concatenate([arr, [arr[-1]]])
+            
+        
+    def append_prediction(self,
+                           arr,
+                           pred):
+        return np.concatenate([arr, [pred]])
+    
+    def append_predictions(self,
+                           d : dict,
+                           preds):
+        new = d.copy()
+        new["input_ids"] = [self.append_prediction(new["input_ids"][i], preds[i]) for i in range(len(preds))]
+        return new
+                
+    def merge_cuts(self, 
+                   input : dict):
+        return {k : np.concatenate(v) for k,v in input.items()}
+        
+    def update_inputs(self,
+                      input,
+                      preds):
+        sliced = self.slice_inputs(input)
+        appended = self.append_predictions(sliced, preds)
+        continued = {k : [self.continue_by_token(e, k) for e in v] for k,v in appended.items()}
+        merged = self.merge_cuts(continued)
+        return merged
+    
+    def generate_sequence(self,
+                          input,
+                          t):
+        new_input = input.copy()
+        for i in range(t):
+            generated = self.generate(new_input)
+            new_input = self.update_inputs(new_input, generated)
+        return new_input
+    
+    def convert_list(self,
+                     d,
+                     keep_original):
+        new_df = d.copy()
+        new_df["start_ids_x"] = [v[0] for v in new_df["start_ids"]]
+        new_df["start_ids_y"] = [v[1] for v in new_df["start_ids"]]
+        new_df["input_ids_x"] = [v[0] for v in new_df["input_ids"]]
+        new_df["input_ids_y"] = [v[1] for v in new_df["input_ids"]]
+        if keep_original == True:
+            return new_df
+        else:
+            return {k : v for k,v in new_df.items() if k not in ["start_ids", "input_ids"]}
+        
+    def remove_pad(self,
+                   seq):
+        df = pd.DataFrame(seq)
+        filtered = df[df["start_ids_x"] != -1000].reset_index(drop=True)
+        filtered = df[df["input_ids_x"] != -1000].reset_index(drop=True)
+        return filtered.to_dict(orient = "list")
+    
+    def _compute_true_sequence(self,
+                          scrimmage_line,
+                          start : list,
+                          moves : list):
+        scrimmage = np.array([scrimmage_line, 26.5])
+        
+        updated_moves = np.array([np.array(start) + np.array(v) for v in moves])
+        
+        appended = np.concatenate([np.expand_dims(start, axis = 0), updated_moves])
+        
+        final = appended + scrimmage
+        return final
+    
+    def compute_true_sequence(self,
+                              scrims,
+                              starts,
+                              moves):
+        return self._compute_true_sequence(np.unique(scrims)[0], self.get_unique_lists(starts)[0], moves)
+    
+    def _resize_variable(self,
+                         x, 
+                         ref: str):
+        
+        if ref in ["pos_ids", "token_type_ids"]:
+            return np.concatenate([[0], x])
+        
+        elif ref in ["input_ids", "start_ids"]:
+            return np.vstack([self.get_unique_lists(x)[0], x])
+        
+        else:
+            return np.concatenate([np.unique(x), x])
+    
+    def prepare_for_plot(self, 
+                         seq):
+        sequence = seq.copy()
+        sequence = self.convert_list(sequence, keep_original = True)
+        sequence = self.remove_pad(sequence)
+        cutted = self.slice_inputs(sequence)
+        moves_updated = [self.compute_true_sequence(cutted["scrim_ids"][i], cutted["start_ids"][i], cutted["input_ids"][i]) for i in range(len(cutted["input_ids"]))]
+        cutted["input_ids"] = moves_updated
+        cutted = {k : [self._resize_variable(e, k) if k != "input_ids" else e for e in v] for k,v in cutted.items()}
+        cutted["ids"] = [[i for e in range(len(cutted["input_ids"][i]))] for i in range(len(cutted["input_ids"]))]
+        merged = self.merge_cuts(cutted)
+        converted = self.convert_list(merged, keep_original = False)
+        structured = {k:v for k,v in converted.items() if k != "labels"}
+        return structured
+
+    def insert_ids(self,
+                   input):
+        
+        cutted = self.slice_inputs(input)
+        cutted["ids"] = [[i for e in range(len(cutted["input_ids"][i]))] for i in range(len(cutted["input_ids"]))]
+        merged = self.merge_cuts(cutted)
+        return merged