change token size limit to 512

main.py

@@ -9,24 +9,28 @@ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from transformers import T5Tokenizer, T5ForConditionalGeneration
 
 
+token_size_limit = None
+
 # FROM: https://huggingface.co/facebook/blenderbot-400M-distill?text=Hey+my+name+is+Thomas%21+How+are+you%3F
 # tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill")
 # model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-400M-distill")
 # tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-1B-distill")
 # model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-1B-distill")
+# token_size_limit = 128
+
+# T5 model can use "any" sequence lenghth, but memory usage is O(L^2).
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-small")
 # model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-small")
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-base")
 # model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-base")
 tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-large")
 model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-large")
+token_size_limit = 512
 
 # Too large for 16GB
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")
 # model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-xl")
 
-token_size_limit = 128
-
 
 app = FastAPI()