add model shards to reduce memory consumption

main.py

@@ -14,7 +14,7 @@ import numpy as np
 from numpy.linalg import norm
 from nltk.tokenize import SpaceTokenizer
 import nltk
-from transformers import pipeline
+from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 from dotenv import load_dotenv
 load_dotenv()
 
@@ -25,7 +25,9 @@ templates = Jinja2Templates(directory="templates/")
 onet = pd.read_csv('static/ONET_JobTitles.csv')
 simdat = pd.read_csv('static/cohere_embeddings.csv')
 
-classifier = pipeline('text-classification', model = 'static/model_shards', tokenizer = 'static/tokenizer_shards')
+model = AutoModelForSequenceClassification.from_pretrained('static/model_shards', low_cpu_mem_usage=True)
+tokenizer = AutoTokenizer.from_pretrained('static/tokenizer_shards', low_cpu_mem_usage=True)
+classifier = pipeline('text-classification', model = model, tokenizer = tokenizer)
 
 ### job information center ###
 # get

requirements.txt

@@ -14,4 +14,5 @@ unidecode==1.3.6
 cohere==3.1.5
 python-dotenv==0.21.1
 transformers==4.25.1
-torch==1.13.1
+torch==1.13.1
+accelerate==0.16.0