llm2vec-metadata

app.py

@@ -341,6 +341,12 @@ EXTERNAL_MODELS = [
     "DanskBERT",
     "LASER2",
     "LaBSE",
+    "LLM2Vec-Llama-supervised",
+    "LLM2Vec-Llama-unsupervised",
+    "LLM2Vec-Mistral-supervised",
+    "LLM2Vec-Mistral-unsupervised",
+    "LLM2Vec-Sheared-Llama-supervised",
+    "LLM2Vec-Sheared-Llama-unsupervised",
     "OpenSearch-text-hybrid",
     "all-MiniLM-L12-v2",
     "all-MiniLM-L6-v2",
@@ -453,6 +459,12 @@ EXTERNAL_MODEL_TO_LINK = {
     "Cohere-embed-english-v3.0": "https://huggingface.co/Cohere/Cohere-embed-english-v3.0",
     "Cohere-embed-multilingual-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-v3.0",
     "Cohere-embed-multilingual-light-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-light-v3.0",
+    "LLM2Vec-Llama-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised",
+    "LLM2Vec-Llama-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp",
+    "LLM2Vec-Mistral-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised",
+    "LLM2Vec-Mistral-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp",
+    "LLM2Vec-Sheared-Llama-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised",
+    "LLM2Vec-Sheared-Llama-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp",
     "allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter",
     "allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter",
     "all-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2",
@@ -577,6 +589,12 @@ EXTERNAL_MODEL_TO_DIM = {
     "Cohere-embed-english-v3.0": 1024,
     "Cohere-embed-multilingual-v3.0": 1024,
     "Cohere-embed-multilingual-light-v3.0": 384,
+    "LLM2Vec-Llama-supervised": 4096,
+    "LLM2Vec-Llama-unsupervised": 4096,
+    "LLM2Vec-Mistral-supervised": 4096,
+    "LLM2Vec-Mistral-unsupervised": 4096,
+    "LLM2Vec-Sheared-Llama-supervised": 2048,
+    "LLM2Vec-Sheared-Llama-unsupervised": 2048,
     "all-MiniLM-L12-v2": 384,
     "all-MiniLM-L6-v2": 384,
     "all-mpnet-base-v2": 768,
@@ -700,6 +718,12 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "Cohere-embed-english-v3.0": 512,
     "Cohere-embed-multilingual-v3.0": 512,
     "Cohere-embed-multilingual-light-v3.0": 512,    
+    "LLM2Vec-Llama-supervised": 512,
+    "LLM2Vec-Llama-unsupervised": 512,
+    "LLM2Vec-Mistral-supervised": 512,
+    "LLM2Vec-Mistral-unsupervised": 512,
+    "LLM2Vec-Sheared-Llama-supervised": 512,
+    "LLM2Vec-Sheared-Llama-unsupervised": 512,    
     "all-MiniLM-L12-v2": 512,
     "all-MiniLM-L6-v2": 512,
     "all-mpnet-base-v2": 514,
@@ -820,6 +844,12 @@ EXTERNAL_MODEL_TO_SEQLEN = {
 }
 
 EXTERNAL_MODEL_TO_SIZE = {
+    "LLM2Vec-Llama-supervised": 6607,
+    "LLM2Vec-Llama-unsupervised": 6607,
+    "LLM2Vec-Mistral-supervised": 7111,
+    "LLM2Vec-Mistral-unsupervised": 7111,
+    "LLM2Vec-Sheared-Llama-supervised": 1280,
+    "LLM2Vec-Sheared-Llama-unsupervised": 1280,
     "allenai-specter": 110,
     "all-MiniLM-L12-v2": 33,
     "all-MiniLM-L6-v2": 23,