Update app.py

app.py

@@ -280,23 +280,14 @@ EXTERNAL_MODELS = [
     "all-mpnet-base-v2",
     "allenai-specter",
     "Baichuan-text-embedding",
-    "bert-base-swedish-cased",
     "bert-base-uncased",
-    "bge-base-zh-v1.5",
-    "bge-large-zh-v1.5",
-    "bge-large-zh-noinstruct",
-    "bge-small-zh-v1.5",
     "contriever-base-msmarco",
-    "cross-en-de-roberta-sentence-transformer",
     "dfm-encoder-large-v1",
     "dfm-sentence-encoder-large-1",
     "distiluse-base-multilingual-cased-v2",
-    "DanskBERT",
     "e5-base",
     "e5-large",
     "e5-small",    
-    "electra-small-nordic",
-    "electra-small-swedish-cased-discriminator",
     "gbert-base",
     "gbert-large",
     "gelectra-base",
@@ -324,29 +315,14 @@ EXTERNAL_MODELS = [
     "norbert3-large",
     "paraphrase-multilingual-MiniLM-L12-v2",
     "paraphrase-multilingual-mpnet-base-v2",
-    "sentence-bert-swedish-cased",
     "sentence-t5-base",
     "sentence-t5-large",
     "sentence-t5-xl",
     "sentence-t5-xxl",
     "sup-simcse-bert-base-uncased",
-    "st-polish-paraphrase-from-distilroberta",
-    "st-polish-paraphrase-from-mpnet",    
-    "text2vec-base-chinese",
-    "text2vec-large-chinese",
     "text-embedding-3-small",
     "text-embedding-3-large",
     "text-embedding-3-large-256",
-    "text-embedding-ada-002",
-    "text-similarity-ada-001",
-    "text-similarity-babbage-001",
-    "text-similarity-curie-001",
-    "text-similarity-davinci-001",
-    "text-search-ada-doc-001",
-    "text-search-ada-001",
-    "text-search-babbage-001",
-    "text-search-curie-001",
-    "text-search-davinci-001",
     "titan-embed-text-v1",
     "unsup-simcse-bert-base-uncased",
     "use-cmlm-multilingual",
@@ -362,24 +338,14 @@ EXTERNAL_MODEL_TO_LINK = {
     "all-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2",
     "all-MiniLM-L6-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2",
     "all-mpnet-base-v2": "https://huggingface.co/sentence-transformers/all-mpnet-base-v2",
-    "Baichuan-text-embedding": "https://platform.baichuan-ai.com/docs/text-Embedding",
-    "bert-base-swedish-cased": "https://huggingface.co/KB/bert-base-swedish-cased",
     "bert-base-uncased": "https://huggingface.co/bert-base-uncased",
-    "bge-base-zh-v1.5": "https://huggingface.co/BAAI/bge-base-zh-v1.5",
-    "bge-large-zh-v1.5": "https://huggingface.co/BAAI/bge-large-zh-v1.5",
-    "bge-large-zh-noinstruct": "https://huggingface.co/BAAI/bge-large-zh-noinstruct",
-    "bge-small-zh-v1.5": "https://huggingface.co/BAAI/bge-small-zh-v1.5",
     "contriever-base-msmarco": "https://huggingface.co/nthakur/contriever-base-msmarco",
-    "cross-en-de-roberta-sentence-transformer": "https://huggingface.co/T-Systems-onsite/cross-en-de-roberta-sentence-transformer",
-    "DanskBERT": "https://huggingface.co/vesteinn/DanskBERT",
     "distiluse-base-multilingual-cased-v2": "https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased-v2",
     "dfm-encoder-large-v1": "https://huggingface.co/chcaa/dfm-encoder-large-v1",
     "dfm-sentence-encoder-large-1": "https://huggingface.co/chcaa/dfm-encoder-large-v1",
     "e5-base": "https://huggingface.co/intfloat/e5-base",
     "e5-large": "https://huggingface.co/intfloat/e5-large",
     "e5-small": "https://huggingface.co/intfloat/e5-small",
-    "electra-small-nordic": "https://huggingface.co/jonfd/electra-small-nordic",
-    "electra-small-swedish-cased-discriminator": "https://huggingface.co/KBLab/electra-small-swedish-cased-discriminator",
     "gbert-base": "https://huggingface.co/deepset/gbert-base",
     "gbert-large": "https://huggingface.co/deepset/gbert-large",
     "gelectra-base": "https://huggingface.co/deepset/gelectra-base",
@@ -407,30 +373,14 @@ EXTERNAL_MODEL_TO_LINK = {
     "norbert3-large": "https://huggingface.co/ltg/norbert3-large",
     "paraphrase-multilingual-mpnet-base-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2",    
     "paraphrase-multilingual-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
-    "sentence-bert-swedish-cased": "https://huggingface.co/KBLab/sentence-bert-swedish-cased",
     "sentence-t5-base": "https://huggingface.co/sentence-transformers/sentence-t5-base",
     "sentence-t5-large": "https://huggingface.co/sentence-transformers/sentence-t5-large",
     "sentence-t5-xl": "https://huggingface.co/sentence-transformers/sentence-t5-xl",
     "sentence-t5-xxl": "https://huggingface.co/sentence-transformers/sentence-t5-xxl",
     "sup-simcse-bert-base-uncased": "https://huggingface.co/princeton-nlp/sup-simcse-bert-base-uncased",
-    "st-polish-paraphrase-from-distilroberta": "https://huggingface.co/sdadas/st-polish-paraphrase-from-distilroberta",
-    "st-polish-paraphrase-from-mpnet": "https://huggingface.co/sdadas/st-polish-paraphrase-from-mpnet",
-    "text2vec-base-chinese": "https://huggingface.co/shibing624/text2vec-base-chinese",
-    "text2vec-large-chinese": "https://huggingface.co/GanymedeNil/text2vec-large-chinese",
     "text-embedding-3-small": "https://openai.com/blog/new-embedding-models-and-api-updates",
     "text-embedding-3-large": "https://openai.com/blog/new-embedding-models-and-api-updates",
     "text-embedding-3-large-256": "https://openai.com/blog/new-embedding-models-and-api-updates",
-    "text-embedding-ada-002": "https://openai.com/blog/new-and-improved-embedding-model",
-    "text-similarity-ada-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-similarity-babbage-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-similarity-curie-001": "https://openai.com/blog/introducing-text-and-code-embeddings",        
-    "text-similarity-davinci-001": "https://openai.com/blog/introducing-text-and-code-embeddings",    
-    "text-search-ada-doc-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-search-ada-query-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-search-ada-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-search-curie-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-search-babbage-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
-    "text-search-davinci-001": "https://openai.com/blog/introducing-text-and-code-embeddings",
     "titan-embed-text-v1": "https://docs.aws.amazon.com/bedrock/latest/userguide/embeddings.html",
     "unsup-simcse-bert-base-uncased": "https://huggingface.co/princeton-nlp/unsup-simcse-bert-base-uncased",
     "use-cmlm-multilingual": "https://huggingface.co/sentence-transformers/use-cmlm-multilingual",
@@ -445,24 +395,14 @@ EXTERNAL_MODEL_TO_DIM = {
     "all-MiniLM-L6-v2": 384,
     "all-mpnet-base-v2": 768,
     "allenai-specter": 768,
-    "Baichuan-text-embedding": 1024,
-    "bert-base-swedish-cased": 768,
     "bert-base-uncased": 768,
-    "bge-base-zh-v1.5": 768,
-    "bge-large-zh-v1.5": 1024,
-    "bge-large-zh-noinstruct": 1024,
-    "bge-small-zh-v1.5": 512,
     "contriever-base-msmarco": 768,
-    "cross-en-de-roberta-sentence-transformer": 768,
-    "DanskBERT": 768,
     "distiluse-base-multilingual-cased-v2": 512,
     "dfm-encoder-large-v1": 1024,
     "dfm-sentence-encoder-large-1": 1024,
     "e5-base": 768,
     "e5-small": 384,
     "e5-large": 1024,    
-    "electra-small-nordic": 256,
-    "electra-small-swedish-cased-discriminator": 256,
     "luotuo-bert-medium": 768,
     "LASER2": 1024,
     "LaBSE": 768,
@@ -490,30 +430,14 @@ EXTERNAL_MODEL_TO_DIM = {
     "norbert3-large": 1024,
     "paraphrase-multilingual-MiniLM-L12-v2": 384,
     "paraphrase-multilingual-mpnet-base-v2": 768,
-    "sentence-bert-swedish-cased": 768,
     "sentence-t5-base": 768,
     "sentence-t5-large": 768,
     "sentence-t5-xl": 768,
     "sentence-t5-xxl": 768,
     "sup-simcse-bert-base-uncased": 768,
-    "st-polish-paraphrase-from-distilroberta": 768,
-    "st-polish-paraphrase-from-mpnet": 768,
-    "text2vec-base-chinese": 768,
-    "text2vec-large-chinese": 1024,
     "text-embedding-3-large": 3072,
     "text-embedding-3-large-256": 256,
     "text-embedding-3-small": 1536,
-    "text-embedding-ada-002": 1536,
-    "text-similarity-ada-001": 1024,
-    "text-similarity-babbage-001": 2048,    
-    "text-similarity-curie-001": 4096,
-    "text-similarity-davinci-001": 12288,    
-    "text-search-ada-doc-001": 1024,
-    "text-search-ada-query-001": 1024,
-    "text-search-ada-001": 1024,   
-    "text-search-babbage-001": 2048,     
-    "text-search-curie-001": 4096,
-    "text-search-davinci-001": 12288,
     "titan-embed-text-v1": 1536,
     "unsup-simcse-bert-base-uncased": 768,
     "use-cmlm-multilingual": 768,
@@ -528,24 +452,14 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "all-MiniLM-L6-v2": 512,
     "all-mpnet-base-v2": 514,
     "allenai-specter": 512,
-    "Baichuan-text-embedding": 512,
-    "bert-base-swedish-cased": 512,
-    "bert-base-uncased": 512,
-    "bge-base-zh-v1.5": 512,
-    "bge-large-zh-v1.5": 512,
-    "bge-large-zh-noinstruct": 512,
-    "bge-small-zh-v1.5": 512,    
-    "contriever-base-msmarco": 512,
-    "cross-en-de-roberta-sentence-transformer": 514,
-    "DanskBERT": 514,
+    "bert-base-uncased": 512,  
+    "contriever-base-msmarco": 512,4,
     "dfm-encoder-large-v1": 512,
     "dfm-sentence-encoder-large-1": 512,
     "distiluse-base-multilingual-cased-v2": 512,
     "e5-base": 512,
     "e5-large": 512,
     "e5-small": 512,
-    "electra-small-nordic": 512,
-    "electra-small-swedish-cased-discriminator": 512,
     "gbert-base": 512,
     "gbert-large": 512,
     "gelectra-base": 512,
@@ -573,30 +487,14 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "norbert3-large": 512,
     "paraphrase-multilingual-MiniLM-L12-v2": 512,
     "paraphrase-multilingual-mpnet-base-v2": 514,
-    "sentence-bert-swedish-cased": 512,
     "sentence-t5-base": 512,
     "sentence-t5-large": 512,
     "sentence-t5-xl": 512,
     "sentence-t5-xxl": 512,
     "sup-simcse-bert-base-uncased": 512,
-    "st-polish-paraphrase-from-distilroberta": 514,
-    "st-polish-paraphrase-from-mpnet": 514,
-    "text2vec-base-chinese": 512,
-    "text2vec-large-chinese": 512,
     "text-embedding-3-large": 8191,
     "text-embedding-3-large-256": 8191,
     "text-embedding-3-small": 8191,
-    "text-embedding-ada-002": 8191,
-    "text-similarity-ada-001": 2046,
-    "text-similarity-babbage-001": 2046,
-    "text-similarity-curie-001": 2046,
-    "text-similarity-davinci-001": 2046,
-    "text-search-ada-doc-001": 2046,
-    "text-search-ada-query-001": 2046,
-    "text-search-ada-001": 2046,
-    "text-search-babbage-001": 2046,
-    "text-search-curie-001": 2046,
-    "text-search-davinci-001": 2046,
     "titan-embed-text-v1": 8000,
     "use-cmlm-multilingual": 512,
     "unsup-simcse-bert-base-uncased": 512,
@@ -611,23 +509,14 @@ EXTERNAL_MODEL_TO_SIZE = {
     "all-MiniLM-L12-v2": 0.13,
     "all-MiniLM-L6-v2": 0.09,
     "all-mpnet-base-v2": 0.44,
-    "bert-base-uncased": 0.44,
-    "bert-base-swedish-cased": 0.50,
-    "bge-base-zh-v1.5": 0.41,
-    "bge-large-zh-v1.5": 1.30,
-    "bge-large-zh-noinstruct": 1.30,
-    "bge-small-zh-v1.5": 0.10,      
-    "cross-en-de-roberta-sentence-transformer": 1.11,
+    "bert-base-uncased": 0.44,     
     "contriever-base-msmarco": 0.44,
-    "DanskBERT": 0.50,
     "distiluse-base-multilingual-cased-v2": 0.54,
     "dfm-encoder-large-v1": 1.42,
     "dfm-sentence-encoder-large-1": 1.63,
     "e5-base": 0.44,
     "e5-small": 0.13,
     "e5-large": 1.34,
-    "electra-small-nordic": 0.09,
-    "electra-small-swedish-cased-discriminator": 0.06,
     "gbert-base": 0.44,
     "gbert-large": 1.35,
     "gelectra-base": 0.44,
@@ -655,16 +544,11 @@ EXTERNAL_MODEL_TO_SIZE = {
     "norbert3-large": 1.47,
     "paraphrase-multilingual-mpnet-base-v2": 1.11,
     "paraphrase-multilingual-MiniLM-L12-v2": 0.47,
-    "sentence-bert-swedish-cased": 0.50,
     "sentence-t5-base": 0.22,
     "sentence-t5-large": 0.67,
     "sentence-t5-xl": 2.48,
     "sentence-t5-xxl": 9.73,
-    "sup-simcse-bert-base-uncased": 0.44,
-    "st-polish-paraphrase-from-distilroberta": 0.50,
-    "st-polish-paraphrase-from-mpnet": 0.50,    
-    "text2vec-base-chinese": 0.41,
-    "text2vec-large-chinese": 1.30,    
+    "sup-simcse-bert-base-uncased": 0.44,  
     "unsup-simcse-bert-base-uncased": 0.44,
     "use-cmlm-multilingual": 1.89,
     "xlm-roberta-base": 1.12,