devMls commited on
Commit
e5a1268
·
1 Parent(s): 26d4422

add jina new models in jina connector (#3770)

Browse files

### What problem does this PR solve?

add new models in jinna connector, to allow use models that support
multilingual models

### Type of change

- [X] Other (please describe): new connectors no breaking change

conf/llm_factories.json CHANGED
@@ -525,6 +525,18 @@
525
  "tags": "TEXT EMBEDDING",
526
  "max_tokens": 8196,
527
  "model_type": "embedding"
 
 
 
 
 
 
 
 
 
 
 
 
528
  }
529
  ]
530
  },
@@ -2432,6 +2444,18 @@
2432
  "max_tokens": 4000,
2433
  "model_type": "embedding"
2434
  },
 
 
 
 
 
 
 
 
 
 
 
 
2435
  {
2436
  "llm_name": "rerank-1",
2437
  "tags": "RE-RANK, 8000",
@@ -2443,6 +2467,18 @@
2443
  "tags": "RE-RANK, 4000",
2444
  "max_tokens": 4000,
2445
  "model_type": "rerank"
 
 
 
 
 
 
 
 
 
 
 
 
2446
  }
2447
  ]
2448
  },
 
525
  "tags": "TEXT EMBEDDING",
526
  "max_tokens": 8196,
527
  "model_type": "embedding"
528
+ },
529
+ {
530
+ "llm_name": "jina-reranker-v2-base-multilingual",
531
+ "tags": "RE-RANK,8k",
532
+ "max_tokens": 8196,
533
+ "model_type": "rerank"
534
+ },
535
+ {
536
+ "llm_name": "jina-embeddings-v3",
537
+ "tags": "TEXT EMBEDDING",
538
+ "max_tokens": 8196,
539
+ "model_type": "embedding"
540
  }
541
  ]
542
  },
 
2444
  "max_tokens": 4000,
2445
  "model_type": "embedding"
2446
  },
2447
+ {
2448
+ "llm_name": "voyage-3",
2449
+ "tags": "TEXT EMBEDDING,32000",
2450
+ "max_tokens": 32000,
2451
+ "model_type": "embedding"
2452
+ },
2453
+ {
2454
+ "llm_name": "voyage-3-lite",
2455
+ "tags": "TEXT EMBEDDING,32000",
2456
+ "max_tokens": 32000,
2457
+ "model_type": "embedding"
2458
+ },
2459
  {
2460
  "llm_name": "rerank-1",
2461
  "tags": "RE-RANK, 8000",
 
2467
  "tags": "RE-RANK, 4000",
2468
  "max_tokens": 4000,
2469
  "model_type": "rerank"
2470
+ },
2471
+ {
2472
+ "llm_name": "rerank-2",
2473
+ "tags": "RE-RANK, 16000",
2474
+ "max_tokens": 16000,
2475
+ "model_type": "rerank"
2476
+ },
2477
+ {
2478
+ "llm_name": "rerank-2-lite",
2479
+ "tags": "RE-RANK, 8000",
2480
+ "max_tokens": 8000,
2481
+ "model_type": "rerank"
2482
  }
2483
  ]
2484
  },
rag/llm/embedding_model.py CHANGED
@@ -322,7 +322,7 @@ class YoudaoEmbed(Base):
322
 
323
 
324
  class JinaEmbed(Base):
325
- def __init__(self, key, model_name="jina-embeddings-v2-base-zh",
326
  base_url="https://api.jina.ai/v1/embeddings"):
327
 
328
  self.base_url = "https://api.jina.ai/v1/embeddings"
 
322
 
323
 
324
  class JinaEmbed(Base):
325
+ def __init__(self, key, model_name="jina-embeddings-v3",
326
  base_url="https://api.jina.ai/v1/embeddings"):
327
 
328
  self.base_url = "https://api.jina.ai/v1/embeddings"
rag/llm/rerank_model.py CHANGED
@@ -92,7 +92,7 @@ class DefaultRerank(Base):
92
 
93
 
94
  class JinaRerank(Base):
95
- def __init__(self, key, model_name="jina-reranker-v1-base-en",
96
  base_url="https://api.jina.ai/v1/rerank"):
97
  self.base_url = "https://api.jina.ai/v1/rerank"
98
  self.headers = {
 
92
 
93
 
94
  class JinaRerank(Base):
95
+ def __init__(self, key, model_name="jina-reranker-v2-base-multilingual",
96
  base_url="https://api.jina.ai/v1/rerank"):
97
  self.base_url = "https://api.jina.ai/v1/rerank"
98
  self.headers = {