localmodelfixed

config-bak.ini

@@ -1,7 +1,7 @@
 [feature_store]
 reject_throttle = 0
-embedding_model_path = "/root/models/bce-embedding-base_v1"
-reranker_model_path = "/root/models/bce-reranker-base_v1"
+embedding_model_path = "bce-embedding-base_v1"
+reranker_model_path = "bce-reranker-base_v1"
 repo_dir = "repodir"
 work_dir = "workdir"
 n_clusters = [20, 50]
@@ -18,7 +18,7 @@ enable_remote = 1
 client_url = "http://127.0.0.1:8888/inference"
 
 [llm.server]
-local_llm_path = "/root/models/Qwen1.5-7B-Chat"
+local_llm_path = "Qwen1.5-7B-Chat"
 local_llm_max_text_length = 32000
 local_llm_bind_port = 8888
 remote_type = ""

config.ini

@@ -1,7 +1,7 @@
 [feature_store]
 reject_throttle = 0
-embedding_model_path = "/root/models/bce-embedding-base_v1"
-reranker_model_path = "/root/models/bce-reranker-base_v1"
+embedding_model_path = "maidalun1020/bce-embedding-base_v1"
+reranker_model_path = "maidalun1020/bce-reranker-base_v1"
 repo_dir = "repodir"
 work_dir = "workdir"
 n_clusters = [20, 50]
@@ -18,7 +18,7 @@ enable_remote = 1
 client_url = "http://127.0.0.1:8888/inference"
 
 [llm.server]
-local_llm_path = "/root/models/Qwen1.5-7B-Chat"
+local_llm_path = "Qwen/Qwen1.5-7B-Chat"
 local_llm_max_text_length = 32000
 local_llm_bind_port = 8888
 remote_type = ""

huixiangdou/service/llm_server_hybrid.py

@@ -92,11 +92,11 @@ class InferenceWrapper:
             )
 
         self.tokenizer = AutoTokenizer.from_pretrained(model_path,
-                                                       trust_remote_code=True, local_files_only=True)
+                                                       trust_remote_code=True)
 
         if 'qwen1.5' in model_path.lower():
             self.model = AutoModelForCausalLM.from_pretrained(
-                model_path, device_map='auto', trust_remote_code=True, local_files_only=True).eval()
+                model_path, device_map='auto', trust_remote_code=True).eval()
         elif 'qwen' in model_path.lower():
             self.model = AutoModelForCausalLM.from_pretrained(
                 model_path,
@@ -104,13 +104,13 @@ class InferenceWrapper:
                 trust_remote_code=True,
                 use_cache_quantization=True,
                 use_cache_kernel=True,
-                use_flash_attn=False, local_files_only=True).eval()
+                use_flash_attn=False).eval()
         else:
             self.model = AutoModelForCausalLM.from_pretrained(
                 model_path,
                 trust_remote_code=True,
                 device_map='auto',
-                torch_dtype='auto', local_files_only=True).eval()
+                torch_dtype='auto').eval()
 
     def chat(self, prompt: str, history=[]):
         """Generate a response from local LLM.