deletepool

config.ini

@@ -13,7 +13,7 @@ domain_partial_order = ["openai.com", "pytorch.org", "readthedocs.io", "nvidia.c
 save_dir = "logs/web_search_result"
 
 [llm]
-enable_local = 1
+enable_local = 0
 enable_remote = 1
 client_url = "http://127.0.0.1:8888/inference"
 
@@ -21,10 +21,10 @@ client_url = "http://127.0.0.1:8888/inference"
 local_llm_path = "Qwen/Qwen1.5-7B-Chat"
 local_llm_max_text_length = 32000
 local_llm_bind_port = 8888
-remote_type = ""
-remote_api_key = ""
+remote_type = "deepseek"
+remote_api_key = "sk-f36f5336010841399abccdfeb6bd1f54"
 remote_llm_max_text_length = 32000
-remote_llm_model = ""
+remote_llm_model = "deepseek-chat"
 rpm = 500
 
 [worker]

huixiangdou/service/feature_store.py

@@ -431,6 +431,7 @@ class FeatureStore:
         preproc_dir = os.path.join(work_dir, 'preprocess')
         if not os.path.exists(preproc_dir):
             os.makedirs(preproc_dir)
+        logger.debug('waiting for preprocess read finish..')
 
         # pool = Pool(processes=16)
         # file_opr = FileOperation()
@@ -473,7 +474,6 @@ class FeatureStore:
                 file.state = False
                 file.reason = 'skip unknown format'
         # pool.close()
-        
         # pool.join()
 
         # check process result