Update bin_public/utils/Pinecone.py

bin_public/utils/Pinecone.py

@@ -104,4 +104,74 @@ def context_construction(api_key, query, model, pinecone_api_key, pinecone_api_e
         # response = chain.run(input_documents=docs, question=str(query))
         for i in docs:
             temp.append(i.page_content)
-        return '用以下资料进行辅助回答\n' + ' '.join(temp), '\n' + ' '.join(temp), "Connecting to Pinecone"
+        return '用以下资料进行辅助回答\n' + ' '.join(temp), '\n' + ' '.join(temp), "Connecting to Pinecone"
+
+def chat_prerequisites(input, filter, embeddings, top_k=4):
+    # filter : dic
+
+    # input_prompt = '只基于以下规范的两种分类对形如 "position_name: xx job_name: xx job_description:  xxx"的描述进行分类，只要回复规范的类别名'
+    input_prompt = '接下来我会给你一段"不规范的招聘职位描述"，以及4个用（选项一，选项二，选项三，选项四）四个选项表示的规范的职业分类描述。' \
+                   '你需要将"不规范的招聘职位描述"归类为”选项一“或“选项二”或“选项三”或“选项四”。' \
+                   '你只需要回复”选项一“或“选项二”或“选项三”或“选项四”，不要回复任何别的东西'
+    query = input_prompt + input
+
+    temp = []
+    docsearch = Pinecone.from_existing_index(index_name=pinecone.list_indexes()[0], embedding=embeddings)
+    docs = docsearch.similarity_search(query, k=top_k, filter=filter)
+    for index, i in enumerate(docs):
+        if index == 0:
+            temp.append("选项一：" + i.page_content + "##")
+        if index == 1:
+            temp.append("选项二：" + i.page_content + "##")
+        if index == 2:
+            temp.append("选项三：" + i.page_content + "##")
+        if index == 3:
+            temp.append("选项四：" + i.page_content + "##")
+
+    system_prompt = '   '.join(temp)
+
+    return system_prompt, query
+
+
+def chat(input, filter, embeddings):
+    system_prompt, query = chat_prerequisites(input, filter, embeddings)
+    logger.info('prerequisites satisfied')
+    completion = openai.ChatCompletion.create(
+    model="gpt-3.5-turbo",
+    messages=[
+      {"role": "system", "content": system_prompt},
+      {"role": "user", "content": query}
+    ])
+    return completion.choices[0].message['content'], system_prompt
+
+def chat_data_cleaning(input):
+    clean_prompt = '我要求你提取出这段文字中的岗位名称、岗位描述（用一句或者两句话概括），去除无关紧要的信息，比如工资，地点等等，并严格遵守"岗位名称: xxx # 岗位描述: xxx # "的格式进行回复'
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": clean_prompt},
+            {"role": "user", "content": clean_prompt + input}
+        ])
+    return completion.choices[0].message['content']
+
+def local_emb2pinecone(PINECONE_API_KEY, PINECONE_API_ENV, level, emb_path, text_path, delete=False):
+    pinecone.init(api_key=PINECONE_API_KEY, environment=PINECONE_API_ENV)
+    logger.info('Pinecone initialized')
+    logger.info(pinecone.list_indexes()[0])
+    l = load_json(emb_path)
+    print(f'level{level} loaded')
+    with open(text_path, 'r', encoding='utf-8') as f:
+        texts = f.readlines()
+    texts = [i.replace('\n', '') for i in texts]
+    index = pinecone.Index(pinecone.list_indexes()[0])
+    if delete:
+        if input('press y to delete all the vectors: ') == 'y':
+            index.delete(delete_all=True)
+            logger.info('delete all')
+        else:
+            pass
+    else:
+        pass
+    for key, value, text in zip(list(l.keys()), list(l.values()), texts):
+        index.upsert([(key, value, {"text": text, "level": level})])
+        logger.info('upload successfully')