remove huggingface

requirements.txt

@@ -1,14 +1,3 @@
 fastapi
 uvicorn[standard]
-huggingface-hub
-langchain
-langchain-community
-langchain-huggingface
-langchain_openai
-transformers
-sentence-transformers
-faiss-cpu
 dashscope
-pdfminer.six
-pdfplumber
-git+https://github.com/east-and-west-magic/pgsoft.git@tag-2024-01-11-a

src/fileservice.py

@@ -1,139 +0,0 @@
-from fastapi import APIRouter
-from pgsoft.pgdate.date_utils import beijing
-from pgsoft.pghash.md5 import md5
-import pgsoft.pgfile as pgfile
-from time import sleep
-import json
-import os
-
-router = APIRouter(prefix="/file", tags=["File Service"])
-dataset_id = "pgsoft/game"
-tempdir = "game"
-pgai_code = os.environ.get("pgai_code")
-db_token = os.environ.get("db_token")
-if db_token:
-    print(db_token[:5])
-    
-
-@router.get("/download")
-def download_file(game: str, token: str, gamecode: str):
-    if token != pgai_code:
-        print(f"[{beijing()}][download file] failed")
-        return {"status": "Failure", "detail": "Invalid token"}
-    game = game.strip().lower()
-    filename = gamecode.strip() + ".json"
-    remotepath = "/".join([game, filename[:2], filename])
-    res = pgfile.download(
-        dataset_id,
-        remotepath=remotepath,
-        repo_type="dataset",
-        localdir=tempdir,
-        token=db_token,
-    )
-    if not res:
-        print(f"[{beijing()}][download file] failed")
-        return {"status": "Failure", "detail": "File not found or server error"}
-    with open(res, "r") as f:
-        outp = json.load(f)
-    print(f"[{beijing()}][download file] OK")
-    return {"status": "OK", "result": outp}
-
-
-@router.post("/upload")
-def upload_file(game: str, token: str, content: str):
-    if token != pgai_code:
-        print(f"[{beijing()}][upload file] failed")
-        return {"status": "Failure", "detail": "Invalid token"}
-    game = game.strip().lower()
-    try:
-        content_dict = json.loads(content)
-    except json.JSONDecodeError as e:
-        print(f"[{beijing()}][upload file] failed, {type(e)}: {e}")
-        return {"status": "Failure", "detail": "Invalid JSON"}
-
-    if not isinstance(content_dict, dict):
-        print(f"[{beijing()}][upload file] failed, not a dict")
-        return {"status": "Failure", "detail": "not a dict"}
-
-    needed_keys = ["game-file", "device-id"]
-    for key in needed_keys:
-        if key not in content_dict:
-            print(f'[{beijing()}][upload file] failed, missed "{key}"')
-            return {"status": "Failure", "detail": f'missed "{key}"'}
-
-    if not isinstance(content_dict["device-id"], str):
-        print(f'[{beijing()}][upload file] failed, "device-id" is not a str')
-        return {"status": "Failure", "detail": '"device-id" is not a str'}
-    if not isinstance(content_dict["game-file"], dict):
-        print(f'[{beijing()}][upload file] failed, "game-file" is not a dict')
-        return {"status": "Failure", "detail": '"game-file" is not a dict'}
-
-    obj = {
-        "upload-time": beijing().__str__(),
-        "game-file": content_dict["game-file"],
-    }
-    maxtry = 5
-    for retry in range(maxtry):
-        md5code = md5(obj)
-        remotepath = "/".join([game, md5code[:2], md5code + ".json"])
-        if not pgfile.api.file_exists(
-            repo_id=dataset_id,
-            filename=remotepath,
-            repo_type="dataset",
-            token=db_token,
-        ):
-            break
-        sleep(0.1)
-        obj["upload-time"] = beijing().__str__()
-        maxtry -= 1
-    if not maxtry and pgfile.api.file_exists(
-        repo_id=dataset_id,
-        filename=remotepath,
-        repo_type="dataset",
-        token=db_token,
-    ):
-        print(f"[{beijing()}][upload file] failed, timeout, please retry")
-        return {"status": "Failure", "detail": "timeout, please retry"}
-    filedir = os.path.join(tempdir, game, md5code[:2])
-    if not os.path.exists(filedir):
-        os.makedirs(filedir)
-    filepath = os.path.join(filedir, md5code + ".json")
-    content_indented = json.dumps(content_dict, indent=4)
-    with open(filepath, "w") as f:
-        f.write(content_indented)
-    res = pgfile.upload(
-        filepath,
-        remotepath,
-        dataset_id,
-        "dataset",
-        db_token,
-        f"Updated at {beijing()}",
-    )
-    if not res:
-        print(f"[{beijing()}][upload file] failed")
-        return {"status": "Failure", "detail": "server error"}
-    print(f"[{beijing()}][upload file] OK")
-    return {"status": "OK", "result": md5code}
-
-
-@router.get("/list")
-def list_files(game: str, token: str):
-    if token != pgai_code:
-        print(f"[{beijing()}][list files] failed")
-        return {"status": "Failure", "detail": "Invalid token"}
-    game = game.strip().lower()
-    games = pgfile.list_files(
-        repo_id=dataset_id,
-        repo_type="dataset",
-        token=db_token,
-    )
-    if games is None:
-        print(f"[{beijing()}][list files] failed")
-        return {"status": "Failure", "detail": "server error"}
-    games = {
-        item.split(".")[0][-32:]: item
-        for item in games
-        if item.endswith(".json") and item.startswith(game)
-    }
-    print(f"[{beijing()}][list files] OK")
-    return {"status": "OK", "result": games}

src/main.py

@@ -1,9 +1,7 @@
 from fastapi import FastAPI
-from . import fileservice
 from . import talk_to_your_manual
 
 app = FastAPI()
-# app.include_router(fileservice.router)
 app.include_router(talk_to_your_manual.router)
 
 

src/talk_to_your_manual/__init__.py

@@ -1,11 +1,9 @@
 from fastapi import APIRouter
 from . import use_aliyun
-from . import use_huggingface
 
 
 router = APIRouter(
     prefix="/talk-to-your-manual",
     tags=["Talk To Your Manual"],
 )
-router.include_router(use_aliyun.router)
-# router.include_router(use_huggingface.router)
+router.include_router(use_aliyun.router)

src/talk_to_your_manual/use_huggingface.py

@@ -1,85 +0,0 @@
-from transformers import BloomForCausalLM, BloomTokenizerFast
-from langchain.chains import RetrievalQA
-from langchain_community.vectorstores import FAISS
-from langchain.text_splitter import CharacterTextSplitter
-from langchain_community.document_loaders import PDFPlumberLoader
-from langchain_huggingface import HuggingFaceEndpointEmbeddings
-from langchain_huggingface import HuggingFacePipeline
-from transformers import pipeline
-from fastapi import APIRouter, status
-from fastapi.responses import JSONResponse
-import os
-
-router = APIRouter()
-
-# 查询文本
-query = "游戏的主要玩法是什么？"
-max_new_tokens = 100
-
-# 加载模型和tokenizer
-db_token = os.environ.get("db_token")
-model = BloomForCausalLM.from_pretrained(
-    "bigscience/bloom-560m",
-    token=db_token,
-)
-tokenizer = BloomTokenizerFast.from_pretrained(
-    "bigscience/bloom-560m",
-    token=db_token,
-)
-
-# 加载文档内容
-loader = PDFPlumberLoader("建造大楼游戏说明.pdf")
-documents = loader.load()
-splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-documents = splitter.split_documents(documents)
-# 使用 Sentence-BERT 模型将文档转化为向量
-embeddings = HuggingFaceEndpointEmbeddings(
-    repo_id="sentence-transformers/all-MiniLM-L6-v2",
-    huggingfacehub_api_token=db_token,
-)
-# 使用 FAISS 创建索引
-faiss_index = FAISS.from_documents(documents, embeddings)
-# 保存索引
-faiss_index.save_local("faiss_index")
-# 从 FAISS 索引中检索相关文档
-retriever = faiss_index.as_retriever()
-
-# 创建 Hugging Face Pipeline 以使用 BLOOM 模型
-hf_pipeline = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
-    max_new_tokens=max_new_tokens,
-)
-
-# 使用 LangChain 的 HuggingFacePipeline 类
-llm = HuggingFacePipeline(pipeline=hf_pipeline)
-
-# 创建问答链
-qa_chain = RetrievalQA.from_chain_type(
-    llm=llm,
-    retriever=retriever,
-    return_source_documents=True,
-)
-
-
-@router.get("/huggingface")
-async def call_huggingface(prompt: str):
-    try:
-        response = qa_chain.invoke({"query": prompt})
-        text: str = response["result"]
-        text = text.split("\nHelpful Answer: ")[1]
-        text = text.split("\n\n")[0]
-        return JSONResponse(
-            status_code=status.HTTP_200_OK,
-            content={
-                "result": text,
-            },
-        )
-    except Exception as e:
-        return JSONResponse(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            content={
-                "error": str(e),
-            },
-        )