Repository: hpcaitech/ColossalAI
File: applications/ColossalQA/examples/webui_demo/server.py (Python)

Code before:
    except Exception as e:
        return {"response": "模型生成回答有误", "error": f"Error in generating answers, details: {e}"}


if __name__ == "__main__":
    args = parseArgs()

    all_config = config.ALL_CONFIG
    model_name = all_config["model"]["model_name"]

    # initialize chatbot
    logger.info(f"Initialize the chatbot from {model_name}")

    if all_config["model"]["mode"] == "local":
        colossal_api = ColossalAPI(model_name, all_config["model"]["model_path"])
        llm = ColossalLLM(n=1, api=colossal_api)
    elif all_config["model"]["mode"] == "api":
        if model_name == "pangu_api":
            from colossalqa.local.pangu_llm import Pangu
            
            gen_config = {
                "user": "User",
                "max_tokens": all_config["chain"]["disambig_llm_kwargs"]["max_new_tokens"],
                "temperature": all_config["chain"]["disambig_llm_kwargs"]["temperature"],
                "n": 1
            }
            llm = Pangu(gen_config=gen_config)
            llm.set_auth_config()  # verify user's auth info here
        elif model_name == "chatgpt_api":
            from langchain.llms import OpenAI
            llm = OpenAI()
    else:
        raise ValueError("Unsupported mode.")

    # initialize chatbot
    chatbot = RAG_ChatBot(llm, all_config)

    app_config = uvicorn.Config(app, host=args.http_host, port=args.http_port)
    server = uvicorn.Server(config=app_config)
    server.run()


Review comment (suggestion):
Same that the `user` can also be customed?

What does the `n` mean here. Please also add comments here.

Thanks

Code after:
    except Exception as e:
        return {"response": "模型生成回答有误", "error": f"Error in generating answers, details: {e}"}


if __name__ == "__main__":
    args = parseArgs()

    all_config = config.ALL_CONFIG
    model_name = all_config["model"]["model_name"]

    # initialize chatbot
    logger.info(f"Initialize the chatbot from {model_name}")

    if all_config["model"]["mode"] == "local":
        colossal_api = ColossalAPI(model_name, all_config["model"]["model_path"])
        llm = ColossalLLM(n=1, api=colossal_api)
    elif all_config["model"]["mode"] == "api":
        if model_name == "pangu_api":
            from colossalqa.local.pangu_llm import Pangu
            
            gen_config = {
                "user": "User",
                "max_tokens": all_config["chain"]["disambig_llm_kwargs"]["max_new_tokens"],
                "temperature": all_config["chain"]["disambig_llm_kwargs"]["temperature"],
                "n": 1   # the number of responses generated
            }
            llm = Pangu(gen_config=gen_config)
            llm.set_auth_config()  # verify user's auth info here
        elif model_name == "chatgpt_api":
            from langchain.llms import OpenAI
            llm = OpenAI()
    else:
        raise ValueError("Unsupported mode.")

    # initialize chatbot
    chatbot = RAG_ChatBot(llm, all_config)

    app_config = uvicorn.Config(app, host=args.http_host, port=args.http_port)
    server = uvicorn.Server(config=app_config)
    server.run()