diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..8449c702e0a9043c83827f23d34c3a0674a15773 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.h linguist-detectable=false +*.cpp linguist-detectable=false +*.tex linguist-detectable=false +*.cs linguist-detectable=false +*.tps linguist-detectable=false diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000000000000000000000000000000000..9c8e5016a783666ee6033569e7fcec5bf6356e34 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,75 @@ +name: Report Bug | 报告BUG +description: "Report bug" +title: "[Bug]: " +labels: [] +body: + - type: dropdown + id: download + attributes: + label: Installation Method | 安装方法与平台 + options: + - Please choose | 请选择 + - Pip Install (I ignored requirements.txt) + - Pip Install (I used latest requirements.txt) + - Anaconda (I ignored requirements.txt) + - Anaconda (I used latest requirements.txt) + - Docker(Windows/Mac) + - Docker(Linux) + - Docker-Compose(Windows/Mac) + - Docker-Compose(Linux) + - Huggingface + - Others (Please Describe) + validations: + required: true + + - type: dropdown + id: version + attributes: + label: Version | 版本 + options: + - Please choose | 请选择 + - Latest | 最新版 + - Others | 非最新版 + validations: + required: true + + - type: dropdown + id: os + attributes: + label: OS | 操作系统 + options: + - Please choose | 请选择 + - Windows + - Mac + - Linux + - Docker + validations: + required: true + + - type: textarea + id: describe + attributes: + label: Describe the bug | 简述 + description: Describe the bug | 简述 + validations: + required: true + + - type: textarea + id: screenshot + attributes: + label: Screen Shot | 有帮助的截图 + description: Screen Shot | 有帮助的截图 + validations: + required: true + + - type: textarea + id: traceback + attributes: + label: Terminal Traceback & Material to Help Reproduce Bugs | 终端traceback(如有) + 帮助我们复现的测试材料样本(如有) + description: Terminal Traceback & Material to Help Reproduce Bugs | 终端traceback(如有) + 帮助我们复现的测试材料样本(如有) + + + + + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000000000000000000000000000000000..80ac7e311c9d191f43f778f3fcbdf9d2585c2db3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,28 @@ +name: Feature Request | 功能请求 +description: "Feature Request" +title: "[Feature]: " +labels: [] +body: + - type: dropdown + id: download + attributes: + label: Class | 类型 + options: + - Please choose | 请选择 + - 其他 + - 函数插件 + - 大语言模型 + - 程序主体 + validations: + required: false + + - type: textarea + id: traceback + attributes: + label: Feature Request | 功能请求 + description: Feature Request | 功能请求 + + + + + diff --git a/.github/workflows/build-with-chatglm.yml b/.github/workflows/build-with-chatglm.yml new file mode 100644 index 0000000000000000000000000000000000000000..f968bb962a026ebb367121607885f8496addfe0e --- /dev/null +++ b/.github/workflows/build-with-chatglm.yml @@ -0,0 +1,44 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages +name: Create and publish a Docker image for ChatGLM support + +on: + push: + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}_chatglm_moss + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + file: docs/GithubAction+ChatGLM+Moss + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-with-jittorllms.yml b/.github/workflows/build-with-jittorllms.yml new file mode 100644 index 0000000000000000000000000000000000000000..c0ce126a9dafa07a176dd5f12f7260f81e20e437 --- /dev/null +++ b/.github/workflows/build-with-jittorllms.yml @@ -0,0 +1,44 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages +name: Create and publish a Docker image for ChatGLM support + +on: + push: + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}_jittorllms + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + file: docs/GithubAction+JittorLLMs + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-with-latex.yml b/.github/workflows/build-with-latex.yml new file mode 100644 index 0000000000000000000000000000000000000000..fb16d2c11fdc7e572bb78a3513b6c91744429a4b --- /dev/null +++ b/.github/workflows/build-with-latex.yml @@ -0,0 +1,44 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages +name: Create and publish a Docker image for Latex support + +on: + push: + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}_with_latex + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + file: docs/GithubAction+NoLocal+Latex + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-without-local-llms.yml b/.github/workflows/build-without-local-llms.yml new file mode 100644 index 0000000000000000000000000000000000000000..b0aed7f6b595bf89bf22d25f7e1fbe966f4f37eb --- /dev/null +++ b/.github/workflows/build-without-local-llms.yml @@ -0,0 +1,44 @@ +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages +name: Create and publish a Docker image + +on: + push: + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}_nolocal + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + file: docs/GithubAction+NoLocal + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..18d3fb84e33168f062969608f9c84d91377b0f4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,152 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot +github +.github +TEMP +TRASH + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.vscode +.idea + +history +ssr_conf +config_private.py +gpt_log +private.md +private_upload +other_llms +cradle* +debug* +private* +crazy_functions/test_project/pdf_and_word +crazy_functions/test_samples +request_llm/jittorllms +multi-language +request_llm/moss +media diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ac47b8e36ea1577c16e2940aaa133dc1774d9348 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# 此Dockerfile适用于“无本地模型”的环境构建,如果需要使用chatglm等本地模型或者latex运行依赖,请参考 docker-compose.yml +# 如何构建: 先修改 `config.py`, 然后 `docker build -t gpt-academic . ` +# 如何运行(Linux下): `docker run --rm -it --net=host gpt-academic ` +# 如何运行(其他操作系统,选择任意一个固定端口50923): `docker run --rm -it -e WEB_PORT=50923 -p 50923:50923 gpt-academic ` +FROM python:3.11 + + +# 非必要步骤,更换pip源 +RUN echo '[global]' > /etc/pip.conf && \ + echo 'index-url = https://mirrors.aliyun.com/pypi/simple/' >> /etc/pip.conf && \ + echo 'trusted-host = mirrors.aliyun.com' >> /etc/pip.conf + + +# 进入工作路径 +WORKDIR /gpt + + +# 安装大部分依赖,利用Docker缓存加速以后的构建 +COPY requirements.txt ./ +COPY ./docs/gradio-3.32.2-py3-none-any.whl ./docs/gradio-3.32.2-py3-none-any.whl +RUN pip3 install -r requirements.txt + + +# 装载项目文件,安装剩余依赖 +COPY . . +RUN pip3 install -r requirements.txt + + +# 非必要步骤,用于预热模块 +RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()' + + +# 启动 +CMD ["python3", "-u", "main.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..3877ae0a7ff6f94ac222fd704e112723db776114 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a1e1f8df3ab973a488d4fa21d19e5eeea17f275 --- /dev/null +++ b/README.md @@ -0,0 +1,371 @@ +--- +title: ChatImprovement +emoji: 😻 +colorFrom: blue +colorTo: blue +sdk: gradio +sdk_version: 3.32.0 +app_file: app.py +pinned: false +duplicated_from: qingxu98/gpt-academic +--- + +# ChatGPT 学术优化 +> **Note** +> +> 2023.7.8: Gradio, Pydantic依赖调整,已修改 `requirements.txt`。请及时**更新代码**,安装依赖时,请严格选择`requirements.txt`中**指定的版本** +> +> `pip install -r requirements.txt` + + +#
GPT 学术优化 (GPT Academic)
+ +**如果喜欢这个项目,请给它一个Star;如果您发明了好用的快捷键或函数插件,欢迎发pull requests!** + +If you like this project, please give it a Star. If you've come up with more useful academic shortcuts or functional plugins, feel free to open an issue or pull request. We also have a README in [English|](docs/README_EN.md)[日本語|](docs/README_JP.md)[한국어|](https://github.com/mldljyh/ko_gpt_academic)[Русский|](docs/README_RS.md)[Français](docs/README_FR.md) translated by this project itself. +To translate this project to arbitary language with GPT, read and run [`multi_language.py`](multi_language.py) (experimental). + +> **Note** +> +> 1.请注意只有 **高亮(如红色)** 标识的函数插件(按钮)才支持读取文件,部分插件位于插件区的**下拉菜单**中。另外我们以**最高优先级**欢迎和处理任何新插件的PR。 +> +> 2.本项目中每个文件的功能都在自译解[`self_analysis.md`](https://github.com/binary-husky/gpt_academic/wiki/chatgpt-academic%E9%A1%B9%E7%9B%AE%E8%87%AA%E8%AF%91%E8%A7%A3%E6%8A%A5%E5%91%8A)详细说明。随着版本的迭代,您也可以随时自行点击相关函数插件,调用GPT重新生成项目的自我解析报告。常见问题汇总在[`wiki`](https://github.com/binary-husky/gpt_academic/wiki/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98)当中。[安装方法](#installation)。 +> +> 3.本项目兼容并鼓励尝试国产大语言模型ChatGLM和Moss等等。支持多个api-key共存,可在配置文件中填写如`API_KEY="openai-key1,openai-key2,azure-key3,api2d-key4"`。需要临时更换`API_KEY`时,在输入区输入临时的`API_KEY`然后回车键提交后即可生效。 + + + + +
+ +功能(⭐= 近期新增功能) | 描述 +--- | --- +⭐[接入新模型](https://github.com/binary-husky/gpt_academic/wiki/%E5%A6%82%E4%BD%95%E5%88%87%E6%8D%A2%E6%A8%A1%E5%9E%8B)! | ⭐阿里达摩院[通义千问](https://modelscope.cn/models/qwen/Qwen-7B-Chat/summary),上海AI-Lab[书生](https://github.com/InternLM/InternLM),讯飞[星火](https://xinghuo.xfyun.cn/) +一键润色 | 支持一键润色、一键查找论文语法错误 +一键中英互译 | 一键中英互译 +一键代码解释 | 显示代码、解释代码、生成代码、给代码加注释 +[自定义快捷键](https://www.bilibili.com/video/BV14s4y1E7jN) | 支持自定义快捷键 +模块化设计 | 支持自定义强大的[函数插件](https://github.com/binary-husky/gpt_academic/tree/master/crazy_functions),插件支持[热更新](https://github.com/binary-husky/gpt_academic/wiki/%E5%87%BD%E6%95%B0%E6%8F%92%E4%BB%B6%E6%8C%87%E5%8D%97) +[自我程序剖析](https://www.bilibili.com/video/BV1cj411A7VW) | [函数插件] [一键读懂](https://github.com/binary-husky/gpt_academic/wiki/chatgpt-academic%E9%A1%B9%E7%9B%AE%E8%87%AA%E8%AF%91%E8%A7%A3%E6%8A%A5%E5%91%8A)本项目的源代码 +[程序剖析](https://www.bilibili.com/video/BV1cj411A7VW) | [函数插件] 一键可以剖析其他Python/C/C++/Java/Lua/...项目树 +读论文、[翻译](https://www.bilibili.com/video/BV1KT411x7Wn)论文 | [函数插件] 一键解读latex/pdf论文全文并生成摘要 +Latex全文[翻译](https://www.bilibili.com/video/BV1nk4y1Y7Js/)、[润色](https://www.bilibili.com/video/BV1FT411H7c5/) | [函数插件] 一键翻译或润色latex论文 +批量注释生成 | [函数插件] 一键批量生成函数注释 +Markdown[中英互译](https://www.bilibili.com/video/BV1yo4y157jV/) | [函数插件] 看到上面5种语言的[README](https://github.com/binary-husky/gpt_academic/blob/master/docs/README_EN.md)了吗? +chat分析报告生成 | [函数插件] 运行后自动生成总结汇报 +[PDF论文全文翻译功能](https://www.bilibili.com/video/BV1KT411x7Wn) | [函数插件] PDF论文提取题目&摘要+翻译全文(多线程) +[Arxiv小助手](https://www.bilibili.com/video/BV1LM4y1279X) | [函数插件] 输入arxiv文章url即可一键翻译摘要+下载PDF +Latex论文一键校对 | [函数插件] 仿Grammarly对Latex文章进行语法、拼写纠错+输出对照PDF +[谷歌学术统合小助手](https://www.bilibili.com/video/BV19L411U7ia) | [函数插件] 给定任意谷歌学术搜索页面URL,让gpt帮你[写relatedworks](https://www.bilibili.com/video/BV1GP411U7Az/) +互联网信息聚合+GPT | [函数插件] 一键[让GPT从互联网获取信息](https://www.bilibili.com/video/BV1om4y127ck)回答问题,让信息永不过时 +⭐Arxiv论文精细翻译 ([Docker](https://github.com/binary-husky/gpt_academic/pkgs/container/gpt_academic_with_latex)) | [函数插件] 一键[以超高质量翻译arxiv论文](https://www.bilibili.com/video/BV1dz4y1v77A/),目前最好的论文翻译工具 +⭐[实时语音对话输入](https://github.com/binary-husky/gpt_academic/blob/master/docs/use_audio.md) | [函数插件] 异步[监听音频](https://www.bilibili.com/video/BV1AV4y187Uy/),自动断句,自动寻找回答时机 +公式/图片/表格显示 | 可以同时显示公式的[tex形式和渲染形式](https://user-images.githubusercontent.com/96192199/230598842-1d7fcddd-815d-40ee-af60-baf488a199df.png),支持公式、代码高亮 +多线程函数插件支持 | 支持多线调用chatgpt,一键处理[海量文本](https://www.bilibili.com/video/BV1FT411H7c5/)或程序 +启动暗色[主题](https://github.com/binary-husky/gpt_academic/issues/173) | 在浏览器url后面添加```/?__theme=dark```可以切换dark主题 +[多LLM模型](https://www.bilibili.com/video/BV1wT411p7yf)支持 | 同时被GPT3.5、GPT4、[清华ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)、[复旦MOSS](https://github.com/OpenLMLab/MOSS)同时伺候的感觉一定会很不错吧? +⭐ChatGLM2微调模型 | 支持加载ChatGLM2微调模型,提供ChatGLM2微调辅助插件 +更多LLM模型接入,支持[huggingface部署](https://huggingface.co/spaces/qingxu98/gpt-academic) | 加入Newbing接口(新必应),引入清华[Jittorllms](https://github.com/Jittor/JittorLLMs)支持[LLaMA](https://github.com/facebookresearch/llama)和[盘古α](https://openi.org.cn/pangu/) +⭐[虚空终端](https://github.com/binary-husky/void-terminal)pip包 | 脱离GUI,在Python中直接调用本项目的函数插件(开发中) +更多新功能展示 (图像生成等) …… | 见本文档结尾处 …… +
+ + +- 新界面(修改`config.py`中的LAYOUT选项即可实现“左右布局”和“上下布局”的切换) +
+ +
+ + +- 所有按钮都通过读取functional.py动态生成,可随意加自定义功能,解放粘贴板 +
+ +
+ +- 润色/纠错 +
+ +
+ +- 如果输出包含公式,会同时以tex形式和渲染形式显示,方便复制和阅读 +
+ +
+ +- 懒得看项目代码?整个工程直接给chatgpt炫嘴里 +
+ +
+ +- 多种大语言模型混合调用(ChatGLM + OpenAI-GPT3.5 + [API2D](https://api2d.com/)-GPT4) +
+ +
+ +# Installation +### 安装方法I:直接运行 (Windows, Linux or MacOS) + +1. 下载项目 +```sh +git clone --depth=1 https://github.com/binary-husky/gpt_academic.git +cd gpt_academic +``` + +2. 配置API_KEY + +在`config.py`中,配置API KEY等设置,[点击查看特殊网络环境设置方法](https://github.com/binary-husky/gpt_academic/issues/1) 。 + +(P.S. 程序运行时会优先检查是否存在名为`config_private.py`的私密配置文件,并用其中的配置覆盖`config.py`的同名配置。因此,如果您能理解我们的配置读取逻辑,我们强烈建议您在`config.py`旁边创建一个名为`config_private.py`的新配置文件,并把`config.py`中的配置转移(复制)到`config_private.py`中。`config_private.py`不受git管控,可以让您的隐私信息更加安全。P.S.项目同样支持通过`环境变量`配置大多数选项,环境变量的书写格式参考`docker-compose`文件。读取优先级: `环境变量` > `config_private.py` > `config.py`) + + +3. 安装依赖 +```sh +# (选择I: 如熟悉python)(python版本3.9以上,越新越好),备注:使用官方pip源或者阿里pip源,临时换源方法:python -m pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +python -m pip install -r requirements.txt + +# (选择II: 如不熟悉python)使用anaconda,步骤也是类似的 (https://www.bilibili.com/video/BV1rc411W7Dr): +conda create -n gptac_venv python=3.11 # 创建anaconda环境 +conda activate gptac_venv # 激活anaconda环境 +python -m pip install -r requirements.txt # 这个步骤和pip安装一样的步骤 +``` + + +
如果需要支持清华ChatGLM2/复旦MOSS/RWKV作为后端,请点击展开此处 +

+ +【可选步骤】如果需要支持清华ChatGLM2/复旦MOSS作为后端,需要额外安装更多依赖(前提条件:熟悉Python + 用过Pytorch + 电脑配置够强): +```sh +# 【可选步骤I】支持清华ChatGLM2。清华ChatGLM备注:如果遇到"Call ChatGLM fail 不能正常加载ChatGLM的参数" 错误,参考如下: 1:以上默认安装的为torch+cpu版,使用cuda需要卸载torch重新安装torch+cuda; 2:如因本机配置不够无法加载模型,可以修改request_llm/bridge_chatglm.py中的模型精度, 将 AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) 都修改为 AutoTokenizer.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True) +python -m pip install -r request_llm/requirements_chatglm.txt + +# 【可选步骤II】支持复旦MOSS +python -m pip install -r request_llm/requirements_moss.txt +git clone --depth=1 https://github.com/OpenLMLab/MOSS.git request_llm/moss # 注意执行此行代码时,必须处于项目根路径 + +# 【可选步骤III】支持RWKV Runner +参考wiki:https://github.com/binary-husky/gpt_academic/wiki/%E9%80%82%E9%85%8DRWKV-Runner + +# 【可选步骤IV】确保config.py配置文件的AVAIL_LLM_MODELS包含了期望的模型,目前支持的全部模型如下(jittorllms系列目前仅支持docker方案): +AVAIL_LLM_MODELS = ["gpt-3.5-turbo", "api2d-gpt-3.5-turbo", "gpt-4", "api2d-gpt-4", "chatglm", "newbing", "moss"] # + ["jittorllms_rwkv", "jittorllms_pangualpha", "jittorllms_llama"] +``` + +

+
+ + + +4. 运行 +```sh +python main.py +``` + +### 安装方法II:使用Docker + +1. 仅ChatGPT(推荐大多数人选择,等价于docker-compose方案1) +[![basic](https://github.com/binary-husky/gpt_academic/actions/workflows/build-without-local-llms.yml/badge.svg?branch=master)](https://github.com/binary-husky/gpt_academic/actions/workflows/build-without-local-llms.yml) +[![basiclatex](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-latex.yml/badge.svg?branch=master)](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-latex.yml) +[![basicaudio](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-audio-assistant.yml/badge.svg?branch=master)](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-audio-assistant.yml) + +``` sh +git clone --depth=1 https://github.com/binary-husky/gpt_academic.git # 下载项目 +cd gpt_academic # 进入路径 +nano config.py # 用任意文本编辑器编辑config.py, 配置 “Proxy”, “API_KEY” 以及 “WEB_PORT” (例如50923) 等 +docker build -t gpt-academic . # 安装 + +#(最后一步-Linux操作系统)用`--net=host`更方便快捷 +docker run --rm -it --net=host gpt-academic +#(最后一步-MacOS/Windows操作系统)只能用-p选项将容器上的端口(例如50923)暴露给主机上的端口 +docker run --rm -it -e WEB_PORT=50923 -p 50923:50923 gpt-academic +``` +P.S. 如果需要依赖Latex的插件功能,请见Wiki。另外,您也可以直接使用docker-compose获取Latex功能(修改docker-compose.yml,保留方案4并删除其他方案)。 + +2. ChatGPT + ChatGLM2 + MOSS(需要熟悉Docker) +[![chatglm](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-chatglm.yml/badge.svg?branch=master)](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-chatglm.yml) + +``` sh +# 修改docker-compose.yml,保留方案2并删除其他方案。修改docker-compose.yml中方案2的配置,参考其中注释即可 +docker-compose up +``` + +3. ChatGPT + LLAMA + 盘古 + RWKV(需要熟悉Docker) +[![jittorllms](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-jittorllms.yml/badge.svg?branch=master)](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-jittorllms.yml) + +``` sh +# 修改docker-compose.yml,保留方案3并删除其他方案。修改docker-compose.yml中方案3的配置,参考其中注释即可 +docker-compose up +``` + + +### 安装方法III:其他部署姿势 +1. 一键运行脚本。 +完全不熟悉python环境的Windows用户可以下载[Release](https://github.com/binary-husky/gpt_academic/releases)中发布的一键运行脚本安装无本地模型的版本。 +脚本的贡献来源是[oobabooga](https://github.com/oobabooga/one-click-installers)。 + +2. 使用docker-compose运行。 +请阅读docker-compose.yml后,按照其中的提示操作即可 + +3. 如何使用反代URL +按照`config.py`中的说明配置API_URL_REDIRECT即可。 + +4. 微软云AzureAPI +按照`config.py`中的说明配置即可(AZURE_ENDPOINT等四个配置) + +5. 远程云服务器部署(需要云服务器知识与经验)。 +请访问[部署wiki-1](https://github.com/binary-husky/gpt_academic/wiki/%E4%BA%91%E6%9C%8D%E5%8A%A1%E5%99%A8%E8%BF%9C%E7%A8%8B%E9%83%A8%E7%BD%B2%E6%8C%87%E5%8D%97) + +6. 使用Sealos[一键部署](https://github.com/binary-husky/gpt_academic/issues/993)。 + +7. 使用WSL2(Windows Subsystem for Linux 子系统)。 +请访问[部署wiki-2](https://github.com/binary-husky/gpt_academic/wiki/%E4%BD%BF%E7%94%A8WSL2%EF%BC%88Windows-Subsystem-for-Linux-%E5%AD%90%E7%B3%BB%E7%BB%9F%EF%BC%89%E9%83%A8%E7%BD%B2) + +8. 如何在二级网址(如`http://localhost/subpath`)下运行。 +请访问[FastAPI运行说明](docs/WithFastapi.md) + + +# Advanced Usage +### I:自定义新的便捷按钮(学术快捷键) +任意文本编辑器打开`core_functional.py`,添加条目如下,然后重启程序即可。(如果按钮已经添加成功并可见,那么前缀、后缀都支持热修改,无需重启程序即可生效。) +例如 +``` +"超级英译中": { + # 前缀,会被加在你的输入之前。例如,用来描述你的要求,例如翻译、解释代码、润色等等 + "Prefix": "请翻译把下面一段内容成中文,然后用一个markdown表格逐一解释文中出现的专有名词:\n\n", + + # 后缀,会被加在你的输入之后。例如,配合前缀可以把你的输入内容用引号圈起来。 + "Suffix": "", +}, +``` +
+ +
+ +### II:自定义函数插件 + +编写强大的函数插件来执行任何你想得到的和想不到的任务。 +本项目的插件编写、调试难度很低,只要您具备一定的python基础知识,就可以仿照我们提供的模板实现自己的插件功能。 +详情请参考[函数插件指南](https://github.com/binary-husky/gpt_academic/wiki/%E5%87%BD%E6%95%B0%E6%8F%92%E4%BB%B6%E6%8C%87%E5%8D%97)。 + + +# Latest Update +### I:新功能动态 + +1. 对话保存功能。在函数插件区调用 `保存当前的对话` 即可将当前对话保存为可读+可复原的html文件, +另外在函数插件区(下拉菜单)调用 `载入对话历史存档` ,即可还原之前的会话。 +Tip:不指定文件直接点击 `载入对话历史存档` 可以查看历史html存档缓存。 +
+ +
+ +2. ⭐Latex/Arxiv论文翻译功能⭐ +
+ ===> + +
+ +3. 生成报告。大部分插件都会在执行结束后,生成工作报告 +
+ + +
+ +4. 模块化功能设计,简单的接口却能支持强大的功能 +
+ + +
+ +5. 译解其他开源项目 +
+ + +
+ +6. 装饰[live2d](https://github.com/fghrsh/live2d_demo)的小功能(默认关闭,需要修改`config.py`) +
+ +
+ +7. 新增MOSS大语言模型支持 +
+ +
+ +8. OpenAI图像生成 +
+ +
+ +9. OpenAI音频解析与总结 +
+ +
+ +10. Latex全文校对纠错 +
+ ===> + +
+ +11. 语言、主题切换 +
+ +
+ + +### II:版本: +- version 3.5(Todo): 使用自然语言调用本项目的所有函数插件(高优先级) +- version 3.46: 支持完全脱手操作的实时语音对话 +- version 3.45: 支持自定义ChatGLM2微调模型 +- version 3.44: 正式支持Azure,优化界面易用性 +- version 3.4: +arxiv论文翻译、latex论文批改功能 +- version 3.3: +互联网信息综合功能 +- version 3.2: 函数插件支持更多参数接口 (保存对话功能, 解读任意语言代码+同时询问任意的LLM组合) +- version 3.1: 支持同时问询多个gpt模型!支持api2d,支持多个apikey负载均衡 +- version 3.0: 对chatglm和其他小型llm的支持 +- version 2.6: 重构了插件结构,提高了交互性,加入更多插件 +- version 2.5: 自更新,解决总结大工程源代码时文本过长、token溢出的问题 +- version 2.4: (1)新增PDF全文翻译功能; (2)新增输入区切换位置的功能; (3)新增垂直布局选项; (4)多线程函数插件优化。 +- version 2.3: 增强多线程交互性 +- version 2.2: 函数插件支持热重载 +- version 2.1: 可折叠式布局 +- version 2.0: 引入模块化函数插件 +- version 1.0: 基础功能 + +gpt_academic开发者QQ群-2:610599535 + +- 已知问题 + - 某些浏览器翻译插件干扰此软件前端的运行 + - 官方Gradio目前有很多兼容性Bug,请务必使用`requirement.txt`安装Gradio + +### III:主题 +可以通过修改`THEME`选项(config.py)变更主题 +1. `Chuanhu-Small-and-Beautiful` [网址](https://github.com/GaiZhenbiao/ChuanhuChatGPT/) + + +### IV:参考与学习 + +``` +代码中参考了很多其他优秀项目中的设计,顺序不分先后: + +# 清华ChatGLM2-6B: +https://github.com/THUDM/ChatGLM2-6B + +# 清华JittorLLMs: +https://github.com/Jittor/JittorLLMs + +# ChatPaper: +https://github.com/kaixindelele/ChatPaper + +# Edge-GPT: +https://github.com/acheong08/EdgeGPT + +# ChuanhuChatGPT: +https://github.com/GaiZhenbiao/ChuanhuChatGPT + +# Oobabooga one-click installer: +https://github.com/oobabooga/one-click-installers + +# More: +https://github.com/gradio-app/gradio +https://github.com/fghrsh/live2d_demo +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e38af7df22f87e6941839ecced4f1e4782d60802 --- /dev/null +++ b/app.py @@ -0,0 +1,238 @@ +import os; os.environ['no_proxy'] = '*' # 避免代理网络产生意外污染 + +def main(): + import subprocess, sys + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'gradio-stable-fork']) + import gradio as gr + if gr.__version__ not in ['3.28.3','3.32.3']: assert False, "需要特殊依赖,请务必用 pip install -r requirements.txt 指令安装依赖,详情信息见requirements.txt" + from request_llm.bridge_all import predict + from toolbox import format_io, find_free_port, on_file_uploaded, on_report_generated, get_conf, ArgsGeneralWrapper, load_chat_cookies, DummyWith + # 建议您复制一个config_private.py放自己的秘密, 如API和代理网址, 避免不小心传github被别人看到 + proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = \ + get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT') + ENABLE_AUDIO, AUTO_CLEAR_TXT = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT') + # 如果WEB_PORT是-1, 则随机选取WEB端口 + PORT = find_free_port() if WEB_PORT <= 0 else WEB_PORT + if not AUTHENTICATION: AUTHENTICATION = None + + from check_proxy import get_current_version + from themes.theme import adjust_theme, advanced_css, theme_declaration + initial_prompt = "Serve me as a writing and programming assistant." + title_html = f"

GPT 学术优化 {get_current_version()}

{theme_declaration}" + description = """代码开源和更新[地址🚀](https://github.com/binary-husky/chatgpt_academic),感谢热情的[开发者们❤️](https://github.com/binary-husky/chatgpt_academic/graphs/contributors)""" + + # 问询记录, python 版本建议3.9+(越新越好) + import logging, uuid + os.makedirs("gpt_log", exist_ok=True) + try:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + except:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + # Disable logging output from the 'httpx' logger + logging.getLogger("httpx").setLevel(logging.WARNING) + print("所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log, 请注意自我隐私保护哦!") + + # 一些普通功能模块 + from core_functional import get_core_functions + functional = get_core_functions() + + # 高级函数插件 + from crazy_functional import get_crazy_functions + crazy_fns = get_crazy_functions() + + # 处理markdown文本格式的转变 + gr.Chatbot.postprocess = format_io + + # 做一些外观色彩上的调整 + set_theme = adjust_theme() + + # 代理与自动更新 + from check_proxy import check_proxy, auto_update, warm_up_modules + proxy_info = check_proxy(proxies) + + gr_L1 = lambda: gr.Row().style() + gr_L2 = lambda scale, elem_id: gr.Column(scale=scale, elem_id=elem_id) + if LAYOUT == "TOP-DOWN": + gr_L1 = lambda: DummyWith() + gr_L2 = lambda scale, elem_id: gr.Row() + CHATBOT_HEIGHT /= 2 + + cancel_handles = [] + with gr.Blocks(title="GPT 学术优化", theme=set_theme, analytics_enabled=False, css=advanced_css) as demo: + gr.HTML(title_html) + gr.HTML('''
Duplicate Space请您打开此页面后务必点击上方的“复制空间”(Duplicate Space)按钮!使用时,先在输入框填入API-KEY然后回车。
切忌在“复制空间”(Duplicate Space)之前填入API_KEY或进行提问,否则您的API_KEY将极可能被空间所有者攫取!
支持任意数量的OpenAI的密钥和API2D的密钥共存,例如输入"OpenAI密钥1,API2D密钥2",然后提交,即可同时使用两种模型接口。
''') + cookies = gr.State(load_chat_cookies()) + with gr_L1(): + with gr_L2(scale=2, elem_id="gpt-chat"): + chatbot = gr.Chatbot(label=f"当前模型:{LLM_MODEL}", elem_id="gpt-chatbot") + if LAYOUT == "TOP-DOWN": chatbot.style(height=CHATBOT_HEIGHT) + history = gr.State([]) + with gr_L2(scale=1, elem_id="gpt-panel"): + with gr.Accordion("输入区", open=True, elem_id="input-panel") as area_input_primary: + with gr.Row(): + txt = gr.Textbox(show_label=False, lines=2, placeholder="输入问题或API密钥,输入多个密钥时,用英文逗号间隔。支持OpenAI密钥和API2D密钥共存。").style(container=False) + with gr.Row(): + submitBtn = gr.Button("提交", variant="primary") + with gr.Row(): + resetBtn = gr.Button("重置", variant="secondary"); resetBtn.style(size="sm") + stopBtn = gr.Button("停止", variant="secondary"); stopBtn.style(size="sm") + clearBtn = gr.Button("清除", variant="secondary", visible=False); clearBtn.style(size="sm") + if ENABLE_AUDIO: + with gr.Row(): + audio_mic = gr.Audio(source="microphone", type="numpy", streaming=True, show_label=False).style(container=False) + with gr.Row(): + status = gr.Markdown(f"Tip: 按Enter提交, 按Shift+Enter换行。当前模型: {LLM_MODEL} \n {proxy_info}", elem_id="state-panel") + with gr.Accordion("基础功能区", open=True, elem_id="basic-panel") as area_basic_fn: + with gr.Row(): + for k in functional: + if ("Visible" in functional[k]) and (not functional[k]["Visible"]): continue + variant = functional[k]["Color"] if "Color" in functional[k] else "secondary" + functional[k]["Button"] = gr.Button(k, variant=variant) + with gr.Accordion("函数插件区", open=True, elem_id="plugin-panel") as area_crazy_fn: + with gr.Row(): + gr.Markdown("插件可读取“输入区”文本/路径作为参数(上传文件自动修正路径)") + with gr.Row(): + for k in crazy_fns: + if not crazy_fns[k].get("AsButton", True): continue + variant = crazy_fns[k]["Color"] if "Color" in crazy_fns[k] else "secondary" + crazy_fns[k]["Button"] = gr.Button(k, variant=variant) + crazy_fns[k]["Button"].style(size="sm") + with gr.Row(): + with gr.Accordion("更多函数插件", open=True): + dropdown_fn_list = [k for k in crazy_fns.keys() if not crazy_fns[k].get("AsButton", True)] + with gr.Row(): + dropdown = gr.Dropdown(dropdown_fn_list, value=r"打开插件列表", label="", show_label=False).style(container=False) + with gr.Row(): + plugin_advanced_arg = gr.Textbox(show_label=True, label="高级参数输入区", visible=False, + placeholder="这里是特殊函数插件的高级参数输入区").style(container=False) + with gr.Row(): + switchy_bt = gr.Button(r"请先从插件列表中选择", variant="secondary") + with gr.Row(): + with gr.Accordion("点击展开“文件上传区”。上传本地文件/压缩包供函数插件调用。", open=False) as area_file_up: + file_upload = gr.Files(label="任何文件, 但推荐上传压缩文件(zip, tar)", file_count="multiple") + with gr.Accordion("更换模型 & SysPrompt & 交互界面布局", open=(LAYOUT == "TOP-DOWN"), elem_id="interact-panel"): + system_prompt = gr.Textbox(show_label=True, placeholder=f"System Prompt", label="System prompt", value=initial_prompt) + top_p = gr.Slider(minimum=-0, maximum=1.0, value=1.0, step=0.01,interactive=True, label="Top-p (nucleus sampling)",) + temperature = gr.Slider(minimum=-0, maximum=2.0, value=1.0, step=0.01, interactive=True, label="Temperature",) + max_length_sl = gr.Slider(minimum=256, maximum=8192, value=4096, step=1, interactive=True, label="Local LLM MaxLength",) + checkboxes = gr.CheckboxGroup(["基础功能区", "函数插件区", "底部输入区", "输入清除键", "插件参数区"], value=["基础功能区", "函数插件区"], label="显示/隐藏功能区") + md_dropdown = gr.Dropdown(AVAIL_LLM_MODELS, value=LLM_MODEL, label="更换LLM模型/请求源").style(container=False) + + gr.Markdown(description) + with gr.Accordion("备选输入区", open=True, visible=False, elem_id="input-panel2") as area_input_secondary: + with gr.Row(): + txt2 = gr.Textbox(show_label=False, placeholder="Input question here.", label="输入区2").style(container=False) + with gr.Row(): + submitBtn2 = gr.Button("提交", variant="primary") + with gr.Row(): + resetBtn2 = gr.Button("重置", variant="secondary"); resetBtn2.style(size="sm") + stopBtn2 = gr.Button("停止", variant="secondary"); stopBtn2.style(size="sm") + clearBtn2 = gr.Button("清除", variant="secondary", visible=False); clearBtn2.style(size="sm") + # 功能区显示开关与功能区的互动 + def fn_area_visibility(a): + ret = {} + ret.update({area_basic_fn: gr.update(visible=("基础功能区" in a))}) + ret.update({area_crazy_fn: gr.update(visible=("函数插件区" in a))}) + ret.update({area_input_primary: gr.update(visible=("底部输入区" not in a))}) + ret.update({area_input_secondary: gr.update(visible=("底部输入区" in a))}) + ret.update({clearBtn: gr.update(visible=("输入清除键" in a))}) + ret.update({clearBtn2: gr.update(visible=("输入清除键" in a))}) + ret.update({plugin_advanced_arg: gr.update(visible=("插件参数区" in a))}) + if "底部输入区" in a: ret.update({txt: gr.update(value="")}) + return ret + checkboxes.select(fn_area_visibility, [checkboxes], [area_basic_fn, area_crazy_fn, area_input_primary, area_input_secondary, txt, txt2, clearBtn, clearBtn2, plugin_advanced_arg] ) + # 整理反复出现的控件句柄组合 + input_combo = [cookies, max_length_sl, md_dropdown, txt, txt2, top_p, temperature, chatbot, history, system_prompt, plugin_advanced_arg] + output_combo = [cookies, chatbot, history, status] + predict_args = dict(fn=ArgsGeneralWrapper(predict), inputs=input_combo, outputs=output_combo) + # 提交按钮、重置按钮 + cancel_handles.append(txt.submit(**predict_args)) + cancel_handles.append(txt2.submit(**predict_args)) + cancel_handles.append(submitBtn.click(**predict_args)) + cancel_handles.append(submitBtn2.click(**predict_args)) + resetBtn.click(lambda: ([], [], "已重置"), None, [chatbot, history, status]) + resetBtn2.click(lambda: ([], [], "已重置"), None, [chatbot, history, status]) + clearBtn.click(lambda: ("",""), None, [txt, txt2]) + clearBtn2.click(lambda: ("",""), None, [txt, txt2]) + if AUTO_CLEAR_TXT: + submitBtn.click(lambda: ("",""), None, [txt, txt2]) + submitBtn2.click(lambda: ("",""), None, [txt, txt2]) + txt.submit(lambda: ("",""), None, [txt, txt2]) + txt2.submit(lambda: ("",""), None, [txt, txt2]) + # 基础功能区的回调函数注册 + for k in functional: + if ("Visible" in functional[k]) and (not functional[k]["Visible"]): continue + click_handle = functional[k]["Button"].click(fn=ArgsGeneralWrapper(predict), inputs=[*input_combo, gr.State(True), gr.State(k)], outputs=output_combo) + cancel_handles.append(click_handle) + # 文件上传区,接收文件后与chatbot的互动 + file_upload.upload(on_file_uploaded, [file_upload, chatbot, txt, txt2, checkboxes], [chatbot, txt, txt2]) + # 函数插件-固定按钮区 + for k in crazy_fns: + if not crazy_fns[k].get("AsButton", True): continue + click_handle = crazy_fns[k]["Button"].click(ArgsGeneralWrapper(crazy_fns[k]["Function"]), [*input_combo, gr.State(PORT)], output_combo) + click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) + cancel_handles.append(click_handle) + # 函数插件-下拉菜单与随变按钮的互动 + def on_dropdown_changed(k): + variant = crazy_fns[k]["Color"] if "Color" in crazy_fns[k] else "secondary" + ret = {switchy_bt: gr.update(value=k, variant=variant)} + if crazy_fns[k].get("AdvancedArgs", False): # 是否唤起高级插件参数区 + ret.update({plugin_advanced_arg: gr.update(visible=True, label=f"插件[{k}]的高级参数说明:" + crazy_fns[k].get("ArgsReminder", [f"没有提供高级参数功能说明"]))}) + else: + ret.update({plugin_advanced_arg: gr.update(visible=False, label=f"插件[{k}]不需要高级参数。")}) + return ret + dropdown.select(on_dropdown_changed, [dropdown], [switchy_bt, plugin_advanced_arg] ) + def on_md_dropdown_changed(k): + return {chatbot: gr.update(label="当前模型:"+k)} + md_dropdown.select(on_md_dropdown_changed, [md_dropdown], [chatbot] ) + # 随变按钮的回调函数注册 + def route(request: gr.Request, k, *args, **kwargs): + if k in [r"打开插件列表", r"请先从插件列表中选择"]: return + yield from ArgsGeneralWrapper(crazy_fns[k]["Function"])(request, *args, **kwargs) + click_handle = switchy_bt.click(route,[switchy_bt, *input_combo, gr.State(PORT)], output_combo) + click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) + cancel_handles.append(click_handle) + # 终止按钮的回调函数注册 + stopBtn.click(fn=None, inputs=None, outputs=None, cancels=cancel_handles) + stopBtn2.click(fn=None, inputs=None, outputs=None, cancels=cancel_handles) + if ENABLE_AUDIO: + from crazy_functions.live_audio.audio_io import RealtimeAudioDistribution + rad = RealtimeAudioDistribution() + def deal_audio(audio, cookies): + rad.feed(cookies['uuid'].hex, audio) + audio_mic.stream(deal_audio, inputs=[audio_mic, cookies]) + + def init_cookie(cookies, chatbot): + # 为每一位访问的用户赋予一个独一无二的uuid编码 + cookies.update({'uuid': uuid.uuid4()}) + return cookies + demo.load(init_cookie, inputs=[cookies, chatbot], outputs=[cookies]) + demo.load(lambda: 0, inputs=None, outputs=None, _js='()=>{ChatBotHeight();}') + + # gradio的inbrowser触发不太稳定,回滚代码到原始的浏览器打开函数 + def auto_opentab_delay(): + import threading, webbrowser, time + print(f"如果浏览器没有自动打开,请复制并转到以下URL:") + print(f"\t(亮色主题): http://localhost:{PORT}") + print(f"\t(暗色主题): http://localhost:{PORT}/?__theme=dark") + def open(): + time.sleep(2) # 打开浏览器 + DARK_MODE, = get_conf('DARK_MODE') + if DARK_MODE: webbrowser.open_new_tab(f"http://localhost:{PORT}/?__theme=dark") + else: webbrowser.open_new_tab(f"http://localhost:{PORT}") + threading.Thread(target=open, name="open-browser", daemon=True).start() + threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start() + threading.Thread(target=warm_up_modules, name="warm-up", daemon=True).start() + + auto_opentab_delay() + demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", share=False, favicon_path="docs/logo.png", blocked_paths=["config.py","config_private.py","docker-compose.yml","Dockerfile"]) + + # 如果需要在二级路径下运行 + # CUSTOM_PATH, = get_conf('CUSTOM_PATH') + # if CUSTOM_PATH != "/": + # from toolbox import run_gradio_in_subpath + # run_gradio_in_subpath(demo, auth=AUTHENTICATION, port=PORT, custom_path=CUSTOM_PATH) + # else: + # demo.launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION, favicon_path="docs/logo.png", + # blocked_paths=["config.py","config_private.py","docker-compose.yml","Dockerfile"]) + +if __name__ == "__main__": + main() diff --git a/check_proxy.py b/check_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..474988c129d51e3f1a4fa59634f073cbda553c09 --- /dev/null +++ b/check_proxy.py @@ -0,0 +1,169 @@ + +def check_proxy(proxies): + import requests + proxies_https = proxies['https'] if proxies is not None else '无' + try: + response = requests.get("https://ipapi.co/json/", proxies=proxies, timeout=4) + data = response.json() + print(f'查询代理的地理位置,返回的结果是{data}') + if 'country_name' in data: + country = data['country_name'] + result = f"代理配置 {proxies_https}, 代理所在地:{country}" + elif 'error' in data: + alternative = _check_with_backup_source(proxies) + if alternative is None: + result = f"代理配置 {proxies_https}, 代理所在地:未知,IP查询频率受限" + else: + result = f"代理配置 {proxies_https}, 代理所在地:{alternative}" + else: + result = f"代理配置 {proxies_https}, 代理数据解析失败:{data}" + print(result) + return result + except: + result = f"代理配置 {proxies_https}, 代理所在地查询超时,代理可能无效" + print(result) + return result + +def _check_with_backup_source(proxies): + import random, string, requests + random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=32)) + try: return requests.get(f"http://{random_string}.edns.ip-api.com/json", proxies=proxies, timeout=4).json()['dns']['geo'] + except: return None + +def backup_and_download(current_version, remote_version): + """ + 一键更新协议:备份和下载 + """ + from toolbox import get_conf + import shutil + import os + import requests + import zipfile + os.makedirs(f'./history', exist_ok=True) + backup_dir = f'./history/backup-{current_version}/' + new_version_dir = f'./history/new-version-{remote_version}/' + if os.path.exists(new_version_dir): + return new_version_dir + os.makedirs(new_version_dir) + shutil.copytree('./', backup_dir, ignore=lambda x, y: ['history']) + proxies, = get_conf('proxies') + r = requests.get( + 'https://github.com/binary-husky/chatgpt_academic/archive/refs/heads/master.zip', proxies=proxies, stream=True) + zip_file_path = backup_dir+'/master.zip' + with open(zip_file_path, 'wb+') as f: + f.write(r.content) + dst_path = new_version_dir + with zipfile.ZipFile(zip_file_path, "r") as zip_ref: + for zip_info in zip_ref.infolist(): + dst_file_path = os.path.join(dst_path, zip_info.filename) + if os.path.exists(dst_file_path): + os.remove(dst_file_path) + zip_ref.extract(zip_info, dst_path) + return new_version_dir + + +def patch_and_restart(path): + """ + 一键更新协议:覆盖和重启 + """ + from distutils import dir_util + import shutil + import os + import sys + import time + import glob + from colorful import print亮黄, print亮绿, print亮红 + # if not using config_private, move origin config.py as config_private.py + if not os.path.exists('config_private.py'): + print亮黄('由于您没有设置config_private.py私密配置,现将您的现有配置移动至config_private.py以防止配置丢失,', + '另外您可以随时在history子文件夹下找回旧版的程序。') + shutil.copyfile('config.py', 'config_private.py') + path_new_version = glob.glob(path + '/*-master')[0] + dir_util.copy_tree(path_new_version, './') + print亮绿('代码已经更新,即将更新pip包依赖……') + for i in reversed(range(5)): time.sleep(1); print(i) + try: + import subprocess + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt']) + except: + print亮红('pip包依赖安装出现问题,需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`,然后在用常规的`python main.py`的方式启动。') + print亮绿('更新完成,您可以随时在history子文件夹下找回旧版的程序,5s之后重启') + print亮红('假如重启失败,您可能需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`,然后在用常规的`python main.py`的方式启动。') + print(' ------------------------------ -----------------------------------') + for i in reversed(range(8)): time.sleep(1); print(i) + os.execl(sys.executable, sys.executable, *sys.argv) + + +def get_current_version(): + import json + try: + with open('./version', 'r', encoding='utf8') as f: + current_version = json.loads(f.read())['version'] + except: + current_version = "" + return current_version + + +def auto_update(raise_error=False): + """ + 一键更新协议:查询版本和用户意见 + """ + try: + from toolbox import get_conf + import requests + import time + import json + proxies, = get_conf('proxies') + response = requests.get( + "https://raw.githubusercontent.com/binary-husky/chatgpt_academic/master/version", proxies=proxies, timeout=5) + remote_json_data = json.loads(response.text) + remote_version = remote_json_data['version'] + if remote_json_data["show_feature"]: + new_feature = "新功能:" + remote_json_data["new_feature"] + else: + new_feature = "" + with open('./version', 'r', encoding='utf8') as f: + current_version = f.read() + current_version = json.loads(current_version)['version'] + if (remote_version - current_version) >= 0.01-1e-5: + from colorful import print亮黄 + print亮黄( + f'\n新版本可用。新版本:{remote_version},当前版本:{current_version}。{new_feature}') + print('(1)Github更新地址:\nhttps://github.com/binary-husky/chatgpt_academic\n') + user_instruction = input('(2)是否一键更新代码(Y+回车=确认,输入其他/无输入+回车=不更新)?') + if user_instruction in ['Y', 'y']: + path = backup_and_download(current_version, remote_version) + try: + patch_and_restart(path) + except: + msg = '更新失败。' + if raise_error: + from toolbox import trimmed_format_exc + msg += trimmed_format_exc() + print(msg) + else: + print('自动更新程序:已禁用') + return + else: + return + except: + msg = '自动更新程序:已禁用。建议排查:代理网络配置。' + if raise_error: + from toolbox import trimmed_format_exc + msg += trimmed_format_exc() + print(msg) + +def warm_up_modules(): + print('正在执行一些模块的预热...') + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + enc.encode("模块预热", disallowed_special=()) + enc = model_info["gpt-4"]['tokenizer'] + enc.encode("模块预热", disallowed_special=()) + +if __name__ == '__main__': + import os + os.environ['no_proxy'] = '*' # 避免代理网络产生意外污染 + from toolbox import get_conf + proxies, = get_conf('proxies') + check_proxy(proxies) diff --git a/colorful.py b/colorful.py new file mode 100644 index 0000000000000000000000000000000000000000..9749861f7e59151cda40ec7b7cbc4ea814b88d71 --- /dev/null +++ b/colorful.py @@ -0,0 +1,61 @@ +import platform +from sys import stdout + +if platform.system()=="Linux": + pass +else: + from colorama import init + init() + +# Do you like the elegance of Chinese characters? +def print红(*kw,**kargs): + print("\033[0;31m",*kw,"\033[0m",**kargs) +def print绿(*kw,**kargs): + print("\033[0;32m",*kw,"\033[0m",**kargs) +def print黄(*kw,**kargs): + print("\033[0;33m",*kw,"\033[0m",**kargs) +def print蓝(*kw,**kargs): + print("\033[0;34m",*kw,"\033[0m",**kargs) +def print紫(*kw,**kargs): + print("\033[0;35m",*kw,"\033[0m",**kargs) +def print靛(*kw,**kargs): + print("\033[0;36m",*kw,"\033[0m",**kargs) + +def print亮红(*kw,**kargs): + print("\033[1;31m",*kw,"\033[0m",**kargs) +def print亮绿(*kw,**kargs): + print("\033[1;32m",*kw,"\033[0m",**kargs) +def print亮黄(*kw,**kargs): + print("\033[1;33m",*kw,"\033[0m",**kargs) +def print亮蓝(*kw,**kargs): + print("\033[1;34m",*kw,"\033[0m",**kargs) +def print亮紫(*kw,**kargs): + print("\033[1;35m",*kw,"\033[0m",**kargs) +def print亮靛(*kw,**kargs): + print("\033[1;36m",*kw,"\033[0m",**kargs) + +# Do you like the elegance of Chinese characters? +def sprint红(*kw): + return "\033[0;31m"+' '.join(kw)+"\033[0m" +def sprint绿(*kw): + return "\033[0;32m"+' '.join(kw)+"\033[0m" +def sprint黄(*kw): + return "\033[0;33m"+' '.join(kw)+"\033[0m" +def sprint蓝(*kw): + return "\033[0;34m"+' '.join(kw)+"\033[0m" +def sprint紫(*kw): + return "\033[0;35m"+' '.join(kw)+"\033[0m" +def sprint靛(*kw): + return "\033[0;36m"+' '.join(kw)+"\033[0m" +def sprint亮红(*kw): + return "\033[1;31m"+' '.join(kw)+"\033[0m" +def sprint亮绿(*kw): + return "\033[1;32m"+' '.join(kw)+"\033[0m" +def sprint亮黄(*kw): + return "\033[1;33m"+' '.join(kw)+"\033[0m" +def sprint亮蓝(*kw): + return "\033[1;34m"+' '.join(kw)+"\033[0m" +def sprint亮紫(*kw): + return "\033[1;35m"+' '.join(kw)+"\033[0m" +def sprint亮靛(*kw): + return "\033[1;36m"+' '.join(kw)+"\033[0m" diff --git a/config.py b/config.py new file mode 100644 index 0000000000000000000000000000000000000000..21e77d50793a6a208ad952118648578d75625dae --- /dev/null +++ b/config.py @@ -0,0 +1,149 @@ +""" + 以下所有配置也都支持利用环境变量覆写,环境变量配置格式见docker-compose.yml。 + 读取优先级:环境变量 > config_private.py > config.py + --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + All the following configurations also support using environment variables to override, + and the environment variable configuration format can be seen in docker-compose.yml. + Configuration reading priority: environment variable > config_private.py > config.py +""" + +# [step 1]>> API_KEY = "sk-123456789xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx123456789"。极少数情况下,还需要填写组织(格式如org-123456789abcdefghijklmno的),请向下翻,找 API_ORG 设置项 +API_KEY = "此处填API密钥" # 可同时填写多个API-KEY,用英文逗号分割,例如API_KEY = "sk-openaikey1,sk-openaikey2,fkxxxx-api2dkey3,azure-apikey4" + + +# [step 2]>> 改为True应用代理,如果直接在海外服务器部署,此处不修改 +USE_PROXY = False +if USE_PROXY: + """ + 填写格式是 [协议]:// [地址] :[端口],填写之前不要忘记把USE_PROXY改成True,如果直接在海外服务器部署,此处不修改 + <配置教程&视频教程> https://github.com/binary-husky/gpt_academic/issues/1> + [协议] 常见协议无非socks5h/http; 例如 v2**y 和 ss* 的默认本地协议是socks5h; 而cl**h 的默认本地协议是http + [地址] 懂的都懂,不懂就填localhost或者127.0.0.1肯定错不了(localhost意思是代理软件安装在本机上) + [端口] 在代理软件的设置里找。虽然不同的代理软件界面不一样,但端口号都应该在最显眼的位置上 + """ + # 代理网络的地址,打开你的*学*网软件查看代理的协议(socks5h / http)、地址(localhost)和端口(11284) + proxies = { + # [协议]:// [地址] :[端口] + "http": "socks5h://localhost:11284", # 再例如 "http": "http://127.0.0.1:7890", + "https": "socks5h://localhost:11284", # 再例如 "https": "http://127.0.0.1:7890", + } +else: + proxies = None + +# ------------------------------------ 以下配置可以优化体验, 但大部分场合下并不需要修改 ------------------------------------ + +# 重新URL重新定向,实现更换API_URL的作用(高危设置! 常规情况下不要修改! 通过修改此设置,您将把您的API-KEY和对话隐私完全暴露给您设定的中间人!) +# 格式: API_URL_REDIRECT = {"https://api.openai.com/v1/chat/completions": "在这里填写重定向的api.openai.com的URL"} +# 举例: API_URL_REDIRECT = {"https://api.openai.com/v1/chat/completions": "https://reverse-proxy-url/v1/chat/completions"} +API_URL_REDIRECT = {} + + +# 多线程函数插件中,默认允许多少路线程同时访问OpenAI。Free trial users的限制是每分钟3次,Pay-as-you-go users的限制是每分钟3500次 +# 一言以蔽之:免费(5刀)用户填3,OpenAI绑了信用卡的用户可以填 16 或者更高。提高限制请查询:https://platform.openai.com/docs/guides/rate-limits/overview +DEFAULT_WORKER_NUM = 3 + + +# 对话窗的高度 +CHATBOT_HEIGHT = 1115 + + +# 代码高亮 +CODE_HIGHLIGHT = True + + +# 窗口布局 +LAYOUT = "LEFT-RIGHT" # "LEFT-RIGHT"(左右布局) # "TOP-DOWN"(上下布局) +DARK_MODE = True # 暗色模式 / 亮色模式 + + +# 发送请求到OpenAI后,等待多久判定为超时 +TIMEOUT_SECONDS = 30 + + +# 网页的端口, -1代表随机端口 +WEB_PORT = -1 + + +# 如果OpenAI不响应(网络卡顿、代理失败、KEY失效),重试的次数限制 +MAX_RETRY = 2 + +# OpenAI模型选择是(gpt4现在只对申请成功的人开放) +LLM_MODEL = "gpt-3.5-turbo" # 可选 "chatglm" +AVAIL_LLM_MODELS = ["newbing-free", "gpt-3.5-turbo", "gpt-4", "api2d-gpt-4", "api2d-gpt-3.5-turbo"] + +# ChatGLM(2) Finetune Model Path (如果使用ChatGLM2微调模型,需要把"chatglmft"加入AVAIL_LLM_MODELS中) +ChatGLM_PTUNING_CHECKPOINT = "" # 例如"/home/hmp/ChatGLM2-6B/ptuning/output/6b-pt-128-1e-2/checkpoint-100" + + +# 本地LLM模型如ChatGLM的执行方式 CPU/GPU +LOCAL_MODEL_DEVICE = "cpu" # 可选 "cuda" +LOCAL_MODEL_QUANT = "FP16" # 默认 "FP16" "INT4" 启用量化INT4版本 "INT8" 启用量化INT8版本 + + +# 设置gradio的并行线程数(不需要修改) +CONCURRENT_COUNT = 100 + + +# 是否在提交时自动清空输入框 +AUTO_CLEAR_TXT = False + + +# 色彩主体,可选 ["Default", "Chuanhu-Small-and-Beautiful"] +THEME = "Default" + + +# 加一个live2d装饰 +ADD_WAIFU = False + + +# 设置用户名和密码(不需要修改)(相关功能不稳定,与gradio版本和网络都相关,如果本地使用不建议加这个) +# [("username", "password"), ("username2", "password2"), ...] +AUTHENTICATION = [] + + +# 如果需要在二级路径下运行(常规情况下,不要修改!!)(需要配合修改main.py才能生效!) +CUSTOM_PATH = "/" + + +# 极少数情况下,openai的官方KEY需要伴随组织编码(格式如org-xxxxxxxxxxxxxxxxxxxxxxxx)使用 +API_ORG = "" + + +# 如果需要使用Slack Claude,使用教程详情见 request_llm/README.md +SLACK_CLAUDE_BOT_ID = '' +SLACK_CLAUDE_USER_TOKEN = '' + + +# 如果需要使用AZURE 详情请见额外文档 docs\use_azure.md +AZURE_ENDPOINT = "https://你亲手写的api名称.openai.azure.com/" +AZURE_API_KEY = "填入azure openai api的密钥" # 建议直接在API_KEY处填写,该选项即将被弃用 +AZURE_ENGINE = "填入你亲手写的部署名" # 读 docs\use_azure.md + + +# 使用Newbing +NEWBING_STYLE = "creative" # ["creative", "balanced", "precise"] +NEWBING_COOKIES = """ +put your new bing cookies here +""" + + +# 阿里云实时语音识别 配置难度较高 仅建议高手用户使用 参考 https://github.com/binary-husky/gpt_academic/blob/master/docs/use_audio.md +ENABLE_AUDIO = False +ALIYUN_TOKEN="" # 例如 f37f30e0f9934c34a992f6f64f7eba4f +ALIYUN_APPKEY="" # 例如 RoPlZrM88DnAFkZK +ALIYUN_ACCESSKEY="" # (无需填写) +ALIYUN_SECRET="" # (无需填写) + + +# 接入讯飞星火大模型 https://console.xfyun.cn/services/iat +XFYUN_APPID = "00000000" +XFYUN_API_SECRET = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +XFYUN_API_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + +# Claude API KEY +ANTHROPIC_API_KEY = "" + + +# 自定义API KEY格式 +CUSTOM_API_KEY_PATTERN = "" diff --git a/core_functional.py b/core_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..b04e1e0df07ce20a9700307bc898934c3f1d90cc --- /dev/null +++ b/core_functional.py @@ -0,0 +1,94 @@ +# 'primary' 颜色对应 theme.py 中的 primary_hue +# 'secondary' 颜色对应 theme.py 中的 neutral_hue +# 'stop' 颜色对应 theme.py 中的 color_er +import importlib +from toolbox import clear_line_break + + +def get_core_functions(): + return { + "英语学术润色": { + # 前缀,会被加在你的输入之前。例如,用来描述你的要求,例如翻译、解释代码、润色等等 + "Prefix": r"Below is a paragraph from an academic paper. Polish the writing to meet the academic style, " + + r"improve the spelling, grammar, clarity, concision and overall readability. When necessary, rewrite the whole sentence. " + + r"Furthermore, list all modification and explain the reasons to do so in markdown table." + "\n\n", + # 后缀,会被加在你的输入之后。例如,配合前缀可以把你的输入内容用引号圈起来 + "Suffix": r"", + # 按钮颜色 (默认 secondary) + "Color": r"secondary", + # 按钮是否可见 (默认 True,即可见) + "Visible": True, + # 是否在触发时清除历史 (默认 False,即不处理之前的对话历史) + "AutoClearHistory": False + }, + "中文学术润色": { + "Prefix": r"作为一名中文学术论文写作改进助理,你的任务是改进所提供文本的拼写、语法、清晰、简洁和整体可读性," + + r"同时分解长句,减少重复,并提供改进建议。请只提供文本的更正版本,避免包括解释。请编辑以下文本" + "\n\n", + "Suffix": r"", + }, + "查找语法错误": { + "Prefix": r"Can you help me ensure that the grammar and the spelling is correct? " + + r"Do not try to polish the text, if no mistake is found, tell me that this paragraph is good." + + r"If you find grammar or spelling mistakes, please list mistakes you find in a two-column markdown table, " + + r"put the original text the first column, " + + r"put the corrected text in the second column and highlight the key words you fixed.""\n" + r"Example:""\n" + r"Paragraph: How is you? Do you knows what is it?""\n" + r"| Original sentence | Corrected sentence |""\n" + r"| :--- | :--- |""\n" + r"| How **is** you? | How **are** you? |""\n" + r"| Do you **knows** what **is** **it**? | Do you **know** what **it** **is** ? |""\n" + r"Below is a paragraph from an academic paper. " + r"You need to report all grammar and spelling mistakes as the example before." + + "\n\n", + "Suffix": r"", + "PreProcess": clear_line_break, # 预处理:清除换行符 + }, + "中译英": { + "Prefix": r"Please translate following sentence to English:" + "\n\n", + "Suffix": r"", + }, + "学术中英互译": { + "Prefix": r"I want you to act as a scientific English-Chinese translator, " + + r"I will provide you with some paragraphs in one language " + + r"and your task is to accurately and academically translate the paragraphs only into the other language. " + + r"Do not repeat the original provided paragraphs after translation. " + + r"You should use artificial intelligence tools, " + + r"such as natural language processing, and rhetorical knowledge " + + r"and experience about effective writing techniques to reply. " + + r"I'll give you my paragraphs as follows, tell me what language it is written in, and then translate:" + "\n\n", + "Suffix": "", + "Color": "secondary", + }, + "英译中": { + "Prefix": r"翻译成地道的中文:" + "\n\n", + "Suffix": r"", + }, + "找图片": { + "Prefix": r"我需要你找一张网络图片。使用Unsplash API(https://source.unsplash.com/960x640/?<英语关键词>)获取图片URL," + + r"然后请使用Markdown格式封装,并且不要有反斜线,不要用代码块。现在,请按以下描述给我发送图片:" + "\n\n", + "Suffix": r"", + "Visible": False, + }, + "解释代码": { + "Prefix": r"请解释以下代码:" + "\n```\n", + "Suffix": "\n```\n", + }, + "参考文献转Bib": { + "Prefix": r"Here are some bibliography items, please transform them into bibtex style." + + r"Note that, reference styles maybe more than one kind, you should transform each item correctly." + + r"Items need to be transformed:", + "Suffix": r"", + } + } + + +def handle_core_functionality(additional_fn, inputs, history, chatbot): + import core_functional + importlib.reload(core_functional) # 热更新prompt + core_functional = core_functional.get_core_functions() + if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数(如果有的话) + inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"] + if core_functional[additional_fn].get("AutoClearHistory", False): + history = [] + return inputs, history diff --git a/crazy_functional.py b/crazy_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..03aaaf5513c04529f188f32bef933b1c46e2e0cb --- /dev/null +++ b/crazy_functional.py @@ -0,0 +1,407 @@ +from toolbox import HotReload # HotReload 的意思是热更新,修改函数插件后,不需要重启程序,代码直接生效 + + +def get_crazy_functions(): + ###################### 第一组插件 ########################### + from crazy_functions.读文章写摘要 import 读文章写摘要 + from crazy_functions.生成函数注释 import 批量生成函数注释 + from crazy_functions.解析项目源代码 import 解析项目本身 + from crazy_functions.解析项目源代码 import 解析一个Python项目 + from crazy_functions.解析项目源代码 import 解析一个C项目的头文件 + from crazy_functions.解析项目源代码 import 解析一个C项目 + from crazy_functions.解析项目源代码 import 解析一个Golang项目 + from crazy_functions.解析项目源代码 import 解析一个Rust项目 + from crazy_functions.解析项目源代码 import 解析一个Java项目 + from crazy_functions.解析项目源代码 import 解析一个前端项目 + from crazy_functions.高级功能函数模板 import 高阶功能模板函数 + from crazy_functions.代码重写为全英文_多线程 import 全项目切换英文 + from crazy_functions.Latex全文润色 import Latex英文润色 + from crazy_functions.询问多个大语言模型 import 同时问询 + from crazy_functions.解析项目源代码 import 解析一个Lua项目 + from crazy_functions.解析项目源代码 import 解析一个CSharp项目 + from crazy_functions.总结word文档 import 总结word文档 + from crazy_functions.解析JupyterNotebook import 解析ipynb文件 + from crazy_functions.对话历史存档 import 对话历史存档 + from crazy_functions.对话历史存档 import 载入对话历史存档 + from crazy_functions.对话历史存档 import 删除所有本地对话历史记录 + + from crazy_functions.批量Markdown翻译 import Markdown英译中 + function_plugins = { + "解析整个Python项目": { + "Color": "stop", # 按钮颜色 + "Function": HotReload(解析一个Python项目) + }, + "载入对话历史存档(先上传存档或输入路径)": { + "Color": "stop", + "AsButton":False, + "Function": HotReload(载入对话历史存档) + }, + "删除所有本地对话历史记录(请谨慎操作)": { + "AsButton":False, + "Function": HotReload(删除所有本地对话历史记录) + }, + "[测试功能] 解析Jupyter Notebook文件": { + "Color": "stop", + "AsButton":False, + "Function": HotReload(解析ipynb文件), + "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False) + "ArgsReminder": "若输入0,则不解析notebook中的Markdown块", # 高级参数输入区的显示提示 + }, + "批量总结Word文档": { + "Color": "stop", + "Function": HotReload(总结word文档) + }, + "解析整个C++项目头文件": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个C项目的头文件) + }, + "解析整个C++项目(.cpp/.hpp/.c/.h)": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个C项目) + }, + "解析整个Go项目": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个Golang项目) + }, + "解析整个Rust项目": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个Rust项目) + }, + "解析整个Java项目": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个Java项目) + }, + "解析整个前端项目(js,ts,css等)": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个前端项目) + }, + "解析整个Lua项目": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个Lua项目) + }, + "解析整个CSharp项目": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析一个CSharp项目) + }, + "读Tex论文写摘要": { + "Color": "stop", # 按钮颜色 + "Function": HotReload(读文章写摘要) + }, + "Markdown/Readme英译中": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "Function": HotReload(Markdown英译中) + }, + "批量生成函数注释": { + "Color": "stop", # 按钮颜色 + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(批量生成函数注释) + }, + "保存当前的对话": { + "Function": HotReload(对话历史存档) + }, + "[多线程Demo] 解析此项目本身(源码自译解)": { + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析项目本身) + }, + # "[老旧的Demo] 把本项目源代码切换成全英文": { + # # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + # "AsButton": False, # 加入下拉菜单中 + # "Function": HotReload(全项目切换英文) + # }, + "[插件demo] 历史上的今天": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Function": HotReload(高阶功能模板函数) + }, + + } + ###################### 第二组插件 ########################### + # [第二组插件]: 经过充分测试 + from crazy_functions.批量总结PDF文档 import 批量总结PDF文档 + # from crazy_functions.批量总结PDF文档pdfminer import 批量总结PDF文档pdfminer + from crazy_functions.批量翻译PDF文档_多线程 import 批量翻译PDF文档 + from crazy_functions.谷歌检索小助手 import 谷歌检索小助手 + from crazy_functions.理解PDF文档内容 import 理解PDF文档内容标准文件输入 + from crazy_functions.Latex全文润色 import Latex中文润色 + from crazy_functions.Latex全文润色 import Latex英文纠错 + from crazy_functions.Latex全文翻译 import Latex中译英 + from crazy_functions.Latex全文翻译 import Latex英译中 + from crazy_functions.批量Markdown翻译 import Markdown中译英 + + function_plugins.update({ + "批量翻译PDF文档(多线程)": { + "Color": "stop", + "AsButton": True, # 加入下拉菜单中 + "Function": HotReload(批量翻译PDF文档) + }, + "询问多个GPT模型": { + "Color": "stop", # 按钮颜色 + "Function": HotReload(同时问询) + }, + "[测试功能] 批量总结PDF文档": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Function": HotReload(批量总结PDF文档) + }, + # "[测试功能] 批量总结PDF文档pdfminer": { + # "Color": "stop", + # "AsButton": False, # 加入下拉菜单中 + # "Function": HotReload(批量总结PDF文档pdfminer) + # }, + "谷歌学术检索助手(输入谷歌学术搜索页url)": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(谷歌检索小助手) + }, + "理解PDF文档内容 (模仿ChatPDF)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(理解PDF文档内容标准文件输入) + }, + "英文Latex项目全文润色(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Latex英文润色) + }, + "英文Latex项目全文纠错(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Latex英文纠错) + }, + "中文Latex项目全文润色(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Latex中文润色) + }, + "Latex项目全文中译英(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Latex中译英) + }, + "Latex项目全文英译中(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Latex英译中) + }, + "批量Markdown中译英(输入路径或上传压缩包)": { + # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效 + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(Markdown中译英) + }, + + + }) + + ###################### 第三组插件 ########################### + # [第三组插件]: 尚未充分测试的函数插件 + + try: + from crazy_functions.下载arxiv论文翻译摘要 import 下载arxiv论文并翻译摘要 + function_plugins.update({ + "一键下载arxiv论文并翻译摘要(先在input输入编号,如1812.10695)": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(下载arxiv论文并翻译摘要) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.联网的ChatGPT import 连接网络回答问题 + function_plugins.update({ + "连接网络回答问题(输入问题后点击该插件,需要访问谷歌)": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(连接网络回答问题) + } + }) + from crazy_functions.联网的ChatGPT_bing版 import 连接bing搜索回答问题 + function_plugins.update({ + "连接网络回答问题(中文Bing版,输入问题后点击该插件)": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(连接bing搜索回答问题) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.解析项目源代码 import 解析任意code项目 + function_plugins.update({ + "解析项目源代码(手动指定和筛选源代码文件类型)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False) + "ArgsReminder": "输入时用逗号隔开, *代表通配符, 加了^代表不匹配; 不输入代表全部匹配。例如: \"*.c, ^*.cpp, config.toml, ^*.toml\"", # 高级参数输入区的显示提示 + "Function": HotReload(解析任意code项目) + }, + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.询问多个大语言模型 import 同时问询_指定模型 + function_plugins.update({ + "询问多个GPT模型(手动指定询问哪些模型)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False) + "ArgsReminder": "支持任意数量的llm接口,用&符号分隔。例如chatglm&gpt-3.5-turbo&api2d-gpt-4", # 高级参数输入区的显示提示 + "Function": HotReload(同时问询_指定模型) + }, + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.图片生成 import 图片生成 + function_plugins.update({ + "图片生成(先切换模型到openai或api2d)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False) + "ArgsReminder": "在这里输入分辨率, 如256x256(默认)", # 高级参数输入区的显示提示 + "Function": HotReload(图片生成) + }, + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.总结音视频 import 总结音视频 + function_plugins.update({ + "批量总结音视频(输入路径或上传压缩包)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": "调用openai api 使用whisper-1模型, 目前支持的格式:mp4, m4a, wav, mpga, mpeg, mp3。此处可以输入解析提示,例如:解析为简体中文(默认)。", + "Function": HotReload(总结音视频) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.数学动画生成manim import 动画生成 + function_plugins.update({ + "数学动画生成(Manim)": { + "Color": "stop", + "AsButton": False, + "Function": HotReload(动画生成) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.批量Markdown翻译 import Markdown翻译指定语言 + function_plugins.update({ + "Markdown翻译(手动指定语言)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": "请输入要翻译成哪种语言,默认为Chinese。", + "Function": HotReload(Markdown翻译指定语言) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.Langchain知识库 import 知识库问答 + function_plugins.update({ + "[功能尚不稳定] 构建知识库(请先上传文件素材)": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": "待注入的知识库名称id, 默认为default", + "Function": HotReload(知识库问答) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.Langchain知识库 import 读取知识库作答 + function_plugins.update({ + "[功能尚不稳定] 知识库问答": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": "待提取的知识库名称id, 默认为default, 您需要首先调用构建知识库", + "Function": HotReload(读取知识库作答) + } + }) + except: + print('Load function plugin failed') + + try: + from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比 + function_plugins.update({ + "Latex英文纠错+高亮修正位置 [需Latex]": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": "如果有必要, 请在此处追加更细致的矫错指令(使用英文)。", + "Function": HotReload(Latex英文纠错加PDF对比) + } + }) + from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF + function_plugins.update({ + "Arixv翻译(输入arxivID)[需Latex]": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": + "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+ + "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ', + "Function": HotReload(Latex翻译中文并重新编译PDF) + } + }) + function_plugins.update({ + "本地论文翻译(上传Latex压缩包)[需Latex]": { + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "ArgsReminder": + "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+ + "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ', + "Function": HotReload(Latex翻译中文并重新编译PDF) + } + }) + except: + print('Load function plugin failed') + + # try: + # from crazy_functions.虚空终端 import 终端 + # function_plugins.update({ + # "超级终端": { + # "Color": "stop", + # "AsButton": False, + # # "AdvancedArgs": True, + # # "ArgsReminder": "", + # "Function": HotReload(终端) + # } + # }) + # except: + # print('Load function plugin failed') + + return function_plugins diff --git "a/crazy_functions/Langchain\347\237\245\350\257\206\345\272\223.py" "b/crazy_functions/Langchain\347\237\245\350\257\206\345\272\223.py" new file mode 100644 index 0000000000000000000000000000000000000000..12735dfd733c345ebba80dc3481734c4282abb15 --- /dev/null +++ "b/crazy_functions/Langchain\347\237\245\350\257\206\345\272\223.py" @@ -0,0 +1,107 @@ +from toolbox import CatchException, update_ui, ProxyNetworkActivate +from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive, get_files_from_everything + + + +@CatchException +def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + """ + txt 输入栏用户输入的文本,例如需要翻译的一段话,再例如一个包含了待处理文件的路径 + llm_kwargs gpt模型参数, 如温度和top_p等, 一般原样传递下去就行 + plugin_kwargs 插件模型的参数,暂时没有用武之地 + chatbot 聊天显示框的句柄,用于显示给用户 + history 聊天历史,前情提要 + system_prompt 给gpt的静默提醒 + web_port 当前软件运行的端口号 + """ + history = [] # 清空历史,以免输入溢出 + chatbot.append(("这是什么功能?", "[Local Message] 从一批文件(txt, md, tex)中读取数据构建知识库, 然后进行问答。")) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # resolve deps + try: + from zh_langchain import construct_vector_store + from langchain.embeddings.huggingface import HuggingFaceEmbeddings + from .crazy_utils import knowledge_archive_interface + except Exception as e: + chatbot.append( + ["依赖不足", + "导入依赖失败。正在尝试自动安装,请查看终端的输出或耐心等待..."] + ) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + from .crazy_utils import try_install_deps + try_install_deps(['zh_langchain==0.2.1', 'pypinyin']) + + # < --------------------读取参数--------------- > + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + + # < --------------------读取文件--------------- > + file_manifest = [] + spl = ["txt", "doc", "docx", "email", "epub", "html", "json", "md", "msg", "pdf", "ppt", "pptx", "rtf"] + for sp in spl: + _, file_manifest_tmp, _ = get_files_from_everything(txt, type=f'.{sp}') + file_manifest += file_manifest_tmp + + if len(file_manifest) == 0: + chatbot.append(["没有找到任何可读取文件", "当前支持的格式包括: txt, md, docx, pptx, pdf, json等"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + # < -------------------预热文本向量化模组--------------- > + chatbot.append(['
'.join(file_manifest), "正在预热文本向量化模组, 如果是第一次运行, 将消耗较长时间下载中文向量化模型..."]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + print('Checking Text2vec ...') + from langchain.embeddings.huggingface import HuggingFaceEmbeddings + with ProxyNetworkActivate(): # 临时地激活代理网络 + HuggingFaceEmbeddings(model_name="GanymedeNil/text2vec-large-chinese") + + # < -------------------构建知识库--------------- > + chatbot.append(['
'.join(file_manifest), "正在构建知识库..."]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + print('Establishing knowledge archive ...') + with ProxyNetworkActivate(): # 临时地激活代理网络 + kai = knowledge_archive_interface() + kai.feed_archive(file_manifest=file_manifest, id=kai_id) + kai_files = kai.get_loaded_file() + kai_files = '
'.join(kai_files) + # chatbot.append(['知识库构建成功', "正在将知识库存储至cookie中"]) + # yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + # chatbot._cookies['langchain_plugin_embedding'] = kai.get_current_archive_id() + # chatbot._cookies['lock_plugin'] = 'crazy_functions.Langchain知识库->读取知识库作答' + # chatbot.append(['完成', "“根据知识库作答”函数插件已经接管问答系统, 提问吧! 但注意, 您接下来不能再使用其他插件了,刷新页面即可以退出知识库问答模式。"]) + chatbot.append(['构建完成', f"当前知识库内的有效文件:\n\n---\n\n{kai_files}\n\n---\n\n请切换至“知识库问答”插件进行知识库访问, 或者使用此插件继续上传更多文件。"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间,我们先及时地做一次界面更新 + +@CatchException +def 读取知识库作答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port=-1): + # resolve deps + try: + from zh_langchain import construct_vector_store + from langchain.embeddings.huggingface import HuggingFaceEmbeddings + from .crazy_utils import knowledge_archive_interface + except Exception as e: + chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装,请查看终端的输出或耐心等待..."]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + from .crazy_utils import try_install_deps + try_install_deps(['zh_langchain==0.2.1']) + + # < ------------------- --------------- > + kai = knowledge_archive_interface() + + if 'langchain_plugin_embedding' in chatbot._cookies: + resp, prompt = kai.answer_with_archive_by_id(txt, chatbot._cookies['langchain_plugin_embedding']) + else: + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + resp, prompt = kai.answer_with_archive_by_id(txt, kai_id) + + chatbot.append((txt, '[Local Message] ' + prompt)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间,我们先及时地做一次界面更新 + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive( + inputs=prompt, inputs_show_user=txt, + llm_kwargs=llm_kwargs, chatbot=chatbot, history=[], + sys_prompt=system_prompt + ) + history.extend((prompt, gpt_say)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间,我们先及时地做一次界面更新 diff --git "a/crazy_functions/Latex\345\205\250\346\226\207\346\266\246\350\211\262.py" "b/crazy_functions/Latex\345\205\250\346\226\207\346\266\246\350\211\262.py" new file mode 100644 index 0000000000000000000000000000000000000000..9e1d4b66a4106ab6056503361d4edfeb5f4b8e99 --- /dev/null +++ "b/crazy_functions/Latex\345\205\250\346\226\207\346\266\246\350\211\262.py" @@ -0,0 +1,243 @@ +from toolbox import update_ui, trimmed_format_exc +from toolbox import CatchException, report_execption, write_results_to_file, zip_folder + + +class PaperFileGroup(): + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + 将长文本分离开来 + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex") + + print('Segmentation: done') + def merge_result(self): + self.file_result = ["" for _ in range(len(self.file_paths))] + for r, k in zip(self.sp_file_result, self.sp_file_index): + self.file_result[k] += r + + def write_result(self): + manifest = [] + for path, res in zip(self.file_paths, self.file_result): + with open(path + '.polish.tex', 'w', encoding='utf8') as f: + manifest.append(path + '.polish.tex') + f.write(res) + return manifest + + def zip_result(self): + import os, time + folder = os.path.dirname(self.file_paths[0]) + t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + + +def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='polish'): + import time, os, re + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + + + # <-------- 读取Latex文件,删除其中的所有注释 ----------> + pfg = PaperFileGroup() + + for index, fp in enumerate(file_manifest): + with open(fp, 'r', encoding='utf-8', errors='replace') as f: + file_content = f.read() + # 定义注释的正则表达式 + comment_pattern = r'(? + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + + # <-------- 多线程润色开始 ----------> + if language == 'en': + if mode == 'polish': + inputs_array = ["Below is a section from an academic paper, polish this section to meet the academic standard, " + + "improve the grammar, clarity and overall readability, do not modify any latex command such as \section, \cite and equations:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + else: + inputs_array = [r"Below is a section from an academic paper, proofread this section." + + r"Do not modify any latex command such as \section, \cite, \begin, \item and equations. " + + r"Answer me only with the revised text:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + inputs_show_user_array = [f"Polish {f}" for f in pfg.sp_file_tag] + sys_prompt_array = ["You are a professional academic paper writer." for _ in range(n_split)] + elif language == 'zh': + if mode == 'polish': + inputs_array = [f"以下是一篇学术论文中的一段内容,请将此部分润色以满足学术标准,提高语法、清晰度和整体可读性,不要修改任何LaTeX命令,例如\section,\cite和方程式:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + else: + inputs_array = [f"以下是一篇学术论文中的一段内容,请对这部分内容进行语法矫正。不要修改任何LaTeX命令,例如\section,\cite和方程式:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + inputs_show_user_array = [f"润色 {f}" for f in pfg.sp_file_tag] + sys_prompt_array=["你是一位专业的中文学术论文作家。" for _ in range(n_split)] + + + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # 并行任务数量限制,最多同时执行5个,其他的排队等待 + scroller_max_len = 80 + ) + + # <-------- 文本碎片重组为完整的tex文件,整理结果为压缩包 ----------> + try: + pfg.sp_file_result = [] + for i_say, gpt_say in zip(gpt_response_collection[0::2], gpt_response_collection[1::2]): + pfg.sp_file_result.append(gpt_say) + pfg.merge_result() + pfg.write_result() + pfg.zip_result() + except: + print(trimmed_format_exc()) + + # <-------- 整理结果,退出 ----------> + create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" + res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + history = gpt_response_collection + chatbot.append((f"{fp}完成了吗?", res)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + +@CatchException +def Latex英文润色(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 尝试导入依赖,如果缺少依赖,则给出安装建议 + try: + import tiktoken + except: + report_execption(chatbot, history, + a=f"解析项目: {txt}", + b=f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + history = [] # 清空历史,以免输入溢出 + import glob, os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en') + + + + + + +@CatchException +def Latex中文润色(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 尝试导入依赖,如果缺少依赖,则给出安装建议 + try: + import tiktoken + except: + report_execption(chatbot, history, + a=f"解析项目: {txt}", + b=f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + history = [] # 清空历史,以免输入溢出 + import glob, os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='zh') + + + + +@CatchException +def Latex英文纠错(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行纠错。函数插件贡献者: Binary-Husky"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 尝试导入依赖,如果缺少依赖,则给出安装建议 + try: + import tiktoken + except: + report_execption(chatbot, history, + a=f"解析项目: {txt}", + b=f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + history = [] # 清空历史,以免输入溢出 + import glob, os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='proofread') + + + diff --git "a/crazy_functions/Latex\345\205\250\346\226\207\347\277\273\350\257\221.py" "b/crazy_functions/Latex\345\205\250\346\226\207\347\277\273\350\257\221.py" new file mode 100644 index 0000000000000000000000000000000000000000..554c485aa0891f74c57cacfcbe076febe7a11029 --- /dev/null +++ "b/crazy_functions/Latex\345\205\250\346\226\207\347\277\273\350\257\221.py" @@ -0,0 +1,175 @@ +from toolbox import update_ui +from toolbox import CatchException, report_execption, write_results_to_file +fast_debug = False + +class PaperFileGroup(): + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + 将长文本分离开来 + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex") + + print('Segmentation: done') + +def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en'): + import time, os, re + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + + # <-------- 读取Latex文件,删除其中的所有注释 ----------> + pfg = PaperFileGroup() + + for index, fp in enumerate(file_manifest): + with open(fp, 'r', encoding='utf-8', errors='replace') as f: + file_content = f.read() + # 定义注释的正则表达式 + comment_pattern = r'(? + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + # <-------- 抽取摘要 ----------> + # if language == 'en': + # abs_extract_inputs = f"Please write an abstract for this paper" + + # # 单线,获取文章meta信息 + # paper_meta_info = yield from request_gpt_model_in_new_thread_with_ui_alive( + # inputs=abs_extract_inputs, + # inputs_show_user=f"正在抽取摘要信息。", + # llm_kwargs=llm_kwargs, + # chatbot=chatbot, history=[], + # sys_prompt="Your job is to collect information from materials。", + # ) + + # <-------- 多线程润色开始 ----------> + if language == 'en->zh': + inputs_array = ["Below is a section from an English academic paper, translate it into Chinese, do not modify any latex command such as \section, \cite and equations:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] + sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] + elif language == 'zh->en': + inputs_array = [f"Below is a section from a Chinese academic paper, translate it into English, do not modify any latex command such as \section, \cite and equations:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] + sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] + + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # OpenAI所允许的最大并行过载 + scroller_max_len = 80 + ) + + # <-------- 整理结果,退出 ----------> + create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" + res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + history = gpt_response_collection + chatbot.append((f"{fp}完成了吗?", res)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + + + + +@CatchException +def Latex英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行翻译。函数插件贡献者: Binary-Husky"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 尝试导入依赖,如果缺少依赖,则给出安装建议 + try: + import tiktoken + except: + report_execption(chatbot, history, + a=f"解析项目: {txt}", + b=f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + history = [] # 清空历史,以免输入溢出 + import glob, os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en->zh') + + + + + +@CatchException +def Latex中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行翻译。函数插件贡献者: Binary-Husky"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 尝试导入依赖,如果缺少依赖,则给出安装建议 + try: + import tiktoken + except: + report_execption(chatbot, history, + a=f"解析项目: {txt}", + b=f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + history = [] # 清空历史,以免输入溢出 + import glob, os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='zh->en') \ No newline at end of file diff --git "a/crazy_functions/Latex\350\276\223\345\207\272PDF\347\273\223\346\236\234.py" "b/crazy_functions/Latex\350\276\223\345\207\272PDF\347\273\223\346\236\234.py" new file mode 100644 index 0000000000000000000000000000000000000000..e79cf8223826e74c31b4cfb0220a395f38b335b2 --- /dev/null +++ "b/crazy_functions/Latex\350\276\223\345\207\272PDF\347\273\223\346\236\234.py" @@ -0,0 +1,300 @@ +from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str +from functools import partial +import glob, os, requests, time +pj = os.path.join +ARXIV_CACHE_DIR = os.path.expanduser(f"~/arxiv_cache/") + +# =================================== 工具函数 =============================================== +专业词汇声明 = 'If the term "agent" is used in this section, it should be translated to "智能体". ' +def switch_prompt(pfg, mode, more_requirement): + """ + Generate prompts and system prompts based on the mode for proofreading or translating. + Args: + - pfg: Proofreader or Translator instance. + - mode: A string specifying the mode, either 'proofread' or 'translate_zh'. + + Returns: + - inputs_array: A list of strings containing prompts for users to respond to. + - sys_prompt_array: A list of strings containing prompts for system prompts. + """ + n_split = len(pfg.sp_file_contents) + if mode == 'proofread_en': + inputs_array = [r"Below is a section from an academic paper, proofread this section." + + r"Do not modify any latex command such as \section, \cite, \begin, \item and equations. " + more_requirement + + r"Answer me only with the revised text:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + sys_prompt_array = ["You are a professional academic paper writer." for _ in range(n_split)] + elif mode == 'translate_zh': + inputs_array = [r"Below is a section from an English academic paper, translate it into Chinese. " + more_requirement + + r"Do not modify any latex command such as \section, \cite, \begin, \item and equations. " + + r"Answer me only with the translated text:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + sys_prompt_array = ["You are a professional translator." for _ in range(n_split)] + else: + assert False, "未知指令" + return inputs_array, sys_prompt_array + +def desend_to_extracted_folder_if_exist(project_folder): + """ + Descend into the extracted folder if it exists, otherwise return the original folder. + + Args: + - project_folder: A string specifying the folder path. + + Returns: + - A string specifying the path to the extracted folder, or the original folder if there is no extracted folder. + """ + maybe_dir = [f for f in glob.glob(f'{project_folder}/*') if os.path.isdir(f)] + if len(maybe_dir) == 0: return project_folder + if maybe_dir[0].endswith('.extract'): return maybe_dir[0] + return project_folder + +def move_project(project_folder, arxiv_id=None): + """ + Create a new work folder and copy the project folder to it. + + Args: + - project_folder: A string specifying the folder path of the project. + + Returns: + - A string specifying the path to the new work folder. + """ + import shutil, time + time.sleep(2) # avoid time string conflict + if arxiv_id is not None: + new_workfolder = pj(ARXIV_CACHE_DIR, arxiv_id, 'workfolder') + else: + new_workfolder = f'gpt_log/{gen_time_str()}' + try: + shutil.rmtree(new_workfolder) + except: + pass + + # align subfolder if there is a folder wrapper + items = glob.glob(pj(project_folder,'*')) + if len(glob.glob(pj(project_folder,'*.tex'))) == 0 and len(items) == 1: + if os.path.isdir(items[0]): project_folder = items[0] + + shutil.copytree(src=project_folder, dst=new_workfolder) + return new_workfolder + +def arxiv_download(chatbot, history, txt): + def check_cached_translation_pdf(arxiv_id): + translation_dir = pj(ARXIV_CACHE_DIR, arxiv_id, 'translation') + if not os.path.exists(translation_dir): + os.makedirs(translation_dir) + target_file = pj(translation_dir, 'translate_zh.pdf') + if os.path.exists(target_file): + promote_file_to_downloadzone(target_file, rename_file=None, chatbot=chatbot) + return target_file + return False + def is_float(s): + try: + float(s) + return True + except ValueError: + return False + if ('.' in txt) and ('/' not in txt) and is_float(txt): # is arxiv ID + txt = 'https://arxiv.org/abs/' + txt.strip() + if ('.' in txt) and ('/' not in txt) and is_float(txt[:10]): # is arxiv ID + txt = 'https://arxiv.org/abs/' + txt[:10] + if not txt.startswith('https://arxiv.org'): + return txt, None + + # <-------------- inspect format -------------> + chatbot.append([f"检测到arxiv文档连接", '尝试下载 ...']) + yield from update_ui(chatbot=chatbot, history=history) + time.sleep(1) # 刷新界面 + + url_ = txt # https://arxiv.org/abs/1707.06690 + if not txt.startswith('https://arxiv.org/abs/'): + msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}" + yield from update_ui_lastest_msg(msg, chatbot=chatbot, history=history) # 刷新界面 + return msg, None + # <-------------- set format -------------> + arxiv_id = url_.split('/abs/')[-1] + if 'v' in arxiv_id: arxiv_id = arxiv_id[:10] + cached_translation_pdf = check_cached_translation_pdf(arxiv_id) + if cached_translation_pdf: return cached_translation_pdf, arxiv_id + + url_tar = url_.replace('/abs/', '/e-print/') + translation_dir = pj(ARXIV_CACHE_DIR, arxiv_id, 'e-print') + extract_dst = pj(ARXIV_CACHE_DIR, arxiv_id, 'extract') + os.makedirs(translation_dir, exist_ok=True) + + # <-------------- download arxiv source file -------------> + dst = pj(translation_dir, arxiv_id+'.tar') + if os.path.exists(dst): + yield from update_ui_lastest_msg("调用缓存", chatbot=chatbot, history=history) # 刷新界面 + else: + yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history) # 刷新界面 + proxies, = get_conf('proxies') + r = requests.get(url_tar, proxies=proxies) + with open(dst, 'wb+') as f: + f.write(r.content) + # <-------------- extract file -------------> + yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history) # 刷新界面 + from toolbox import extract_archive + extract_archive(file_path=dst, dest_dir=extract_dst) + return extract_dst, arxiv_id +# ========================================= 插件主程序1 ===================================================== + + +@CatchException +def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # <-------------- information about this plugin -------------> + chatbot.append([ "函数插件功能?", + "对整个Latex项目进行纠错, 用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky。注意事项: 目前仅支持GPT3.5/GPT4,其他模型转化效果未知。目前对机器学习类文献转化效果最好,其他类型文献转化效果未知。仅在Windows系统进行了测试,其他操作系统表现未知。"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------------- more requirements -------------> + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + more_req = plugin_kwargs.get("advanced_arg", "") + _switch_prompt_ = partial(switch_prompt, more_requirement=more_req) + + # <-------------- check deps -------------> + try: + import glob, os, time, subprocess + subprocess.Popen(['pdflatex', '-version']) + from .latex_fns.latex_actions import Latex精细分解与转化, 编译Latex + except Exception as e: + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。安装方法https://tug.org/texlive/。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- clear history and read input -------------> + history = [] + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder, arxiv_id=None) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> + if not os.path.exists(project_folder + '/merge_proofread_en.tex'): + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, + chatbot, history, system_prompt, mode='proofread_en', switch_prompt=_switch_prompt_) + + + # <-------------- compile PDF -------------> + success = yield from 编译Latex(chatbot, history, main_file_original='merge', main_file_modified='merge_proofread_en', + work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) + + + # <-------------- zip PDF -------------> + zip_res = zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果(压缩包)...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + promote_file_to_downloadzone(file=zip_res, chatbot=chatbot) + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + promote_file_to_downloadzone(file=zip_res, chatbot=chatbot) + + # <-------------- we are done -------------> + return success + + +# ========================================= 插件主程序2 ===================================================== + +@CatchException +def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # <-------------- information about this plugin -------------> + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行翻译, 生成中文PDF。函数插件贡献者: Binary-Husky。注意事项: 此插件Windows支持最佳,Linux下必须使用Docker安装,详见项目主README.md。目前仅支持GPT3.5/GPT4,其他模型转化效果未知。目前对机器学习类文献转化效果最好,其他类型文献转化效果未知。"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------------- more requirements -------------> + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + more_req = plugin_kwargs.get("advanced_arg", "") + _switch_prompt_ = partial(switch_prompt, more_requirement=more_req) + + # <-------------- check deps -------------> + try: + import glob, os, time, subprocess + subprocess.Popen(['pdflatex', '-version']) + from .latex_fns.latex_actions import Latex精细分解与转化, 编译Latex + except Exception as e: + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。安装方法https://tug.org/texlive/。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- clear history and read input -------------> + history = [] + txt, arxiv_id = yield from arxiv_download(chatbot, history, txt) + if txt.endswith('.pdf'): + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"发现已经存在翻译好的PDF文档") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder, arxiv_id) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> + if not os.path.exists(project_folder + '/merge_translate_zh.tex'): + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, + chatbot, history, system_prompt, mode='translate_zh', switch_prompt=_switch_prompt_) + + + # <-------------- compile PDF -------------> + success = yield from 编译Latex(chatbot, history, main_file_original='merge', main_file_modified='merge_translate_zh', mode='translate_zh', + work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) + + # <-------------- zip PDF -------------> + zip_res = zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果(压缩包)...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + promote_file_to_downloadzone(file=zip_res, chatbot=chatbot) + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + promote_file_to_downloadzone(file=zip_res, chatbot=chatbot) + + + # <-------------- we are done -------------> + return success diff --git a/crazy_functions/__init__.py b/crazy_functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git "a/crazy_functions/chatglm\345\276\256\350\260\203\345\267\245\345\205\267.py" "b/crazy_functions/chatglm\345\276\256\350\260\203\345\267\245\345\205\267.py" new file mode 100644 index 0000000000000000000000000000000000000000..336d7cfc85ac159841758123fa057bd20a0bbbec --- /dev/null +++ "b/crazy_functions/chatglm\345\276\256\350\260\203\345\267\245\345\205\267.py" @@ -0,0 +1,141 @@ +from toolbox import CatchException, update_ui, promote_file_to_downloadzone +from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency +import datetime, json + +def fetch_items(list_of_items, batch_size): + for i in range(0, len(list_of_items), batch_size): + yield list_of_items[i:i + batch_size] + +def string_to_options(arguments): + import argparse + import shlex + + # Create an argparse.ArgumentParser instance + parser = argparse.ArgumentParser() + + # Add command-line arguments + parser.add_argument("--llm_to_learn", type=str, help="LLM model to learn", default="gpt-3.5-turbo") + parser.add_argument("--prompt_prefix", type=str, help="Prompt prefix", default='') + parser.add_argument("--system_prompt", type=str, help="System prompt", default='') + parser.add_argument("--batch", type=int, help="System prompt", default=50) + parser.add_argument("--pre_seq_len", type=int, help="pre_seq_len", default=50) + parser.add_argument("--learning_rate", type=float, help="learning_rate", default=2e-2) + parser.add_argument("--num_gpus", type=int, help="num_gpus", default=1) + parser.add_argument("--json_dataset", type=str, help="json_dataset", default="") + parser.add_argument("--ptuning_directory", type=str, help="ptuning_directory", default="") + + + + # Parse the arguments + args = parser.parse_args(shlex.split(arguments)) + + return args + +@CatchException +def 微调数据集生成(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + """ + txt 输入栏用户输入的文本,例如需要翻译的一段话,再例如一个包含了待处理文件的路径 + llm_kwargs gpt模型参数,如温度和top_p等,一般原样传递下去就行 + plugin_kwargs 插件模型的参数 + chatbot 聊天显示框的句柄,用于显示给用户 + history 聊天历史,前情提要 + system_prompt 给gpt的静默提醒 + web_port 当前软件运行的端口号 + """ + history = [] # 清空历史,以免输入溢出 + chatbot.append(("这是什么功能?", "[Local Message] 微调数据集生成")) + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + args = plugin_kwargs.get("advanced_arg", None) + if args is None: + chatbot.append(("没给定指令", "退出")) + yield from update_ui(chatbot=chatbot, history=history); return + else: + arguments = string_to_options(arguments=args) + + dat = [] + with open(txt, 'r', encoding='utf8') as f: + for line in f.readlines(): + json_dat = json.loads(line) + dat.append(json_dat["content"]) + + llm_kwargs['llm_model'] = arguments.llm_to_learn + for batch in fetch_items(dat, arguments.batch): + res = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=[f"{arguments.prompt_prefix}\n\n{b}" for b in (batch)], + inputs_show_user_array=[f"Show Nothing" for _ in (batch)], + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[] for _ in (batch)], + sys_prompt_array=[arguments.system_prompt for _ in (batch)], + max_workers=10 # OpenAI所允许的最大并行过载 + ) + + with open(txt+'.generated.json', 'a+', encoding='utf8') as f: + for b, r in zip(batch, res[1::2]): + f.write(json.dumps({"content":b, "summary":r}, ensure_ascii=False)+'\n') + + promote_file_to_downloadzone(txt+'.generated.json', rename_file='generated.json', chatbot=chatbot) + return + + + +@CatchException +def 启动微调(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + """ + txt 输入栏用户输入的文本,例如需要翻译的一段话,再例如一个包含了待处理文件的路径 + llm_kwargs gpt模型参数,如温度和top_p等,一般原样传递下去就行 + plugin_kwargs 插件模型的参数 + chatbot 聊天显示框的句柄,用于显示给用户 + history 聊天历史,前情提要 + system_prompt 给gpt的静默提醒 + web_port 当前软件运行的端口号 + """ + import subprocess + history = [] # 清空历史,以免输入溢出 + chatbot.append(("这是什么功能?", "[Local Message] 微调数据集生成")) + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + args = plugin_kwargs.get("advanced_arg", None) + if args is None: + chatbot.append(("没给定指令", "退出")) + yield from update_ui(chatbot=chatbot, history=history); return + else: + arguments = string_to_options(arguments=args) + + + + pre_seq_len = arguments.pre_seq_len # 128 + learning_rate = arguments.learning_rate # 2e-2 + num_gpus = arguments.num_gpus # 1 + json_dataset = arguments.json_dataset # 't_code.json' + ptuning_directory = arguments.ptuning_directory # '/home/hmp/ChatGLM2-6B/ptuning' + + command = f"torchrun --standalone --nnodes=1 --nproc-per-node={num_gpus} main.py \ + --do_train \ + --train_file AdvertiseGen/{json_dataset} \ + --validation_file AdvertiseGen/{json_dataset} \ + --preprocessing_num_workers 20 \ + --prompt_column content \ + --response_column summary \ + --overwrite_cache \ + --model_name_or_path THUDM/chatglm2-6b \ + --output_dir output/clothgen-chatglm2-6b-pt-{pre_seq_len}-{learning_rate} \ + --overwrite_output_dir \ + --max_source_length 256 \ + --max_target_length 256 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 1 \ + --gradient_accumulation_steps 16 \ + --predict_with_generate \ + --max_steps 100 \ + --logging_steps 10 \ + --save_steps 20 \ + --learning_rate {learning_rate} \ + --pre_seq_len {pre_seq_len} \ + --quantization_bit 4" + + process = subprocess.Popen(command, shell=True, cwd=ptuning_directory) + try: + process.communicate(timeout=3600*24) + except subprocess.TimeoutExpired: + process.kill() + return diff --git a/crazy_functions/crazy_functions_test.py b/crazy_functions/crazy_functions_test.py new file mode 100644 index 0000000000000000000000000000000000000000..0c623b8e027858b2579a021769bb304e34c4e373 --- /dev/null +++ b/crazy_functions/crazy_functions_test.py @@ -0,0 +1,231 @@ +""" +这是什么? + 这个文件用于函数插件的单元测试 + 运行方法 python crazy_functions/crazy_functions_test.py +""" + +# ============================================================================================================================== + +def validate_path(): + import os, sys + dir_name = os.path.dirname(__file__) + root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..') + os.chdir(root_dir_assume) + sys.path.append(root_dir_assume) +validate_path() # validate path so you can run from base directory + +# ============================================================================================================================== + +from colorful import * +from toolbox import get_conf, ChatBotWithCookies +import contextlib +import os +import sys +from functools import wraps +proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \ + get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY') + +llm_kwargs = { + 'api_key': API_KEY, + 'llm_model': LLM_MODEL, + 'top_p':1.0, + 'max_length': None, + 'temperature':1.0, +} +plugin_kwargs = { } +chatbot = ChatBotWithCookies(llm_kwargs) +history = [] +system_prompt = "Serve me as a writing and programming assistant." +web_port = 1024 + +# ============================================================================================================================== + +def silence_stdout(func): + @wraps(func) + def wrapper(*args, **kwargs): + _original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + for q in func(*args, **kwargs): + sys.stdout = _original_stdout + yield q + sys.stdout = open(os.devnull, 'w') + sys.stdout.close() + sys.stdout = _original_stdout + return wrapper + +class CLI_Printer(): + def __init__(self) -> None: + self.pre_buf = "" + + def print(self, buf): + bufp = "" + for index, chat in enumerate(buf): + a, b = chat + bufp += sprint亮靛('[Me]:' + a) + '\n' + bufp += '[GPT]:' + b + if index < len(buf)-1: + bufp += '\n' + + if self.pre_buf!="" and bufp.startswith(self.pre_buf): + print(bufp[len(self.pre_buf):], end='') + else: + print('\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n'+bufp, end='') + self.pre_buf = bufp + return + +cli_printer = CLI_Printer() +# ============================================================================================================================== +def test_解析一个Python项目(): + from crazy_functions.解析项目源代码 import 解析一个Python项目 + txt = "crazy_functions/test_project/python/dqn" + for cookies, cb, hist, msg in 解析一个Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_解析一个Cpp项目(): + from crazy_functions.解析项目源代码 import 解析一个C项目 + txt = "crazy_functions/test_project/cpp/cppipc" + for cookies, cb, hist, msg in 解析一个C项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_Latex英文润色(): + from crazy_functions.Latex全文润色 import Latex英文润色 + txt = "crazy_functions/test_project/latex/attention" + for cookies, cb, hist, msg in Latex英文润色(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_Markdown中译英(): + from crazy_functions.批量Markdown翻译 import Markdown中译英 + txt = "README.md" + for cookies, cb, hist, msg in Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_批量翻译PDF文档(): + from crazy_functions.批量翻译PDF文档_多线程 import 批量翻译PDF文档 + txt = "crazy_functions/test_project/pdf_and_word" + for cookies, cb, hist, msg in 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_谷歌检索小助手(): + from crazy_functions.谷歌检索小助手 import 谷歌检索小助手 + txt = "https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=auto+reinforcement+learning&btnG=" + for cookies, cb, hist, msg in 谷歌检索小助手(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_总结word文档(): + from crazy_functions.总结word文档 import 总结word文档 + txt = "crazy_functions/test_project/pdf_and_word" + for cookies, cb, hist, msg in 总结word文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_下载arxiv论文并翻译摘要(): + from crazy_functions.下载arxiv论文翻译摘要 import 下载arxiv论文并翻译摘要 + txt = "1812.10695" + for cookies, cb, hist, msg in 下载arxiv论文并翻译摘要(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_联网回答问题(): + from crazy_functions.联网的ChatGPT import 连接网络回答问题 + # txt = "谁是应急食品?" + # >> '根据以上搜索结果可以得知,应急食品是“原神”游戏中的角色派蒙的外号。' + # txt = "道路千万条,安全第一条。后面两句是?" + # >> '行车不规范,亲人两行泪。' + # txt = "You should have gone for the head. What does that mean?" + # >> The phrase "You should have gone for the head" is a quote from the Marvel movies, Avengers: Infinity War and Avengers: Endgame. It was spoken by the character Thanos in Infinity War and by Thor in Endgame. + txt = "AutoGPT是什么?" + for cookies, cb, hist, msg in 连接网络回答问题(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print("当前问答:", cb[-1][-1].replace("\n"," ")) + for i, it in enumerate(cb): print亮蓝(it[0]); print亮黄(it[1]) + +def test_解析ipynb文件(): + from crazy_functions.解析JupyterNotebook import 解析ipynb文件 + txt = "crazy_functions/test_samples" + for cookies, cb, hist, msg in 解析ipynb文件(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + + +def test_数学动画生成manim(): + from crazy_functions.数学动画生成manim import 动画生成 + txt = "A ball split into 2, and then split into 4, and finally split into 8." + for cookies, cb, hist, msg in 动画生成(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + + + +def test_Markdown多语言(): + from crazy_functions.批量Markdown翻译 import Markdown翻译指定语言 + txt = "README.md" + history = [] + for lang in ["English", "French", "Japanese", "Korean", "Russian", "Italian", "German", "Portuguese", "Arabic"]: + plugin_kwargs = {"advanced_arg": lang} + for cookies, cb, hist, msg in Markdown翻译指定语言(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + +def test_Langchain知识库(): + from crazy_functions.Langchain知识库 import 知识库问答 + txt = "./" + chatbot = ChatBotWithCookies(llm_kwargs) + for cookies, cb, hist, msg in silence_stdout(知识库问答)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + cli_printer.print(cb) # print(cb) + + chatbot = ChatBotWithCookies(cookies) + from crazy_functions.Langchain知识库 import 读取知识库作答 + txt = "What is the installation method?" + for cookies, cb, hist, msg in silence_stdout(读取知识库作答)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + cli_printer.print(cb) # print(cb) + +def test_Langchain知识库读取(): + from crazy_functions.Langchain知识库 import 读取知识库作答 + txt = "远程云服务器部署?" + for cookies, cb, hist, msg in silence_stdout(读取知识库作答)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + cli_printer.print(cb) # print(cb) + +def test_Latex(): + from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比, Latex翻译中文并重新编译PDF + + # txt = r"https://arxiv.org/abs/1706.03762" + # txt = r"https://arxiv.org/abs/1902.03185" + # txt = r"https://arxiv.org/abs/2305.18290" + # txt = r"https://arxiv.org/abs/2305.17608" + # txt = r"https://arxiv.org/abs/2211.16068" # ACE + # txt = r"C:\Users\x\arxiv_cache\2211.16068\workfolder" # ACE + # txt = r"https://arxiv.org/abs/2002.09253" + # txt = r"https://arxiv.org/abs/2306.07831" + # txt = r"https://arxiv.org/abs/2212.10156" + # txt = r"https://arxiv.org/abs/2211.11559" + # txt = r"https://arxiv.org/abs/2303.08774" + txt = r"https://arxiv.org/abs/2303.12712" + # txt = r"C:\Users\fuqingxu\arxiv_cache\2303.12712\workfolder" + + + for cookies, cb, hist, msg in (Latex翻译中文并重新编译PDF)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + cli_printer.print(cb) # print(cb) + + + + # txt = "2302.02948.tar" + # print(txt) + # main_tex, work_folder = Latex预处理(txt) + # print('main tex:', main_tex) + # res = 编译Latex(main_tex, work_folder) + # # for cookies, cb, hist, msg in silence_stdout(编译Latex)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # cli_printer.print(cb) # print(cb) + + + +# test_解析一个Python项目() +# test_Latex英文润色() +# test_Markdown中译英() +# test_批量翻译PDF文档() +# test_谷歌检索小助手() +# test_总结word文档() +# test_下载arxiv论文并翻译摘要() +# test_解析一个Cpp项目() +# test_联网回答问题() +# test_解析ipynb文件() +# test_数学动画生成manim() +# test_Langchain知识库() +# test_Langchain知识库读取() +if __name__ == "__main__": + test_Latex() + input("程序完成,回车退出。") + print("退出。") \ No newline at end of file diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ffe95e2be56e969c787f5d2895fa502540501660 --- /dev/null +++ b/crazy_functions/crazy_utils.py @@ -0,0 +1,753 @@ +from toolbox import update_ui, get_conf, trimmed_format_exc +import threading + +def input_clipping(inputs, history, max_token_limit): + import numpy as np + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + + mode = 'input-and-history' + # 当 输入部分的token占比 小于 全文的一半时,只裁剪历史 + input_token_num = get_token_num(inputs) + if input_token_num < max_token_limit//2: + mode = 'only-history' + max_token_limit = max_token_limit - input_token_num + + everything = [inputs] if mode == 'input-and-history' else [''] + everything.extend(history) + n_token = get_token_num('\n'.join(everything)) + everything_token = [get_token_num(e) for e in everything] + delta = max(everything_token) // 16 # 截断时的颗粒度 + + while n_token > max_token_limit: + where = np.argmax(everything_token) + encoded = enc.encode(everything[where], disallowed_special=()) + clipped_encoded = encoded[:len(encoded)-delta] + everything[where] = enc.decode(clipped_encoded)[:-1] # -1 to remove the may-be illegal char + everything_token[where] = get_token_num(everything[where]) + n_token = get_token_num('\n'.join(everything)) + + if mode == 'input-and-history': + inputs = everything[0] + else: + pass + history = everything[1:] + return inputs, history + +def request_gpt_model_in_new_thread_with_ui_alive( + inputs, inputs_show_user, llm_kwargs, + chatbot, history, sys_prompt, refresh_interval=0.2, + handle_token_exceed=True, + retry_times_at_unknown_error=2, + ): + """ + Request GPT model,请求GPT模型同时维持用户界面活跃。 + + 输入参数 Args (以_array结尾的输入变量都是列表,列表长度为子任务的数量,执行时,会把列表拆解,放到每个子线程中分别执行): + inputs (string): List of inputs (输入) + inputs_show_user (string): List of inputs to show user(展现在报告中的输入,借助此参数,在汇总报告中隐藏啰嗦的真实输入,增强报告的可读性) + top_p (float): Top p value for sampling from model distribution (GPT参数,浮点数) + temperature (float): Temperature value for sampling from model distribution(GPT参数,浮点数) + chatbot: chatbot inputs and outputs (用户界面对话窗口句柄,用于数据流可视化) + history (list): List of chat history (历史,对话历史列表) + sys_prompt (string): List of system prompts (系统输入,列表,用于输入给GPT的前提提示,比如你是翻译官怎样怎样) + refresh_interval (float, optional): Refresh interval for UI (default: 0.2) (刷新时间间隔频率,建议低于1,不可高于3,仅仅服务于视觉效果) + handle_token_exceed:是否自动处理token溢出的情况,如果选择自动处理,则会在溢出时暴力截断,默认开启 + retry_times_at_unknown_error:失败时的重试次数 + + 输出 Returns: + future: 输出,GPT返回的结果 + """ + import time + from concurrent.futures import ThreadPoolExecutor + from request_llm.bridge_all import predict_no_ui_long_connection + # 用户反馈 + chatbot.append([inputs_show_user, ""]) + yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面 + executor = ThreadPoolExecutor(max_workers=16) + mutable = ["", time.time(), ""] + def _req_gpt(inputs, history, sys_prompt): + retry_op = retry_times_at_unknown_error + exceeded_cnt = 0 + while True: + # watchdog error + if len(mutable) >= 2 and (time.time()-mutable[1]) > 5: + raise RuntimeError("检测到程序终止。") + try: + # 【第一种情况】:顺利完成 + result = predict_no_ui_long_connection( + inputs=inputs, llm_kwargs=llm_kwargs, + history=history, sys_prompt=sys_prompt, observe_window=mutable) + return result + except ConnectionAbortedError as token_exceeded_error: + # 【第二种情况】:Token溢出 + if handle_token_exceed: + exceeded_cnt += 1 + # 【选择处理】 尝试计算比例,尽可能多地保留文本 + from toolbox import get_reduce_token_percent + p_ratio, n_exceed = get_reduce_token_percent(str(token_exceeded_error)) + MAX_TOKEN = 4096 + EXCEED_ALLO = 512 + 512 * exceeded_cnt + inputs, history = input_clipping(inputs, history, max_token_limit=MAX_TOKEN-EXCEED_ALLO) + mutable[0] += f'[Local Message] 警告,文本过长将进行截断,Token溢出数:{n_exceed}。\n\n' + continue # 返回重试 + else: + # 【选择放弃】 + tb_str = '```\n' + trimmed_format_exc() + '```' + mutable[0] += f"[Local Message] 警告,在执行过程中遭遇问题, Traceback:\n\n{tb_str}\n\n" + return mutable[0] # 放弃 + except: + # 【第三种情况】:其他错误:重试几次 + tb_str = '```\n' + trimmed_format_exc() + '```' + print(tb_str) + mutable[0] += f"[Local Message] 警告,在执行过程中遭遇问题, Traceback:\n\n{tb_str}\n\n" + if retry_op > 0: + retry_op -= 1 + mutable[0] += f"[Local Message] 重试中,请稍等 {retry_times_at_unknown_error-retry_op}/{retry_times_at_unknown_error}:\n\n" + if ("Rate limit reached" in tb_str) or ("Too Many Requests" in tb_str): + time.sleep(30) + time.sleep(5) + continue # 返回重试 + else: + time.sleep(5) + return mutable[0] # 放弃 + + # 提交任务 + future = executor.submit(_req_gpt, inputs, history, sys_prompt) + while True: + # yield一次以刷新前端页面 + time.sleep(refresh_interval) + # “喂狗”(看门狗) + mutable[1] = time.time() + if future.done(): + break + chatbot[-1] = [chatbot[-1][0], mutable[0]] + yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面 + + final_result = future.result() + chatbot[-1] = [chatbot[-1][0], final_result] + yield from update_ui(chatbot=chatbot, history=[]) # 如果最后成功了,则删除报错信息 + return final_result + +def can_multi_process(llm): + if llm.startswith('gpt-'): return True + if llm.startswith('api2d-'): return True + if llm.startswith('azure-'): return True + return False + +def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array, inputs_show_user_array, llm_kwargs, + chatbot, history_array, sys_prompt_array, + refresh_interval=0.2, max_workers=-1, scroller_max_len=30, + handle_token_exceed=True, show_user_at_complete=False, + retry_times_at_unknown_error=2, + ): + """ + Request GPT model using multiple threads with UI and high efficiency + 请求GPT模型的[多线程]版。 + 具备以下功能: + 实时在UI上反馈远程数据流 + 使用线程池,可调节线程池的大小避免openai的流量限制错误 + 处理中途中止的情况 + 网络等出问题时,会把traceback和已经接收的数据转入输出 + + 输入参数 Args (以_array结尾的输入变量都是列表,列表长度为子任务的数量,执行时,会把列表拆解,放到每个子线程中分别执行): + inputs_array (list): List of inputs (每个子任务的输入) + inputs_show_user_array (list): List of inputs to show user(每个子任务展现在报告中的输入,借助此参数,在汇总报告中隐藏啰嗦的真实输入,增强报告的可读性) + llm_kwargs: llm_kwargs参数 + chatbot: chatbot (用户界面对话窗口句柄,用于数据流可视化) + history_array (list): List of chat history (历史对话输入,双层列表,第一层列表是子任务分解,第二层列表是对话历史) + sys_prompt_array (list): List of system prompts (系统输入,列表,用于输入给GPT的前提提示,比如你是翻译官怎样怎样) + refresh_interval (float, optional): Refresh interval for UI (default: 0.2) (刷新时间间隔频率,建议低于1,不可高于3,仅仅服务于视觉效果) + max_workers (int, optional): Maximum number of threads (default: see config.py) (最大线程数,如果子任务非常多,需要用此选项防止高频地请求openai导致错误) + scroller_max_len (int, optional): Maximum length for scroller (default: 30)(数据流的显示最后收到的多少个字符,仅仅服务于视觉效果) + handle_token_exceed (bool, optional): (是否在输入过长时,自动缩减文本) + handle_token_exceed:是否自动处理token溢出的情况,如果选择自动处理,则会在溢出时暴力截断,默认开启 + show_user_at_complete (bool, optional): (在结束时,把完整输入-输出结果显示在聊天框) + retry_times_at_unknown_error:子任务失败时的重试次数 + + 输出 Returns: + list: List of GPT model responses (每个子任务的输出汇总,如果某个子任务出错,response中会携带traceback报错信息,方便调试和定位问题。) + """ + import time, random + from concurrent.futures import ThreadPoolExecutor + from request_llm.bridge_all import predict_no_ui_long_connection + assert len(inputs_array) == len(history_array) + assert len(inputs_array) == len(sys_prompt_array) + if max_workers == -1: # 读取配置文件 + try: max_workers, = get_conf('DEFAULT_WORKER_NUM') + except: max_workers = 8 + if max_workers <= 0: max_workers = 3 + # 屏蔽掉 chatglm的多线程,可能会导致严重卡顿 + if not can_multi_process(llm_kwargs['llm_model']): + max_workers = 1 + + executor = ThreadPoolExecutor(max_workers=max_workers) + n_frag = len(inputs_array) + # 用户反馈 + chatbot.append(["请开始多线程操作。", ""]) + yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面 + # 跨线程传递 + mutable = [["", time.time(), "等待中"] for _ in range(n_frag)] + + # 子线程任务 + def _req_gpt(index, inputs, history, sys_prompt): + gpt_say = "" + retry_op = retry_times_at_unknown_error + exceeded_cnt = 0 + mutable[index][2] = "执行中" + while True: + # watchdog error + if len(mutable[index]) >= 2 and (time.time()-mutable[index][1]) > 5: + raise RuntimeError("检测到程序终止。") + try: + # 【第一种情况】:顺利完成 + # time.sleep(10); raise RuntimeError("测试") + gpt_say = predict_no_ui_long_connection( + inputs=inputs, llm_kwargs=llm_kwargs, history=history, + sys_prompt=sys_prompt, observe_window=mutable[index], console_slience=True + ) + mutable[index][2] = "已成功" + return gpt_say + except ConnectionAbortedError as token_exceeded_error: + # 【第二种情况】:Token溢出, + if handle_token_exceed: + exceeded_cnt += 1 + # 【选择处理】 尝试计算比例,尽可能多地保留文本 + from toolbox import get_reduce_token_percent + p_ratio, n_exceed = get_reduce_token_percent(str(token_exceeded_error)) + MAX_TOKEN = 4096 + EXCEED_ALLO = 512 + 512 * exceeded_cnt + inputs, history = input_clipping(inputs, history, max_token_limit=MAX_TOKEN-EXCEED_ALLO) + gpt_say += f'[Local Message] 警告,文本过长将进行截断,Token溢出数:{n_exceed}。\n\n' + mutable[index][2] = f"截断重试" + continue # 返回重试 + else: + # 【选择放弃】 + tb_str = '```\n' + trimmed_format_exc() + '```' + gpt_say += f"[Local Message] 警告,线程{index}在执行过程中遭遇问题, Traceback:\n\n{tb_str}\n\n" + if len(mutable[index][0]) > 0: gpt_say += "此线程失败前收到的回答:\n\n" + mutable[index][0] + mutable[index][2] = "输入过长已放弃" + return gpt_say # 放弃 + except: + # 【第三种情况】:其他错误 + tb_str = '```\n' + trimmed_format_exc() + '```' + print(tb_str) + gpt_say += f"[Local Message] 警告,线程{index}在执行过程中遭遇问题, Traceback:\n\n{tb_str}\n\n" + if len(mutable[index][0]) > 0: gpt_say += "此线程失败前收到的回答:\n\n" + mutable[index][0] + if retry_op > 0: + retry_op -= 1 + wait = random.randint(5, 20) + if ("Rate limit reached" in tb_str) or ("Too Many Requests" in tb_str): + wait = wait * 3 + fail_info = "OpenAI绑定信用卡可解除频率限制 " + else: + fail_info = "" + # 也许等待十几秒后,情况会好转 + for i in range(wait): + mutable[index][2] = f"{fail_info}等待重试 {wait-i}"; time.sleep(1) + # 开始重试 + mutable[index][2] = f"重试中 {retry_times_at_unknown_error-retry_op}/{retry_times_at_unknown_error}" + continue # 返回重试 + else: + mutable[index][2] = "已失败" + wait = 5 + time.sleep(5) + return gpt_say # 放弃 + + # 异步任务开始 + futures = [executor.submit(_req_gpt, index, inputs, history, sys_prompt) for index, inputs, history, sys_prompt in zip( + range(len(inputs_array)), inputs_array, history_array, sys_prompt_array)] + cnt = 0 + while True: + # yield一次以刷新前端页面 + time.sleep(refresh_interval) + cnt += 1 + worker_done = [h.done() for h in futures] + # 更好的UI视觉效果 + observe_win = [] + # 每个线程都要“喂狗”(看门狗) + for thread_index, _ in enumerate(worker_done): + mutable[thread_index][1] = time.time() + # 在前端打印些好玩的东西 + for thread_index, _ in enumerate(worker_done): + print_something_really_funny = "[ ...`"+mutable[thread_index][0][-scroller_max_len:].\ + replace('\n', '').replace('```', '...').replace( + ' ', '.').replace('
', '.....').replace('$', '.')+"`... ]" + observe_win.append(print_something_really_funny) + # 在前端打印些好玩的东西 + stat_str = ''.join([f'`{mutable[thread_index][2]}`: {obs}\n\n' + if not done else f'`{mutable[thread_index][2]}`\n\n' + for thread_index, done, obs in zip(range(len(worker_done)), worker_done, observe_win)]) + # 在前端打印些好玩的东西 + chatbot[-1] = [chatbot[-1][0], f'多线程操作已经开始,完成情况: \n\n{stat_str}' + ''.join(['.']*(cnt % 10+1))] + yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面 + if all(worker_done): + executor.shutdown() + break + + # 异步任务结束 + gpt_response_collection = [] + for inputs_show_user, f in zip(inputs_show_user_array, futures): + gpt_res = f.result() + gpt_response_collection.extend([inputs_show_user, gpt_res]) + + # 是否在结束时,在界面上显示结果 + if show_user_at_complete: + for inputs_show_user, f in zip(inputs_show_user_array, futures): + gpt_res = f.result() + chatbot.append([inputs_show_user, gpt_res]) + yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面 + time.sleep(0.3) + return gpt_response_collection + + +def breakdown_txt_to_satisfy_token_limit(txt, get_token_fn, limit): + def cut(txt_tocut, must_break_at_empty_line): # 递归 + if get_token_fn(txt_tocut) <= limit: + return [txt_tocut] + else: + lines = txt_tocut.split('\n') + estimated_line_cut = limit / get_token_fn(txt_tocut) * len(lines) + estimated_line_cut = int(estimated_line_cut) + for cnt in reversed(range(estimated_line_cut)): + if must_break_at_empty_line: + if lines[cnt] != "": + continue + print(cnt) + prev = "\n".join(lines[:cnt]) + post = "\n".join(lines[cnt:]) + if get_token_fn(prev) < limit: + break + if cnt == 0: + raise RuntimeError("存在一行极长的文本!") + # print(len(post)) + # 列表递归接龙 + result = [prev] + result.extend(cut(post, must_break_at_empty_line)) + return result + try: + return cut(txt, must_break_at_empty_line=True) + except RuntimeError: + return cut(txt, must_break_at_empty_line=False) + + +def force_breakdown(txt, limit, get_token_fn): + """ + 当无法用标点、空行分割时,我们用最暴力的方法切割 + """ + for i in reversed(range(len(txt))): + if get_token_fn(txt[:i]) < limit: + return txt[:i], txt[i:] + return "Tiktoken未知错误", "Tiktoken未知错误" + +def breakdown_txt_to_satisfy_token_limit_for_pdf(txt, get_token_fn, limit): + # 递归 + def cut(txt_tocut, must_break_at_empty_line, break_anyway=False): + if get_token_fn(txt_tocut) <= limit: + return [txt_tocut] + else: + lines = txt_tocut.split('\n') + estimated_line_cut = limit / get_token_fn(txt_tocut) * len(lines) + estimated_line_cut = int(estimated_line_cut) + cnt = 0 + for cnt in reversed(range(estimated_line_cut)): + if must_break_at_empty_line: + if lines[cnt] != "": + continue + prev = "\n".join(lines[:cnt]) + post = "\n".join(lines[cnt:]) + if get_token_fn(prev) < limit: + break + if cnt == 0: + if break_anyway: + prev, post = force_breakdown(txt_tocut, limit, get_token_fn) + else: + raise RuntimeError(f"存在一行极长的文本!{txt_tocut}") + # print(len(post)) + # 列表递归接龙 + result = [prev] + result.extend(cut(post, must_break_at_empty_line, break_anyway=break_anyway)) + return result + try: + # 第1次尝试,将双空行(\n\n)作为切分点 + return cut(txt, must_break_at_empty_line=True) + except RuntimeError: + try: + # 第2次尝试,将单空行(\n)作为切分点 + return cut(txt, must_break_at_empty_line=False) + except RuntimeError: + try: + # 第3次尝试,将英文句号(.)作为切分点 + res = cut(txt.replace('.', '。\n'), must_break_at_empty_line=False) # 这个中文的句号是故意的,作为一个标识而存在 + return [r.replace('。\n', '.') for r in res] + except RuntimeError as e: + try: + # 第4次尝试,将中文句号(。)作为切分点 + res = cut(txt.replace('。', '。。\n'), must_break_at_empty_line=False) + return [r.replace('。。\n', '。') for r in res] + except RuntimeError as e: + # 第5次尝试,没办法了,随便切一下敷衍吧 + return cut(txt, must_break_at_empty_line=False, break_anyway=True) + + + +def read_and_clean_pdf_text(fp): + """ + 这个函数用于分割pdf,用了很多trick,逻辑较乱,效果奇好 + + **输入参数说明** + - `fp`:需要读取和清理文本的pdf文件路径 + + **输出参数说明** + - `meta_txt`:清理后的文本内容字符串 + - `page_one_meta`:第一页清理后的文本内容列表 + + **函数功能** + 读取pdf文件并清理其中的文本内容,清理规则包括: + - 提取所有块元的文本信息,并合并为一个字符串 + - 去除短块(字符数小于100)并替换为回车符 + - 清理多余的空行 + - 合并小写字母开头的段落块并替换为空格 + - 清除重复的换行 + - 将每个换行符替换为两个换行符,使每个段落之间有两个换行符分隔 + """ + import fitz, copy + import re + import numpy as np + from colorful import print亮黄, print亮绿 + fc = 0 # Index 0 文本 + fs = 1 # Index 1 字体 + fb = 2 # Index 2 框框 + REMOVE_FOOT_NOTE = True # 是否丢弃掉 不是正文的内容 (比正文字体小,如参考文献、脚注、图注等) + REMOVE_FOOT_FFSIZE_PERCENT = 0.95 # 小于正文的?时,判定为不是正文(有些文章的正文部分字体大小不是100%统一的,有肉眼不可见的小变化) + def primary_ffsize(l): + """ + 提取文本块主字体 + """ + fsize_statiscs = {} + for wtf in l['spans']: + if wtf['size'] not in fsize_statiscs: fsize_statiscs[wtf['size']] = 0 + fsize_statiscs[wtf['size']] += len(wtf['text']) + return max(fsize_statiscs, key=fsize_statiscs.get) + + def ffsize_same(a,b): + """ + 提取字体大小是否近似相等 + """ + return abs((a-b)/max(a,b)) < 0.02 + + with fitz.open(fp) as doc: + meta_txt = [] + meta_font = [] + + meta_line = [] + meta_span = [] + ############################## <第 1 步,搜集初始信息> ################################## + for index, page in enumerate(doc): + # file_content += page.get_text() + text_areas = page.get_text("dict") # 获取页面上的文本信息 + for t in text_areas['blocks']: + if 'lines' in t: + pf = 998 + for l in t['lines']: + txt_line = "".join([wtf['text'] for wtf in l['spans']]) + if len(txt_line) == 0: continue + pf = primary_ffsize(l) + meta_line.append([txt_line, pf, l['bbox'], l]) + for wtf in l['spans']: # for l in t['lines']: + meta_span.append([wtf['text'], wtf['size'], len(wtf['text'])]) + # meta_line.append(["NEW_BLOCK", pf]) + # 块元提取 for each word segment with in line for each line cross-line words for each block + meta_txt.extend([" ".join(["".join([wtf['text'] for wtf in l['spans']]) for l in t['lines']]).replace( + '- ', '') for t in text_areas['blocks'] if 'lines' in t]) + meta_font.extend([np.mean([np.mean([wtf['size'] for wtf in l['spans']]) + for l in t['lines']]) for t in text_areas['blocks'] if 'lines' in t]) + if index == 0: + page_one_meta = [" ".join(["".join([wtf['text'] for wtf in l['spans']]) for l in t['lines']]).replace( + '- ', '') for t in text_areas['blocks'] if 'lines' in t] + + ############################## <第 2 步,获取正文主字体> ################################## + fsize_statiscs = {} + for span in meta_span: + if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 + fsize_statiscs[span[1]] += span[2] + main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) + if REMOVE_FOOT_NOTE: + give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT + + ############################## <第 3 步,切分和重新整合> ################################## + mega_sec = [] + sec = [] + for index, line in enumerate(meta_line): + if index == 0: + sec.append(line[fc]) + continue + if REMOVE_FOOT_NOTE: + if meta_line[index][fs] <= give_up_fize_threshold: + continue + if ffsize_same(meta_line[index][fs], meta_line[index-1][fs]): + # 尝试识别段落 + if meta_line[index][fc].endswith('.') and\ + (meta_line[index-1][fc] != 'NEW_BLOCK') and \ + (meta_line[index][fb][2] - meta_line[index][fb][0]) < (meta_line[index-1][fb][2] - meta_line[index-1][fb][0]) * 0.7: + sec[-1] += line[fc] + sec[-1] += "\n\n" + else: + sec[-1] += " " + sec[-1] += line[fc] + else: + if (index+1 < len(meta_line)) and \ + meta_line[index][fs] > main_fsize: + # 单行 + 字体大 + mega_sec.append(copy.deepcopy(sec)) + sec = [] + sec.append("# " + line[fc]) + else: + # 尝试识别section + if meta_line[index-1][fs] > meta_line[index][fs]: + sec.append("\n" + line[fc]) + else: + sec.append(line[fc]) + mega_sec.append(copy.deepcopy(sec)) + + finals = [] + for ms in mega_sec: + final = " ".join(ms) + final = final.replace('- ', ' ') + finals.append(final) + meta_txt = finals + + ############################## <第 4 步,乱七八糟的后处理> ################################## + def 把字符太少的块清除为回车(meta_txt): + for index, block_txt in enumerate(meta_txt): + if len(block_txt) < 100: + meta_txt[index] = '\n' + return meta_txt + meta_txt = 把字符太少的块清除为回车(meta_txt) + + def 清理多余的空行(meta_txt): + for index in reversed(range(1, len(meta_txt))): + if meta_txt[index] == '\n' and meta_txt[index-1] == '\n': + meta_txt.pop(index) + return meta_txt + meta_txt = 清理多余的空行(meta_txt) + + def 合并小写开头的段落块(meta_txt): + def starts_with_lowercase_word(s): + pattern = r"^[a-z]+" + match = re.match(pattern, s) + if match: + return True + else: + return False + for _ in range(100): + for index, block_txt in enumerate(meta_txt): + if starts_with_lowercase_word(block_txt): + if meta_txt[index-1] != '\n': + meta_txt[index-1] += ' ' + else: + meta_txt[index-1] = '' + meta_txt[index-1] += meta_txt[index] + meta_txt[index] = '\n' + return meta_txt + meta_txt = 合并小写开头的段落块(meta_txt) + meta_txt = 清理多余的空行(meta_txt) + + meta_txt = '\n'.join(meta_txt) + # 清除重复的换行 + for _ in range(5): + meta_txt = meta_txt.replace('\n\n', '\n') + + # 换行 -> 双换行 + meta_txt = meta_txt.replace('\n', '\n\n') + + ############################## <第 5 步,展示分割效果> ################################## + # for f in finals: + # print亮黄(f) + # print亮绿('***************************') + + return meta_txt, page_one_meta + + +def get_files_from_everything(txt, type): # type='.md' + """ + 这个函数是用来获取指定目录下所有指定类型(如.md)的文件,并且对于网络上的文件,也可以获取它。 + 下面是对每个参数和返回值的说明: + 参数 + - txt: 路径或网址,表示要搜索的文件或者文件夹路径或网络上的文件。 + - type: 字符串,表示要搜索的文件类型。默认是.md。 + 返回值 + - success: 布尔值,表示函数是否成功执行。 + - file_manifest: 文件路径列表,里面包含以指定类型为后缀名的所有文件的绝对路径。 + - project_folder: 字符串,表示文件所在的文件夹路径。如果是网络上的文件,就是临时文件夹的路径。 + 该函数详细注释已添加,请确认是否满足您的需要。 + """ + import glob, os + + success = True + if txt.startswith('http'): + # 网络的远程文件 + import requests + from toolbox import get_conf + proxies, = get_conf('proxies') + r = requests.get(txt, proxies=proxies) + with open('./gpt_log/temp'+type, 'wb+') as f: f.write(r.content) + project_folder = './gpt_log/' + file_manifest = ['./gpt_log/temp'+type] + elif txt.endswith(type): + # 直接给定文件 + file_manifest = [txt] + project_folder = os.path.dirname(txt) + elif os.path.exists(txt): + # 本地路径,递归搜索 + project_folder = txt + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*'+type, recursive=True)] + if len(file_manifest) == 0: + success = False + else: + project_folder = None + file_manifest = [] + success = False + + return success, file_manifest, project_folder + + + + +def Singleton(cls): + _instance = {} + + def _singleton(*args, **kargs): + if cls not in _instance: + _instance[cls] = cls(*args, **kargs) + return _instance[cls] + + return _singleton + + +@Singleton +class knowledge_archive_interface(): + def __init__(self) -> None: + self.threadLock = threading.Lock() + self.current_id = "" + self.kai_path = None + self.qa_handle = None + self.text2vec_large_chinese = None + + def get_chinese_text2vec(self): + if self.text2vec_large_chinese is None: + # < -------------------预热文本向量化模组--------------- > + from toolbox import ProxyNetworkActivate + print('Checking Text2vec ...') + from langchain.embeddings.huggingface import HuggingFaceEmbeddings + with ProxyNetworkActivate(): # 临时地激活代理网络 + self.text2vec_large_chinese = HuggingFaceEmbeddings(model_name="GanymedeNil/text2vec-large-chinese") + + return self.text2vec_large_chinese + + + def feed_archive(self, file_manifest, id="default"): + self.threadLock.acquire() + # import uuid + self.current_id = id + from zh_langchain import construct_vector_store + self.qa_handle, self.kai_path = construct_vector_store( + vs_id=self.current_id, + files=file_manifest, + sentence_size=100, + history=[], + one_conent="", + one_content_segmentation="", + text2vec = self.get_chinese_text2vec(), + ) + self.threadLock.release() + + def get_current_archive_id(self): + return self.current_id + + def get_loaded_file(self): + return self.qa_handle.get_loaded_file() + + def answer_with_archive_by_id(self, txt, id): + self.threadLock.acquire() + if not self.current_id == id: + self.current_id = id + from zh_langchain import construct_vector_store + self.qa_handle, self.kai_path = construct_vector_store( + vs_id=self.current_id, + files=[], + sentence_size=100, + history=[], + one_conent="", + one_content_segmentation="", + text2vec = self.get_chinese_text2vec(), + ) + VECTOR_SEARCH_SCORE_THRESHOLD = 0 + VECTOR_SEARCH_TOP_K = 4 + CHUNK_SIZE = 512 + resp, prompt = self.qa_handle.get_knowledge_based_conent_test( + query = txt, + vs_path = self.kai_path, + score_threshold=VECTOR_SEARCH_SCORE_THRESHOLD, + vector_search_top_k=VECTOR_SEARCH_TOP_K, + chunk_conent=True, + chunk_size=CHUNK_SIZE, + text2vec = self.get_chinese_text2vec(), + ) + self.threadLock.release() + return resp, prompt + +def try_install_deps(deps): + for dep in deps: + import subprocess, sys + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', dep]) + + +class construct_html(): + def __init__(self) -> None: + self.css = """ +.row { + display: flex; + flex-wrap: wrap; +} + +.column { + flex: 1; + padding: 10px; +} + +.table-header { + font-weight: bold; + border-bottom: 1px solid black; +} + +.table-row { + border-bottom: 1px solid lightgray; +} + +.table-cell { + padding: 5px; +} + """ + self.html_string = f'翻译结果' + + + def add_row(self, a, b): + tmp = """ +
+
REPLACE_A
+
REPLACE_B
+
+ """ + from toolbox import markdown_convertion + tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) + tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) + self.html_string += tmp + + + def save_file(self, file_name): + with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + f.write(self.html_string.encode('utf-8', 'ignore').decode()) + diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py new file mode 100644 index 0000000000000000000000000000000000000000..c59bc31d4b445f69ac58b1e84db789cc4bbe4b43 --- /dev/null +++ b/crazy_functions/latex_fns/latex_actions.py @@ -0,0 +1,447 @@ +from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone +from .latex_toolbox import PRESERVE, TRANSFORM +from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace +from .latex_toolbox import reverse_forbidden_text_careful_brace, reverse_forbidden_text, convert_to_linklist, post_process +from .latex_toolbox import fix_content, find_main_tex_file, merge_tex_files, compile_latex_with_timeout + +import os, shutil +import re +import numpy as np + +pj = os.path.join + + +def split_subprocess(txt, project_folder, return_dict, opts): + """ + break down latex file to a linked list, + each node use a preserve flag to indicate whether it should + be proccessed by GPT. + """ + text = txt + mask = np.zeros(len(txt), dtype=np.uint8) + TRANSFORM + + # 吸收title与作者以上的部分 + text, mask = set_forbidden_text(text, mask, r"^(.*?)\\maketitle", re.DOTALL) + text, mask = set_forbidden_text(text, mask, r"^(.*?)\\begin{document}", re.DOTALL) + # 吸收iffalse注释 + text, mask = set_forbidden_text(text, mask, r"\\iffalse(.*?)\\fi", re.DOTALL) + # 吸收在42行以内的begin-end组合 + text, mask = set_forbidden_text_begin_end(text, mask, r"\\begin\{([a-z\*]*)\}(.*?)\\end\{\1\}", re.DOTALL, limit_n_lines=42) + # 吸收匿名公式 + text, mask = set_forbidden_text(text, mask, [ r"\$\$([^$]+)\$\$", r"\\\[.*?\\\]" ], re.DOTALL) + # 吸收其他杂项 + text, mask = set_forbidden_text(text, mask, [ r"\\section\{(.*?)\}", r"\\section\*\{(.*?)\}", r"\\subsection\{(.*?)\}", r"\\subsubsection\{(.*?)\}" ]) + text, mask = set_forbidden_text(text, mask, [ r"\\bibliography\{(.*?)\}", r"\\bibliographystyle\{(.*?)\}" ]) + text, mask = set_forbidden_text(text, mask, r"\\begin\{thebibliography\}.*?\\end\{thebibliography\}", re.DOTALL) + text, mask = set_forbidden_text(text, mask, r"\\begin\{lstlisting\}(.*?)\\end\{lstlisting\}", re.DOTALL) + text, mask = set_forbidden_text(text, mask, r"\\begin\{wraptable\}(.*?)\\end\{wraptable\}", re.DOTALL) + text, mask = set_forbidden_text(text, mask, r"\\begin\{algorithm\}(.*?)\\end\{algorithm\}", re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{wrapfigure\}(.*?)\\end\{wrapfigure\}", r"\\begin\{wrapfigure\*\}(.*?)\\end\{wrapfigure\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{figure\}(.*?)\\end\{figure\}", r"\\begin\{figure\*\}(.*?)\\end\{figure\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{multline\}(.*?)\\end\{multline\}", r"\\begin\{multline\*\}(.*?)\\end\{multline\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{table\}(.*?)\\end\{table\}", r"\\begin\{table\*\}(.*?)\\end\{table\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{minipage\}(.*?)\\end\{minipage\}", r"\\begin\{minipage\*\}(.*?)\\end\{minipage\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{align\*\}(.*?)\\end\{align\*\}", r"\\begin\{align\}(.*?)\\end\{align\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\begin\{equation\}(.*?)\\end\{equation\}", r"\\begin\{equation\*\}(.*?)\\end\{equation\*\}"], re.DOTALL) + text, mask = set_forbidden_text(text, mask, [r"\\includepdf\[(.*?)\]\{(.*?)\}", r"\\clearpage", r"\\newpage", r"\\appendix", r"\\tableofcontents", r"\\include\{(.*?)\}"]) + text, mask = set_forbidden_text(text, mask, [r"\\vspace\{(.*?)\}", r"\\hspace\{(.*?)\}", r"\\label\{(.*?)\}", r"\\begin\{(.*?)\}", r"\\end\{(.*?)\}", r"\\item "]) + text, mask = set_forbidden_text_careful_brace(text, mask, r"\\hl\{(.*?)\}", re.DOTALL) + # reverse 操作必须放在最后 + text, mask = reverse_forbidden_text_careful_brace(text, mask, r"\\caption\{(.*?)\}", re.DOTALL, forbid_wrapper=True) + text, mask = reverse_forbidden_text_careful_brace(text, mask, r"\\abstract\{(.*?)\}", re.DOTALL, forbid_wrapper=True) + text, mask = reverse_forbidden_text(text, mask, r"\\begin\{abstract\}(.*?)\\end\{abstract\}", re.DOTALL, forbid_wrapper=True) + root = convert_to_linklist(text, mask) + + # 最后一步处理,增强稳健性 + root = post_process(root) + + # 输出html调试文件,用红色标注处保留区(PRESERVE),用黑色标注转换区(TRANSFORM) + with open(pj(project_folder, 'debug_log.html'), 'w', encoding='utf8') as f: + segment_parts_for_gpt = [] + nodes = [] + node = root + while True: + nodes.append(node) + show_html = node.string.replace('\n','
') + if not node.preserve: + segment_parts_for_gpt.append(node.string) + f.write(f'

#{node.range}{show_html}#

') + else: + f.write(f'

{show_html}

') + node = node.next + if node is None: break + + for n in nodes: n.next = None # break + return_dict['nodes'] = nodes + return_dict['segment_parts_for_gpt'] = segment_parts_for_gpt + return return_dict + +class LatexPaperSplit(): + """ + break down latex file to a linked list, + each node use a preserve flag to indicate whether it should + be proccessed by GPT. + """ + def __init__(self) -> None: + self.nodes = None + self.msg = "*{\\scriptsize\\textbf{警告:该PDF由GPT-Academic开源项目调用大语言模型+Latex翻译插件一键生成," + \ + "版权归原文作者所有。翻译内容可靠性无保障,请仔细鉴别并以原文为准。" + \ + "项目Github地址 \\url{https://github.com/binary-husky/gpt_academic/}。" + # 请您不要删除或修改这行警告,除非您是论文的原作者(如果您是论文原作者,欢迎加REAME中的QQ联系开发者) + self.msg_declare = "为了防止大语言模型的意外谬误产生扩散影响,禁止移除或修改此警告。}}\\\\" + + + def merge_result(self, arr, mode, msg, buggy_lines=[], buggy_line_surgery_n_lines=10): + """ + Merge the result after the GPT process completed + """ + result_string = "" + node_cnt = 0 + line_cnt = 0 + + for node in self.nodes: + if node.preserve: + line_cnt += node.string.count('\n') + result_string += node.string + else: + translated_txt = fix_content(arr[node_cnt], node.string) + begin_line = line_cnt + end_line = line_cnt + translated_txt.count('\n') + + # reverse translation if any error + if any([begin_line-buggy_line_surgery_n_lines <= b_line <= end_line+buggy_line_surgery_n_lines for b_line in buggy_lines]): + translated_txt = node.string + + result_string += translated_txt + node_cnt += 1 + line_cnt += translated_txt.count('\n') + + if mode == 'translate_zh': + pattern = re.compile(r'\\begin\{abstract\}.*\n') + match = pattern.search(result_string) + if not match: + # match \abstract{xxxx} + pattern_compile = re.compile(r"\\abstract\{(.*?)\}", flags=re.DOTALL) + match = pattern_compile.search(result_string) + position = match.regs[1][0] + else: + # match \begin{abstract}xxxx\end{abstract} + position = match.end() + result_string = result_string[:position] + self.msg + msg + self.msg_declare + result_string[position:] + return result_string + + + def split(self, txt, project_folder, opts): + """ + break down latex file to a linked list, + each node use a preserve flag to indicate whether it should + be proccessed by GPT. + P.S. use multiprocessing to avoid timeout error + """ + import multiprocessing + manager = multiprocessing.Manager() + return_dict = manager.dict() + p = multiprocessing.Process( + target=split_subprocess, + args=(txt, project_folder, return_dict, opts)) + p.start() + p.join() + p.close() + self.nodes = return_dict['nodes'] + self.sp = return_dict['segment_parts_for_gpt'] + return self.sp + + +class LatexPaperFileGroup(): + """ + use tokenizer to break down text according to max_token_limit + """ + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + use tokenizer to break down text according to max_token_limit + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from ..crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex") + print('Segmentation: done') + + def merge_result(self): + self.file_result = ["" for _ in range(len(self.file_paths))] + for r, k in zip(self.sp_file_result, self.sp_file_index): + self.file_result[k] += r + + def write_result(self): + manifest = [] + for path, res in zip(self.file_paths, self.file_result): + with open(path + '.polish.tex', 'w', encoding='utf8') as f: + manifest.append(path + '.polish.tex') + f.write(res) + return manifest + + +def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None, opts=[]): + import time, os, re + from ..crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + from .latex_actions import LatexPaperFileGroup, LatexPaperSplit + + # <-------- 寻找主tex文件 ----------> + maintex = find_main_tex_file(file_manifest, mode) + chatbot.append((f"定位主Latex文件", f'[Local Message] 分析结果:该项目的Latex主文件是{maintex}, 如果分析错误, 请立即终止程序, 删除或修改歧义文件, 然后重试。主程序即将开始, 请稍候。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + time.sleep(3) + + # <-------- 读取Latex文件, 将多文件tex工程融合为一个巨型tex ----------> + main_tex_basename = os.path.basename(maintex) + assert main_tex_basename.endswith('.tex') + main_tex_basename_bare = main_tex_basename[:-4] + may_exist_bbl = pj(project_folder, f'{main_tex_basename_bare}.bbl') + if os.path.exists(may_exist_bbl): + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge.bbl')) + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge_{mode}.bbl')) + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge_diff.bbl')) + + with open(maintex, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + merged_content = merge_tex_files(project_folder, content, mode) + + with open(project_folder + '/merge.tex', 'w', encoding='utf-8', errors='replace') as f: + f.write(merged_content) + + # <-------- 精细切分latex文件 ----------> + chatbot.append((f"Latex文件融合完成", f'[Local Message] 正在精细切分latex文件,这需要一段时间计算,文档越长耗时越长,请耐心等待。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + lps = LatexPaperSplit() + res = lps.split(merged_content, project_folder, opts) # 消耗时间的函数 + + # <-------- 拆分过长的latex片段 ----------> + pfg = LatexPaperFileGroup() + for index, r in enumerate(res): + pfg.file_paths.append('segment-' + str(index)) + pfg.file_contents.append(r) + + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + # <-------- 根据需要切换prompt ----------> + inputs_array, sys_prompt_array = switch_prompt(pfg, mode) + inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag] + + if os.path.exists(pj(project_folder,'temp.pkl')): + + # <-------- 【仅调试】如果存在调试缓存文件,则跳过GPT请求环节 ----------> + pfg = objload(file=pj(project_folder,'temp.pkl')) + + else: + # <-------- gpt 多线程请求 ----------> + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待 + scroller_max_len = 40 + ) + + # <-------- 文本碎片重组为完整的tex片段 ----------> + pfg.sp_file_result = [] + for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents): + pfg.sp_file_result.append(gpt_say) + pfg.merge_result() + + # <-------- 临时存储用于调试 ----------> + pfg.get_token_num = None + objdump(pfg, file=pj(project_folder,'temp.pkl')) + + write_html(pfg.sp_file_contents, pfg.sp_file_result, chatbot=chatbot, project_folder=project_folder) + + # <-------- 写出文件 ----------> + msg = f"当前大语言模型: {llm_kwargs['llm_model']},当前语言模型温度设定: {llm_kwargs['temperature']}。" + final_tex = lps.merge_result(pfg.file_result, mode, msg) + objdump((lps, pfg.file_result, mode, msg), file=pj(project_folder,'merge_result.pkl')) + + with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f: + if mode != 'translate_zh' or "binary" in final_tex: f.write(final_tex) + + + # <-------- 整理结果, 退出 ----------> + chatbot.append((f"完成了吗?", 'GPT结果已输出, 即将编译PDF')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------- 返回 ----------> + return project_folder + f'/merge_{mode}.tex' + + +def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified, fixed_line=[]): + try: + with open(log_path, 'r', encoding='utf-8', errors='replace') as f: + log = f.read() + import re + buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log) + buggy_lines = [int(l) for l in buggy_lines] + buggy_lines = sorted(buggy_lines) + buggy_line = buggy_lines[0]-1 + print("reversing tex line that has errors", buggy_line) + + # 重组,逆转出错的段落 + if buggy_line not in fixed_line: + fixed_line.append(buggy_line) + + lps, file_result, mode, msg = objload(file=pj(work_folder_modified,'merge_result.pkl')) + final_tex = lps.merge_result(file_result, mode, msg, buggy_lines=fixed_line, buggy_line_surgery_n_lines=5*n_fix) + + with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f: + f.write(final_tex) + + return True, f"{tex_name_pure}_fix_{n_fix}", buggy_lines + except: + print("Fatal error occurred, but we cannot identify error, please download zip, read latex log, and compile manually.") + return False, -1, [-1] + + +def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder, mode='default'): + import os, time + n_fix = 1 + fixed_line = [] + max_try = 32 + chatbot.append([f"正在编译PDF文档", f'编译已经开始。当前工作路径为{work_folder},如果程序停顿5分钟以上,请直接去该路径下取回翻译结果,或者重启之后再度尝试 ...']); yield from update_ui(chatbot=chatbot, history=history) + chatbot.append([f"正在编译PDF文档", '...']); yield from update_ui(chatbot=chatbot, history=history); time.sleep(1); chatbot[-1] = list(chatbot[-1]) # 刷新界面 + yield from update_ui_lastest_msg('编译已经开始...', chatbot, history) # 刷新Gradio前端界面 + + while True: + import os + may_exist_bbl = pj(work_folder_modified, f'merge.bbl') + target_bbl = pj(work_folder_modified, f'{main_file_modified}.bbl') + if os.path.exists(may_exist_bbl) and not os.path.exists(target_bbl): + shutil.copyfile(may_exist_bbl, target_bbl) + + # https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译原始PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译转化后的PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + + if ok and os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')): + # 只有第二步成功,才能继续下面的步骤 + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译BibTex ...', chatbot, history) # 刷新Gradio前端界面 + if not os.path.exists(pj(work_folder_original, f'{main_file_original}.bbl')): + ok = compile_latex_with_timeout(f'bibtex {main_file_original}.aux', work_folder_original) + if not os.path.exists(pj(work_folder_modified, f'{main_file_modified}.bbl')): + ok = compile_latex_with_timeout(f'bibtex {main_file_modified}.aux', work_folder_modified) + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译文献交叉引用 ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + + if mode!='translate_zh': + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 + print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + ok = compile_latex_with_timeout(f'bibtex merge_diff.aux', work_folder) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + + # <---------- 检查结果 -----------> + results_ = "" + original_pdf_success = os.path.exists(pj(work_folder_original, f'{main_file_original}.pdf')) + modified_pdf_success = os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')) + diff_pdf_success = os.path.exists(pj(work_folder, f'merge_diff.pdf')) + results_ += f"原始PDF编译是否成功: {original_pdf_success};" + results_ += f"转化PDF编译是否成功: {modified_pdf_success};" + results_ += f"对比PDF编译是否成功: {diff_pdf_success};" + yield from update_ui_lastest_msg(f'第{n_fix}编译结束:
{results_}...', chatbot, history) # 刷新Gradio前端界面 + + if diff_pdf_success: + result_pdf = pj(work_folder_modified, f'merge_diff.pdf') # get pdf path + promote_file_to_downloadzone(result_pdf, rename_file=None, chatbot=chatbot) # promote file to web UI + if modified_pdf_success: + yield from update_ui_lastest_msg(f'转化PDF编译已经成功, 即将退出 ...', chatbot, history) # 刷新Gradio前端界面 + result_pdf = pj(work_folder_modified, f'{main_file_modified}.pdf') # get pdf path + origin_pdf = pj(work_folder_original, f'{main_file_original}.pdf') # get pdf path + if os.path.exists(pj(work_folder, '..', 'translation')): + shutil.copyfile(result_pdf, pj(work_folder, '..', 'translation', 'translate_zh.pdf')) + promote_file_to_downloadzone(result_pdf, rename_file=None, chatbot=chatbot) # promote file to web UI + # 将两个PDF拼接 + if original_pdf_success: + try: + from .latex_toolbox import merge_pdfs + concat_pdf = pj(work_folder_modified, f'comparison.pdf') + merge_pdfs(origin_pdf, result_pdf, concat_pdf) + promote_file_to_downloadzone(concat_pdf, rename_file=None, chatbot=chatbot) # promote file to web UI + except Exception as e: + pass + return True # 成功啦 + else: + if n_fix>=max_try: break + n_fix += 1 + can_retry, main_file_modified, buggy_lines = remove_buggy_lines( + file_path=pj(work_folder_modified, f'{main_file_modified}.tex'), + log_path=pj(work_folder_modified, f'{main_file_modified}.log'), + tex_name=f'{main_file_modified}.tex', + tex_name_pure=f'{main_file_modified}', + n_fix=n_fix, + work_folder_modified=work_folder_modified, + fixed_line=fixed_line + ) + yield from update_ui_lastest_msg(f'由于最为关键的转化PDF编译失败, 将根据报错信息修正tex源文件并重试, 当前报错的latex代码处于第{buggy_lines}行 ...', chatbot, history) # 刷新Gradio前端界面 + if not can_retry: break + + return False # 失败啦 + + +def write_html(sp_file_contents, sp_file_result, chatbot, project_folder): + # write html + try: + import shutil + from ..crazy_utils import construct_html + from toolbox import gen_time_str + ch = construct_html() + orig = "" + trans = "" + final = [] + for c,r in zip(sp_file_contents, sp_file_result): + final.append(c) + final.append(r) + for i, k in enumerate(final): + if i%2==0: + orig = k + if i%2==1: + trans = k + ch.add_row(a=orig, b=trans) + create_report_file_name = f"{gen_time_str()}.trans.html" + ch.save_file(create_report_file_name) + shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name)) + promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot) + except: + from toolbox import trimmed_format_exc + print('writing html result failed:', trimmed_format_exc()) diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c889a80867903efab84498fea71b4c7120fd21 --- /dev/null +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -0,0 +1,456 @@ +import os, shutil +import re +import numpy as np +PRESERVE = 0 +TRANSFORM = 1 + +pj = os.path.join + +class LinkedListNode(): + """ + Linked List Node + """ + def __init__(self, string, preserve=True) -> None: + self.string = string + self.preserve = preserve + self.next = None + self.range = None + # self.begin_line = 0 + # self.begin_char = 0 + +def convert_to_linklist(text, mask): + root = LinkedListNode("", preserve=True) + current_node = root + for c, m, i in zip(text, mask, range(len(text))): + if (m==PRESERVE and current_node.preserve) \ + or (m==TRANSFORM and not current_node.preserve): + # add + current_node.string += c + else: + current_node.next = LinkedListNode(c, preserve=(m==PRESERVE)) + current_node = current_node.next + return root + +def post_process(root): + # 修复括号 + node = root + while True: + string = node.string + if node.preserve: + node = node.next + if node is None: break + continue + def break_check(string): + str_stack = [""] # (lv, index) + for i, c in enumerate(string): + if c == '{': + str_stack.append('{') + elif c == '}': + if len(str_stack) == 1: + print('stack fix') + return i + str_stack.pop(-1) + else: + str_stack[-1] += c + return -1 + bp = break_check(string) + + if bp == -1: + pass + elif bp == 0: + node.string = string[:1] + q = LinkedListNode(string[1:], False) + q.next = node.next + node.next = q + else: + node.string = string[:bp] + q = LinkedListNode(string[bp:], False) + q.next = node.next + node.next = q + + node = node.next + if node is None: break + + # 屏蔽空行和太短的句子 + node = root + while True: + if len(node.string.strip('\n').strip(''))==0: node.preserve = True + if len(node.string.strip('\n').strip(''))<42: node.preserve = True + node = node.next + if node is None: break + node = root + while True: + if node.next and node.preserve and node.next.preserve: + node.string += node.next.string + node.next = node.next.next + node = node.next + if node is None: break + + # 将前后断行符脱离 + node = root + prev_node = None + while True: + if not node.preserve: + lstriped_ = node.string.lstrip().lstrip('\n') + if (prev_node is not None) and (prev_node.preserve) and (len(lstriped_)!=len(node.string)): + prev_node.string += node.string[:-len(lstriped_)] + node.string = lstriped_ + rstriped_ = node.string.rstrip().rstrip('\n') + if (node.next is not None) and (node.next.preserve) and (len(rstriped_)!=len(node.string)): + node.next.string = node.string[len(rstriped_):] + node.next.string + node.string = rstriped_ + # ===== + prev_node = node + node = node.next + if node is None: break + + # 标注节点的行数范围 + node = root + n_line = 0 + expansion = 2 + while True: + n_l = node.string.count('\n') + node.range = [n_line-expansion, n_line+n_l+expansion] # 失败时,扭转的范围 + n_line = n_line+n_l + node = node.next + if node is None: break + return root + + +""" +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +Latex segmentation with a binary mask (PRESERVE=0, TRANSFORM=1) +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +""" + + +def set_forbidden_text(text, mask, pattern, flags=0): + """ + Add a preserve text area in this paper + e.g. with pattern = r"\\begin\{algorithm\}(.*?)\\end\{algorithm\}" + you can mask out (mask = PRESERVE so that text become untouchable for GPT) + everything between "\begin{equation}" and "\end{equation}" + """ + if isinstance(pattern, list): pattern = '|'.join(pattern) + pattern_compile = re.compile(pattern, flags) + for res in pattern_compile.finditer(text): + mask[res.span()[0]:res.span()[1]] = PRESERVE + return text, mask + +def reverse_forbidden_text(text, mask, pattern, flags=0, forbid_wrapper=True): + """ + Move area out of preserve area (make text editable for GPT) + count the number of the braces so as to catch compelete text area. + e.g. + \begin{abstract} blablablablablabla. \end{abstract} + """ + if isinstance(pattern, list): pattern = '|'.join(pattern) + pattern_compile = re.compile(pattern, flags) + for res in pattern_compile.finditer(text): + if not forbid_wrapper: + mask[res.span()[0]:res.span()[1]] = TRANSFORM + else: + mask[res.regs[0][0]: res.regs[1][0]] = PRESERVE # '\\begin{abstract}' + mask[res.regs[1][0]: res.regs[1][1]] = TRANSFORM # abstract + mask[res.regs[1][1]: res.regs[0][1]] = PRESERVE # abstract + return text, mask + +def set_forbidden_text_careful_brace(text, mask, pattern, flags=0): + """ + Add a preserve text area in this paper (text become untouchable for GPT). + count the number of the braces so as to catch compelete text area. + e.g. + \caption{blablablablabla\texbf{blablabla}blablabla.} + """ + pattern_compile = re.compile(pattern, flags) + for res in pattern_compile.finditer(text): + brace_level = -1 + p = begin = end = res.regs[0][0] + for _ in range(1024*16): + if text[p] == '}' and brace_level == 0: break + elif text[p] == '}': brace_level -= 1 + elif text[p] == '{': brace_level += 1 + p += 1 + end = p+1 + mask[begin:end] = PRESERVE + return text, mask + +def reverse_forbidden_text_careful_brace(text, mask, pattern, flags=0, forbid_wrapper=True): + """ + Move area out of preserve area (make text editable for GPT) + count the number of the braces so as to catch compelete text area. + e.g. + \caption{blablablablabla\texbf{blablabla}blablabla.} + """ + pattern_compile = re.compile(pattern, flags) + for res in pattern_compile.finditer(text): + brace_level = 0 + p = begin = end = res.regs[1][0] + for _ in range(1024*16): + if text[p] == '}' and brace_level == 0: break + elif text[p] == '}': brace_level -= 1 + elif text[p] == '{': brace_level += 1 + p += 1 + end = p + mask[begin:end] = TRANSFORM + if forbid_wrapper: + mask[res.regs[0][0]:begin] = PRESERVE + mask[end:res.regs[0][1]] = PRESERVE + return text, mask + +def set_forbidden_text_begin_end(text, mask, pattern, flags=0, limit_n_lines=42): + """ + Find all \begin{} ... \end{} text block that with less than limit_n_lines lines. + Add it to preserve area + """ + pattern_compile = re.compile(pattern, flags) + def search_with_line_limit(text, mask): + for res in pattern_compile.finditer(text): + cmd = res.group(1) # begin{what} + this = res.group(2) # content between begin and end + this_mask = mask[res.regs[2][0]:res.regs[2][1]] + white_list = ['document', 'abstract', 'lemma', 'definition', 'sproof', + 'em', 'emph', 'textit', 'textbf', 'itemize', 'enumerate'] + if (cmd in white_list) or this.count('\n') >= limit_n_lines: # use a magical number 42 + this, this_mask = search_with_line_limit(this, this_mask) + mask[res.regs[2][0]:res.regs[2][1]] = this_mask + else: + mask[res.regs[0][0]:res.regs[0][1]] = PRESERVE + return text, mask + return search_with_line_limit(text, mask) + + + +""" +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +Latex Merge File +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +""" + +def find_main_tex_file(file_manifest, mode): + """ + 在多Tex文档中,寻找主文件,必须包含documentclass,返回找到的第一个。 + P.S. 但愿没人把latex模板放在里面传进来 (6.25 加入判定latex模板的代码) + """ + canidates = [] + for texf in file_manifest: + if os.path.basename(texf).startswith('merge'): + continue + with open(texf, 'r', encoding='utf8', errors='ignore') as f: + file_content = f.read() + if r'\documentclass' in file_content: + canidates.append(texf) + else: + continue + + if len(canidates) == 0: + raise RuntimeError('无法找到一个主Tex文件(包含documentclass关键字)') + elif len(canidates) == 1: + return canidates[0] + else: # if len(canidates) >= 2 通过一些Latex模板中常见(但通常不会出现在正文)的单词,对不同latex源文件扣分,取评分最高者返回 + canidates_score = [] + # 给出一些判定模板文档的词作为扣分项 + unexpected_words = ['\LaTeX', 'manuscript', 'Guidelines', 'font', 'citations', 'rejected', 'blind review', 'reviewers'] + expected_words = ['\input', '\ref', '\cite'] + for texf in canidates: + canidates_score.append(0) + with open(texf, 'r', encoding='utf8', errors='ignore') as f: + file_content = f.read() + for uw in unexpected_words: + if uw in file_content: + canidates_score[-1] -= 1 + for uw in expected_words: + if uw in file_content: + canidates_score[-1] += 1 + select = np.argmax(canidates_score) # 取评分最高者返回 + return canidates[select] + +def rm_comments(main_file): + new_file_remove_comment_lines = [] + for l in main_file.splitlines(): + # 删除整行的空注释 + if l.lstrip().startswith("%"): + pass + else: + new_file_remove_comment_lines.append(l) + main_file = '\n'.join(new_file_remove_comment_lines) + # main_file = re.sub(r"\\include{(.*?)}", r"\\input{\1}", main_file) # 将 \include 命令转换为 \input 命令 + main_file = re.sub(r'(? 0 and node_string.count('\_') > final_tex.count('\_'): + # walk and replace any _ without \ + final_tex = re.sub(r"(?= limit_n_lines: # use a magical number 42 + this, this_mask = search_with_line_limit(this, this_mask) + mask[res.regs[2][0]:res.regs[2][1]] = this_mask + else: + mask[res.regs[0][0]:res.regs[0][1]] = PRESERVE + return text, mask + return search_with_line_limit(text, mask) + +class LinkedListNode(): + """ + Linked List Node + """ + def __init__(self, string, preserve=True) -> None: + self.string = string + self.preserve = preserve + self.next = None + # self.begin_line = 0 + # self.begin_char = 0 + +def convert_to_linklist(text, mask): + root = LinkedListNode("", preserve=True) + current_node = root + for c, m, i in zip(text, mask, range(len(text))): + if (m==PRESERVE and current_node.preserve) \ + or (m==TRANSFORM and not current_node.preserve): + # add + current_node.string += c + else: + current_node.next = LinkedListNode(c, preserve=(m==PRESERVE)) + current_node = current_node.next + return root +""" +======================================================================== +Latex Merge File +======================================================================== +""" + +def 寻找Latex主文件(file_manifest, mode): + """ + 在多Tex文档中,寻找主文件,必须包含documentclass,返回找到的第一个。 + P.S. 但愿没人把latex模板放在里面传进来 (6.25 加入判定latex模板的代码) + """ + canidates = [] + for texf in file_manifest: + if os.path.basename(texf).startswith('merge'): + continue + with open(texf, 'r', encoding='utf8') as f: + file_content = f.read() + if r'\documentclass' in file_content: + canidates.append(texf) + else: + continue + + if len(canidates) == 0: + raise RuntimeError('无法找到一个主Tex文件(包含documentclass关键字)') + elif len(canidates) == 1: + return canidates[0] + else: # if len(canidates) >= 2 通过一些Latex模板中常见(但通常不会出现在正文)的单词,对不同latex源文件扣分,取评分最高者返回 + canidates_score = [] + # 给出一些判定模板文档的词作为扣分项 + unexpected_words = ['\LaTeX', 'manuscript', 'Guidelines', 'font', 'citations', 'rejected', 'blind review', 'reviewers'] + expected_words = ['\input', '\ref', '\cite'] + for texf in canidates: + canidates_score.append(0) + with open(texf, 'r', encoding='utf8') as f: + file_content = f.read() + for uw in unexpected_words: + if uw in file_content: + canidates_score[-1] -= 1 + for uw in expected_words: + if uw in file_content: + canidates_score[-1] += 1 + select = np.argmax(canidates_score) # 取评分最高者返回 + return canidates[select] + +def rm_comments(main_file): + new_file_remove_comment_lines = [] + for l in main_file.splitlines(): + # 删除整行的空注释 + if l.lstrip().startswith("%"): + pass + else: + new_file_remove_comment_lines.append(l) + main_file = '\n'.join(new_file_remove_comment_lines) + # main_file = re.sub(r"\\include{(.*?)}", r"\\input{\1}", main_file) # 将 \include 命令转换为 \input 命令 + main_file = re.sub(r'(? 0 and node_string.count('\_') > final_tex.count('\_'): + # walk and replace any _ without \ + final_tex = re.sub(r"(?') + if not node.preserve: + segment_parts_for_gpt.append(node.string) + f.write(f'

#{show_html}#

') + else: + f.write(f'

{show_html}

') + node = node.next + if node is None: break + + for n in nodes: n.next = None # break + return_dict['nodes'] = nodes + return_dict['segment_parts_for_gpt'] = segment_parts_for_gpt + return return_dict + + + +class LatexPaperSplit(): + """ + break down latex file to a linked list, + each node use a preserve flag to indicate whether it should + be proccessed by GPT. + """ + def __init__(self) -> None: + self.nodes = None + self.msg = "*{\\scriptsize\\textbf{警告:该PDF由GPT-Academic开源项目调用大语言模型+Latex翻译插件一键生成," + \ + "版权归原文作者所有。翻译内容可靠性无保障,请仔细鉴别并以原文为准。" + \ + "项目Github地址 \\url{https://github.com/binary-husky/gpt_academic/}。" + # 请您不要删除或修改这行警告,除非您是论文的原作者(如果您是论文原作者,欢迎加REAME中的QQ联系开发者) + self.msg_declare = "为了防止大语言模型的意外谬误产生扩散影响,禁止移除或修改此警告。}}\\\\" + + def merge_result(self, arr, mode, msg): + """ + Merge the result after the GPT process completed + """ + result_string = "" + p = 0 + for node in self.nodes: + if node.preserve: + result_string += node.string + else: + result_string += fix_content(arr[p], node.string) + p += 1 + if mode == 'translate_zh': + pattern = re.compile(r'\\begin\{abstract\}.*\n') + match = pattern.search(result_string) + if not match: + # match \abstract{xxxx} + pattern_compile = re.compile(r"\\abstract\{(.*?)\}", flags=re.DOTALL) + match = pattern_compile.search(result_string) + position = match.regs[1][0] + else: + # match \begin{abstract}xxxx\end{abstract} + position = match.end() + result_string = result_string[:position] + self.msg + msg + self.msg_declare + result_string[position:] + return result_string + + def split(self, txt, project_folder, opts): + """ + break down latex file to a linked list, + each node use a preserve flag to indicate whether it should + be proccessed by GPT. + P.S. use multiprocessing to avoid timeout error + """ + import multiprocessing + manager = multiprocessing.Manager() + return_dict = manager.dict() + p = multiprocessing.Process( + target=split_subprocess, + args=(txt, project_folder, return_dict, opts)) + p.start() + p.join() + p.close() + self.nodes = return_dict['nodes'] + self.sp = return_dict['segment_parts_for_gpt'] + return self.sp + + + +class LatexPaperFileGroup(): + """ + use tokenizer to break down text according to max_token_limit + """ + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + use tokenizer to break down text according to max_token_limit + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex") + print('Segmentation: done') + + def merge_result(self): + self.file_result = ["" for _ in range(len(self.file_paths))] + for r, k in zip(self.sp_file_result, self.sp_file_index): + self.file_result[k] += r + + def write_result(self): + manifest = [] + for path, res in zip(self.file_paths, self.file_result): + with open(path + '.polish.tex', 'w', encoding='utf8') as f: + manifest.append(path + '.polish.tex') + f.write(res) + return manifest + +def write_html(sp_file_contents, sp_file_result, chatbot, project_folder): + + # write html + try: + import shutil + from .crazy_utils import construct_html + from toolbox import gen_time_str + ch = construct_html() + orig = "" + trans = "" + final = [] + for c,r in zip(sp_file_contents, sp_file_result): + final.append(c) + final.append(r) + for i, k in enumerate(final): + if i%2==0: + orig = k + if i%2==1: + trans = k + ch.add_row(a=orig, b=trans) + create_report_file_name = f"{gen_time_str()}.trans.html" + ch.save_file(create_report_file_name) + shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name)) + promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot) + except: + from toolbox import trimmed_format_exc + print('writing html result failed:', trimmed_format_exc()) + +def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None, opts=[]): + import time, os, re + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件 + + # <-------- 寻找主tex文件 ----------> + maintex = 寻找Latex主文件(file_manifest, mode) + chatbot.append((f"定位主Latex文件", f'[Local Message] 分析结果:该项目的Latex主文件是{maintex}, 如果分析错误, 请立即终止程序, 删除或修改歧义文件, 然后重试。主程序即将开始, 请稍候。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + time.sleep(3) + + # <-------- 读取Latex文件, 将多文件tex工程融合为一个巨型tex ----------> + main_tex_basename = os.path.basename(maintex) + assert main_tex_basename.endswith('.tex') + main_tex_basename_bare = main_tex_basename[:-4] + may_exist_bbl = pj(project_folder, f'{main_tex_basename_bare}.bbl') + if os.path.exists(may_exist_bbl): + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge.bbl')) + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge_{mode}.bbl')) + shutil.copyfile(may_exist_bbl, pj(project_folder, f'merge_diff.bbl')) + + with open(maintex, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + merged_content = merge_tex_files(project_folder, content, mode) + + with open(project_folder + '/merge.tex', 'w', encoding='utf-8', errors='replace') as f: + f.write(merged_content) + + # <-------- 精细切分latex文件 ----------> + chatbot.append((f"Latex文件融合完成", f'[Local Message] 正在精细切分latex文件,这需要一段时间计算,文档越长耗时越长,请耐心等待。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + lps = LatexPaperSplit() + res = lps.split(merged_content, project_folder, opts) # 消耗时间的函数 + + # <-------- 拆分过长的latex片段 ----------> + pfg = LatexPaperFileGroup() + for index, r in enumerate(res): + pfg.file_paths.append('segment-' + str(index)) + pfg.file_contents.append(r) + + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + # <-------- 根据需要切换prompt ----------> + inputs_array, sys_prompt_array = switch_prompt(pfg, mode) + inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag] + + if os.path.exists(pj(project_folder,'temp.pkl')): + + # <-------- 【仅调试】如果存在调试缓存文件,则跳过GPT请求环节 ----------> + pfg = objload(file=pj(project_folder,'temp.pkl')) + + else: + # <-------- gpt 多线程请求 ----------> + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待 + scroller_max_len = 40 + ) + + # <-------- 文本碎片重组为完整的tex片段 ----------> + pfg.sp_file_result = [] + for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents): + pfg.sp_file_result.append(gpt_say) + pfg.merge_result() + + # <-------- 临时存储用于调试 ----------> + pfg.get_token_num = None + objdump(pfg, file=pj(project_folder,'temp.pkl')) + + write_html(pfg.sp_file_contents, pfg.sp_file_result, chatbot=chatbot, project_folder=project_folder) + + # <-------- 写出文件 ----------> + msg = f"当前大语言模型: {llm_kwargs['llm_model']},当前语言模型温度设定: {llm_kwargs['temperature']}。" + final_tex = lps.merge_result(pfg.file_result, mode, msg) + with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f: + if mode != 'translate_zh' or "binary" in final_tex: f.write(final_tex) + + + # <-------- 整理结果, 退出 ----------> + chatbot.append((f"完成了吗?", 'GPT结果已输出, 正在编译PDF')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------- 返回 ----------> + return project_folder + f'/merge_{mode}.tex' + + + +def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified): + try: + with open(log_path, 'r', encoding='utf-8', errors='replace') as f: + log = f.read() + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + file_lines = f.readlines() + import re + buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log) + buggy_lines = [int(l) for l in buggy_lines] + buggy_lines = sorted(buggy_lines) + print("removing lines that has errors", buggy_lines) + file_lines.pop(buggy_lines[0]-1) + with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f: + f.writelines(file_lines) + return True, f"{tex_name_pure}_fix_{n_fix}", buggy_lines + except: + print("Fatal error occurred, but we cannot identify error, please download zip, read latex log, and compile manually.") + return False, -1, [-1] + +def compile_latex_with_timeout(command, cwd, timeout=60): + import subprocess + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + print("Process timed out!") + return False + return True + +def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder, mode='default'): + import os, time + current_dir = os.getcwd() + n_fix = 1 + max_try = 32 + chatbot.append([f"正在编译PDF文档", f'编译已经开始。当前工作路径为{work_folder},如果程序停顿5分钟以上,请直接去该路径下取回翻译结果,或者重启之后再度尝试 ...']); yield from update_ui(chatbot=chatbot, history=history) + chatbot.append([f"正在编译PDF文档", '...']); yield from update_ui(chatbot=chatbot, history=history); time.sleep(1); chatbot[-1] = list(chatbot[-1]) # 刷新界面 + yield from update_ui_lastest_msg('编译已经开始...', chatbot, history) # 刷新Gradio前端界面 + + while True: + import os + + # https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译原始PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译转化后的PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + + if ok and os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')): + # 只有第二步成功,才能继续下面的步骤 + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译BibTex ...', chatbot, history) # 刷新Gradio前端界面 + if not os.path.exists(pj(work_folder_original, f'{main_file_original}.bbl')): + ok = compile_latex_with_timeout(f'bibtex {main_file_original}.aux', work_folder_original) + if not os.path.exists(pj(work_folder_modified, f'{main_file_modified}.bbl')): + ok = compile_latex_with_timeout(f'bibtex {main_file_modified}.aux', work_folder_modified) + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 编译文献交叉引用 ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex', work_folder_original) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex', work_folder_modified) + + if mode!='translate_zh': + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 + print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + + yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + ok = compile_latex_with_timeout(f'bibtex merge_diff.aux', work_folder) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) + + + # <---------- 检查结果 -----------> + results_ = "" + original_pdf_success = os.path.exists(pj(work_folder_original, f'{main_file_original}.pdf')) + modified_pdf_success = os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')) + diff_pdf_success = os.path.exists(pj(work_folder, f'merge_diff.pdf')) + results_ += f"原始PDF编译是否成功: {original_pdf_success};" + results_ += f"转化PDF编译是否成功: {modified_pdf_success};" + results_ += f"对比PDF编译是否成功: {diff_pdf_success};" + yield from update_ui_lastest_msg(f'第{n_fix}编译结束:
{results_}...', chatbot, history) # 刷新Gradio前端界面 + + if diff_pdf_success: + result_pdf = pj(work_folder_modified, f'merge_diff.pdf') # get pdf path + promote_file_to_downloadzone(result_pdf, rename_file=None, chatbot=chatbot) # promote file to web UI + if modified_pdf_success: + yield from update_ui_lastest_msg(f'转化PDF编译已经成功, 即将退出 ...', chatbot, history) # 刷新Gradio前端界面 + result_pdf = pj(work_folder_modified, f'{main_file_modified}.pdf') # get pdf path + if os.path.exists(pj(work_folder, '..', 'translation')): + shutil.copyfile(result_pdf, pj(work_folder, '..', 'translation', 'translate_zh.pdf')) + promote_file_to_downloadzone(result_pdf, rename_file=None, chatbot=chatbot) # promote file to web UI + return True # 成功啦 + else: + if n_fix>=max_try: break + n_fix += 1 + can_retry, main_file_modified, buggy_lines = remove_buggy_lines( + file_path=pj(work_folder_modified, f'{main_file_modified}.tex'), + log_path=pj(work_folder_modified, f'{main_file_modified}.log'), + tex_name=f'{main_file_modified}.tex', + tex_name_pure=f'{main_file_modified}', + n_fix=n_fix, + work_folder_modified=work_folder_modified, + ) + yield from update_ui_lastest_msg(f'由于最为关键的转化PDF编译失败, 将根据报错信息修正tex源文件并重试, 当前报错的latex代码处于第{buggy_lines}行 ...', chatbot, history) # 刷新Gradio前端界面 + if not can_retry: break + + return False # 失败啦 + + + diff --git a/crazy_functions/live_audio/aliyunASR.py b/crazy_functions/live_audio/aliyunASR.py new file mode 100644 index 0000000000000000000000000000000000000000..964105779f15c92e8cb87dbd5ad7c883ed32538d --- /dev/null +++ b/crazy_functions/live_audio/aliyunASR.py @@ -0,0 +1,130 @@ +import time, threading, json + + +class AliyunASR(): + + def test_on_sentence_begin(self, message, *args): + # print("test_on_sentence_begin:{}".format(message)) + pass + + def test_on_sentence_end(self, message, *args): + # print("test_on_sentence_end:{}".format(message)) + message = json.loads(message) + self.parsed_sentence = message['payload']['result'] + self.event_on_entence_end.set() + print(self.parsed_sentence) + + def test_on_start(self, message, *args): + # print("test_on_start:{}".format(message)) + pass + + def test_on_error(self, message, *args): + print("on_error args=>{}".format(args)) + pass + + def test_on_close(self, *args): + self.aliyun_service_ok = False + pass + + def test_on_result_chg(self, message, *args): + # print("test_on_chg:{}".format(message)) + message = json.loads(message) + self.parsed_text = message['payload']['result'] + self.event_on_result_chg.set() + + def test_on_completed(self, message, *args): + # print("on_completed:args=>{} message=>{}".format(args, message)) + pass + + + def audio_convertion_thread(self, uuid): + # 在一个异步线程中采集音频 + import nls # pip install git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git + import tempfile + from scipy import io + from toolbox import get_conf + from .audio_io import change_sample_rate + from .audio_io import RealtimeAudioDistribution + NEW_SAMPLERATE = 16000 + rad = RealtimeAudioDistribution() + rad.clean_up() + temp_folder = tempfile.gettempdir() + TOKEN, APPKEY = get_conf('ALIYUN_TOKEN', 'ALIYUN_APPKEY') + if len(TOKEN) == 0: + TOKEN = self.get_token() + self.aliyun_service_ok = True + URL="wss://nls-gateway.aliyuncs.com/ws/v1" + sr = nls.NlsSpeechTranscriber( + url=URL, + token=TOKEN, + appkey=APPKEY, + on_sentence_begin=self.test_on_sentence_begin, + on_sentence_end=self.test_on_sentence_end, + on_start=self.test_on_start, + on_result_changed=self.test_on_result_chg, + on_completed=self.test_on_completed, + on_error=self.test_on_error, + on_close=self.test_on_close, + callback_args=[uuid.hex] + ) + + r = sr.start(aformat="pcm", + enable_intermediate_result=True, + enable_punctuation_prediction=True, + enable_inverse_text_normalization=True) + + while not self.stop: + # time.sleep(self.capture_interval) + audio = rad.read(uuid.hex) + if audio is not None: + # convert to pcm file + temp_file = f'{temp_folder}/{uuid.hex}.pcm' # + dsdata = change_sample_rate(audio, rad.rate, NEW_SAMPLERATE) # 48000 --> 16000 + io.wavfile.write(temp_file, NEW_SAMPLERATE, dsdata) + # read pcm binary + with open(temp_file, "rb") as f: data = f.read() + # print('audio len:', len(audio), '\t ds len:', len(dsdata), '\t need n send:', len(data)//640) + slices = zip(*(iter(data),) * 640) # 640个字节为一组 + for i in slices: sr.send_audio(bytes(i)) + else: + time.sleep(0.1) + + if not self.aliyun_service_ok: + self.stop = True + self.stop_msg = 'Aliyun音频服务异常,请检查ALIYUN_TOKEN和ALIYUN_APPKEY是否过期。' + r = sr.stop() + + def get_token(self): + from toolbox import get_conf + import json + from aliyunsdkcore.request import CommonRequest + from aliyunsdkcore.client import AcsClient + AccessKey_ID, AccessKey_secret = get_conf('ALIYUN_ACCESSKEY', 'ALIYUN_SECRET') + + # 创建AcsClient实例 + client = AcsClient( + AccessKey_ID, + AccessKey_secret, + "cn-shanghai" + ) + + # 创建request,并设置参数。 + request = CommonRequest() + request.set_method('POST') + request.set_domain('nls-meta.cn-shanghai.aliyuncs.com') + request.set_version('2019-02-28') + request.set_action_name('CreateToken') + + try: + response = client.do_action_with_exception(request) + print(response) + jss = json.loads(response) + if 'Token' in jss and 'Id' in jss['Token']: + token = jss['Token']['Id'] + expireTime = jss['Token']['ExpireTime'] + print("token = " + token) + print("expireTime = " + str(expireTime)) + except Exception as e: + print(e) + + return token diff --git a/crazy_functions/live_audio/audio_io.py b/crazy_functions/live_audio/audio_io.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff83a66e8d9f0bb15250f1c3c2b5ea36745ff55 --- /dev/null +++ b/crazy_functions/live_audio/audio_io.py @@ -0,0 +1,51 @@ +import numpy as np +from scipy import interpolate + +def Singleton(cls): + _instance = {} + + def _singleton(*args, **kargs): + if cls not in _instance: + _instance[cls] = cls(*args, **kargs) + return _instance[cls] + + return _singleton + + +@Singleton +class RealtimeAudioDistribution(): + def __init__(self) -> None: + self.data = {} + self.max_len = 1024*1024 + self.rate = 48000 # 只读,每秒采样数量 + + def clean_up(self): + self.data = {} + + def feed(self, uuid, audio): + self.rate, audio_ = audio + # print('feed', len(audio_), audio_[-25:]) + if uuid not in self.data: + self.data[uuid] = audio_ + else: + new_arr = np.concatenate((self.data[uuid], audio_)) + if len(new_arr) > self.max_len: new_arr = new_arr[-self.max_len:] + self.data[uuid] = new_arr + + def read(self, uuid): + if uuid in self.data: + res = self.data.pop(uuid) + print('\r read-', len(res), '-', max(res), end='', flush=True) + else: + res = None + return res + +def change_sample_rate(audio, old_sr, new_sr): + duration = audio.shape[0] / old_sr + + time_old = np.linspace(0, duration, audio.shape[0]) + time_new = np.linspace(0, duration, int(audio.shape[0] * new_sr / old_sr)) + + interpolator = interpolate.interp1d(time_old, audio.T) + new_audio = interpolator(time_new).T + return new_audio.astype(np.int16) \ No newline at end of file diff --git a/crazy_functions/test_project/cpp/cppipc/buffer.cpp b/crazy_functions/test_project/cpp/cppipc/buffer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..084b8153e9401f4e9dc5a6a67cfb5f48b0183ccb --- /dev/null +++ b/crazy_functions/test_project/cpp/cppipc/buffer.cpp @@ -0,0 +1,87 @@ +#include "libipc/buffer.h" +#include "libipc/utility/pimpl.h" + +#include + +namespace ipc { + +bool operator==(buffer const & b1, buffer const & b2) { + return (b1.size() == b2.size()) && (std::memcmp(b1.data(), b2.data(), b1.size()) == 0); +} + +bool operator!=(buffer const & b1, buffer const & b2) { + return !(b1 == b2); +} + +class buffer::buffer_ : public pimpl { +public: + void* p_; + std::size_t s_; + void* a_; + buffer::destructor_t d_; + + buffer_(void* p, std::size_t s, buffer::destructor_t d, void* a) + : p_(p), s_(s), a_(a), d_(d) { + } + + ~buffer_() { + if (d_ == nullptr) return; + d_((a_ == nullptr) ? p_ : a_, s_); + } +}; + +buffer::buffer() + : buffer(nullptr, 0, nullptr, nullptr) { +} + +buffer::buffer(void* p, std::size_t s, destructor_t d) + : p_(p_->make(p, s, d, nullptr)) { +} + +buffer::buffer(void* p, std::size_t s, destructor_t d, void* additional) + : p_(p_->make(p, s, d, additional)) { +} + +buffer::buffer(void* p, std::size_t s) + : buffer(p, s, nullptr) { +} + +buffer::buffer(char const & c) + : buffer(const_cast(&c), 1) { +} + +buffer::buffer(buffer&& rhs) + : buffer() { + swap(rhs); +} + +buffer::~buffer() { + p_->clear(); +} + +void buffer::swap(buffer& rhs) { + std::swap(p_, rhs.p_); +} + +buffer& buffer::operator=(buffer rhs) { + swap(rhs); + return *this; +} + +bool buffer::empty() const noexcept { + return (impl(p_)->p_ == nullptr) || (impl(p_)->s_ == 0); +} + +void* buffer::data() noexcept { + return impl(p_)->p_; +} + +void const * buffer::data() const noexcept { + return impl(p_)->p_; +} + +std::size_t buffer::size() const noexcept { + return impl(p_)->s_; +} + +} // namespace ipc diff --git a/crazy_functions/test_project/cpp/cppipc/ipc.cpp b/crazy_functions/test_project/cpp/cppipc/ipc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4dc71c071c524906205cc4e2eae9ca8bac8b2d2c --- /dev/null +++ b/crazy_functions/test_project/cpp/cppipc/ipc.cpp @@ -0,0 +1,701 @@ + +#include +#include +#include +#include // std::pair, std::move, std::forward +#include +#include // aligned_storage_t +#include +#include +#include +#include + +#include "libipc/ipc.h" +#include "libipc/def.h" +#include "libipc/shm.h" +#include "libipc/pool_alloc.h" +#include "libipc/queue.h" +#include "libipc/policy.h" +#include "libipc/rw_lock.h" +#include "libipc/waiter.h" + +#include "libipc/utility/log.h" +#include "libipc/utility/id_pool.h" +#include "libipc/utility/scope_guard.h" +#include "libipc/utility/utility.h" + +#include "libipc/memory/resource.h" +#include "libipc/platform/detail.h" +#include "libipc/circ/elem_array.h" + +namespace { + +using msg_id_t = std::uint32_t; +using acc_t = std::atomic; + +template +struct msg_t; + +template +struct msg_t<0, AlignSize> { + msg_id_t cc_id_; + msg_id_t id_; + std::int32_t remain_; + bool storage_; +}; + +template +struct msg_t : msg_t<0, AlignSize> { + std::aligned_storage_t data_ {}; + + msg_t() = default; + msg_t(msg_id_t cc_id, msg_id_t id, std::int32_t remain, void const * data, std::size_t size) + : msg_t<0, AlignSize> {cc_id, id, remain, (data == nullptr) || (size == 0)} { + if (this->storage_) { + if (data != nullptr) { + // copy storage-id + *reinterpret_cast(&data_) = + *static_cast(data); + } + } + else std::memcpy(&data_, data, size); + } +}; + +template +ipc::buff_t make_cache(T& data, std::size_t size) { + auto ptr = ipc::mem::alloc(size); + std::memcpy(ptr, &data, (ipc::detail::min)(sizeof(data), size)); + return { ptr, size, ipc::mem::free }; +} + +struct cache_t { + std::size_t fill_; + ipc::buff_t buff_; + + cache_t(std::size_t f, ipc::buff_t && b) + : fill_(f), buff_(std::move(b)) + {} + + void append(void const * data, std::size_t size) { + if (fill_ >= buff_.size() || data == nullptr || size == 0) return; + auto new_fill = (ipc::detail::min)(fill_ + size, buff_.size()); + std::memcpy(static_cast(buff_.data()) + fill_, data, new_fill - fill_); + fill_ = new_fill; + } +}; + +auto cc_acc() { + static ipc::shm::handle acc_h("__CA_CONN__", sizeof(acc_t)); + return static_cast(acc_h.get()); +} + +IPC_CONSTEXPR_ std::size_t align_chunk_size(std::size_t size) noexcept { + return (((size - 1) / ipc::large_msg_align) + 1) * ipc::large_msg_align; +} + +IPC_CONSTEXPR_ std::size_t calc_chunk_size(std::size_t size) noexcept { + return ipc::make_align(alignof(std::max_align_t), align_chunk_size( + ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic)) + size)); +} + +struct chunk_t { + std::atomic &conns() noexcept { + return *reinterpret_cast *>(this); + } + + void *data() noexcept { + return reinterpret_cast(this) + + ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic)); + } +}; + +struct chunk_info_t { + ipc::id_pool<> pool_; + ipc::spin_lock lock_; + + IPC_CONSTEXPR_ static std::size_t chunks_mem_size(std::size_t chunk_size) noexcept { + return ipc::id_pool<>::max_count * chunk_size; + } + + ipc::byte_t *chunks_mem() noexcept { + return reinterpret_cast(this + 1); + } + + chunk_t *at(std::size_t chunk_size, ipc::storage_id_t id) noexcept { + if (id < 0) return nullptr; + return reinterpret_cast(chunks_mem() + (chunk_size * id)); + } +}; + +auto& chunk_storages() { + class chunk_handle_t { + ipc::shm::handle handle_; + + public: + chunk_info_t *get_info(std::size_t chunk_size) { + if (!handle_.valid() && + !handle_.acquire( ("__CHUNK_INFO__" + ipc::to_string(chunk_size)).c_str(), + sizeof(chunk_info_t) + chunk_info_t::chunks_mem_size(chunk_size) )) { + ipc::error("[chunk_storages] chunk_shm.id_info_.acquire failed: chunk_size = %zd\n", chunk_size); + return nullptr; + } + auto info = static_cast(handle_.get()); + if (info == nullptr) { + ipc::error("[chunk_storages] chunk_shm.id_info_.get failed: chunk_size = %zd\n", chunk_size); + return nullptr; + } + return info; + } + }; + static ipc::map chunk_hs; + return chunk_hs; +} + +chunk_info_t *chunk_storage_info(std::size_t chunk_size) { + auto &storages = chunk_storages(); + std::decay_t::iterator it; + { + static ipc::rw_lock lock; + IPC_UNUSED_ std::shared_lock guard {lock}; + if ((it = storages.find(chunk_size)) == storages.end()) { + using chunk_handle_t = std::decay_t::value_type::second_type; + guard.unlock(); + IPC_UNUSED_ std::lock_guard guard {lock}; + it = storages.emplace(chunk_size, chunk_handle_t{}).first; + } + } + return it->second.get_info(chunk_size); +} + +std::pair acquire_storage(std::size_t size, ipc::circ::cc_t conns) { + std::size_t chunk_size = calc_chunk_size(size); + auto info = chunk_storage_info(chunk_size); + if (info == nullptr) return {}; + + info->lock_.lock(); + info->pool_.prepare(); + // got an unique id + auto id = info->pool_.acquire(); + info->lock_.unlock(); + + auto chunk = info->at(chunk_size, id); + if (chunk == nullptr) return {}; + chunk->conns().store(conns, std::memory_order_relaxed); + return { id, chunk->data() }; +} + +void *find_storage(ipc::storage_id_t id, std::size_t size) { + if (id < 0) { + ipc::error("[find_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); + return nullptr; + } + std::size_t chunk_size = calc_chunk_size(size); + auto info = chunk_storage_info(chunk_size); + if (info == nullptr) return nullptr; + return info->at(chunk_size, id)->data(); +} + +void release_storage(ipc::storage_id_t id, std::size_t size) { + if (id < 0) { + ipc::error("[release_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); + return; + } + std::size_t chunk_size = calc_chunk_size(size); + auto info = chunk_storage_info(chunk_size); + if (info == nullptr) return; + info->lock_.lock(); + info->pool_.release(id); + info->lock_.unlock(); +} + +template +bool sub_rc(ipc::wr, + std::atomic &/*conns*/, ipc::circ::cc_t /*curr_conns*/, ipc::circ::cc_t /*conn_id*/) noexcept { + return true; +} + +template +bool sub_rc(ipc::wr, + std::atomic &conns, ipc::circ::cc_t curr_conns, ipc::circ::cc_t conn_id) noexcept { + auto last_conns = curr_conns & ~conn_id; + for (unsigned k = 0;;) { + auto chunk_conns = conns.load(std::memory_order_acquire); + if (conns.compare_exchange_weak(chunk_conns, chunk_conns & last_conns, std::memory_order_release)) { + return (chunk_conns & last_conns) == 0; + } + ipc::yield(k); + } +} + +template +void recycle_storage(ipc::storage_id_t id, std::size_t size, ipc::circ::cc_t curr_conns, ipc::circ::cc_t conn_id) { + if (id < 0) { + ipc::error("[recycle_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); + return; + } + std::size_t chunk_size = calc_chunk_size(size); + auto info = chunk_storage_info(chunk_size); + if (info == nullptr) return; + + auto chunk = info->at(chunk_size, id); + if (chunk == nullptr) return; + + if (!sub_rc(Flag{}, chunk->conns(), curr_conns, conn_id)) { + return; + } + info->lock_.lock(); + info->pool_.release(id); + info->lock_.unlock(); +} + +template +bool clear_message(void* p) { + auto msg = static_cast(p); + if (msg->storage_) { + std::int32_t r_size = static_cast(ipc::data_length) + msg->remain_; + if (r_size <= 0) { + ipc::error("[clear_message] invalid msg size: %d\n", (int)r_size); + return true; + } + release_storage( + *reinterpret_cast(&msg->data_), + static_cast(r_size)); + } + return true; +} + +struct conn_info_head { + + ipc::string name_; + msg_id_t cc_id_; // connection-info id + ipc::detail::waiter cc_waiter_, wt_waiter_, rd_waiter_; + ipc::shm::handle acc_h_; + + conn_info_head(char const * name) + : name_ {name} + , cc_id_ {(cc_acc() == nullptr) ? 0 : cc_acc()->fetch_add(1, std::memory_order_relaxed)} + , cc_waiter_{("__CC_CONN__" + name_).c_str()} + , wt_waiter_{("__WT_CONN__" + name_).c_str()} + , rd_waiter_{("__RD_CONN__" + name_).c_str()} + , acc_h_ {("__AC_CONN__" + name_).c_str(), sizeof(acc_t)} { + } + + void quit_waiting() { + cc_waiter_.quit_waiting(); + wt_waiter_.quit_waiting(); + rd_waiter_.quit_waiting(); + } + + auto acc() { + return static_cast(acc_h_.get()); + } + + auto& recv_cache() { + thread_local ipc::unordered_map tls; + return tls; + } +}; + +template +bool wait_for(W& waiter, F&& pred, std::uint64_t tm) { + if (tm == 0) return !pred(); + for (unsigned k = 0; pred();) { + bool ret = true; + ipc::sleep(k, [&k, &ret, &waiter, &pred, tm] { + ret = waiter.wait_if(std::forward(pred), tm); + k = 0; + }); + if (!ret) return false; // timeout or fail + if (k == 0) break; // k has been reset + } + return true; +} + +template +struct queue_generator { + + using queue_t = ipc::queue, Policy>; + + struct conn_info_t : conn_info_head { + queue_t que_; + + conn_info_t(char const * name) + : conn_info_head{name} + , que_{("__QU_CONN__" + + ipc::to_string(DataSize) + "__" + + ipc::to_string(AlignSize) + "__" + name).c_str()} { + } + + void disconnect_receiver() { + bool dis = que_.disconnect(); + this->quit_waiting(); + if (dis) { + this->recv_cache().clear(); + } + } + }; +}; + +template +struct detail_impl { + +using policy_t = Policy; +using flag_t = typename policy_t::flag_t; +using queue_t = typename queue_generator::queue_t; +using conn_info_t = typename queue_generator::conn_info_t; + +constexpr static conn_info_t* info_of(ipc::handle_t h) noexcept { + return static_cast(h); +} + +constexpr static queue_t* queue_of(ipc::handle_t h) noexcept { + return (info_of(h) == nullptr) ? nullptr : &(info_of(h)->que_); +} + +/* API implementations */ + +static void disconnect(ipc::handle_t h) { + auto que = queue_of(h); + if (que == nullptr) { + return; + } + que->shut_sending(); + assert(info_of(h) != nullptr); + info_of(h)->disconnect_receiver(); +} + +static bool reconnect(ipc::handle_t * ph, bool start_to_recv) { + assert(ph != nullptr); + assert(*ph != nullptr); + auto que = queue_of(*ph); + if (que == nullptr) { + return false; + } + if (start_to_recv) { + que->shut_sending(); + if (que->connect()) { // wouldn't connect twice + info_of(*ph)->cc_waiter_.broadcast(); + return true; + } + return false; + } + // start_to_recv == false + if (que->connected()) { + info_of(*ph)->disconnect_receiver(); + } + return que->ready_sending(); +} + +static bool connect(ipc::handle_t * ph, char const * name, bool start_to_recv) { + assert(ph != nullptr); + if (*ph == nullptr) { + *ph = ipc::mem::alloc(name); + } + return reconnect(ph, start_to_recv); +} + +static void destroy(ipc::handle_t h) { + disconnect(h); + ipc::mem::free(info_of(h)); +} + +static std::size_t recv_count(ipc::handle_t h) noexcept { + auto que = queue_of(h); + if (que == nullptr) { + return ipc::invalid_value; + } + return que->conn_count(); +} + +static bool wait_for_recv(ipc::handle_t h, std::size_t r_count, std::uint64_t tm) { + auto que = queue_of(h); + if (que == nullptr) { + return false; + } + return wait_for(info_of(h)->cc_waiter_, [que, r_count] { + return que->conn_count() < r_count; + }, tm); +} + +template +static bool send(F&& gen_push, ipc::handle_t h, void const * data, std::size_t size) { + if (data == nullptr || size == 0) { + ipc::error("fail: send(%p, %zd)\n", data, size); + return false; + } + auto que = queue_of(h); + if (que == nullptr) { + ipc::error("fail: send, queue_of(h) == nullptr\n"); + return false; + } + if (que->elems() == nullptr) { + ipc::error("fail: send, queue_of(h)->elems() == nullptr\n"); + return false; + } + if (!que->ready_sending()) { + ipc::error("fail: send, que->ready_sending() == false\n"); + return false; + } + ipc::circ::cc_t conns = que->elems()->connections(std::memory_order_relaxed); + if (conns == 0) { + ipc::error("fail: send, there is no receiver on this connection.\n"); + return false; + } + // calc a new message id + auto acc = info_of(h)->acc(); + if (acc == nullptr) { + ipc::error("fail: send, info_of(h)->acc() == nullptr\n"); + return false; + } + auto msg_id = acc->fetch_add(1, std::memory_order_relaxed); + auto try_push = std::forward(gen_push)(info_of(h), que, msg_id); + if (size > ipc::large_msg_limit) { + auto dat = acquire_storage(size, conns); + void * buf = dat.second; + if (buf != nullptr) { + std::memcpy(buf, data, size); + return try_push(static_cast(size) - + static_cast(ipc::data_length), &(dat.first), 0); + } + // try using message fragment + //ipc::log("fail: shm::handle for big message. msg_id: %zd, size: %zd\n", msg_id, size); + } + // push message fragment + std::int32_t offset = 0; + for (std::int32_t i = 0; i < static_cast(size / ipc::data_length); ++i, offset += ipc::data_length) { + if (!try_push(static_cast(size) - offset - static_cast(ipc::data_length), + static_cast(data) + offset, ipc::data_length)) { + return false; + } + } + // if remain > 0, this is the last message fragment + std::int32_t remain = static_cast(size) - offset; + if (remain > 0) { + if (!try_push(remain - static_cast(ipc::data_length), + static_cast(data) + offset, + static_cast(remain))) { + return false; + } + } + return true; +} + +static bool send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { + return send([tm](auto info, auto que, auto msg_id) { + return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) { + if (!wait_for(info->wt_waiter_, [&] { + return !que->push( + [](void*) { return true; }, + info->cc_id_, msg_id, remain, data, size); + }, tm)) { + ipc::log("force_push: msg_id = %zd, remain = %d, size = %zd\n", msg_id, remain, size); + if (!que->force_push( + clear_message, + info->cc_id_, msg_id, remain, data, size)) { + return false; + } + } + info->rd_waiter_.broadcast(); + return true; + }; + }, h, data, size); +} + +static bool try_send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { + return send([tm](auto info, auto que, auto msg_id) { + return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) { + if (!wait_for(info->wt_waiter_, [&] { + return !que->push( + [](void*) { return true; }, + info->cc_id_, msg_id, remain, data, size); + }, tm)) { + return false; + } + info->rd_waiter_.broadcast(); + return true; + }; + }, h, data, size); +} + +static ipc::buff_t recv(ipc::handle_t h, std::uint64_t tm) { + auto que = queue_of(h); + if (que == nullptr) { + ipc::error("fail: recv, queue_of(h) == nullptr\n"); + return {}; + } + if (!que->connected()) { + // hasn't connected yet, just return. + return {}; + } + auto& rc = info_of(h)->recv_cache(); + for (;;) { + // pop a new message + typename queue_t::value_t msg; + if (!wait_for(info_of(h)->rd_waiter_, [que, &msg] { + return !que->pop(msg); + }, tm)) { + // pop failed, just return. + return {}; + } + info_of(h)->wt_waiter_.broadcast(); + if ((info_of(h)->acc() != nullptr) && (msg.cc_id_ == info_of(h)->cc_id_)) { + continue; // ignore message to self + } + // msg.remain_ may minus & abs(msg.remain_) < data_length + std::int32_t r_size = static_cast(ipc::data_length) + msg.remain_; + if (r_size <= 0) { + ipc::error("fail: recv, r_size = %d\n", (int)r_size); + return {}; + } + std::size_t msg_size = static_cast(r_size); + // large message + if (msg.storage_) { + ipc::storage_id_t buf_id = *reinterpret_cast(&msg.data_); + void* buf = find_storage(buf_id, msg_size); + if (buf != nullptr) { + struct recycle_t { + ipc::storage_id_t storage_id; + ipc::circ::cc_t curr_conns; + ipc::circ::cc_t conn_id; + } *r_info = ipc::mem::alloc(recycle_t{ + buf_id, que->elems()->connections(std::memory_order_relaxed), que->connected_id() + }); + if (r_info == nullptr) { + ipc::log("fail: ipc::mem::alloc.\n"); + return ipc::buff_t{buf, msg_size}; // no recycle + } else { + return ipc::buff_t{buf, msg_size, [](void* p_info, std::size_t size) { + auto r_info = static_cast(p_info); + IPC_UNUSED_ auto finally = ipc::guard([r_info] { + ipc::mem::free(r_info); + }); + recycle_storage(r_info->storage_id, size, r_info->curr_conns, r_info->conn_id); + }, r_info}; + } + } else { + ipc::log("fail: shm::handle for large message. msg_id: %zd, buf_id: %zd, size: %zd\n", msg.id_, buf_id, msg_size); + continue; + } + } + // find cache with msg.id_ + auto cac_it = rc.find(msg.id_); + if (cac_it == rc.end()) { + if (msg_size <= ipc::data_length) { + return make_cache(msg.data_, msg_size); + } + // gc + if (rc.size() > 1024) { + std::vector need_del; + for (auto const & pair : rc) { + auto cmp = std::minmax(msg.id_, pair.first); + if (cmp.second - cmp.first > 8192) { + need_del.push_back(pair.first); + } + } + for (auto id : need_del) rc.erase(id); + } + // cache the first message fragment + rc.emplace(msg.id_, cache_t { ipc::data_length, make_cache(msg.data_, msg_size) }); + } + // has cached before this message + else { + auto& cac = cac_it->second; + // this is the last message fragment + if (msg.remain_ <= 0) { + cac.append(&(msg.data_), msg_size); + // finish this message, erase it from cache + auto buff = std::move(cac.buff_); + rc.erase(cac_it); + return buff; + } + // there are remain datas after this message + cac.append(&(msg.data_), ipc::data_length); + } + } +} + +static ipc::buff_t try_recv(ipc::handle_t h) { + return recv(h, 0); +} + +}; // detail_impl + +template +using policy_t = ipc::policy::choose; + +} // internal-linkage + +namespace ipc { + +template +ipc::handle_t chan_impl::inited() { + ipc::detail::waiter::init(); + return nullptr; +} + +template +bool chan_impl::connect(ipc::handle_t * ph, char const * name, unsigned mode) { + return detail_impl>::connect(ph, name, mode & receiver); +} + +template +bool chan_impl::reconnect(ipc::handle_t * ph, unsigned mode) { + return detail_impl>::reconnect(ph, mode & receiver); +} + +template +void chan_impl::disconnect(ipc::handle_t h) { + detail_impl>::disconnect(h); +} + +template +void chan_impl::destroy(ipc::handle_t h) { + detail_impl>::destroy(h); +} + +template +char const * chan_impl::name(ipc::handle_t h) { + auto info = detail_impl>::info_of(h); + return (info == nullptr) ? nullptr : info->name_.c_str(); +} + +template +std::size_t chan_impl::recv_count(ipc::handle_t h) { + return detail_impl>::recv_count(h); +} + +template +bool chan_impl::wait_for_recv(ipc::handle_t h, std::size_t r_count, std::uint64_t tm) { + return detail_impl>::wait_for_recv(h, r_count, tm); +} + +template +bool chan_impl::send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { + return detail_impl>::send(h, data, size, tm); +} + +template +buff_t chan_impl::recv(ipc::handle_t h, std::uint64_t tm) { + return detail_impl>::recv(h, tm); +} + +template +bool chan_impl::try_send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { + return detail_impl>::try_send(h, data, size, tm); +} + +template +buff_t chan_impl::try_recv(ipc::handle_t h) { + return detail_impl>::try_recv(h); +} + +template struct chan_impl>; +// template struct chan_impl>; // TBD +// template struct chan_impl>; // TBD +template struct chan_impl>; +template struct chan_impl>; + +} // namespace ipc diff --git a/crazy_functions/test_project/cpp/cppipc/policy.h b/crazy_functions/test_project/cpp/cppipc/policy.h new file mode 100644 index 0000000000000000000000000000000000000000..89596079e2cbb3ffa4ce68264a9b67a4c0f363b5 --- /dev/null +++ b/crazy_functions/test_project/cpp/cppipc/policy.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include "libipc/def.h" +#include "libipc/prod_cons.h" + +#include "libipc/circ/elem_array.h" + +namespace ipc { +namespace policy { + +template