Duplicate from mikeee/WizardLM-13B-V1.0-Uncensored-GGML

.flake8-

@@ -0,0 +1,21 @@
+[flake8]
+ignore =
+  # E203 whitespace before ':'
+  E203
+  D203
+  # line too long
+  E501
+per-file-ignores =
+  # imported but unused
+  # __init__.py: F401
+  test_*.py: F401
+exclude =
+  .git
+  __pycache__
+  docs/source/conf.py
+  old
+  build
+  dist
+  .venv
+  pad*.py app-.py
+max-complexity = 25

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+call-activate.bat
+okteto.yml
+okteto-up.bat
+install-sw.sh
+install-sw1.sh
+start-sshd.sh
+pyproject.toml
+models
+.ruff_cache
+run-nodemon.sh
+app-.py

.ruff.toml

@@ -0,0 +1,17 @@
+# Assume Python 3.10.
+target-version = "py310"
+# Decrease the maximum line length to 79 characters.
+line-length = 300
+
+# pyflakes, pycodestyle, isort
+# flake8 YTT, pydocstyle D, pylint PLC
+select = ["F", "E", "W", "I001", "YTT", "D", "PLC"]
+# select = ["ALL"]
+
+# D103 Missing docstring in public function
+# D101 Missing docstring in public class
+# `multi-line-summary-first-line` (D212)
+# `one-blank-line-before-class` (D203)
+extend-ignore = ["D103", "D101", "D212", "D203"]
+
+exclude = [".venv"]

.stignore

@@ -0,0 +1,102 @@
+models
+.git
+# Byte-compiled / optimized / DLL files
+__pycache__
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build
+develop-eggs
+dist
+downloads
+eggs
+.eggs
+lib
+lib64
+parts
+sdist
+var
+wheels
+pip-wheel-metadata
+share/python-wheels
+*.egg-info
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build
+
+# PyBuilder
+target
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env
+venv
+ENV
+env.bak
+venv.bak
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre

README.md

@@ -0,0 +1,13 @@
+---
+title: TheBloke/WizardLM-13B-V1.0-Uncensored-GGML
+emoji: 🚀
+colorFrom: green
+colorTo: green
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+duplicated_from: mikeee/WizardLM-13B-V1.0-Uncensored-GGML
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,485 @@
+"""Run codes."""
+# pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
+# ruff: noqa: E501
+import os
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from types import SimpleNamespace
+
+import gradio as gr
+from about_time import about_time
+
+# from ctransformers import AutoConfig, AutoModelForCausalLM
+from ctransformers import AutoModelForCausalLM
+from huggingface_hub import hf_hub_download
+from loguru import logger
+
+os.environ["TZ"] = "Asia/Shanghai"
+try:
+    time.tzset()  # type: ignore # pylint: disable=no-member
+except Exception:
+    # Windows
+    logger.warning("Windows, cant run time.tzset()")
+
+ns = SimpleNamespace(
+    response="",
+    generator=[],
+)
+
+default_system_prompt = "A conversation between a user and an LLM-based AI assistant named Local Assistant. Local Assistant gives helpful and honest answers."
+
+user_prefix = "[user]: "
+assistant_prefix = "[assistant]: "
+
+
+def predict_str(prompt, bot):  # bot is in fact bot_history
+    # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
+
+    if bot is None:
+        bot = []
+
+    logger.debug(f"{prompt=}, {bot=}")
+
+    try:
+        # user_prompt = prompt
+        generator = generate(
+            LLM,
+            GENERATION_CONFIG,
+            system_prompt=default_system_prompt,
+            user_prompt=prompt.strip(),
+        )
+
+        ns.generator = generator  # for .then
+
+    except Exception as exc:
+        logger.error(exc)
+
+    # bot.append([prompt, f"{response} {_}"])
+    # return prompt, bot
+
+    return prompt, bot + [[prompt, None]]
+
+
+def bot_str(bot):
+    if bot:
+        bot[-1][1] = ""
+    else:
+        bot = [["Something is wrong", ""]]
+
+    print(assistant_prefix, end=" ", flush=True)
+
+    response = ""
+
+    flag = 1
+    then = time.time()
+    for word in ns.generator:
+        # record first response time
+        if flag:
+            logger.debug(f"\t {time.time() - then:.1f}s")
+            flag = 0
+        print(word, end="", flush=True)
+        # print(word, flush=True)  # vertical stream
+        response += word
+        bot[-1][1] = response
+        yield bot
+
+
+def predict(prompt, bot):
+    # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
+    logger.debug(f"{prompt=}, {bot=}")
+
+    ns.response = ""
+    then = time.time()
+    with about_time() as atime:  # type: ignore
+        try:
+            # user_prompt = prompt
+            generator = generate(
+                LLM,
+                GENERATION_CONFIG,
+                system_prompt=default_system_prompt,
+                user_prompt=prompt.strip(),
+            )
+
+            ns.generator = generator  # for .then
+
+            print(assistant_prefix, end=" ", flush=True)
+
+            response = ""
+            buff.update(value="diggin...")
+
+            flag = 1
+            for word in generator:
+                # record first response time
+                if flag:
+                    logger.debug(f"\t {time.time() - then:.1f}s")
+                    flag = 0
+                # print(word, end="", flush=True)
+                print(word, flush=True)  # vertical stream
+                response += word
+                ns.response = response
+                buff.update(value=response)
+            print("")
+            logger.debug(f"{response=}")
+        except Exception as exc:
+            logger.error(exc)
+            response = f"{exc=}"
+
+    # bot = {"inputs": [response]}
+    _ = (
+        f"(time elapsed: {atime.duration_human}, "  # type: ignore
+        f"{atime.duration/(len(prompt) + len(response)):.1f}s/char)"  # type: ignore
+    )
+
+    bot.append([prompt, f"{response} {_}"])
+
+    return prompt, bot
+
+
+def predict_api(prompt):
+    logger.debug(f"{prompt=}")
+    ns.response = ""
+    try:
+        # user_prompt = prompt
+        _ = GenerationConfig(
+            temperature=0.2,
+            top_k=0,
+            top_p=0.9,
+            repetition_penalty=1.0,
+            max_new_tokens=512,  # adjust as needed
+            seed=42,
+            reset=False,  # reset history (cache)
+            stream=True,  # TODO stream=False and generator
+            threads=os.cpu_count() // 2,  # type: ignore  # adjust for your CPU
+            stop=["<|im_end|>", "|<"],
+        )
+
+        # TODO: stream does not make sense in api?
+        generator = generate(
+            LLM, _, system_prompt=default_system_prompt, user_prompt=prompt.strip()
+        )
+        print(assistant_prefix, end=" ", flush=True)
+
+        response = ""
+        buff.update(value="diggin...")
+        for word in generator:
+            print(word, end="", flush=True)
+            response += word
+            ns.response = response
+            buff.update(value=response)
+        print("")
+        logger.debug(f"{response=}")
+    except Exception as exc:
+        logger.error(exc)
+        response = f"{exc=}"
+    # bot = {"inputs": [response]}
+    # bot = [(prompt, response)]
+
+    return response
+
+
+def download_quant(destination_folder: str, repo_id: str, model_filename: str):
+    local_path = os.path.abspath(destination_folder)
+    return hf_hub_download(
+        repo_id=repo_id,
+        filename=model_filename,
+        local_dir=local_path,
+        local_dir_use_symlinks=True,
+    )
+
+
+@dataclass
+class GenerationConfig:
+    temperature: float
+    top_k: int
+    top_p: float
+    repetition_penalty: float
+    max_new_tokens: int
+    seed: int
+    reset: bool
+    stream: bool
+    threads: int
+    stop: list[str]
+
+
+def format_prompt(system_prompt: str, user_prompt: str):
+    """Format prompt based on: https://huggingface.co/spaces/mosaicml/mpt-30b-chat/blob/main/app.py."""
+    # TODO im_start/im_end possible fix for WizardCoder
+
+    system_prompt = f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+    user_prompt = f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
+    assistant_prompt = "<|im_start|>assistant\n"
+
+    return f"{system_prompt}{user_prompt}{assistant_prompt}"
+
+
+def generate(
+    llm: AutoModelForCausalLM,
+    generation_config: GenerationConfig,
+    system_prompt: str = default_system_prompt,
+    user_prompt: str = "",
+):
+    """Run model inference, will return a Generator if streaming is true."""
+    # if not user_prompt.strip():
+    return llm(
+        format_prompt(
+            system_prompt,
+            user_prompt,
+        ),
+        **asdict(generation_config),
+    )
+
+
+_ = """full url: https://huggingface.co/TheBloke/mpt-30B-chat-GGML/blob/main/mpt-30b-chat.ggmlv0.q4_1.bin"""
+
+# https://huggingface.co/TheBloke/mpt-30B-chat-GGML
+_ = """
+mpt-30b-chat.ggmlv0.q4_0.bin 	q4_0 	4 	16.85 GB 	19.35 GB 	4-bit.
+mpt-30b-chat.ggmlv0.q4_1.bin 	q4_1 	4 	18.73 GB 	21.23 GB 	4-bit. Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.
+mpt-30b-chat.ggmlv0.q5_0.bin 	q5_0 	5 	20.60 GB 	23.10 GB
+mpt-30b-chat.ggmlv0.q5_1.bin 	q5_1 	5 	22.47 GB 	24.97 GB
+mpt-30b-chat.ggmlv0.q8_0.bin 	q8_0 	8 	31.83 GB 	34.33 GB
+"""
+MODEL_FILENAME = "mpt-30b-chat.ggmlv0.q4_1.bin"
+MODEL_FILENAME = "WizardCoder-15B-1.0.ggmlv3.q4_0.bin"  # 10.7G
+MODEL_FILENAME = "WizardCoder-15B-1.0.ggmlv3.q4_1.bin"  # 11.9G
+MODEL_FILENAME = "WizardCoder-15B-1.0.ggmlv3.q4_1.bin"  # 11.9G
+
+# https://huggingface.co/TheBloke/WizardLM-13B-V1.0-Uncensored-GGML
+MODEL_FILENAME = "wizardlm-13b-v1.0-uncensored.ggmlv3.q4_1.bin"  # 8.4G
+
+DESTINATION_FOLDER = "models"
+
+REPO_ID = "TheBloke/mpt-30B-chat-GGML"
+if "WizardCoder" in MODEL_FILENAME:
+    REPO_ID = "TheBloke/WizardCoder-15B-1.0-GGML"
+
+if "uncensored" in MODEL_FILENAME.lower():
+    REPO_ID = "TheBloke/WizardLM-13B-V1.0-Uncensored-GGML"
+
+logger.info(f"start dl, {REPO_ID=}, {MODEL_FILENAME=}, {DESTINATION_FOLDER=}")
+download_quant(DESTINATION_FOLDER, REPO_ID, MODEL_FILENAME)
+logger.info("done dl")
+
+# if "mpt" in model_filename:
+#     config = AutoConfig.from_pretrained("mosaicml/mpt-30b-cha t", context_length=8192)
+#     llm = AutoModelForCausalLM.from_pretrained(
+#         os.path.abspath(f"models/{model_filename}"),
+#         model_type="mpt",
+#         config=config,
+#     )
+
+# https://huggingface.co/spaces/matthoffner/wizardcoder-ggml/blob/main/main.py
+_ = """
+llm = AutoModelForCausalLM.from_pretrained(
+    "TheBloke/WizardCoder-15B-1.0-GGML",
+    model_file="WizardCoder-15B-1.0.ggmlv3.q4_0.bin",
+    model_type="starcoder",
+    threads=8
+)
+# """
+
+logger.debug(f"{os.cpu_count()=}")
+logger.info("load llm")
+
+_ = Path("models", MODEL_FILENAME).absolute().as_posix()
+logger.debug(f"model_file: {_}, exists: {Path(_).exists()}")
+LLM = AutoModelForCausalLM.from_pretrained(
+    # "TheBloke/WizardCoder-15B-1.0-GGML",
+    REPO_ID,  # DESTINATION_FOLDER,  # model_path_or_repo_id: str required
+    model_file=_,
+    model_type="llama",   # "starcoder",  AutoConfig.from_pretrained("TheBloke/WizardLM-13B-V1.0-Uncensored-GGML")
+    threads=os.cpu_count() // 2,  # type: ignore
+)
+
+logger.info("done load llm")
+
+cpu_count = os.cpu_count() // 2  # type: ignore
+logger.debug(f"{cpu_count=}")
+
+GENERATION_CONFIG = GenerationConfig(
+    temperature=0.2,
+    top_k=0,
+    top_p=0.9,
+    repetition_penalty=1.0,
+    max_new_tokens=512,  # adjust as needed
+    seed=42,
+    reset=False,  # reset history (cache)
+    stream=True,  # streaming per word/token
+    threads=cpu_count,
+    stop=["<|im_end|>", "|<"],  # TODO possible fix of stop
+)
+
+css = """
+    .importantButton {
+        background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
+        border: none !important;
+    }
+    .importantButton:hover {
+        background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
+        border: none !important;
+    }
+    .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
+    .xsmall {font-size: x-small;}
+"""
+etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
+examples = [
+    ["How to pick a lock? Provide detailed steps."],
+    ["Explain the plot of Cinderella in a sentence."],
+    [
+        "How long does it take to become proficient in French, and what are the best methods for retaining information?"
+    ],
+    ["What are some common mistakes to avoid when writing code?"],
+    ["Build a prompt to generate a beautiful portrait of a horse"],
+    ["Suggest four metaphors to describe the benefits of AI"],
+    ["Write a pop song about leaving home for the sandy beaches."],
+    ["Write a summary demonstrating my ability to tame lions"],
+    ["鲁迅和周树人什么关系 说中文"],
+    ["鲁迅和周树人什么关系"],
+    ["鲁迅和周树人什么关系 用英文回答"],
+    ["从前有一头牛，这头牛后面有什么？"],
+    ["正无穷大加一大于正无穷大吗？"],
+    ["正无穷大加正无穷大大于正无穷大吗？"],
+    ["-2的平方根等于什么"],
+    ["树上有5只鸟，猎人开枪打死了一只。树上还有几只鸟？"],
+    ["树上有11只鸟，猎人开枪打死了一只。树上还有几只鸟？提示：需考虑鸟可能受惊吓飞走。"],
+    ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
+    [f"{etext} 翻成中文，列出3个版本"],
+    [f"{etext} \n 翻成中文，保留原意，但使用文学性的语言。不要写解释。列出3个版本"],
+    ["假定 1 + 2 = 4, 试求 7 + 8"],
+    ["判断一个数是不是质数的 javascript 码"],
+    ["实现python 里 range(10)的 javascript 码"],
+    ["实现python 里 [*(range(10)]的 javascript 码"],
+    ["Erkläre die Handlung von Cinderella in einem Satz."],
+    ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
+]
+
+with gr.Blocks(
+    # title="mpt-30b-chat-ggml",
+    title=f"{MODEL_FILENAME}",
+    theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
+    css=css,
+) as block:
+    with gr.Accordion("🎈 Info", open=False):
+        # gr.HTML(
+        #     """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
+        # )
+        gr.Markdown(
+            f"""<h4><center>{MODEL_FILENAME}</center></h4>
+            It takes about 100 seconds for the initial reply
+            message to appear. Average streaming rate ~1 sec/chat. The bot only speaks English.
+
+            Most examples are meant for another model. You probably should try to test
+            some related prompts.
+
+            Try to refresh the browser and try again when occasionally errors occur.
+
+            It takes about >100 seconds to get a response. Restarting the space takes about 2 minutes if the space is asleep due to inactivity. If the space crashes for some reason, it will also take about 2 minutes to restart. You need to refresh the browser to reload the new space.
+            """,
+            elem_classes="xsmall",
+        )
+
+    # chatbot = gr.Chatbot().style(height=700)  # 500
+    chatbot = gr.Chatbot(height=700)  # 500
+    buff = gr.Textbox(show_label=False, visible=False)
+    with gr.Row():
+        with gr.Column(scale=5):
+            msg = gr.Textbox(
+                label="Chat Message Box",
+                placeholder="Ask me anything (press Enter or click Submit to send)",
+                show_label=False,
+            ).style(container=False)
+        with gr.Column(scale=1, min_width=80):
+            with gr.Row():
+                submit = gr.Button("Submit", elem_classes="xsmall")
+                stop = gr.Button("Stop", visible=False)
+                clear = gr.Button("Clear History", visible=True)
+    with gr.Row(visible=False):
+        with gr.Accordion("Advanced Options:", open=False):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    system = gr.Textbox(
+                        label="System Prompt",
+                        value=default_system_prompt,
+                        show_label=False,
+                    ).style(container=False)
+                with gr.Column():
+                    with gr.Row():
+                        change = gr.Button("Change System Prompt")
+                        reset = gr.Button("Reset System Prompt")
+
+    with gr.Accordion("Example Inputs", open=True):
+        examples = gr.Examples(
+            examples=examples,
+            inputs=[msg],
+            examples_per_page=40,
+        )
+
+    # with gr.Row():
+    with gr.Accordion("Disclaimer", open=False):
+        _ = "-".join(MODEL_FILENAME.split("-")[:2])
+        gr.Markdown(
+            f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
+            "factually accurate information. {_} was trained on various public datasets; while great efforts "
+            "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
+            "biased, or otherwise offensive outputs.",
+            elem_classes=["disclaimer"],
+        )
+    _ = """
+    msg.submit(
+        # fn=conversation.user_turn,
+        fn=predict,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        # queue=True,
+        show_progress="full",
+        api_name="predict",
+    )
+    submit.click(
+        fn=lambda x, y: ("",) + predict(x, y)[1:],  # clear msg
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=True,
+        show_progress="full",
+    )
+    # """
+    msg.submit(
+        # fn=conversation.user_turn,
+        fn=predict_str,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=True,
+        show_progress="full",
+        api_name="predict",
+    ).then(bot_str, chatbot, chatbot)
+    submit.click(
+        fn=lambda x, y: ("",) + predict_str(x, y)[1:],  # clear msg
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=True,
+        show_progress="full",
+    ).then(bot_str, chatbot, chatbot)
+
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+    # update buff Textbox, every: units in seconds)
+    # https://huggingface.co/spaces/julien-c/nvidia-smi/discussions
+    # does not work
+    # AttributeError: 'Blocks' object has no attribute 'run_forever'
+    # block.run_forever(lambda: ns.response, None, [buff], every=1)
+
+    with gr.Accordion("For Chat/Translation API", open=False, visible=False):
+        input_text = gr.Text()
+        api_btn = gr.Button("Go", variant="primary")
+        out_text = gr.Text()
+    api_btn.click(
+        predict_api,
+        input_text,
+        out_text,
+        # show_progress="full",
+        api_name="api",
+    )
+
+# concurrency_count=5, max_size=20
+# max_size=36, concurrency_count=14
+block.queue(concurrency_count=5, max_size=20).launch(debug=True)

requirements.txt

@@ -0,0 +1,6 @@
+ctransformers==0.2.10
+transformers==4.30.2
+huggingface_hub
+gradio
+loguru
+about-time