snphs

/

dolly-v2-12b-q4

like 13

this doesnt works.
u can compile cformers and check it by yourself.

this doesnt works.
u can compile cformers and check it by yourself.

I compiled cformers, but have no idea how to put the model there.

Hello, you need to add this in MAP_MODEL_TO_URL (https://github.com/NolanoOrg/cformers/blob/master/cformers/interface.py#L90):

    'databricks/dolly-v2-12b': ModelUrlMap(
        cpp_model_name="gptneox",
        int4_fixed_zero="https://huggingface.co/snphs/dolly-v2-12b-q4/resolve/main/int4_fixed_zero.bin"),

Then you can use it like the other models in the library.

from interface import AutoInference as AI
ai = AI("databricks/dolly-v2-12b")
prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n### Instruction:\nExplain to me the difference between nuclear fission and fusion.\n### Response:\n'
x = ai.generate(prompt, num_tokens_to_generate=100)
print(x['token_str'])

maybe this way, should be good to get started.

pip install cformers

then

import torch
import os
import wget
import sys
from cformers import AutoInference
from cformers.interface import MAP_MODEL_TO_URL, ModelUrlMap,compare_file_hash_sha256

import transformers as tf # RIP TensorFlow

if "CFORMERS_CACHE_PATH" in os.environ:
    CFORMERS_CACHE_PATH = os.environ["CFORMERS_CACHE_PATH"]
else:
    CFORMERS_CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cformers")

class AI(AutoInference):
    """A wrapper for the C++ model."""
    def __init__(self, model_name, hash_sum="", mode="int4_fixed_zero"):
        self.model_name = model_name
        self.mode = mode
        self.hash_sum = hash_sum
        self.cpp_model_name = "gptneox"
        self.model_url = "https://huggingface.co/snphs/dolly-v2-12b-q4/resolve/main/int4_fixed_zero.bin"
        self.model_save_path = os.path.join(CFORMERS_CACHE_PATH, "models", model_name, mode)
        self.tokenizer = tf.AutoTokenizer.from_pretrained(model_name)

        # Download the model if it doesn't exist
        if not os.path.exists(self.model_save_path):
            # Create the directory if it doesn't exist
            parent_dir = os.path.dirname(self.model_save_path)
            if not os.path.exists(parent_dir):
                os.makedirs(parent_dir)
            print("Downloading model...")
            def bar_progress(current, total, width=80):
                progress_message = "Downloading: %d%% [%d / %d] bytes" % (current / total * 100, current, total)
                sys.stdout.write("\r" + progress_message)
                sys.stdout.flush()
            wget.download(self.model_url, self.model_save_path, bar=bar_progress)

            print("Download complete!")
            compare_file_hash_sha256(self.model_save_path, self.model_url.replace("resolve", "blob"))

ai = AI("databricks/dolly-v2-12b")

prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n### Instruction:\nExplain to me the difference between nuclear fission and fusion.\n### Response:\n'
x = ai.generate(prompt, num_tokens_to_generate=100)
print(x['token_str'])

Deleted my message.
I find my error!
thank you snphs!