Spaces:
Runtime error
Runtime error
| import torch | |
| from peft import PeftModel | |
| import transformers | |
| import gradio as gr | |
| from scrape_website import process_webpages | |
| import os | |
| print("Setting the PYTORCH_CUDA_ALLOC_CONF env variable.") | |
| os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:512" | |
| assert ( | |
| "LlamaTokenizer" in transformers._import_structure["models.llama"] | |
| ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git" | |
| from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig | |
| tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf") | |
| BASE_MODEL = "decapoda-research/llama-7b-hf" | |
| LORA_WEIGHTS = "tloen/alpaca-lora-7b" | |
| if torch.cuda.is_available(): | |
| device = "cuda" | |
| else: | |
| device = "cpu" | |
| try: | |
| if torch.backends.mps.is_available(): | |
| device = "mps" | |
| except: | |
| pass | |
| if device == "cuda": | |
| model = LlamaForCausalLM.from_pretrained( | |
| BASE_MODEL, | |
| load_in_8bit=False, | |
| torch_dtype=torch.float16, | |
| device_map="auto", | |
| ) | |
| model = PeftModel.from_pretrained( | |
| model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True | |
| ) | |
| elif device == "mps": | |
| model = LlamaForCausalLM.from_pretrained( | |
| BASE_MODEL, | |
| device_map={"": device}, | |
| torch_dtype=torch.float16, | |
| ) | |
| model = PeftModel.from_pretrained( | |
| model, | |
| LORA_WEIGHTS, | |
| device_map={"": device}, | |
| torch_dtype=torch.float16, | |
| ) | |
| else: | |
| model = LlamaForCausalLM.from_pretrained( | |
| BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True | |
| ) | |
| model = PeftModel.from_pretrained( | |
| model, | |
| LORA_WEIGHTS, | |
| device_map={"": device}, | |
| ) | |
| def generate_prompt(instruction, input=None): | |
| if input: | |
| return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. | |
| ### Instruction: | |
| {instruction} | |
| ### Input: | |
| {input} | |
| ### Response:""" | |
| else: | |
| return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request. | |
| ### Instruction: | |
| {instruction} | |
| ### Response:""" | |
| if device != "cpu": | |
| print("Using model.half() since device != cpu") | |
| model.half() | |
| model.eval() | |
| if torch.__version__ >= "2": | |
| model = torch.compile(model) | |
| def evaluate( | |
| instruction, | |
| urls_string, | |
| temperature=0.1, | |
| top_p=0.75, | |
| top_k=40, | |
| num_beams=4, | |
| max_new_tokens=128, | |
| **kwargs, | |
| ): | |
| print("Setting the PYTORCH_CUDA_ALLOC_CONF env variable in evaluate.") | |
| os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:512" | |
| content = process_webpages(urls=urls_string.split()) | |
| # avoid GPU memory overflow | |
| with torch.no_grad(): | |
| torch.cuda.empty_cache() | |
| print(f"memory_allocated() {torch.cuda.memory_allocated()}") | |
| print(f"max_memory_allocated() {torch.cuda.max_memory_allocated()}") | |
| print(f"memory_reserved() {torch.cuda.memory_reserved()}") | |
| print(f"max_memory_reserved() {torch.cuda.max_memory_reserved()}") | |
| prompt = generate_prompt(instruction, content) | |
| inputs = tokenizer(prompt, return_tensors="pt") | |
| input_ids = inputs["input_ids"].to(device) | |
| generation_config = GenerationConfig( | |
| temperature=temperature, | |
| top_p=top_p, | |
| top_k=top_k, | |
| num_beams=num_beams, | |
| **kwargs, | |
| ) | |
| generation_output = model.generate( | |
| input_ids=input_ids, | |
| generation_config=generation_config, | |
| return_dict_in_generate=True, | |
| output_scores=True, | |
| max_new_tokens=max_new_tokens, | |
| ) | |
| s = generation_output.sequences[0] | |
| output = tokenizer.decode(s) | |
| # avoid GPU memory overflow | |
| torch.cuda.empty_cache() | |
| return output.split("### Response:")[1].strip() | |
| g = gr.Interface( | |
| fn=evaluate, | |
| inputs=[ | |
| gr.components.Textbox( | |
| lines=2, label="FAQ", placeholder="Ask me anything about this website?" | |
| ), | |
| gr.components.Textbox( | |
| lines=2, | |
| label="Website URLs", | |
| placeholder="https://www.example.org/ https://www.example.com/", | |
| ), | |
| gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"), | |
| # gr.components.Slider(minimum=0, maximum=1, value=0.75, label="Top p"), | |
| # gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k"), | |
| # gr.components.Slider(minimum=1, maximum=4, step=1, value=4, label="Beams"), | |
| # gr.components.Slider( | |
| # minimum=1, maximum=512, step=1, value=128, label="Max tokens" | |
| # ), | |
| ], | |
| outputs=[ | |
| gr.inputs.Textbox( | |
| lines=5, | |
| label="Output", | |
| ) | |
| ], | |
| title="FAQ A Website", | |
| examples=[ | |
| [ | |
| "Which actions can we take to reduce climate change?", | |
| "https://www.imperial.ac.uk/stories/climate-action/", | |
| ], | |
| [ | |
| "Which actions can we take to reduce climate change?", | |
| "https://support.worldwildlife.org/site/SPageNavigator/ActionsToFightClimateChange.html", | |
| ], | |
| ] | |
| # description="Alpaca-LoRA is a 7B-parameter LLaMA model finetuned to follow instructions. It is trained on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset and makes use of the Huggingface LLaMA implementation. For more information, please visit [the project's website](https://github.com/tloen/alpaca-lora).", | |
| ) | |
| g.queue(concurrency_count=1) | |
| g.launch() | |