Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| from transformers import AutoTokenizer, GemmaForCausalLM | |
| import torch | |
| model = GemmaForCausalLM.from_pretrained("google/gemma-2b-it", device_map="auto", torch_dtype=torch.bfloat16) | |
| tokenizer = AutoTokenizer.from_pretrained("google/gemma-2b-it") | |
| # from transformers import AutoTokenizer, GemmaForCausalLM, BitsAndBytesConfig | |
| # quantization_config = BitsAndBytesConfig(load_in_4bit=True) | |
| # tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b") | |
| # model = GemmaForCausalLM.from_pretrained("google/gemma-7b", quantization_config=quantization_config) | |
| #prompt = "What is your favorite condiment?" | |
| def generate(prompt): | |
| input_ids = tokenizer(prompt, return_tensors="pt") | |
| # Generate | |
| #generate_ids = model.generate(inputs.input_ids, max_length=100) | |
| #return tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] | |
| outputs = model.generate(**input_ids) | |
| return(tokenizer.decode(outputs[0])) | |
| demo = gr.Interface( | |
| fn=generate, | |
| inputs=gr.Textbox(lines=5, label="Input Text"), | |
| outputs=gr.Textbox(label="Generated Text") | |
| ) | |
| demo.launch(share=True) | |
| # # pip install bitsandbytes accelerate | |
| # from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig | |
| # quantization_config = BitsAndBytesConfig(load_in_4bit=True) | |
| # tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b") | |
| # model = AutoModelForCausalLM.from_pretrained("google/gemma-7b", quantization_config=quantization_config) | |
| # input_text = "Write me a poem about Machine Learning." | |
| # input_ids = tokenizer(input_text, return_tensors="pt").to("cuda") | |
| # outputs = model.generate(**input_ids) | |
| # print(tokenizer.decode(outputs[0])) | |