Spaces:
Runtime error
Runtime error
File size: 2,501 Bytes
228fd17 e9bcaca 228fd17 e9bcaca 228fd17 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
#@title 2. Launch the web UI
#@markdown If unsure about the branch, write "main" or leave it blank.
import torch
from pathlib import Path
if Path.cwd().name != 'text-generation-webui':
print("Installing the webui...")
# !git clone https://github.com/oobabooga/text-generation-webui
# %cd text-generation-webui
torver = torch.__version__
print(f"TORCH: {torver}")
is_cuda118 = '+cu118' in torver # 2.1.0+cu118
is_cuda117 = '+cu117' in torver # 2.0.1+cu117
textgen_requirements = open('text-generation-webui\requirements.txt').read().splitlines()
if is_cuda117:
textgen_requirements = [req.replace('+cu121', '+cu117').replace('+cu122', '+cu117').replace('torch2.1', 'torch2.0') for req in textgen_requirements]
elif is_cuda118:
textgen_requirements = [req.replace('+cu121', '+cu118').replace('+cu122', '+cu118') for req in textgen_requirements]
with open('temp_requirements.txt', 'w') as file:
file.write('\n'.join(textgen_requirements))
!pip install -r extensions/openai/requirements.txt --upgrade
!pip install -r temp_requirements.txt --upgrade
print("\033[1;32;1m\n --> If you see a warning about \"previously imported packages\", just ignore it.\033[0;37;0m")
print("\033[1;32;1m\n --> There is no need to restart the runtime.\n\033[0;37;0m")
try:
import flash_attn
except:
!pip uninstall -y flash_attn
# Parameters
model_url = "https://huggingface.co/Mr-Bhaskar/fbt-mistral-7b" #@param {type:"string"}
branch = "main" #@param {type:"string"}
command_line_flags = "--n-gpu-layers 128 --load-in-4bit --use_double_quant" #@param {type:"string"}
api = False #@param {type:"boolean"}
if api:
for param in ['--api', '--public-api']:
if param not in command_line_flags:
command_line_flags += f" {param}"
model_url = model_url.strip()
if model_url != "":
if not model_url.startswith('http'):
model_url = 'https://huggingface.co/' + model_url
# Download the model
url_parts = model_url.strip('/').strip().split('/')
output_folder = f"{url_parts[-2]}_{url_parts[-1]}"
branch = branch.strip('"\' ')
if branch.strip() not in ['', 'main']:
output_folder += f"_{branch}"
!python download-model.py {model_url} --branch {branch}
else:
!python download-model.py {model_url}
else:
output_folder = ""
# Start the web UI
cmd = f"python server.py --share"
if output_folder != "":
cmd += f" --model {output_folder}"
cmd += f" {command_line_flags}"
print(cmd)
!$cmd |