import gradio as gr import pandas as pd import requests as r import time from random import * import datasets import os hf_token = os.getenv("hf_token") wl_win_buk = [] wl_lose_buk = [] eval_count = 0 wl = datasets.load_dataset("kopaser/model_win_lose") wl_win = list(map(str,wl.data["train"]["win"])) wl_lose = list(map(str,wl.data["train"]["lose"])) load = datasets.load_dataset("kopaser/elo_rating") elo_list = {"model": list(map(str,load.data["train"]["model"])), "elo": load.data["train"]["elo"].to_pylist()} #elo_list = {"model": [], "elo": []} elo_upd_list = {"model" : [], "elo": [], "times_rated": []} elo_upd_list2 = {"model" : [], "elo": [], "times_rated": []} last = [] def eval_ask(inp,temp,lenght): if len(inp) == 0: return 0 mod_list = list(map(lambda x: x["models"][0],r.get("https://stablehorde.net/api/v2/workers?type=text").json())) js1 = {"prompt": inp, "params": {"n": 1, "max_context_length": 1024, "max_length": lenght, "rep_pen": 1.1, "temperature": temp, "top_p": 0.92, "top_k": 100, "top_a": 0, "typical": 1, "tfs": 1, "rep_pen_range": 256, "rep_pen_slope": 0.7, "sampler_order": [6, 0, 1, 3, 4, 2, 5], "memory": "", "genkey": "KCPP8320", "min_p": 0, "dynatemp_range": 0, "dynatemp_exponent": 1, "smoothing_factor": 0, "presence_penalty": 0, "stop_sequence": ["### Instruction:", "### Response:"], "use_default_badwordsids": True} ,"models": [mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)]]} js2 = {"prompt": inp, "params": {"n": 1, "max_context_length": 1024, "max_length": lenght, "rep_pen": 1.1, "temperature": temp, "top_p": 0.92, "top_k": 100, "top_a": 0, "typical": 1, "tfs": 1, "rep_pen_range": 256, "rep_pen_slope": 0.7, "sampler_order": [6, 0, 1, 3, 4, 2, 5], "memory": "", "genkey": "KCPP8320", "min_p": 0, "dynatemp_range": 0, "dynatemp_exponent": 1, "smoothing_factor": 0, "presence_penalty": 0, "stop_sequence": ["### Instruction:", "### Response:"], "use_default_badwordsids": True} ,"models": [mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)],mod_list[randint(0,len(mod_list)-1)]]} #CHANGE MODELS TO FATCH ALL NOT ONLY 0 head = {"apikey": "0000000000"} a = r.post("https://stablehorde.net/api/v2/generate/text/async", json = js1, headers = head) b = r.post("https://stablehorde.net/api/v2/generate/text/async", json = js2, headers = head) print("posted") tn = time.time() time.sleep(1) while True: aa = r.get("https://stablehorde.net/api/v2/generate/text/status/"+a.json()["id"]).json() bb = r.get("https://stablehorde.net/api/v2/generate/text/status/"+b.json()["id"]).json() #print(aa['wait_time'], bb['wait_time']) if aa["done"] == True and bb["done"] == True: print(time.time() - tn) print(aa['generations'][0]["model"]) print(inp+aa['generations'][0]["text"]) print("") print(bb['generations'][0]["model"]) print(inp+bb['generations'][0]["text"]) break time.sleep(1) return [inp+aa['generations'][0]["text"],inp+bb['generations'][0]["text"],aa,bb] def evalm(rate): global eval_count global elo_list global last global wl_win_buk global wl_lose_buk global wl_win global wl_lose global textb2 Q = 28 if len(last) > 2 or len(last) == 0: return 0 eval_count += 1 choos = rate == 1 aa_name = last[0]["generations"][0]["worker_name"]+"/"+last[0]["generations"][0]["model"] bb_name = last[1]["generations"][0]["worker_name"]+"/"+last[1]["generations"][0]["model"] if choos: wl_win_buk.append(aa_name) wl_lose_buk.append(bb_name) else: wl_win_buk.append(bb_name) wl_lose_buk.append(aa_name) if eval_count % 3 == 1: wl = datasets.load_dataset("kopaser/model_win_lose") wl_win = list(map(str,wl.data["train"]["win"])) wl_lose = list(map(str,wl.data["train"]["lose"])) wl_win.extend(wl_win_buk) wl_lose.extend(wl_lose_buk) wl_win_buk = [] wl_lose_buk = [] tp = ["general"] * len(wl_lose) wl_dataset = datasets.Dataset.from_dict({"win": wl_win, "lose": wl_lose, "type": tp}) wl_dataset.push_to_hub('kopaser/model_win_lose', token=hf_token) elo_list = {"model": [], "elo": []} for i in range(len(wl_win)): aa_name = wl_win[i] bb_name = wl_lose[i] if aa_name not in elo_list["model"]: elo_list["model"].append(aa_name) elo_list["elo"].append(800) if bb_name not in elo_list["model"]: elo_list["model"].append(bb_name) elo_list["elo"].append(800) aind = elo_list["model"].index(aa_name) bind = elo_list["model"].index(bb_name) diff = 1 / (1 + 10 ** ((elo_list["elo"][aind]/elo_list["elo"][bind]))/400) elo_list["elo"][aind] += round(Q * (1 - (1 - diff))) elo_list["elo"][bind] += round(Q * (0 - (diff))) dataset = datasets.Dataset.from_dict({"model": elo_list["model"], "elo": elo_list["elo"]}) dataset.push_to_hub('kopaser/elo_rating', token=hf_token) last.append(0) def b_fn(text,temp,lenght): global elo_list global last df = pd.DataFrame(elo_list) out = eval_ask(text,temp,lenght) last = [out[2],out[3]] return [out[0],out[1]] #table = gr.DataFrame(pd.DataFrame(elo_list)) def inp1(): evalm(1) def inp2(): evalm(2) greeting = """### Instruction: Which weighs more, a kilogram of feathers or a pound of steel? ### Response: """ textb1 = gr.Textbox(label = "Model 1") textb2 = gr.Textbox(label = "Model 2") inff1 = gr.Interface( fn=b_fn, inputs=[gr.Textbox(value = greeting, lines=3),gr.Slider(0, 2, value=0.3, label="Temperature"), gr.Slider(20, 2048, value=50, label="Amount to Generate")], outputs=[textb1,textb2], title="This is horde text model elo battle." ) def upt(): return pd.DataFrame(elo_list) def set_label(): return pd.DataFrame({"model1": [last[0]["generations"][0]["worker_name"]+"/"+last[0]["generations"][0]["model"]], "model2": [last[1]["generations"][0]["worker_name"]+"/"+last[1]["generations"][0]["model"]]}) def upt_r(): load = datasets.load_dataset("kopaser/elo_rating") elo_list = {"model": list(map(str,load.data["train"]["model"])), "elo": load.data["train"]["elo"].to_pylist()} #FIX THIS TOO mod_list = list(map(lambda x: x["name"]+"/"+x["models"][0],r.get("https://stablehorde.net/api/v2/workers?type=text").json())) elo_upd_list = {"model" : [], "elo": [], "times_rated": []} print("WORK") for i in mod_list: #print() if i in elo_list["model"]: elo_upd_list["model"].append(i) elo_upd_list["elo"].append(elo_list["elo"][elo_list["model"].index(i)]) elo_upd_list["times_rated"].append(wl_win.count(i) + wl_lose.count(i)) #print(i, elo_list["elo"][elo_list["model"].index(i)]) return pd.DataFrame(elo_upd_list) with gr.Blocks() as inff2: b3 = gr.Button() table2 = gr.DataFrame(pd.DataFrame(elo_upd_list)) b3.click(upt_r,None, table2) title="Elo for online models, with workers." with inff1: with gr.Row(): b1 = gr.Button(value = "1 win") b2 = gr.Button(value = "2 win") model_names = gr.DataFrame(pd.DataFrame({"model1": [""] ,"model2": [""] })) b1.click(set_label, None, model_names) b2.click(set_label, None, model_names) b1.click(fn = inp1) b2.click(fn = inp2) table = gr.DataFrame(pd.DataFrame(elo_list)) b1.click(upt, None, table) b2.click(upt, None, table) def upt_r_nowork(): Q = 28 wl = datasets.load_dataset("kopaser/model_win_lose") #wl_win = list(map(lambda x: str(x).split("/")[-2] + "/" + str(x).split("/")[-1],wl.data["train"]["win"])) #wl_lose = list(map(lambda x: str(x).split("/")[-2] + "/" + str(x).split("/")[-1],wl.data["train"]["lose"])) wl_win = list(map(lambda x: str(x).split("/")[-1],wl.data["train"]["win"])) wl_lose = list(map(lambda x: str(x).split("/")[-1],wl.data["train"]["lose"])) elo_list_tr = {"model": [], "elo": []} for i in range(len(wl_win)): aa_name = wl_win[i] bb_name = wl_lose[i] if aa_name not in elo_list_tr["model"]: elo_list_tr["model"].append(aa_name) elo_list_tr["elo"].append(800) if bb_name not in elo_list_tr["model"]: elo_list_tr["model"].append(bb_name) elo_list_tr["elo"].append(800) aind = elo_list_tr["model"].index(aa_name) bind = elo_list_tr["model"].index(bb_name) diff = 1 / (1 + 10 ** ((elo_list_tr["elo"][aind]/elo_list_tr["elo"][bind]))/400) elo_list_tr["elo"][aind] += round(Q * (1 - (1 - diff))) elo_list_tr["elo"][bind] += round(Q * (0 - (diff))) #mod_list = set(list(map(lambda x: x["models"][0],r.get("https://stablehorde.net/api/v2/workers?type=text").json()))) mod_list = set(list(map(lambda x: x["models"][0].split("/")[-1],r.get("https://stablehorde.net/api/v2/workers?type=text").json()))) #print(mod_list) #print(elo_list_tr) elo_upd_list2 = {"model" : [], "elo": [], "times_rated": []} print("WORK") for i in mod_list: #print() if i in elo_list_tr["model"]: elo_upd_list2["model"].append(i) elo_upd_list2["elo"].append(elo_list_tr["elo"][elo_list_tr["model"].index(i)]) elo_upd_list2["times_rated"].append(wl_win.count(i) + wl_lose.count(i)) return pd.DataFrame(elo_upd_list2) with gr.Blocks() as inff3: b4 = gr.Button() table3 = gr.DataFrame(pd.DataFrame(elo_upd_list2)) b4.click(upt_r_nowork, None, table3) title="Elo for online models" demo = gr.TabbedInterface([inff1, inff2, inff3], ["Rate page", "online worker/model elo", "online model elo"]) # Run the interface demo.queue().launch(share = False)