File size: 8,249 Bytes
d512d2d 58629f0 cc521be 58629f0 f4a098d c7e44c9 e94b477 c7e44c9 0704646 c7e44c9 67f7621 e94b477 67f7621 e94b477 5ebe6d2 67f7621 cc521be 67f7621 c7e44c9 e94b477 9977893 e94b477 9977893 e94b477 9977893 e94b477 9977893 e94b477 c7e44c9 9977893 e94b477 c7e44c9 9977893 cc521be 9977893 e94b477 cc521be e94b477 ed19e77 f4a098d ed19e77 cc521be ee1c835 cc521be ee1c835 9977893 cc521be 9977893 c7e44c9 e94b477 c7e44c9 cc521be c7e44c9 cc521be c7e44c9 cc521be c7e44c9 cc521be c7e44c9 e94b477 d512d2d ed19e77 e94b477 c7e44c9 cc521be c7e44c9 e94b477 c7e44c9 e94b477 cc521be c7e44c9 cc521be ed19e77 cc521be ed19e77 cc521be ed19e77 cc521be ed19e77 d6fb3c3 cc521be c7e44c9 123ce74 cc521be 123ce74 cc521be e94b477 cc521be 67f7621 c7e44c9 5513bbd cc521be 5513bbd 9977893 f4a098d cc521be 9977893 ed19e77 cc521be ed19e77 123ce74 cc521be 123ce74 5513bbd 67f7621 cc521be 67f7621 cc521be 67f7621 e3173bf cc521be e3173bf cc521be bd0b4a3 c4d6746 bd0b4a3 ed19e77 bd0b4a3 cc521be e94b477 ffa92d9 cc521be 5513bbd c7e44c9 cc521be 5513bbd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 |
#!/usr/bin/env python3
# Copyright 2023 Dmitry Ustalov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__author__ = "Dmitry Ustalov"
__license__ = "Apache 2.0"
from typing import BinaryIO, cast
import evalica
import gradio as gr
import networkx as nx
import numpy as np
import pandas as pd
import plotly.express as px
from evalica import Winner
from plotly.graph_objects import Figure
TOLERANCE, LIMIT = 1e-6, 100
def visualize(df_pairwise: pd.DataFrame) -> Figure:
fig = px.imshow(df_pairwise, color_continuous_scale="RdBu", text_auto=".2f")
fig.update_layout(xaxis_title="Loser", yaxis_title="Winner", xaxis_side="top")
fig.update_traces(hovertemplate="Winner: %{y}<br>Loser: %{x}<br>Fraction of Wins: %{z}<extra></extra>")
return fig
def counting(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.counting(xs, ys, ws)
return result.scores, result.index
def bradley_terry(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.bradley_terry(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
return result.scores, result.index
def elo(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.elo(xs, ys, ws)
return result.scores, result.index
def eigen(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.eigen(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
return result.scores, result.index
def pagerank(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.pagerank(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
return result.scores, result.index
def newman(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
result = evalica.newman(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
return result.scores, result.index
ALGORITHMS = {
"Counting": counting,
"Bradley-Terry (1952)": bradley_terry,
"Elo (1960)": elo,
"Eigenvector (1987)": eigen,
"PageRank (1998)": pagerank,
"Newman (2023)": newman,
}
def largest_strongly_connected_component(df_pairs: pd.DataFrame) -> set[str]:
G = nx.from_pandas_edgelist(df_pairs, source="left", target="right", create_using=nx.DiGraph)
H = nx.from_pandas_edgelist(df_pairs[df_pairs["winner"] == "tie"], source="right", target="left",
create_using=nx.DiGraph)
F = nx.compose(G, H)
largest = max(nx.strongly_connected_components(F), key=len)
return cast(set[str], largest)
def handler(
file: BinaryIO,
algorithm: str,
filtered: bool,
truncated: bool,
) -> tuple[pd.DataFrame, Figure]:
if file is None:
raise gr.Error("File must be uploaded")
if algorithm not in ALGORITHMS:
raise gr.Error(f"Unknown algorithm: {algorithm}")
try:
df_pairs = pd.read_csv(file.name, dtype=str)
except ValueError as e:
raise gr.Error(f"Parsing error: {e}") from e
if not pd.Series(["left", "right", "winner"]).isin(df_pairs.columns).all():
raise gr.Error("Columns must exist: left, right, winner")
if not df_pairs["winner"].isin(pd.Series(["left", "right", "tie"])).all():
raise gr.Error("Allowed winner values: left, right, tie")
df_pairs = df_pairs[["left", "right", "winner"]]
df_pairs.dropna(axis=0, inplace=True)
if filtered:
largest = largest_strongly_connected_component(df_pairs)
df_pairs.drop(df_pairs[~(df_pairs["left"].isin(largest) & df_pairs["right"].isin(largest))].index, inplace=True)
xs, ys = df_pairs["left"], df_pairs["right"]
ws = df_pairs["winner"].map({"left": Winner.X, "right": Winner.Y, "tie": Winner.Draw})
scores, index = ALGORITHMS[algorithm](xs, ys, ws)
index.name = "item"
df_result = pd.DataFrame(data={"score": scores}, index=index)
df_result["pairs"] = pd.Series(0, dtype=int, index=index).add(
df_pairs.groupby("left")["left"].count(), fill_value=0,
).add(
df_pairs.groupby("right")["right"].count(), fill_value=0,
).astype(int)
df_result["rank"] = df_result["score"].rank(na_option="bottom", ascending=False).astype(int)
df_result.fillna(-np.inf, inplace=True)
df_result.sort_values(by=["rank", "score"], ascending=[True, False], inplace=True)
df_result.reset_index(inplace=True)
if truncated:
df_result = pd.concat((df_result.head(5), df_result.tail(5)), copy=False)
df_result = df_result[~df_result.index.duplicated(keep="last")]
pairwise = evalica.pairwise_scores(df_result["score"].to_numpy())
df_pairwise = pd.DataFrame(data=pairwise, index=df_result["item"], columns=df_result["item"])
fig = visualize(df_pairwise)
return df_result, fig
def main() -> None:
iface = gr.Interface(
fn=handler,
inputs=[
gr.File(
file_types=[".tsv", ".csv"],
label="Comparisons",
),
gr.Dropdown(
choices=cast(list[str], ALGORITHMS),
value="Bradley-Terry (1952)",
label="Algorithm",
),
gr.Checkbox(
value=False,
label="Largest SCC",
info="Bradley-Terry, Eigenvector, and Newman algorithms require the comparison graph "
"to be strongly-connected. "
"This option keeps only the largest strongly-connected component (SCC) of the input graph. "
"Some items might be missing as a result of this filtering.",
),
gr.Checkbox(
value=False,
label="Truncate Output",
info="Perform the entire computation but output only five head and five tail items, "
"avoiding overlap.",
),
],
outputs=[
gr.Dataframe(
headers=["item", "score", "pairs", "rank"],
label="Ranking",
),
gr.Plot(
label="Pairwise Chances of Winning the Comparison",
),
],
examples=[
["food.csv", "Counting", False, False],
["food.csv", "Bradley-Terry (1952)", False, False],
["food.csv", "Eigenvector (1987)", False, False],
["food.csv", "PageRank (1998)", False, False],
["food.csv", "Newman (2023)", False, False],
["llmfao.csv", "Bradley-Terry (1952)", False, True],
["llmfao.csv", "Elo (1960)", False, True],
],
title="Pair2Rank: Turn Your Side-by-Side Comparisons into Ranking!",
description="""
This easy-to-use tool transforms pairwise comparisons (aka side-by-side) to a meaningful ranking of items.
As an input, it expects a comma-separated (CSV) file with a header containing the following columns:
- `left`: the first compared item
- `right`: the second compared item
- `winner`: the label indicating the winning item
Possible values for `winner` are `left`, `right`, or `tie`. The provided examples might be a good starting point.
As the output, this tool provides a table with items, their estimated scores, and ranks.
""".strip(),
article="""
Pair2Rank uses the [Evalica](https://pypi.org/p/evalica) library for computing the scores: <https://github.com/dustalov/evalica>.
Read more about Pair2Rank at <https://evalovernite.substack.com/p/llmfao-human-ranking>.
""".strip(),
allow_flagging="never",
)
iface.launch()
if __name__ == "__main__":
main()
|