Spaces:
Runtime error
Runtime error
# gradio run.pyでHotReload開発 | |
import os | |
from dotenv import load_dotenv | |
import json | |
import datetime | |
from pathlib import Path | |
import uuid | |
from typing import Tuple | |
import pandas as pd | |
import gradio as gr | |
from datasets import load_dataset | |
from huggingface_hub import CommitScheduler | |
from huggingface_hub import HfFolder | |
# .envファイルから環境変数をロードする(ローカル開発環境) | |
load_dotenv() | |
# HF_Spaceでプライベート関連にアクセスするための環境変数 | |
# SecretKey をSpaceのSettingsに設定 | |
HF_TOKEN = os.getenv("HF_TOKEN") | |
if HF_TOKEN: | |
HfFolder.save_token(HF_TOKEN) | |
else: | |
print("Warning: HF_TOKEN not found. Please set it in your Space secrets.") | |
# HFデータセット アップロード先 | |
# (切り替えてテストする用に配列) | |
OUTPUT_DATASET = [ | |
"team-hatakeyama-phase2/annotation_tanuki_phase2", | |
"kevineen/Tanuki-Phase2-annotation-dataset", # 出力テスト | |
] | |
# アノテーション対象データセット | |
ANNOTATION_DATASET = [ | |
"hatakeyama-llm-team/AutoGeneratedJapaneseQA", | |
"hatakeyama-llm-team/AutoGeneratedJapaneseQA-other", | |
"kanhatakeyama/ChatbotArenaJaMixtral8x22b", | |
"kanhatakeyama/OrcaJaMixtral8x22b", | |
"kanhatakeyama/LogicalDatasetsByMixtral8x22b", | |
# データ形式未対応(対応予定 | |
# "hatakeyama-llm-team/WikiBookJa", | |
# "kanhatakeyama/AutoWikiQA", | |
# "susumuota/SyntheticTextWikiTranslate-askllm-v1", # Ask-LLM 翻訳 | |
# Nomotron データ (将来用 | |
] | |
MULTITURN_ANNOTATION_DATASET = [ | |
# マルチターン 未対応 | |
"kanhatakeyama/AutoMultiTurnByMixtral8x22b", | |
] | |
# Session State : (ブラウザセッション単位の変数管理) =========================== | |
# UIのEnable/Disable用State | |
is_selected_dataset = gr.State(False) | |
is_loaded_dataset = gr.State(False) | |
# 選択中のデータセットリスト | |
dropdown_dataset_list = gr.State(value=ANNOTATION_DATASET) | |
# 現在の対象データセット 初期値は"hatakeyama-llm-team/AutoGeneratedJapaneseQA", | |
select_dropdown_dataset = gr.State(dropdown_dataset_list.value[0]) | |
select_dataset = gr.State(None) # 現在のデータセット | |
select_dataset_total_len = gr.State(0) # 現在のデータセットの長さ | |
select_idx = gr.State(0) # 現在のインデックス (ランダムモードにするなら不要? | |
# random_mode = gr.State(False) # ランダムモード | |
# 回答者がアノテーションしたデータセット | |
annotated_dataset = gr.State( | |
pd.DataFrame({ | |
'dataset_name': [], # 対象データセット | |
'dataset_id': [], # データセットindex | |
'who': [], # アノテーション者名 | |
'unknown_quality': [], # 不明能 | |
'good': [], # 良 | |
'bad': [], # 悪 | |
'is_proofreading_1': [], # テキストに修正があったか | |
'answer_text_1': [], | |
'is_proofreading_2': [], # 2ターン目用 | |
'answer_text_2': [], # 2ターン目用 | |
# 5段階評価 | |
'score': [], # 総合 (5段階評価) | |
'helphulness': [], # 有用性 (5段階評価) | |
'correcthness': [], # 正確さ (5段階評価) | |
'coherence': [], # 一貫性 (5段階評価) | |
'complexity': [], # 複雑さ (5段階評価) | |
'verbosity': [], # 冗長性 (5段階評価) | |
'humor': [], # ユーモア (SteerLM) | |
'creativity': [], # 創造性 (SteerLM) | |
'appropriate': [], # 適切性 (SteerLMでは不適) | |
'following_instructions': [], # 指示が合った場合、従っているか (SteerLMのfails_taskに該当) | |
'politeness': [], # 礼儀正しさ (minnadechat) | |
'harmfulness': [], # 有性 (minnadechat) | |
# 0: 分からない(未設定) 1: 無 2: 有 | |
'typos': [], # 誤字脱字 (minnadechat) | |
'hate': [], # ヘイト | |
'sexual': [], # 性的 | |
'violence': [], # 暴力 | |
'suicide': [], # 自殺 | |
'threat': [], # 犯罪 | |
'gun': [], # 重火器 | |
'controlled_substance': [], # 規制対象物質 | |
'climinal_planing': [], # 犯罪計画 | |
'privacy': [], # 個人情報 | |
'harassment': [], # ハラスメント | |
'profanity': [], # 冒涜 | |
'political_content': [], # 政治的内容 (SteerLM) | |
'moral_judgement': [], # 非道徳 (SteerLM) | |
}) | |
) | |
is_dataset_loaded = gr.State(False) # データセットロード状態 | |
you_dataset_id = gr.State(0) # 回答者がアノテーションしているデータのID | |
dataset_name = gr.State("") # 編集に使用したデータセット名 | |
dataset_id = gr.State(0) # 加工元データセットのindex | |
who = gr.State("") # アノテーション者名 | |
# シンプルモード | |
unknown_quality = gr.State(False) # 分からない(未設定) | |
good = gr.State(False) # 良 | |
bad = gr.State(False) # 悪 | |
initial_answer_text_1 = gr.State("") # 回答1を整形したかチェック用 | |
initial_answer_text_2 = gr.State("") # 回答2を整形したかチェック用 | |
is_proofreading_1 = gr.State(False) # 回答1を整形したか_1 | |
answer_text_1 = gr.State("") # answer_1 回答 | |
is_proofreading_2 = gr.State(False) # 回答2を整形したか_2 | |
answer_text_2 = gr.State("") # answer_2 回答 | |
# (5段階評価) | |
# 0は判断していない、又は判断が難しいという特徴量にする | |
score = gr.State(3) # 総合スコア 初期値は3 (最低限必須) | |
helphulness = gr.State(0) # 有用性 | |
correcthness = gr.State(0) # 正確さ | |
coherenece = gr.State(0) # 一貫性 | |
complexity = gr.State(0) # 複雑さ | |
verbosity = gr.State(0) # 冗長性 | |
humor = gr.State(0) # ユーモア (SteerLM) | |
creativity = gr.State(0) # 創造性 (SteerLM) | |
appropriate = gr.State(0) # 適切性 (SteerLMではnot_appropriate) | |
following_instructions = gr.State(0) # 指示への忠実度 (SteerLMのfails_task) | |
politeness = gr.State(0) # 礼儀正しさ (minnadechat) | |
harmfulness = gr.State(0) # 有度 (minnadechat) | |
# (有無評価) | |
# 0は判断していない、又は判断が難しいという特徴量にする | |
hate = gr.State(0) # ヘイト | |
sexual = gr.State(0) # 性的 | |
violence = gr.State(0) # 暴力 | |
suicide = gr.State(0) # 自殺 | |
threat = gr.State(0) # 犯罪 | |
gun = gr.State(0) # 銃・重火器 | |
controlled_substance = gr.State(0) # 規制対象物質 | |
climinal_planing = gr.State(0) # 犯罪計画 | |
privacy = gr.State(0) # 個人情報・プライバシー情報 | |
harassment = gr.State(0) # ハラスメント | |
profanity = gr.State(0) # 冒涜 | |
political_content = gr.State(0) # 政治的内容 (SteerLM) | |
moral_judgement = gr.State(0) # 非道徳 (SteerLM) | |
typos = gr.State(0) # 誤字脱字(minnadechat) | |
# 未整理 ======================================== | |
# データセット読み込み | |
def dataset_load_fn() -> Tuple[ | |
str, | |
str, | |
str, | |
str, | |
str, | |
str, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
]: | |
is_dataset_loaded.value = False # ロード状態初期化 | |
select_dataset.value = load_dataset( | |
select_dropdown_dataset.value | |
) | |
# DatasetオブジェクトをPandas DataFrameに変換 | |
df = select_dataset.value["train"].to_pandas() | |
# ランダム化実施 | |
# index列を追加し、シャッフル | |
df = df.reset_index(drop=False) # 元のindexを保持 | |
df = df.sample(frac=1).reset_index(drop=True) # シャッフル | |
select_dataset.value["train"] = df # シャッフルされたDataFrameを格納 | |
select_idx.value = 0 # index初期化 | |
select_dataset_total_len.value = len(df) # 長さを取得 | |
is_dataset_loaded.value = True # ロード完了 | |
# データロード時にテキストの初期値を設定 | |
initial_answer_text_1.value = df.iloc[select_idx.value]["answer"] | |
initial_answer_text_2.value = df.iloc[select_idx.value]["answer"] | |
return df.iloc[select_idx.value]["question"], \ | |
df.iloc[select_idx.value]["answer"], \ | |
df.iloc[select_idx.value]["question"], \ | |
df.iloc[select_idx.value]["answer"], \ | |
df.iloc[select_idx.value]["question"], \ | |
df.iloc[select_idx.value]["answer"], \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True), \ | |
gr.update(interactive=True) | |
# データの保存処理 ======================================== | |
# Spaceの場合の保存先はCommitSchedulerのpath_in_repoフォルダ | |
# (ローカル開発の場合./user_annotationフォルダにjsonファイルが作成される) | |
annotation_file = Path("user_annotation/") / f"data_{uuid.uuid4()}.json" | |
annotated_folder = annotation_file.parent | |
scheduler = CommitScheduler( | |
repo_id=OUTPUT_DATASET[1], | |
repo_type="dataset", | |
folder_path=annotated_folder, | |
path_in_repo="data", # Spaceの場合の保存先フォルダー | |
private=True, | |
every=5, # 5分毎にアップロード HuggingFAce_Documentの最低推奨値 | |
) | |
# チェックボックスをintに変換 | |
def checkbox_to_int(checkbox_value) -> int: | |
if checkbox_value == "不明": | |
output = 0 | |
elif checkbox_value == "有": | |
output = 1 | |
elif checkbox_value == "無": | |
output = 2 | |
else : | |
output = 0 | |
print("error: ") | |
return output | |
# CommitScheduler (HFへのデータアップロード | |
def save_annotation( | |
dataset_name: str, | |
dataset_id: int, | |
who: str, | |
unknown_quality: bool, | |
good: bool, | |
bad: bool, | |
is_proofreading_1: bool, | |
answer_text_1: str, | |
is_proofreading_2: bool, | |
answer_text_2: str, | |
score: int, | |
helphulness: int, | |
correcthness: int, | |
coherence: int, | |
complexity: int, | |
verbosity: int, | |
humor: int, | |
creativity: int, | |
appropriate: int, | |
following_instructions: int, | |
politeness: int, | |
harmfulness: int, | |
hate: int, | |
sexual: int, | |
violence: int, | |
suicide: int, | |
threat: int, | |
gun: int, | |
controlled_substance: int, | |
climinal_planing: int, | |
privacy: int, | |
harassment: int, | |
profanity: int, | |
political_content: int, | |
moral_judgement: int, | |
typos: int, | |
) -> None: | |
annotated_dataset.value = pd.concat([ | |
annotated_dataset.value, | |
pd.DataFrame({ | |
'dataset_name': [dataset_name], | |
'dataset_id': [dataset_id], | |
'who': [who], | |
'unknown_quality': [unknown_quality], | |
'good': [good], | |
'bad': [bad], | |
'is_proofreading_1': [is_proofreading_1], | |
"answer_text_1": [answer_text_1], | |
'is_proofreading_2': [is_proofreading_2], | |
'answer_text_2': [answer_text_2], | |
'score': [score], | |
'helphullness': [helphulness], | |
'correcthness': [correcthness], | |
'complexity': [complexity], | |
'verbosity': [verbosity], | |
'humor': [humor], | |
'creativity': [creativity], | |
'appropriate': [appropriate], | |
'following_finstructions': [following_instructions], | |
'politeness': [politeness], | |
'harmfulness': [harmfulness], | |
'hate': [hate], | |
'sextual': [sexual], | |
'violence': [violence], | |
'suicide': [suicide], | |
'threat': [threat], | |
'gun': [gun], | |
'controlled_substance': [controlled_substance], | |
'climinal_planing': [climinal_planing], | |
'privacy': [privacy], | |
'harassment': [harassment], | |
'profanity': [profanity], | |
'political_content': [political_content], | |
'moral_judgement': [moral_judgement], | |
'types': [typos] | |
})], ignore_index=True).reset_index(drop=True) | |
# 書き込み | |
with scheduler.lock: | |
with annotation_file.open("a", encoding='utf-8') as f: | |
data_to_write = { | |
# "id": , CommitSchedulerは、取得して末尾idを付与することが不可能? | |
"datetime": str(datetime.datetime.now().isoformat()), | |
"dataset_name": dataset_name, | |
"dataset_id": int(dataset_id), | |
"who": who, | |
"unknown_quality": unknown_quality, | |
"good": good, | |
"bad": bad, | |
"is_proofreading_1": is_proofreading_1, | |
"answer_text_1": answer_text_1, | |
"is_proofreading_2": is_proofreading_2, | |
"answer_text_2": answer_text_2, | |
"score": int(score), | |
"helphulness": int(helphulness), | |
"correcthness": int(correcthness), | |
"coherence": int(coherence), | |
"complexity": int(complexity), | |
"verbosity": int(verbosity), | |
"humor": int(humor), | |
"creativity": int(creativity), | |
"appropriate": int(appropriate), | |
"following_instructions": int(following_instructions), | |
"politeness": int(politeness), | |
"harmfulness": int(harmfulness), | |
"hate": int(hate), | |
"sexual": int(sexual), | |
"violence": int(violence), | |
"suicide": int(suicide), | |
"threat": int(threat), | |
"gun": int(gun), | |
"controlled_substance": int(controlled_substance), | |
"climinal_planing": int(climinal_planing), | |
"privacy": int(privacy), | |
"harassment": int(harassment), | |
"profanity": int(profanity), | |
"policital_content": int(political_content), | |
"moral_judgement": int(moral_judgement), | |
"typos": int(typos) | |
} | |
f.write(json.dumps(data_to_write, ensure_ascii=False)) | |
f.write("\n") | |
# アノテーションの追加処理 ======================================== | |
# UI処理 ======================================== | |
# ユーザー名表示 | |
def hello(profile: gr.OAuthProfile | None) -> Tuple[str, str]: | |
if profile is None: | |
return "プライベートデータセット取得のためにログインしてください。", who.value | |
who.value = profile.username | |
return f'{profile.username} さん、よろしくお願いいたします。', who.value | |
# テーマの状態 | |
theme_ = gr.themes.Default() | |
# 後のCSSデザイン変更用 | |
def load_css(): | |
with open("style.css", "r") as file: | |
css_content = file.read() | |
return css_content | |
# Gradio 画面 ============================================ | |
with gr.Blocks(theme=theme_, css=load_css()) as demo: | |
gr.Markdown("# データセット アノテーション for Tanuki (Phase2)") | |
with gr.Row(): | |
gr.Markdown("### GENIACにて開発中のLLM用データセットに対してアノテーションするSpaceです\n \ | |
入力されたデータは使用される可能性があるため、個人情報・秘匿情報などは入力しないでください。\n \ | |
テスト中です。") | |
gr_who = gr.TextArea(value=who.value, lines=1, max_lines=1, label="ユーザー名 (入力してください)") | |
def change_name(name: str): | |
who.value = name | |
gr_who.change( | |
change_name, | |
inputs=[gr_who], | |
outputs=[] | |
) | |
with gr.Tab("アノテーション (シングルターン)"): | |
# HF login 機能 | |
# with gr.Row(equal_height=True): | |
# gr.LoginButton(value="HuggingFace ログイン", | |
# logout_value="HuggingFace ログアウト", scale=1) | |
# ユーザー名 | |
# gr_profile_name = gr.Markdown() | |
# demo.load(hello, inputs=None, outputs=[gr_profile_name, who]) | |
with gr.Row(): | |
def dropdown_select(select_value) -> None: | |
select_dropdown_dataset.value = select_value | |
# 対象データセットの選択 | |
gr_dropdown_dataset = gr.Dropdown( | |
label="データセット選択 ①", | |
choices=dropdown_dataset_list.value, | |
value=select_dropdown_dataset.value, | |
elem_id="dataset_sel", | |
scale=2) | |
gr_dropdown_dataset.change( | |
dropdown_select, | |
inputs=[gr_dropdown_dataset] | |
) | |
gr_data_load_btn = gr.Button("② データセットを読み込む") | |
with gr.Column() as content_column: | |
with gr.Tab("③ シンプル(良・悪)"): | |
with gr.Column(): | |
with gr.Row(equal_height=True): | |
gr_good_btn = gr.Button("良い", interactive=False) | |
gr_unknown_btn = gr.Button( | |
"分からない", interactive=False) # 「分からない」ボタンを追加 | |
gr_bad_btn = gr.Button("悪い", interactive=False) | |
gr_question_text_1_1 = gr.Textbox( | |
label="質問: ", lines=5, interactive=False) | |
gr_answer_text_1_1 = gr.Textbox( | |
label="回答: 訂正して頂けると、品質が上がります。", | |
lines=20, | |
interactive=False) | |
with gr.Tab("③ 5段階評価(シンプル)"): | |
with gr.Row() as simple_score_btn: | |
gr_score_5_btn = gr.Button("5: 高品質", interactive=False) | |
gr_score_4_btn = gr.Button("4: 良い", interactive=False) | |
gr_score_3_btn = gr.Button("3: 普通", interactive=False) | |
gr_score_2_btn = gr.Button("2: 悪い", interactive=False) | |
gr_score_1_btn = gr.Button("1: 低品質", interactive=False) | |
gr_question_text_2_1 = gr.Textbox( | |
label="質問: ", lines=5, interactive=False) | |
gr_answer_text_2_1 = gr.Textbox( | |
label="回答: 訂正して頂けると、品質が上がります。", lines=20, interactive=False) | |
with gr.Tab("③ 5段階評価 (詳細)"): | |
with gr.Row(): | |
with gr.Column() as EvalFive: | |
gr_question_text_3_1 = gr.Textbox( | |
label="質問: ", lines=10, interactive=False) | |
# gr_explain = gr.Markdown("回答") | |
gr_answer_text_3_1 = gr.Textbox( | |
label="回答: 訂正して頂けると、品質が上がります。", lines=35, interactive=False) | |
with gr.Column() as EvalFiveDetail: | |
with gr.Row(): | |
gr_submit_score = gr.Button("評価送信", interactive=False) | |
gr_score_reset = gr.Button("スコアリセット", interactive=False) | |
gr_score_detail = gr.Slider(label="総合スコア 【必須】",value=3, minimum=1, maximum=5, step=1, interactive=False) | |
gr_evalannotation_explain = gr.Markdown("詳細アノテーション (5点満点)") | |
with gr.Row() : | |
gr_helphulness = gr.Slider(label="有用性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_correcthness = gr.Slider(label="正確さ", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_coherence = gr.Slider(label="一貫性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_complexity = gr.Slider(label="複雑さ", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
with gr.Row() : | |
gr_verbosity = gr.Slider(label="冗長性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_humor = gr.Slider(label="ユーモア", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_creativity = gr.Slider(label="創造性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_appropriate = gr.Slider(label="適切性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
with gr.Row() : | |
gr_following_instructions = gr.Slider(label="忠実性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_politeness = gr.Slider(label="礼儀正しさ", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_harmfulness = gr.Slider(label="有害性", minimum=0, value=0, maximum=5, step=1, interactive=False) | |
gr_text_4 = gr.Markdown("判断可能な物のみ\n\nスコアを付けてください。") | |
gr_text_5 = gr.Markdown("") | |
with gr.Row(): | |
gr_typos = gr.Radio(label="誤字・脱字", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_moral_judgement = gr.Radio(label="非道徳", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_hate = gr.Radio(label="ヘイト", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_sexual = gr.Radio(label="性的内容", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_violence = gr.Radio(label="暴力的", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_suicide = gr.Radio(label="自殺行為", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_threat = gr.Radio(label="犯罪", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_gun = gr.Radio(label="銃等", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_controlled_substance = gr.Radio(label="規制対象物質", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_climinal_planing = gr.Radio(label="犯罪計画", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_privacy = gr.Radio(label="個人情報", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_harassment = gr.Radio(label="ハラスメント", choices=["不明", "有", "無"], value="不明", interactive=False) | |
with gr.Row(): | |
gr_profanity = gr.Radio(label="冒涜行為", choices=["不明", "有", "無"], value="不明", interactive=False) | |
gr_political_content = gr.Radio(label="政治的内容", choices=["不明", "有", "無"], value="不明", interactive=False) | |
# 5段階評価 詳細のgr.State Update | |
def eval_submit( | |
# text | |
g_answer_text_3_1: str, | |
# slicer | |
g_score: int, | |
g_helphulness: int, | |
g_correcthness: int, | |
g_coherenece: int, | |
g_complexity: int, | |
g_verbosity: int, | |
g_humor: int, | |
g_creativity: int, | |
g_appropriate: int, | |
g_following_instructions: int, | |
g_politeness: int, | |
g_harmfulness: int, | |
# checkbox | |
g_hate: str, | |
g_sexual: str, | |
g_violence: str, | |
g_suicide: str, | |
g_threat: str, | |
g_gun: str, | |
g_controlled_substance: str, | |
g_climinal_planing: str, | |
g_privacy: str, | |
g_harassment: str, | |
g_profanity: str, | |
g_political_content: str, | |
g_moral_judgement: str, | |
g_typos: str, | |
): | |
# 状態初期化 | |
good.value = False | |
bad.value = False | |
unknown_quality.value = False | |
# テキストに更新があったかどうかチェック | |
if initial_answer_text_1.value != g_answer_text_3_1: | |
is_proofreading_1.value = True | |
answer_text_1.value = g_answer_text_3_1 | |
else: | |
is_proofreading_1.value = False | |
answer_text_1.value = "" | |
# 送信用データ更新 | |
score.value = g_score | |
helphulness.value = g_helphulness | |
correcthness.value = g_correcthness | |
coherenece.value = g_coherenece | |
complexity.value = g_complexity | |
verbosity.value = g_verbosity | |
humor.value = g_humor | |
creativity.value = g_creativity | |
appropriate.value = g_appropriate | |
following_instructions.value = g_following_instructions | |
politeness.value = g_politeness | |
harmfulness.value = g_harmfulness | |
hate.value = checkbox_to_int(g_hate) | |
sexual.value = checkbox_to_int(g_sexual) | |
violence.value = checkbox_to_int(g_violence) | |
suicide.value = checkbox_to_int(g_suicide) | |
threat.value = checkbox_to_int(g_threat) | |
gun.value = checkbox_to_int(g_gun) | |
controlled_substance.value = checkbox_to_int(g_controlled_substance) | |
climinal_planing.value = checkbox_to_int(g_climinal_planing) | |
privacy.value = checkbox_to_int(g_privacy) | |
harassment.value = checkbox_to_int(g_harassment) | |
profanity.value = checkbox_to_int(g_profanity) | |
political_content.value = checkbox_to_int(g_political_content) | |
moral_judgement.value = checkbox_to_int(g_moral_judgement) | |
typos.value = checkbox_to_int(g_typos) | |
# データ送信 | |
# 表示更新 | |
# indexを進める | |
select_idx.value += 1 | |
df = select_dataset.value["train"] | |
# ループさせるか、エラー処理を行う | |
if select_idx.value >= len(df): | |
select_idx.value = 0 | |
# データセットに追加 | |
df = select_dataset.value["train"] | |
save_annotation( | |
select_dropdown_dataset.value, | |
df.iloc[select_idx.value]['index'], | |
who.value, | |
unknown_quality.value, | |
good.value, | |
bad.value, | |
is_proofreading_1.value, | |
answer_text_1.value, | |
is_proofreading_2.value, | |
answer_text_2.value, | |
score.value, | |
helphulness.value, | |
correcthness.value, | |
coherenece.value, | |
complexity.value, | |
verbosity.value, | |
humor.value, | |
creativity.value, | |
appropriate.value, | |
following_instructions.value, | |
politeness.value, | |
harmfulness.value, | |
hate.value, | |
sexual.value, | |
violence.value, | |
suicide.value, | |
threat.value, | |
gun.value, | |
controlled_substance.value, | |
climinal_planing.value, | |
privacy.value, | |
harassment.value, | |
profanity.value, | |
political_content.value, | |
moral_judgement.value, | |
typos.value, | |
) | |
# Nextデータ初期化 | |
initialize_next_data(df) | |
# 5段階評価をリセット | |
score_reset_display() | |
# UI更新のための返り値 | |
return gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
*score_reset_display() # スコアリセット用の返り値 | |
gr_submit_score.click( | |
eval_submit, | |
inputs=[ | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
], | |
# UI更新 | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
], | |
) | |
# 5段階評価ボタンのクリックイベントを定義 | |
def score_button_clicked(button_value): | |
# 状態初期化 | |
good.value = False | |
bad.value = False | |
unknown_quality.value = False | |
score.value = button_value | |
# 5段階評価クリック | |
def update_annotation( | |
input_ans_1: str = None, | |
input_ans_2: str = None, | |
is_good: bool = None, # good/bad を表すフラグを追加 | |
is_unknown: bool = None, # 「分からない」を表すフラグを追加 | |
is_simple: bool = None, | |
score_value: int = None # 5段階評価の値、good/badの場合はNone | |
) -> Tuple[ | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
gr.update, | |
]: | |
# good/bad/unknown と score の状態を更新 | |
update_evaluation_state(is_good, is_unknown, score_value) | |
# 変更を検知 (5段階評価の場合も処理するように変更) | |
update_answer_state(input_ans_1, input_ans_2) | |
# 表示更新 | |
# indexを進める | |
select_idx.value += 1 | |
df = select_dataset.value["train"] | |
# ループさせるか、エラー処理を行う | |
if select_idx.value >= len(df): | |
select_idx.value = 0 | |
# シンプルモードクリックで、scoreを0に設定 | |
if is_simple == True: | |
score.value = 0 | |
# データセットに追加 | |
# 元のindex番号(dataset_id)を指定して保存 | |
save_annotation_data(df) | |
# Nextデータ初期化 | |
initialize_next_data(df) | |
# 5段階評価をリセット | |
reset_values = score_reset_display() | |
return gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
gr.update(value=df.iloc[select_idx.value]["question"]), \ | |
gr.update(value=df.iloc[select_idx.value]["answer"]), \ | |
*reset_values | |
# 評価状態リセット | |
def update_evaluation_state(is_good, is_unknown, score_value): | |
if score_value is not None: # 5段階評価の場合 | |
good.value = False | |
bad.value = False | |
unknown_quality.value = False | |
score.value = score_value | |
else: # good/bad/unknown 評価の場合 | |
good.value = is_good | |
bad.value = not is_good if not is_unknown else False | |
unknown_quality.value = is_unknown # 「分からない」の状態を設定 | |
# 変更を検知して値を設定 | |
def update_answer_state(input_ans_1, input_ans_2): | |
if input_ans_1 is not None and initial_answer_text_1.value != input_ans_1: | |
is_proofreading_1.value = True | |
answer_text_1.value = input_ans_1 | |
else: | |
answer_text_1.value = "" | |
# 2ターン目用 | |
if input_ans_2 is not None and initial_answer_text_2.value != input_ans_2: | |
is_proofreading_2.value = True | |
answer_text_2.value = input_ans_2 | |
else: | |
answer_text_2.value = "" | |
# データ保存 | |
def save_annotation_data(df): | |
save_annotation( | |
select_dropdown_dataset.value, | |
# datasetIdは元のindex番号を使用 | |
df.iloc[select_idx.value]['index'], | |
who.value, | |
unknown_quality.value, | |
good.value, | |
bad.value, | |
is_proofreading_1.value, | |
answer_text_1.value, | |
is_proofreading_2.value, | |
answer_text_2.value, | |
score.value, | |
helphulness.value, | |
correcthness.value, | |
coherenece.value, | |
complexity.value, | |
verbosity.value, | |
humor.value, | |
creativity.value, | |
appropriate.value, | |
following_instructions.value, | |
politeness.value, | |
harmfulness.value, | |
hate.value, | |
sexual.value, | |
violence.value, | |
suicide.value, | |
threat.value, | |
gun.value, | |
controlled_substance.value, | |
climinal_planing.value, | |
privacy.value, | |
harassment.value, | |
profanity.value, | |
political_content.value, | |
moral_judgement.value, | |
typos.value, | |
) | |
# 次データ読込時の初期化 | |
def initialize_next_data(df): | |
is_proofreading_1.value = False | |
is_proofreading_2.value = False | |
initial_answer_text_1.value = df.iloc[select_idx.value]["answer"] | |
initial_answer_text_2.value = df.iloc[select_idx.value]["answer"] | |
# 評価不可 | |
def unknown_click(input_ans_1, input_ans_2): | |
good.value = False | |
bad.value = False | |
score_reset_display() | |
return update_annotation( | |
input_ans_1=input_ans_1, | |
input_ans_2=input_ans_2, | |
is_good=False, | |
is_unknown=True, | |
is_simple=True, | |
) | |
# 評価不可クリックイベント | |
gr_unknown_btn.click( | |
unknown_click, | |
inputs=[ | |
gr_answer_text_1_1, | |
gr_answer_text_2_1 | |
], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
] | |
) | |
# 良いクリック | |
def good_click(input_ans_1, input_ans_2): | |
score_reset_display() | |
return update_annotation( | |
input_ans_1=input_ans_1, | |
input_ans_2=input_ans_2, | |
is_good=True, | |
is_unknown=False, | |
is_simple=True, | |
) | |
# 良いクリック | |
gr_good_btn.click( | |
good_click, | |
inputs=[ | |
gr_answer_text_1_1, | |
gr_answer_text_2_1 | |
], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
] | |
) | |
# 低評価クリック | |
def bad_click(input_ans_1, input_ans_2): | |
score_reset_display() | |
return update_annotation( | |
input_ans_1=input_ans_1, | |
input_ans_2=input_ans_2, | |
is_good=False, | |
is_unknown=False, | |
is_simple=True, | |
) | |
# 低評価クリックイベント | |
gr_bad_btn.click( | |
bad_click, | |
inputs=[ | |
gr_answer_text_1_1, | |
gr_answer_text_2_1 | |
], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
] | |
) | |
# 5段階評価ボタンのクリックイベント | |
gr_score_1_btn.click(lambda x: update_annotation(input_ans_1=x, is_unknown=False, score_value=1), | |
inputs=[gr_answer_text_2_1], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
]) | |
gr_score_2_btn.click(lambda x: update_annotation(input_ans_1=x, is_unknown=False, score_value=2), | |
inputs=[gr_answer_text_2_1], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
]) | |
gr_score_3_btn.click(lambda x: update_annotation(input_ans_1=x, is_unknown=False, score_value=3), | |
inputs=[gr_answer_text_2_1], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
]) | |
gr_score_4_btn.click(lambda x: update_annotation(input_ans_1=x, is_unknown=False, score_value=4), | |
inputs=[gr_answer_text_2_1], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
]) | |
gr_score_5_btn.click(lambda x: update_annotation(input_ans_1=x, is_unknown=False, score_value=5), | |
inputs=[gr_answer_text_2_1], | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
]) | |
# Scoreリセット | |
def score_reset_display(): | |
# gr.State | |
score.value = 3 | |
helphulness.value = 0 | |
correcthness.value = 0 | |
coherenece.value = 0 | |
complexity.value = 0 | |
verbosity.value = 0 | |
humor.value = 0 | |
creativity.value = 0 | |
appropriate.value = 0 | |
following_instructions.value = 0 | |
politeness.value = 0 | |
harmfulness.value = 0 | |
hate.value = 0 | |
sexual.value = 0 | |
violence.value = 0 | |
suicide.value = 0 | |
threat.value = 0 | |
gun.value = 0 | |
controlled_substance.vlaue = 0 | |
climinal_planing.value = 0 | |
privacy.value = 0 | |
harassment.value = 0 | |
profanity.value = 0 | |
political_content.value = 0 | |
moral_judgement.value = 0 | |
typos.value = 0 | |
return 3,0,0,0,0,0,0,0,0,0,0,0, \ | |
"不明", "不明", "不明", "不明", "不明", \ | |
"不明", "不明", "不明", "不明", "不明", \ | |
"不明", "不明", "不明", "不明" | |
# Scoreリセット | |
gr_score_reset.click( | |
score_reset_display, | |
inputs=[], | |
outputs=[ | |
gr_score_detail, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
] | |
) | |
# データ読込 | |
gr_data_load_btn.click( | |
dataset_load_fn, | |
inputs=None, | |
# textをセットし、intractiveをTrueにする (UI更新) | |
outputs=[ | |
gr_question_text_1_1, | |
gr_answer_text_1_1, | |
gr_question_text_2_1, | |
gr_answer_text_2_1, | |
gr_question_text_3_1, | |
gr_answer_text_3_1, | |
gr_answer_text_1_1, | |
gr_answer_text_2_1, | |
gr_answer_text_3_1, | |
gr_unknown_btn, | |
gr_good_btn, | |
gr_bad_btn, | |
gr_score_5_btn, | |
gr_score_4_btn, | |
gr_score_3_btn, | |
gr_score_2_btn, | |
gr_score_1_btn, | |
gr_submit_score, | |
gr_score_detail, | |
gr_score_reset, | |
gr_helphulness, | |
gr_correcthness, | |
gr_coherence, | |
gr_complexity, | |
gr_verbosity, | |
gr_humor, | |
gr_creativity, | |
gr_appropriate, | |
gr_following_instructions, | |
gr_politeness, | |
gr_harmfulness, | |
gr_hate, | |
gr_sexual, | |
gr_violence, | |
gr_suicide, | |
gr_threat, | |
gr_gun, | |
gr_controlled_substance, | |
gr_climinal_planing, | |
gr_privacy, | |
gr_harassment, | |
gr_profanity, | |
gr_political_content, | |
gr_moral_judgement, | |
gr_typos | |
] | |
) | |
# TODO Tab切り替えで、アノテ済みの一覧を表示する | |
# with gr.Tab("アノテ済みデータセット(管理画面)"): | |
# タブを切り替えた時にデータ表示を更新する | |
if __name__ == "__main__": | |
demo.launch() |