kevineen commited on
Commit
19ee3e1
1 Parent(s): 509018d

typo 修正

Browse files
Files changed (2) hide show
  1. app.py +84 -99
  2. run.py +84 -84
app.py CHANGED
@@ -1,6 +1,3 @@
1
- import os
2
- from dotenv import load_dotenv
3
-
4
  import json
5
  import datetime
6
  from pathlib import Path
@@ -12,24 +9,12 @@ import pandas as pd
12
  import gradio as gr
13
  from datasets import load_dataset
14
  from huggingface_hub import CommitScheduler
15
- from huggingface_hub import HfFolder
16
-
17
- # .envファイルから環境変数をロードする(ローカル開発環境)
18
- load_dotenv()
19
-
20
- # HF_Spaceでプライベート関連にアクセスするための環境変数
21
- # SecretKey をSpaceのSettingsに設定
22
- HF_TOKEN = os.getenv("HF_TOKEN")
23
- if HF_TOKEN:
24
- HfFolder.save_token(HF_TOKEN)
25
- else:
26
- print("Warning: HF_TOKEN not found. Please set it in your Space secrets.")
27
 
28
  # HFデータセット アップロード先
29
  # (切り替えてテストする用に配列)
30
  OUTPUT_DATASET = [
31
  "team-hatakeyama-phase2/annotation_tanuki_phase2",
32
- "kevineen/Tanuki-Phase2-annotation-dataset", # 出力テスト
33
  ]
34
 
35
  # アノテーション対象データセット
@@ -45,10 +30,10 @@ ANNOTATION_DATASET = [
45
  # "kanhatakeyama/AutoWikiQA",
46
  # "susumuota/SyntheticTextWikiTranslate-askllm-v1", # Ask-LLM 翻訳
47
 
48
- # Nomotron データ (将来用
49
  ]
50
 
51
- MULTITURN_ANNOTATION_DATASET = [
52
  # マルチターン 未対応
53
  "kanhatakeyama/AutoMultiTurnByMixtral8x22b",
54
  ]
@@ -85,8 +70,8 @@ annotated_dataset = gr.State(
85
 
86
  # 5段階評価
87
  'score': [], # 総合 (5段階評価)
88
- 'helphulness': [], # 有用性 (5段階評価)
89
- 'correcthness': [], # 正確さ (5段階評価)
90
  'coherence': [], # 一貫性 (5段階評価)
91
  'complexity': [], # 複雑さ (5段階評価)
92
  'verbosity': [], # 冗長性 (5段階評価)
@@ -106,7 +91,7 @@ annotated_dataset = gr.State(
106
  'threat': [], # 犯罪
107
  'gun': [], # 重火器
108
  'controlled_substance': [], # 規制対象物質
109
- 'climinal_planing': [], # 犯罪計画
110
  'privacy': [], # 個人情報
111
  'harassment': [], # ハラスメント
112
  'profanity': [], # 冒涜
@@ -138,9 +123,9 @@ answer_text_2 = gr.State("") # answer_2 回答
138
  # (5段階評価)
139
  # 0は判断していない、又は判断が難しいという特徴量にする
140
  score = gr.State(3) # 総合スコア 初期値は3 (最低限必須)
141
- helphulness = gr.State(0) # 有用性
142
- correcthness = gr.State(0) # 正確さ
143
- coherenece = gr.State(0) # 一貫性
144
  complexity = gr.State(0) # 複雑さ
145
  verbosity = gr.State(0) # 冗長性
146
  humor = gr.State(0) # ユーモア (SteerLM)
@@ -159,7 +144,7 @@ suicide = gr.State(0) # 自殺
159
  threat = gr.State(0) # 犯罪
160
  gun = gr.State(0) # 銃・重火器
161
  controlled_substance = gr.State(0) # 規制対象物質
162
- climinal_planing = gr.State(0) # 犯罪計画
163
  privacy = gr.State(0) # 個人情報・プライバシー情報
164
  harassment = gr.State(0) # ハラスメント
165
  profanity = gr.State(0) # 冒涜
@@ -330,8 +315,8 @@ def save_annotation(
330
  is_proofreading_2: bool,
331
  answer_text_2: str,
332
  score: int,
333
- helphulness: int,
334
- correcthness: int,
335
  coherence: int,
336
  complexity: int,
337
  verbosity: int,
@@ -348,7 +333,7 @@ def save_annotation(
348
  threat: int,
349
  gun: int,
350
  controlled_substance: int,
351
- climinal_planing: int,
352
  privacy: int,
353
  harassment: int,
354
  profanity: int,
@@ -371,24 +356,24 @@ def save_annotation(
371
  'is_proofreading_2': [is_proofreading_2],
372
  'answer_text_2': [answer_text_2],
373
  'score': [score],
374
- 'helphullness': [helphulness],
375
- 'correcthness': [correcthness],
376
  'complexity': [complexity],
377
  'verbosity': [verbosity],
378
  'humor': [humor],
379
  'creativity': [creativity],
380
  'appropriate': [appropriate],
381
- 'following_finstructions': [following_instructions],
382
  'politeness': [politeness],
383
  'harmfulness': [harmfulness],
384
  'hate': [hate],
385
- 'sextual': [sexual],
386
  'violence': [violence],
387
  'suicide': [suicide],
388
  'threat': [threat],
389
  'gun': [gun],
390
  'controlled_substance': [controlled_substance],
391
- 'climinal_planing': [climinal_planing],
392
  'privacy': [privacy],
393
  'harassment': [harassment],
394
  'profanity': [profanity],
@@ -414,8 +399,8 @@ def save_annotation(
414
  "is_proofreading_2": is_proofreading_2,
415
  "answer_text_2": answer_text_2,
416
  "score": int(score),
417
- "helphulness": int(helphulness),
418
- "correcthness": int(correcthness),
419
  "coherence": int(coherence),
420
  "complexity": int(complexity),
421
  "verbosity": int(verbosity),
@@ -432,11 +417,11 @@ def save_annotation(
432
  "threat": int(threat),
433
  "gun": int(gun),
434
  "controlled_substance": int(controlled_substance),
435
- "climinal_planing": int(climinal_planing),
436
  "privacy": int(privacy),
437
  "harassment": int(harassment),
438
  "profanity": int(profanity),
439
- "policital_content": int(political_content),
440
  "moral_judgement": int(moral_judgement),
441
  "typos": int(typos)
442
  }
@@ -570,11 +555,11 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
570
  gr_score_reset = gr.Button("スコアリセット", interactive=False)
571
 
572
  gr_score_detail = gr.Slider(label="総合スコア 【必須】",value=3, minimum=1, maximum=5, step=1, interactive=False)
573
- gr_evalannotation_explain = gr.Markdown("詳細アノテーション (5点満点)")
574
 
575
  with gr.Row() :
576
- gr_helphulness = gr.Slider(label="有用性", minimum=0, value=0, maximum=5, step=1, interactive=False)
577
- gr_correcthness = gr.Slider(label="正確さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
578
  gr_coherence = gr.Slider(label="一貫性", minimum=0, value=0, maximum=5, step=1, interactive=False)
579
  gr_complexity = gr.Slider(label="複雑さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
580
 
@@ -610,7 +595,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
610
 
611
  with gr.Row():
612
  gr_controlled_substance = gr.Radio(label="規制対象物質", choices=["不明", "有", "無"], value="不明", interactive=False)
613
- gr_climinal_planing = gr.Radio(label="犯罪計画", choices=["不明", "有", "無"], value="不明", interactive=False)
614
 
615
  with gr.Row():
616
  gr_privacy = gr.Radio(label="個人情報", choices=["不明", "有", "無"], value="不明", interactive=False)
@@ -628,9 +613,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
628
 
629
  # slicer
630
  g_score: int,
631
- g_helphulness: int,
632
- g_correcthness: int,
633
- g_coherenece: int,
634
  g_complexity: int,
635
  g_verbosity: int,
636
  g_humor: int,
@@ -648,7 +633,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
648
  g_threat: str,
649
  g_gun: str,
650
  g_controlled_substance: str,
651
- g_climinal_planing: str,
652
  g_privacy: str,
653
  g_harassment: str,
654
  g_profanity: str,
@@ -671,9 +656,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
671
 
672
  # 送信用データ更新
673
  score.value = g_score
674
- helphulness.value = g_helphulness
675
- correcthness.value = g_correcthness
676
- coherenece.value = g_coherenece
677
  complexity.value = g_complexity
678
  verbosity.value = g_verbosity
679
  humor.value = g_humor
@@ -690,7 +675,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
690
  threat.value = checkbox_to_int(g_threat)
691
  gun.value = checkbox_to_int(g_gun)
692
  controlled_substance.value = checkbox_to_int(g_controlled_substance)
693
- climinal_planing.value = checkbox_to_int(g_climinal_planing)
694
  privacy.value = checkbox_to_int(g_privacy)
695
  harassment.value = checkbox_to_int(g_harassment)
696
  profanity.value = checkbox_to_int(g_profanity)
@@ -724,9 +709,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
724
  is_proofreading_2.value,
725
  answer_text_2.value,
726
  score.value,
727
- helphulness.value,
728
- correcthness.value,
729
- coherenece.value,
730
  complexity.value,
731
  verbosity.value,
732
  humor.value,
@@ -742,7 +727,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
742
  threat.value,
743
  gun.value,
744
  controlled_substance.value,
745
- climinal_planing.value,
746
  privacy.value,
747
  harassment.value,
748
  profanity.value,
@@ -771,8 +756,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
771
  inputs=[
772
  gr_answer_text_3_1,
773
  gr_score_detail,
774
- gr_helphulness,
775
- gr_correcthness,
776
  gr_coherence,
777
  gr_complexity,
778
  gr_verbosity,
@@ -789,7 +774,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
789
  gr_threat,
790
  gr_gun,
791
  gr_controlled_substance,
792
- gr_climinal_planing,
793
  gr_privacy,
794
  gr_harassment,
795
  gr_profanity,
@@ -806,8 +791,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
806
  gr_question_text_3_1,
807
  gr_answer_text_3_1,
808
  gr_score_detail,
809
- gr_helphulness,
810
- gr_correcthness,
811
  gr_coherence,
812
  gr_complexity,
813
  gr_verbosity,
@@ -824,7 +809,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
824
  gr_threat,
825
  gr_gun,
826
  gr_controlled_substance,
827
- gr_climinal_planing,
828
  gr_privacy,
829
  gr_harassment,
830
  gr_profanity,
@@ -939,9 +924,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
939
  is_proofreading_2.value,
940
  answer_text_2.value,
941
  score.value,
942
- helphulness.value,
943
- correcthness.value,
944
- coherenece.value,
945
  complexity.value,
946
  verbosity.value,
947
  humor.value,
@@ -957,7 +942,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
957
  threat.value,
958
  gun.value,
959
  controlled_substance.value,
960
- climinal_planing.value,
961
  privacy.value,
962
  harassment.value,
963
  profanity.value,
@@ -1001,8 +986,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1001
  gr_question_text_3_1,
1002
  gr_answer_text_3_1,
1003
  gr_score_detail,
1004
- gr_helphulness,
1005
- gr_correcthness,
1006
  gr_coherence,
1007
  gr_complexity,
1008
  gr_verbosity,
@@ -1019,7 +1004,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1019
  gr_threat,
1020
  gr_gun,
1021
  gr_controlled_substance,
1022
- gr_climinal_planing,
1023
  gr_privacy,
1024
  gr_harassment,
1025
  gr_profanity,
@@ -1055,8 +1040,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1055
  gr_question_text_3_1,
1056
  gr_answer_text_3_1,
1057
  gr_score_detail,
1058
- gr_helphulness,
1059
- gr_correcthness,
1060
  gr_coherence,
1061
  gr_complexity,
1062
  gr_verbosity,
@@ -1073,7 +1058,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1073
  gr_threat,
1074
  gr_gun,
1075
  gr_controlled_substance,
1076
- gr_climinal_planing,
1077
  gr_privacy,
1078
  gr_harassment,
1079
  gr_profanity,
@@ -1109,8 +1094,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1109
  gr_question_text_3_1,
1110
  gr_answer_text_3_1,
1111
  gr_score_detail,
1112
- gr_helphulness,
1113
- gr_correcthness,
1114
  gr_coherence,
1115
  gr_complexity,
1116
  gr_verbosity,
@@ -1127,7 +1112,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1127
  gr_threat,
1128
  gr_gun,
1129
  gr_controlled_substance,
1130
- gr_climinal_planing,
1131
  gr_privacy,
1132
  gr_harassment,
1133
  gr_profanity,
@@ -1148,8 +1133,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1148
  gr_question_text_3_1,
1149
  gr_answer_text_3_1,
1150
  gr_score_detail,
1151
- gr_helphulness,
1152
- gr_correcthness,
1153
  gr_coherence,
1154
  gr_complexity,
1155
  gr_verbosity,
@@ -1166,7 +1151,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1166
  gr_threat,
1167
  gr_gun,
1168
  gr_controlled_substance,
1169
- gr_climinal_planing,
1170
  gr_privacy,
1171
  gr_harassment,
1172
  gr_profanity,
@@ -1184,8 +1169,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1184
  gr_question_text_3_1,
1185
  gr_answer_text_3_1,
1186
  gr_score_detail,
1187
- gr_helphulness,
1188
- gr_correcthness,
1189
  gr_coherence,
1190
  gr_complexity,
1191
  gr_verbosity,
@@ -1202,7 +1187,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1202
  gr_threat,
1203
  gr_gun,
1204
  gr_controlled_substance,
1205
- gr_climinal_planing,
1206
  gr_privacy,
1207
  gr_harassment,
1208
  gr_profanity,
@@ -1220,8 +1205,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1220
  gr_question_text_3_1,
1221
  gr_answer_text_3_1,
1222
  gr_score_detail,
1223
- gr_helphulness,
1224
- gr_correcthness,
1225
  gr_coherence,
1226
  gr_complexity,
1227
  gr_verbosity,
@@ -1238,7 +1223,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1238
  gr_threat,
1239
  gr_gun,
1240
  gr_controlled_substance,
1241
- gr_climinal_planing,
1242
  gr_privacy,
1243
  gr_harassment,
1244
  gr_profanity,
@@ -1256,8 +1241,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1256
  gr_question_text_3_1,
1257
  gr_answer_text_3_1,
1258
  gr_score_detail,
1259
- gr_helphulness,
1260
- gr_correcthness,
1261
  gr_coherence,
1262
  gr_complexity,
1263
  gr_verbosity,
@@ -1274,7 +1259,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1274
  gr_threat,
1275
  gr_gun,
1276
  gr_controlled_substance,
1277
- gr_climinal_planing,
1278
  gr_privacy,
1279
  gr_harassment,
1280
  gr_profanity,
@@ -1292,8 +1277,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1292
  gr_question_text_3_1,
1293
  gr_answer_text_3_1,
1294
  gr_score_detail,
1295
- gr_helphulness,
1296
- gr_correcthness,
1297
  gr_coherence,
1298
  gr_complexity,
1299
  gr_verbosity,
@@ -1310,7 +1295,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1310
  gr_threat,
1311
  gr_gun,
1312
  gr_controlled_substance,
1313
- gr_climinal_planing,
1314
  gr_privacy,
1315
  gr_harassment,
1316
  gr_profanity,
@@ -1323,9 +1308,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1323
  def score_reset_display():
1324
  # gr.State
1325
  score.value = 3
1326
- helphulness.value = 0
1327
- correcthness.value = 0
1328
- coherenece.value = 0
1329
  complexity.value = 0
1330
  verbosity.value = 0
1331
  humor.value = 0
@@ -1340,8 +1325,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1340
  suicide.value = 0
1341
  threat.value = 0
1342
  gun.value = 0
1343
- controlled_substance.vlaue = 0
1344
- climinal_planing.value = 0
1345
  privacy.value = 0
1346
  harassment.value = 0
1347
  profanity.value = 0
@@ -1360,8 +1345,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1360
  inputs=[],
1361
  outputs=[
1362
  gr_score_detail,
1363
- gr_helphulness,
1364
- gr_correcthness,
1365
  gr_coherence,
1366
  gr_complexity,
1367
  gr_verbosity,
@@ -1378,7 +1363,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1378
  gr_threat,
1379
  gr_gun,
1380
  gr_controlled_substance,
1381
- gr_climinal_planing,
1382
  gr_privacy,
1383
  gr_harassment,
1384
  gr_profanity,
@@ -1392,7 +1377,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1392
  gr_data_load_btn.click(
1393
  dataset_load_fn,
1394
  inputs=None,
1395
- # textをセットし、intractiveをTrueにする (UI更新)
1396
  outputs=[
1397
  gr_question_text_1_1,
1398
  gr_answer_text_1_1,
@@ -1414,8 +1399,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1414
  gr_submit_score,
1415
  gr_score_detail,
1416
  gr_score_reset,
1417
- gr_helphulness,
1418
- gr_correcthness,
1419
  gr_coherence,
1420
  gr_complexity,
1421
  gr_verbosity,
@@ -1432,7 +1417,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1432
  gr_threat,
1433
  gr_gun,
1434
  gr_controlled_substance,
1435
- gr_climinal_planing,
1436
  gr_privacy,
1437
  gr_harassment,
1438
  gr_profanity,
 
 
 
 
1
  import json
2
  import datetime
3
  from pathlib import Path
 
9
  import gradio as gr
10
  from datasets import load_dataset
11
  from huggingface_hub import CommitScheduler
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  # HFデータセット アップロード先
14
  # (切り替えてテストする用に配列)
15
  OUTPUT_DATASET = [
16
  "team-hatakeyama-phase2/annotation_tanuki_phase2",
17
+ "kevineen/Tanuki-Phase2-annotation-dataset", # 出力テスト
18
  ]
19
 
20
  # アノテーション対象データセット
 
30
  # "kanhatakeyama/AutoWikiQA",
31
  # "susumuota/SyntheticTextWikiTranslate-askllm-v1", # Ask-LLM 翻訳
32
 
33
+ # Nemotron データ (将来用
34
  ]
35
 
36
+ MULTI_TURN_ANNOTATION_DATASET = [
37
  # マルチターン 未対応
38
  "kanhatakeyama/AutoMultiTurnByMixtral8x22b",
39
  ]
 
70
 
71
  # 5段階評価
72
  'score': [], # 総合 (5段階評価)
73
+ 'helpfulness': [], # 有用性 (5段階評価)
74
+ 'correctness': [], # 正確さ (5段階評価)
75
  'coherence': [], # 一貫性 (5段階評価)
76
  'complexity': [], # 複雑さ (5段階評価)
77
  'verbosity': [], # 冗長性 (5段階評価)
 
91
  'threat': [], # 犯罪
92
  'gun': [], # 重火器
93
  'controlled_substance': [], # 規制対象物質
94
+ 'criminal_planing': [], # 犯罪計画
95
  'privacy': [], # 個人情報
96
  'harassment': [], # ハラスメント
97
  'profanity': [], # 冒涜
 
123
  # (5段階評価)
124
  # 0は判断していない、又は判断が難しいという特徴量にする
125
  score = gr.State(3) # 総合スコア 初期値は3 (最低限必須)
126
+ helpfulness = gr.State(0) # 有用性
127
+ correctness = gr.State(0) # 正確さ
128
+ coherence = gr.State(0) # 一貫性
129
  complexity = gr.State(0) # 複雑さ
130
  verbosity = gr.State(0) # 冗長性
131
  humor = gr.State(0) # ユーモア (SteerLM)
 
144
  threat = gr.State(0) # 犯罪
145
  gun = gr.State(0) # 銃・重火器
146
  controlled_substance = gr.State(0) # 規制対象物質
147
+ criminal_planing = gr.State(0) # 犯罪計画
148
  privacy = gr.State(0) # 個人情報・プライバシー情報
149
  harassment = gr.State(0) # ハラスメント
150
  profanity = gr.State(0) # 冒涜
 
315
  is_proofreading_2: bool,
316
  answer_text_2: str,
317
  score: int,
318
+ helpfulness: int,
319
+ correctness: int,
320
  coherence: int,
321
  complexity: int,
322
  verbosity: int,
 
333
  threat: int,
334
  gun: int,
335
  controlled_substance: int,
336
+ criminal_planing: int,
337
  privacy: int,
338
  harassment: int,
339
  profanity: int,
 
356
  'is_proofreading_2': [is_proofreading_2],
357
  'answer_text_2': [answer_text_2],
358
  'score': [score],
359
+ 'helpfulness': [helpfulness],
360
+ 'correctness': [correctness],
361
  'complexity': [complexity],
362
  'verbosity': [verbosity],
363
  'humor': [humor],
364
  'creativity': [creativity],
365
  'appropriate': [appropriate],
366
+ 'following_instructions': [following_instructions],
367
  'politeness': [politeness],
368
  'harmfulness': [harmfulness],
369
  'hate': [hate],
370
+ 'sexual': [sexual],
371
  'violence': [violence],
372
  'suicide': [suicide],
373
  'threat': [threat],
374
  'gun': [gun],
375
  'controlled_substance': [controlled_substance],
376
+ 'criminal_planing': [criminal_planing],
377
  'privacy': [privacy],
378
  'harassment': [harassment],
379
  'profanity': [profanity],
 
399
  "is_proofreading_2": is_proofreading_2,
400
  "answer_text_2": answer_text_2,
401
  "score": int(score),
402
+ "helpfulness": int(helpfulness),
403
+ "correctness": int(correctness),
404
  "coherence": int(coherence),
405
  "complexity": int(complexity),
406
  "verbosity": int(verbosity),
 
417
  "threat": int(threat),
418
  "gun": int(gun),
419
  "controlled_substance": int(controlled_substance),
420
+ "criminal_planing": int(criminal_planing),
421
  "privacy": int(privacy),
422
  "harassment": int(harassment),
423
  "profanity": int(profanity),
424
+ "political_content": int(political_content),
425
  "moral_judgement": int(moral_judgement),
426
  "typos": int(typos)
427
  }
 
555
  gr_score_reset = gr.Button("スコアリセット", interactive=False)
556
 
557
  gr_score_detail = gr.Slider(label="総合スコア 【必須】",value=3, minimum=1, maximum=5, step=1, interactive=False)
558
+ gr_eval_annotation_explain = gr.Markdown("詳細アノテーション (5点満点)")
559
 
560
  with gr.Row() :
561
+ gr_helpfulness = gr.Slider(label="有用性", minimum=0, value=0, maximum=5, step=1, interactive=False)
562
+ gr_correctness = gr.Slider(label="正確さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
563
  gr_coherence = gr.Slider(label="一貫性", minimum=0, value=0, maximum=5, step=1, interactive=False)
564
  gr_complexity = gr.Slider(label="複雑さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
565
 
 
595
 
596
  with gr.Row():
597
  gr_controlled_substance = gr.Radio(label="規制対象物質", choices=["不明", "有", "無"], value="不明", interactive=False)
598
+ gr_criminal_planing = gr.Radio(label="犯罪計画", choices=["不明", "有", "無"], value="不明", interactive=False)
599
 
600
  with gr.Row():
601
  gr_privacy = gr.Radio(label="個人情報", choices=["不明", "有", "無"], value="不明", interactive=False)
 
613
 
614
  # slicer
615
  g_score: int,
616
+ g_helpfulness: int,
617
+ g_correctness: int,
618
+ g_coherence: int,
619
  g_complexity: int,
620
  g_verbosity: int,
621
  g_humor: int,
 
633
  g_threat: str,
634
  g_gun: str,
635
  g_controlled_substance: str,
636
+ g_criminal_planing: str,
637
  g_privacy: str,
638
  g_harassment: str,
639
  g_profanity: str,
 
656
 
657
  # 送信用データ更新
658
  score.value = g_score
659
+ helpfulness.value = g_helpfulness
660
+ correctness.value = g_correctness
661
+ coherence.value = g_coherence
662
  complexity.value = g_complexity
663
  verbosity.value = g_verbosity
664
  humor.value = g_humor
 
675
  threat.value = checkbox_to_int(g_threat)
676
  gun.value = checkbox_to_int(g_gun)
677
  controlled_substance.value = checkbox_to_int(g_controlled_substance)
678
+ criminal_planing.value = checkbox_to_int(g_criminal_planing)
679
  privacy.value = checkbox_to_int(g_privacy)
680
  harassment.value = checkbox_to_int(g_harassment)
681
  profanity.value = checkbox_to_int(g_profanity)
 
709
  is_proofreading_2.value,
710
  answer_text_2.value,
711
  score.value,
712
+ helpfulness.value,
713
+ correctness.value,
714
+ coherence.value,
715
  complexity.value,
716
  verbosity.value,
717
  humor.value,
 
727
  threat.value,
728
  gun.value,
729
  controlled_substance.value,
730
+ criminal_planing.value,
731
  privacy.value,
732
  harassment.value,
733
  profanity.value,
 
756
  inputs=[
757
  gr_answer_text_3_1,
758
  gr_score_detail,
759
+ gr_helpfulness,
760
+ gr_correctness,
761
  gr_coherence,
762
  gr_complexity,
763
  gr_verbosity,
 
774
  gr_threat,
775
  gr_gun,
776
  gr_controlled_substance,
777
+ gr_criminal_planing,
778
  gr_privacy,
779
  gr_harassment,
780
  gr_profanity,
 
791
  gr_question_text_3_1,
792
  gr_answer_text_3_1,
793
  gr_score_detail,
794
+ gr_helpfulness,
795
+ gr_correctness,
796
  gr_coherence,
797
  gr_complexity,
798
  gr_verbosity,
 
809
  gr_threat,
810
  gr_gun,
811
  gr_controlled_substance,
812
+ gr_criminal_planing,
813
  gr_privacy,
814
  gr_harassment,
815
  gr_profanity,
 
924
  is_proofreading_2.value,
925
  answer_text_2.value,
926
  score.value,
927
+ helpfulness.value,
928
+ correctness.value,
929
+ coherence.value,
930
  complexity.value,
931
  verbosity.value,
932
  humor.value,
 
942
  threat.value,
943
  gun.value,
944
  controlled_substance.value,
945
+ criminal_planing.value,
946
  privacy.value,
947
  harassment.value,
948
  profanity.value,
 
986
  gr_question_text_3_1,
987
  gr_answer_text_3_1,
988
  gr_score_detail,
989
+ gr_helpfulness,
990
+ gr_correctness,
991
  gr_coherence,
992
  gr_complexity,
993
  gr_verbosity,
 
1004
  gr_threat,
1005
  gr_gun,
1006
  gr_controlled_substance,
1007
+ gr_criminal_planing,
1008
  gr_privacy,
1009
  gr_harassment,
1010
  gr_profanity,
 
1040
  gr_question_text_3_1,
1041
  gr_answer_text_3_1,
1042
  gr_score_detail,
1043
+ gr_helpfulness,
1044
+ gr_correctness,
1045
  gr_coherence,
1046
  gr_complexity,
1047
  gr_verbosity,
 
1058
  gr_threat,
1059
  gr_gun,
1060
  gr_controlled_substance,
1061
+ gr_criminal_planing,
1062
  gr_privacy,
1063
  gr_harassment,
1064
  gr_profanity,
 
1094
  gr_question_text_3_1,
1095
  gr_answer_text_3_1,
1096
  gr_score_detail,
1097
+ gr_helpfulness,
1098
+ gr_correctness,
1099
  gr_coherence,
1100
  gr_complexity,
1101
  gr_verbosity,
 
1112
  gr_threat,
1113
  gr_gun,
1114
  gr_controlled_substance,
1115
+ gr_criminal_planing,
1116
  gr_privacy,
1117
  gr_harassment,
1118
  gr_profanity,
 
1133
  gr_question_text_3_1,
1134
  gr_answer_text_3_1,
1135
  gr_score_detail,
1136
+ gr_helpfulness,
1137
+ gr_correctness,
1138
  gr_coherence,
1139
  gr_complexity,
1140
  gr_verbosity,
 
1151
  gr_threat,
1152
  gr_gun,
1153
  gr_controlled_substance,
1154
+ gr_criminal_planing,
1155
  gr_privacy,
1156
  gr_harassment,
1157
  gr_profanity,
 
1169
  gr_question_text_3_1,
1170
  gr_answer_text_3_1,
1171
  gr_score_detail,
1172
+ gr_helpfulness,
1173
+ gr_correctness,
1174
  gr_coherence,
1175
  gr_complexity,
1176
  gr_verbosity,
 
1187
  gr_threat,
1188
  gr_gun,
1189
  gr_controlled_substance,
1190
+ gr_criminal_planing,
1191
  gr_privacy,
1192
  gr_harassment,
1193
  gr_profanity,
 
1205
  gr_question_text_3_1,
1206
  gr_answer_text_3_1,
1207
  gr_score_detail,
1208
+ gr_helpfulness,
1209
+ gr_correctness,
1210
  gr_coherence,
1211
  gr_complexity,
1212
  gr_verbosity,
 
1223
  gr_threat,
1224
  gr_gun,
1225
  gr_controlled_substance,
1226
+ gr_criminal_planing,
1227
  gr_privacy,
1228
  gr_harassment,
1229
  gr_profanity,
 
1241
  gr_question_text_3_1,
1242
  gr_answer_text_3_1,
1243
  gr_score_detail,
1244
+ gr_helpfulness,
1245
+ gr_correctness,
1246
  gr_coherence,
1247
  gr_complexity,
1248
  gr_verbosity,
 
1259
  gr_threat,
1260
  gr_gun,
1261
  gr_controlled_substance,
1262
+ gr_criminal_planing,
1263
  gr_privacy,
1264
  gr_harassment,
1265
  gr_profanity,
 
1277
  gr_question_text_3_1,
1278
  gr_answer_text_3_1,
1279
  gr_score_detail,
1280
+ gr_helpfulness,
1281
+ gr_correctness,
1282
  gr_coherence,
1283
  gr_complexity,
1284
  gr_verbosity,
 
1295
  gr_threat,
1296
  gr_gun,
1297
  gr_controlled_substance,
1298
+ gr_criminal_planing,
1299
  gr_privacy,
1300
  gr_harassment,
1301
  gr_profanity,
 
1308
  def score_reset_display():
1309
  # gr.State
1310
  score.value = 3
1311
+ helpfulness.value = 0
1312
+ correctness.value = 0
1313
+ coherence.value = 0
1314
  complexity.value = 0
1315
  verbosity.value = 0
1316
  humor.value = 0
 
1325
  suicide.value = 0
1326
  threat.value = 0
1327
  gun.value = 0
1328
+ controlled_substance.value = 0
1329
+ criminal_planing.value = 0
1330
  privacy.value = 0
1331
  harassment.value = 0
1332
  profanity.value = 0
 
1345
  inputs=[],
1346
  outputs=[
1347
  gr_score_detail,
1348
+ gr_helpfulness,
1349
+ gr_correctness,
1350
  gr_coherence,
1351
  gr_complexity,
1352
  gr_verbosity,
 
1363
  gr_threat,
1364
  gr_gun,
1365
  gr_controlled_substance,
1366
+ gr_criminal_planing,
1367
  gr_privacy,
1368
  gr_harassment,
1369
  gr_profanity,
 
1377
  gr_data_load_btn.click(
1378
  dataset_load_fn,
1379
  inputs=None,
1380
+ # textをセットし、interactiveをTrueにする (UI更新)
1381
  outputs=[
1382
  gr_question_text_1_1,
1383
  gr_answer_text_1_1,
 
1399
  gr_submit_score,
1400
  gr_score_detail,
1401
  gr_score_reset,
1402
+ gr_helpfulness,
1403
+ gr_correctness,
1404
  gr_coherence,
1405
  gr_complexity,
1406
  gr_verbosity,
 
1417
  gr_threat,
1418
  gr_gun,
1419
  gr_controlled_substance,
1420
+ gr_criminal_planing,
1421
  gr_privacy,
1422
  gr_harassment,
1423
  gr_profanity,
run.py CHANGED
@@ -31,7 +31,7 @@ else:
31
  # (切り替えてテストする用に配列)
32
  OUTPUT_DATASET = [
33
  "team-hatakeyama-phase2/annotation_tanuki_phase2",
34
- "kevineen/Tanuki-Phase2-annotation-dataset", # 出力テスト
35
  ]
36
 
37
  # アノテーション対象データセット
@@ -47,10 +47,10 @@ ANNOTATION_DATASET = [
47
  # "kanhatakeyama/AutoWikiQA",
48
  # "susumuota/SyntheticTextWikiTranslate-askllm-v1", # Ask-LLM 翻訳
49
 
50
- # Nomotron データ (将来用
51
  ]
52
 
53
- MULTITURN_ANNOTATION_DATASET = [
54
  # マルチターン 未対応
55
  "kanhatakeyama/AutoMultiTurnByMixtral8x22b",
56
  ]
@@ -87,8 +87,8 @@ annotated_dataset = gr.State(
87
 
88
  # 5段階評価
89
  'score': [], # 総合 (5段階評価)
90
- 'helphulness': [], # 有用性 (5段階評価)
91
- 'correcthness': [], # 正確さ (5段階評価)
92
  'coherence': [], # 一貫性 (5段階評価)
93
  'complexity': [], # 複雑さ (5段階評価)
94
  'verbosity': [], # 冗長性 (5段階評価)
@@ -108,7 +108,7 @@ annotated_dataset = gr.State(
108
  'threat': [], # 犯罪
109
  'gun': [], # 重火器
110
  'controlled_substance': [], # 規制対象物質
111
- 'climinal_planing': [], # 犯罪計画
112
  'privacy': [], # 個人情報
113
  'harassment': [], # ハラスメント
114
  'profanity': [], # 冒涜
@@ -140,9 +140,9 @@ answer_text_2 = gr.State("") # answer_2 回答
140
  # (5段階評価)
141
  # 0は判断していない、又は判断が難しいという特徴量にする
142
  score = gr.State(3) # 総合スコア 初期値は3 (最低限必須)
143
- helphulness = gr.State(0) # 有用性
144
- correcthness = gr.State(0) # 正確さ
145
- coherenece = gr.State(0) # 一貫性
146
  complexity = gr.State(0) # 複雑さ
147
  verbosity = gr.State(0) # 冗長性
148
  humor = gr.State(0) # ユーモア (SteerLM)
@@ -161,7 +161,7 @@ suicide = gr.State(0) # 自殺
161
  threat = gr.State(0) # 犯罪
162
  gun = gr.State(0) # 銃・重火器
163
  controlled_substance = gr.State(0) # 規制対象物質
164
- climinal_planing = gr.State(0) # 犯罪計画
165
  privacy = gr.State(0) # 個人情報・プライバシー情報
166
  harassment = gr.State(0) # ハラスメント
167
  profanity = gr.State(0) # 冒涜
@@ -332,8 +332,8 @@ def save_annotation(
332
  is_proofreading_2: bool,
333
  answer_text_2: str,
334
  score: int,
335
- helphulness: int,
336
- correcthness: int,
337
  coherence: int,
338
  complexity: int,
339
  verbosity: int,
@@ -350,7 +350,7 @@ def save_annotation(
350
  threat: int,
351
  gun: int,
352
  controlled_substance: int,
353
- climinal_planing: int,
354
  privacy: int,
355
  harassment: int,
356
  profanity: int,
@@ -373,24 +373,24 @@ def save_annotation(
373
  'is_proofreading_2': [is_proofreading_2],
374
  'answer_text_2': [answer_text_2],
375
  'score': [score],
376
- 'helphullness': [helphulness],
377
- 'correcthness': [correcthness],
378
  'complexity': [complexity],
379
  'verbosity': [verbosity],
380
  'humor': [humor],
381
  'creativity': [creativity],
382
  'appropriate': [appropriate],
383
- 'following_finstructions': [following_instructions],
384
  'politeness': [politeness],
385
  'harmfulness': [harmfulness],
386
  'hate': [hate],
387
- 'sextual': [sexual],
388
  'violence': [violence],
389
  'suicide': [suicide],
390
  'threat': [threat],
391
  'gun': [gun],
392
  'controlled_substance': [controlled_substance],
393
- 'climinal_planing': [climinal_planing],
394
  'privacy': [privacy],
395
  'harassment': [harassment],
396
  'profanity': [profanity],
@@ -416,8 +416,8 @@ def save_annotation(
416
  "is_proofreading_2": is_proofreading_2,
417
  "answer_text_2": answer_text_2,
418
  "score": int(score),
419
- "helphulness": int(helphulness),
420
- "correcthness": int(correcthness),
421
  "coherence": int(coherence),
422
  "complexity": int(complexity),
423
  "verbosity": int(verbosity),
@@ -434,11 +434,11 @@ def save_annotation(
434
  "threat": int(threat),
435
  "gun": int(gun),
436
  "controlled_substance": int(controlled_substance),
437
- "climinal_planing": int(climinal_planing),
438
  "privacy": int(privacy),
439
  "harassment": int(harassment),
440
  "profanity": int(profanity),
441
- "policital_content": int(political_content),
442
  "moral_judgement": int(moral_judgement),
443
  "typos": int(typos)
444
  }
@@ -572,11 +572,11 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
572
  gr_score_reset = gr.Button("スコアリセット", interactive=False)
573
 
574
  gr_score_detail = gr.Slider(label="総合スコア 【必須】",value=3, minimum=1, maximum=5, step=1, interactive=False)
575
- gr_evalannotation_explain = gr.Markdown("詳細アノテーション (5点満点)")
576
 
577
  with gr.Row() :
578
- gr_helphulness = gr.Slider(label="有用性", minimum=0, value=0, maximum=5, step=1, interactive=False)
579
- gr_correcthness = gr.Slider(label="正確さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
580
  gr_coherence = gr.Slider(label="一貫性", minimum=0, value=0, maximum=5, step=1, interactive=False)
581
  gr_complexity = gr.Slider(label="複雑さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
582
 
@@ -612,7 +612,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
612
 
613
  with gr.Row():
614
  gr_controlled_substance = gr.Radio(label="規制対象物質", choices=["不明", "有", "無"], value="不明", interactive=False)
615
- gr_climinal_planing = gr.Radio(label="犯罪計画", choices=["不明", "有", "無"], value="不明", interactive=False)
616
 
617
  with gr.Row():
618
  gr_privacy = gr.Radio(label="個人情報", choices=["不明", "有", "無"], value="不明", interactive=False)
@@ -630,9 +630,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
630
 
631
  # slicer
632
  g_score: int,
633
- g_helphulness: int,
634
- g_correcthness: int,
635
- g_coherenece: int,
636
  g_complexity: int,
637
  g_verbosity: int,
638
  g_humor: int,
@@ -650,7 +650,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
650
  g_threat: str,
651
  g_gun: str,
652
  g_controlled_substance: str,
653
- g_climinal_planing: str,
654
  g_privacy: str,
655
  g_harassment: str,
656
  g_profanity: str,
@@ -673,9 +673,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
673
 
674
  # 送信用データ更新
675
  score.value = g_score
676
- helphulness.value = g_helphulness
677
- correcthness.value = g_correcthness
678
- coherenece.value = g_coherenece
679
  complexity.value = g_complexity
680
  verbosity.value = g_verbosity
681
  humor.value = g_humor
@@ -692,7 +692,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
692
  threat.value = checkbox_to_int(g_threat)
693
  gun.value = checkbox_to_int(g_gun)
694
  controlled_substance.value = checkbox_to_int(g_controlled_substance)
695
- climinal_planing.value = checkbox_to_int(g_climinal_planing)
696
  privacy.value = checkbox_to_int(g_privacy)
697
  harassment.value = checkbox_to_int(g_harassment)
698
  profanity.value = checkbox_to_int(g_profanity)
@@ -726,9 +726,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
726
  is_proofreading_2.value,
727
  answer_text_2.value,
728
  score.value,
729
- helphulness.value,
730
- correcthness.value,
731
- coherenece.value,
732
  complexity.value,
733
  verbosity.value,
734
  humor.value,
@@ -744,7 +744,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
744
  threat.value,
745
  gun.value,
746
  controlled_substance.value,
747
- climinal_planing.value,
748
  privacy.value,
749
  harassment.value,
750
  profanity.value,
@@ -773,8 +773,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
773
  inputs=[
774
  gr_answer_text_3_1,
775
  gr_score_detail,
776
- gr_helphulness,
777
- gr_correcthness,
778
  gr_coherence,
779
  gr_complexity,
780
  gr_verbosity,
@@ -791,7 +791,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
791
  gr_threat,
792
  gr_gun,
793
  gr_controlled_substance,
794
- gr_climinal_planing,
795
  gr_privacy,
796
  gr_harassment,
797
  gr_profanity,
@@ -808,8 +808,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
808
  gr_question_text_3_1,
809
  gr_answer_text_3_1,
810
  gr_score_detail,
811
- gr_helphulness,
812
- gr_correcthness,
813
  gr_coherence,
814
  gr_complexity,
815
  gr_verbosity,
@@ -826,7 +826,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
826
  gr_threat,
827
  gr_gun,
828
  gr_controlled_substance,
829
- gr_climinal_planing,
830
  gr_privacy,
831
  gr_harassment,
832
  gr_profanity,
@@ -941,9 +941,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
941
  is_proofreading_2.value,
942
  answer_text_2.value,
943
  score.value,
944
- helphulness.value,
945
- correcthness.value,
946
- coherenece.value,
947
  complexity.value,
948
  verbosity.value,
949
  humor.value,
@@ -959,7 +959,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
959
  threat.value,
960
  gun.value,
961
  controlled_substance.value,
962
- climinal_planing.value,
963
  privacy.value,
964
  harassment.value,
965
  profanity.value,
@@ -1003,8 +1003,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1003
  gr_question_text_3_1,
1004
  gr_answer_text_3_1,
1005
  gr_score_detail,
1006
- gr_helphulness,
1007
- gr_correcthness,
1008
  gr_coherence,
1009
  gr_complexity,
1010
  gr_verbosity,
@@ -1021,7 +1021,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1021
  gr_threat,
1022
  gr_gun,
1023
  gr_controlled_substance,
1024
- gr_climinal_planing,
1025
  gr_privacy,
1026
  gr_harassment,
1027
  gr_profanity,
@@ -1057,8 +1057,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1057
  gr_question_text_3_1,
1058
  gr_answer_text_3_1,
1059
  gr_score_detail,
1060
- gr_helphulness,
1061
- gr_correcthness,
1062
  gr_coherence,
1063
  gr_complexity,
1064
  gr_verbosity,
@@ -1075,7 +1075,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1075
  gr_threat,
1076
  gr_gun,
1077
  gr_controlled_substance,
1078
- gr_climinal_planing,
1079
  gr_privacy,
1080
  gr_harassment,
1081
  gr_profanity,
@@ -1111,8 +1111,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1111
  gr_question_text_3_1,
1112
  gr_answer_text_3_1,
1113
  gr_score_detail,
1114
- gr_helphulness,
1115
- gr_correcthness,
1116
  gr_coherence,
1117
  gr_complexity,
1118
  gr_verbosity,
@@ -1129,7 +1129,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1129
  gr_threat,
1130
  gr_gun,
1131
  gr_controlled_substance,
1132
- gr_climinal_planing,
1133
  gr_privacy,
1134
  gr_harassment,
1135
  gr_profanity,
@@ -1150,8 +1150,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1150
  gr_question_text_3_1,
1151
  gr_answer_text_3_1,
1152
  gr_score_detail,
1153
- gr_helphulness,
1154
- gr_correcthness,
1155
  gr_coherence,
1156
  gr_complexity,
1157
  gr_verbosity,
@@ -1168,7 +1168,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1168
  gr_threat,
1169
  gr_gun,
1170
  gr_controlled_substance,
1171
- gr_climinal_planing,
1172
  gr_privacy,
1173
  gr_harassment,
1174
  gr_profanity,
@@ -1186,8 +1186,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1186
  gr_question_text_3_1,
1187
  gr_answer_text_3_1,
1188
  gr_score_detail,
1189
- gr_helphulness,
1190
- gr_correcthness,
1191
  gr_coherence,
1192
  gr_complexity,
1193
  gr_verbosity,
@@ -1204,7 +1204,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1204
  gr_threat,
1205
  gr_gun,
1206
  gr_controlled_substance,
1207
- gr_climinal_planing,
1208
  gr_privacy,
1209
  gr_harassment,
1210
  gr_profanity,
@@ -1222,8 +1222,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1222
  gr_question_text_3_1,
1223
  gr_answer_text_3_1,
1224
  gr_score_detail,
1225
- gr_helphulness,
1226
- gr_correcthness,
1227
  gr_coherence,
1228
  gr_complexity,
1229
  gr_verbosity,
@@ -1240,7 +1240,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1240
  gr_threat,
1241
  gr_gun,
1242
  gr_controlled_substance,
1243
- gr_climinal_planing,
1244
  gr_privacy,
1245
  gr_harassment,
1246
  gr_profanity,
@@ -1258,8 +1258,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1258
  gr_question_text_3_1,
1259
  gr_answer_text_3_1,
1260
  gr_score_detail,
1261
- gr_helphulness,
1262
- gr_correcthness,
1263
  gr_coherence,
1264
  gr_complexity,
1265
  gr_verbosity,
@@ -1276,7 +1276,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1276
  gr_threat,
1277
  gr_gun,
1278
  gr_controlled_substance,
1279
- gr_climinal_planing,
1280
  gr_privacy,
1281
  gr_harassment,
1282
  gr_profanity,
@@ -1294,8 +1294,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1294
  gr_question_text_3_1,
1295
  gr_answer_text_3_1,
1296
  gr_score_detail,
1297
- gr_helphulness,
1298
- gr_correcthness,
1299
  gr_coherence,
1300
  gr_complexity,
1301
  gr_verbosity,
@@ -1312,7 +1312,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1312
  gr_threat,
1313
  gr_gun,
1314
  gr_controlled_substance,
1315
- gr_climinal_planing,
1316
  gr_privacy,
1317
  gr_harassment,
1318
  gr_profanity,
@@ -1325,9 +1325,9 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1325
  def score_reset_display():
1326
  # gr.State
1327
  score.value = 3
1328
- helphulness.value = 0
1329
- correcthness.value = 0
1330
- coherenece.value = 0
1331
  complexity.value = 0
1332
  verbosity.value = 0
1333
  humor.value = 0
@@ -1342,8 +1342,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1342
  suicide.value = 0
1343
  threat.value = 0
1344
  gun.value = 0
1345
- controlled_substance.vlaue = 0
1346
- climinal_planing.value = 0
1347
  privacy.value = 0
1348
  harassment.value = 0
1349
  profanity.value = 0
@@ -1362,8 +1362,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1362
  inputs=[],
1363
  outputs=[
1364
  gr_score_detail,
1365
- gr_helphulness,
1366
- gr_correcthness,
1367
  gr_coherence,
1368
  gr_complexity,
1369
  gr_verbosity,
@@ -1380,7 +1380,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1380
  gr_threat,
1381
  gr_gun,
1382
  gr_controlled_substance,
1383
- gr_climinal_planing,
1384
  gr_privacy,
1385
  gr_harassment,
1386
  gr_profanity,
@@ -1394,7 +1394,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1394
  gr_data_load_btn.click(
1395
  dataset_load_fn,
1396
  inputs=None,
1397
- # textをセットし、intractiveをTrueにする (UI更新)
1398
  outputs=[
1399
  gr_question_text_1_1,
1400
  gr_answer_text_1_1,
@@ -1416,8 +1416,8 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1416
  gr_submit_score,
1417
  gr_score_detail,
1418
  gr_score_reset,
1419
- gr_helphulness,
1420
- gr_correcthness,
1421
  gr_coherence,
1422
  gr_complexity,
1423
  gr_verbosity,
@@ -1434,7 +1434,7 @@ with gr.Blocks(theme=theme_, css=load_css()) as demo:
1434
  gr_threat,
1435
  gr_gun,
1436
  gr_controlled_substance,
1437
- gr_climinal_planing,
1438
  gr_privacy,
1439
  gr_harassment,
1440
  gr_profanity,
 
31
  # (切り替えてテストする用に配列)
32
  OUTPUT_DATASET = [
33
  "team-hatakeyama-phase2/annotation_tanuki_phase2",
34
+ "kevineen/Tanuki-Phase2-annotation-dataset", # 出力テスト
35
  ]
36
 
37
  # アノテーション対象データセット
 
47
  # "kanhatakeyama/AutoWikiQA",
48
  # "susumuota/SyntheticTextWikiTranslate-askllm-v1", # Ask-LLM 翻訳
49
 
50
+ # Nemotron データ (将来用
51
  ]
52
 
53
+ MULTI_TURN_ANNOTATION_DATASET = [
54
  # マルチターン 未対応
55
  "kanhatakeyama/AutoMultiTurnByMixtral8x22b",
56
  ]
 
87
 
88
  # 5段階評価
89
  'score': [], # 総合 (5段階評価)
90
+ 'helpfulness': [], # 有用性 (5段階評価)
91
+ 'correctness': [], # 正確さ (5段階評価)
92
  'coherence': [], # 一貫性 (5段階評価)
93
  'complexity': [], # 複雑さ (5段階評価)
94
  'verbosity': [], # 冗長性 (5段階評価)
 
108
  'threat': [], # 犯罪
109
  'gun': [], # 重火器
110
  'controlled_substance': [], # 規制対象物質
111
+ 'criminal_planing': [], # 犯罪計画
112
  'privacy': [], # 個人情報
113
  'harassment': [], # ハラスメント
114
  'profanity': [], # 冒涜
 
140
  # (5段階評価)
141
  # 0は判断していない、又は判断が難しいという特徴量にする
142
  score = gr.State(3) # 総合スコア 初期値は3 (最低限必須)
143
+ helpfulness = gr.State(0) # 有用性
144
+ correctness = gr.State(0) # 正確さ
145
+ coherence = gr.State(0) # 一貫性
146
  complexity = gr.State(0) # 複雑さ
147
  verbosity = gr.State(0) # 冗長性
148
  humor = gr.State(0) # ユーモア (SteerLM)
 
161
  threat = gr.State(0) # 犯罪
162
  gun = gr.State(0) # 銃・重火器
163
  controlled_substance = gr.State(0) # 規制対象物質
164
+ criminal_planing = gr.State(0) # 犯罪計画
165
  privacy = gr.State(0) # 個人情報・プライバシー情報
166
  harassment = gr.State(0) # ハラスメント
167
  profanity = gr.State(0) # 冒涜
 
332
  is_proofreading_2: bool,
333
  answer_text_2: str,
334
  score: int,
335
+ helpfulness: int,
336
+ correctness: int,
337
  coherence: int,
338
  complexity: int,
339
  verbosity: int,
 
350
  threat: int,
351
  gun: int,
352
  controlled_substance: int,
353
+ criminal_planing: int,
354
  privacy: int,
355
  harassment: int,
356
  profanity: int,
 
373
  'is_proofreading_2': [is_proofreading_2],
374
  'answer_text_2': [answer_text_2],
375
  'score': [score],
376
+ 'helpfulness': [helpfulness],
377
+ 'correctness': [correctness],
378
  'complexity': [complexity],
379
  'verbosity': [verbosity],
380
  'humor': [humor],
381
  'creativity': [creativity],
382
  'appropriate': [appropriate],
383
+ 'following_instructions': [following_instructions],
384
  'politeness': [politeness],
385
  'harmfulness': [harmfulness],
386
  'hate': [hate],
387
+ 'sexual': [sexual],
388
  'violence': [violence],
389
  'suicide': [suicide],
390
  'threat': [threat],
391
  'gun': [gun],
392
  'controlled_substance': [controlled_substance],
393
+ 'criminal_planing': [criminal_planing],
394
  'privacy': [privacy],
395
  'harassment': [harassment],
396
  'profanity': [profanity],
 
416
  "is_proofreading_2": is_proofreading_2,
417
  "answer_text_2": answer_text_2,
418
  "score": int(score),
419
+ "helpfulness": int(helpfulness),
420
+ "correctness": int(correctness),
421
  "coherence": int(coherence),
422
  "complexity": int(complexity),
423
  "verbosity": int(verbosity),
 
434
  "threat": int(threat),
435
  "gun": int(gun),
436
  "controlled_substance": int(controlled_substance),
437
+ "criminal_planing": int(criminal_planing),
438
  "privacy": int(privacy),
439
  "harassment": int(harassment),
440
  "profanity": int(profanity),
441
+ "political_content": int(political_content),
442
  "moral_judgement": int(moral_judgement),
443
  "typos": int(typos)
444
  }
 
572
  gr_score_reset = gr.Button("スコアリセット", interactive=False)
573
 
574
  gr_score_detail = gr.Slider(label="総合スコア 【必須】",value=3, minimum=1, maximum=5, step=1, interactive=False)
575
+ gr_eval_annotation_explain = gr.Markdown("詳細アノテーション (5点満点)")
576
 
577
  with gr.Row() :
578
+ gr_helpfulness = gr.Slider(label="有用性", minimum=0, value=0, maximum=5, step=1, interactive=False)
579
+ gr_correctness = gr.Slider(label="正確さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
580
  gr_coherence = gr.Slider(label="一貫性", minimum=0, value=0, maximum=5, step=1, interactive=False)
581
  gr_complexity = gr.Slider(label="複雑さ", minimum=0, value=0, maximum=5, step=1, interactive=False)
582
 
 
612
 
613
  with gr.Row():
614
  gr_controlled_substance = gr.Radio(label="規制対象物質", choices=["不明", "有", "無"], value="不明", interactive=False)
615
+ gr_criminal_planing = gr.Radio(label="犯罪計画", choices=["不明", "有", "無"], value="不明", interactive=False)
616
 
617
  with gr.Row():
618
  gr_privacy = gr.Radio(label="個人情報", choices=["不明", "有", "無"], value="不明", interactive=False)
 
630
 
631
  # slicer
632
  g_score: int,
633
+ g_helpfulness: int,
634
+ g_correctness: int,
635
+ g_coherence: int,
636
  g_complexity: int,
637
  g_verbosity: int,
638
  g_humor: int,
 
650
  g_threat: str,
651
  g_gun: str,
652
  g_controlled_substance: str,
653
+ g_criminal_planing: str,
654
  g_privacy: str,
655
  g_harassment: str,
656
  g_profanity: str,
 
673
 
674
  # 送信用データ更新
675
  score.value = g_score
676
+ helpfulness.value = g_helpfulness
677
+ correctness.value = g_correctness
678
+ coherence.value = g_coherence
679
  complexity.value = g_complexity
680
  verbosity.value = g_verbosity
681
  humor.value = g_humor
 
692
  threat.value = checkbox_to_int(g_threat)
693
  gun.value = checkbox_to_int(g_gun)
694
  controlled_substance.value = checkbox_to_int(g_controlled_substance)
695
+ criminal_planing.value = checkbox_to_int(g_criminal_planing)
696
  privacy.value = checkbox_to_int(g_privacy)
697
  harassment.value = checkbox_to_int(g_harassment)
698
  profanity.value = checkbox_to_int(g_profanity)
 
726
  is_proofreading_2.value,
727
  answer_text_2.value,
728
  score.value,
729
+ helpfulness.value,
730
+ correctness.value,
731
+ coherence.value,
732
  complexity.value,
733
  verbosity.value,
734
  humor.value,
 
744
  threat.value,
745
  gun.value,
746
  controlled_substance.value,
747
+ criminal_planing.value,
748
  privacy.value,
749
  harassment.value,
750
  profanity.value,
 
773
  inputs=[
774
  gr_answer_text_3_1,
775
  gr_score_detail,
776
+ gr_helpfulness,
777
+ gr_correctness,
778
  gr_coherence,
779
  gr_complexity,
780
  gr_verbosity,
 
791
  gr_threat,
792
  gr_gun,
793
  gr_controlled_substance,
794
+ gr_criminal_planing,
795
  gr_privacy,
796
  gr_harassment,
797
  gr_profanity,
 
808
  gr_question_text_3_1,
809
  gr_answer_text_3_1,
810
  gr_score_detail,
811
+ gr_helpfulness,
812
+ gr_correctness,
813
  gr_coherence,
814
  gr_complexity,
815
  gr_verbosity,
 
826
  gr_threat,
827
  gr_gun,
828
  gr_controlled_substance,
829
+ gr_criminal_planing,
830
  gr_privacy,
831
  gr_harassment,
832
  gr_profanity,
 
941
  is_proofreading_2.value,
942
  answer_text_2.value,
943
  score.value,
944
+ helpfulness.value,
945
+ correctness.value,
946
+ coherence.value,
947
  complexity.value,
948
  verbosity.value,
949
  humor.value,
 
959
  threat.value,
960
  gun.value,
961
  controlled_substance.value,
962
+ criminal_planing.value,
963
  privacy.value,
964
  harassment.value,
965
  profanity.value,
 
1003
  gr_question_text_3_1,
1004
  gr_answer_text_3_1,
1005
  gr_score_detail,
1006
+ gr_helpfulness,
1007
+ gr_correctness,
1008
  gr_coherence,
1009
  gr_complexity,
1010
  gr_verbosity,
 
1021
  gr_threat,
1022
  gr_gun,
1023
  gr_controlled_substance,
1024
+ gr_criminal_planing,
1025
  gr_privacy,
1026
  gr_harassment,
1027
  gr_profanity,
 
1057
  gr_question_text_3_1,
1058
  gr_answer_text_3_1,
1059
  gr_score_detail,
1060
+ gr_helpfulness,
1061
+ gr_correctness,
1062
  gr_coherence,
1063
  gr_complexity,
1064
  gr_verbosity,
 
1075
  gr_threat,
1076
  gr_gun,
1077
  gr_controlled_substance,
1078
+ gr_criminal_planing,
1079
  gr_privacy,
1080
  gr_harassment,
1081
  gr_profanity,
 
1111
  gr_question_text_3_1,
1112
  gr_answer_text_3_1,
1113
  gr_score_detail,
1114
+ gr_helpfulness,
1115
+ gr_correctness,
1116
  gr_coherence,
1117
  gr_complexity,
1118
  gr_verbosity,
 
1129
  gr_threat,
1130
  gr_gun,
1131
  gr_controlled_substance,
1132
+ gr_criminal_planing,
1133
  gr_privacy,
1134
  gr_harassment,
1135
  gr_profanity,
 
1150
  gr_question_text_3_1,
1151
  gr_answer_text_3_1,
1152
  gr_score_detail,
1153
+ gr_helpfulness,
1154
+ gr_correctness,
1155
  gr_coherence,
1156
  gr_complexity,
1157
  gr_verbosity,
 
1168
  gr_threat,
1169
  gr_gun,
1170
  gr_controlled_substance,
1171
+ gr_criminal_planing,
1172
  gr_privacy,
1173
  gr_harassment,
1174
  gr_profanity,
 
1186
  gr_question_text_3_1,
1187
  gr_answer_text_3_1,
1188
  gr_score_detail,
1189
+ gr_helpfulness,
1190
+ gr_correctness,
1191
  gr_coherence,
1192
  gr_complexity,
1193
  gr_verbosity,
 
1204
  gr_threat,
1205
  gr_gun,
1206
  gr_controlled_substance,
1207
+ gr_criminal_planing,
1208
  gr_privacy,
1209
  gr_harassment,
1210
  gr_profanity,
 
1222
  gr_question_text_3_1,
1223
  gr_answer_text_3_1,
1224
  gr_score_detail,
1225
+ gr_helpfulness,
1226
+ gr_correctness,
1227
  gr_coherence,
1228
  gr_complexity,
1229
  gr_verbosity,
 
1240
  gr_threat,
1241
  gr_gun,
1242
  gr_controlled_substance,
1243
+ gr_criminal_planing,
1244
  gr_privacy,
1245
  gr_harassment,
1246
  gr_profanity,
 
1258
  gr_question_text_3_1,
1259
  gr_answer_text_3_1,
1260
  gr_score_detail,
1261
+ gr_helpfulness,
1262
+ gr_correctness,
1263
  gr_coherence,
1264
  gr_complexity,
1265
  gr_verbosity,
 
1276
  gr_threat,
1277
  gr_gun,
1278
  gr_controlled_substance,
1279
+ gr_criminal_planing,
1280
  gr_privacy,
1281
  gr_harassment,
1282
  gr_profanity,
 
1294
  gr_question_text_3_1,
1295
  gr_answer_text_3_1,
1296
  gr_score_detail,
1297
+ gr_helpfulness,
1298
+ gr_correctness,
1299
  gr_coherence,
1300
  gr_complexity,
1301
  gr_verbosity,
 
1312
  gr_threat,
1313
  gr_gun,
1314
  gr_controlled_substance,
1315
+ gr_criminal_planing,
1316
  gr_privacy,
1317
  gr_harassment,
1318
  gr_profanity,
 
1325
  def score_reset_display():
1326
  # gr.State
1327
  score.value = 3
1328
+ helpfulness.value = 0
1329
+ correctness.value = 0
1330
+ coherence.value = 0
1331
  complexity.value = 0
1332
  verbosity.value = 0
1333
  humor.value = 0
 
1342
  suicide.value = 0
1343
  threat.value = 0
1344
  gun.value = 0
1345
+ controlled_substance.value = 0
1346
+ criminal_planing.value = 0
1347
  privacy.value = 0
1348
  harassment.value = 0
1349
  profanity.value = 0
 
1362
  inputs=[],
1363
  outputs=[
1364
  gr_score_detail,
1365
+ gr_helpfulness,
1366
+ gr_correctness,
1367
  gr_coherence,
1368
  gr_complexity,
1369
  gr_verbosity,
 
1380
  gr_threat,
1381
  gr_gun,
1382
  gr_controlled_substance,
1383
+ gr_criminal_planing,
1384
  gr_privacy,
1385
  gr_harassment,
1386
  gr_profanity,
 
1394
  gr_data_load_btn.click(
1395
  dataset_load_fn,
1396
  inputs=None,
1397
+ # textをセットし、interactiveをTrueにする (UI更新)
1398
  outputs=[
1399
  gr_question_text_1_1,
1400
  gr_answer_text_1_1,
 
1416
  gr_submit_score,
1417
  gr_score_detail,
1418
  gr_score_reset,
1419
+ gr_helpfulness,
1420
+ gr_correctness,
1421
  gr_coherence,
1422
  gr_complexity,
1423
  gr_verbosity,
 
1434
  gr_threat,
1435
  gr_gun,
1436
  gr_controlled_substance,
1437
+ gr_criminal_planing,
1438
  gr_privacy,
1439
  gr_harassment,
1440
  gr_profanity,