Dingli Yu commited on
Commit
92fb594
1 Parent(s): 59315a1
Files changed (1) hide show
  1. app.py +433 -0
app.py ADDED
@@ -0,0 +1,433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from functools import partial
2
+ import gradio as gr
3
+ import os
4
+ import csv
5
+ import pandas as pd
6
+ import pickle
7
+
8
+ def load_results(gen_file, grader_file, exp_name, model_name, grader):
9
+ record = []
10
+ if 'llama' in model_name:
11
+ version = exp_name.split('_')[2]
12
+ k = exp_name.split('_')[0]
13
+ if k == 'k2' and version == 'v8':
14
+ return []
15
+ if k != 'k2' and version == 'v9':
16
+ return []
17
+ # read csv file
18
+ with open(gen_file, 'r') as file:
19
+ reader = csv.reader(file)
20
+ gen = list(reader)
21
+ with open(grader_file, 'r') as file:
22
+ reader = csv.reader(file)
23
+ grade = list(reader)
24
+ # print(gen_file, grader_file)
25
+ # print(gen[0], grade[0])
26
+ if len(gen) != len(grade):
27
+ # print("ERROR: len(gen) != len(grade)")
28
+ # print(gen_file, grader_file)
29
+ return []
30
+ else:
31
+ # print(gen[0], grade[0])
32
+ # exit(0)
33
+ #['k', 'skills', 'topic', 'system prompt', '[INST]_0', '[/INST]_0', 'model_input_0', 'text_0', '[INST]_1', '[/INST]_1', 'model_input_1', 'text_1']
34
+ # ['k', 'skills', 'topic', 'system prompt', 'user_0', 'assistant_0', 'model_input_0', 'score_0', 'score_extracted_0', 'points_0', 'num_sentences_manual_in_student_answer_0', 'true_sentence_lim_pt_0', 'num_sentences_extracted_eq_num_sentences_model_0']
35
+ for i in range(1, len(gen)):
36
+ skills = [skill.strip() for skill in gen[i][1].split(',')]
37
+ topic = gen[i][2]
38
+ assert(skills == [skill.strip() for skill in grade[i][1].split(',')])
39
+ assert(topic == grade[i][2])
40
+ points = grade[i][9].split(',')
41
+ if len(points) < len(skills):
42
+ points = points + ['0.0'] * (len(skills) - len(points))
43
+ points[-1] = grade[i][11]
44
+ points_no_skill_name = points.copy()
45
+ answer = gen[i][-1]
46
+ for skill_id, skill in enumerate(skills):
47
+ simple_skill = skill.split('(')[0].strip()
48
+ if simple_skill in answer:
49
+ # print(skill_id, skill, simple_skill, answer, points_no_skill_name, points)
50
+ points_no_skill_name[skill_id] = '0.0'
51
+ record.append({
52
+ 'k': gen[i][0],
53
+ 'exp_name': exp_name,
54
+ 'model': model_name,
55
+ 'grader': 'gpt-4' if 'gpt-4' in grader else 'llama-2-70b',
56
+ 'grade_run': grader,
57
+ 'skills': '\n\n'.join(skills),
58
+ 'topic': topic,
59
+ 'topic+skills': '+'.join([topic] + sorted(skills)),
60
+ 'gen_prompt': gen[i][4].split('examples for the concepts:')[1].split('Please start the minimal natural')[0].replace('\n', '\n\n'),
61
+ 'gen': gen[i][-3].replace('\n', '\n\n'),
62
+ 'grade': grade[i][5].replace('\n', '\n\n'),
63
+ 'points': ' '.join([(g[:-2] if g[-2:] == '.0' else g) for g in points]),
64
+ 'points_no_skill_name': ' '.join([(g[:-2] if g[-2:] == '.0' else g) for g in points_no_skill_name]),
65
+ # 'sent_limit_point': grade[i][11]
66
+ })
67
+ return record
68
+
69
+
70
+ def load_all_results(path='final'):
71
+ all_results = []
72
+ for exp_name in os.listdir(path):
73
+ if os.path.isfile(os.path.join(path, exp_name)):
74
+ continue
75
+ for model_name in os.listdir(os.path.join(path, exp_name)):
76
+ gen_file = os.path.join(path, exp_name, model_name, "records.csv")
77
+ if os.path.exists(gen_file) and os.path.isdir(os.path.join(path, exp_name, model_name, 'graded')):
78
+ for grader in os.listdir(os.path.join(path, exp_name, model_name, 'graded')):
79
+ grader_file = os.path.join(path, exp_name, model_name, 'graded', grader, "records.csv")
80
+ if os.path.exists(grader_file):
81
+ all_results += load_results(gen_file, grader_file, exp_name, model_name, grader)
82
+ return pd.DataFrame(all_results)
83
+
84
+
85
+ block_css = """
86
+ #a {
87
+ background-color: #DEEBF7;
88
+ font-size: 20px;
89
+ }
90
+ #b {
91
+ background-color: #E2F0D9;
92
+ font-size: 20px;
93
+ }
94
+ #c {
95
+ background-color: #FFF2CC;
96
+ font-size: 20px;
97
+ }
98
+ #d {
99
+ background-color: #FBE5D6;
100
+ font-size: 20px;
101
+ }
102
+ """
103
+
104
+ from Levenshtein import distance
105
+
106
+ def best_match(comb, comb_list):
107
+ if comb == '':
108
+ return comb_list[0]
109
+ dist = [distance(comb.split('+'), comb_.split('+')) for comb_ in comb_list]
110
+ return comb_list[dist.index(min(dist))]
111
+
112
+ class Tracker:
113
+ def __init__(self, df) -> None:
114
+ self.df = df
115
+ self.value = {k: '' for k in ['k', 'k_list', 'comb', 'comb_list', 'model', 'model_list', 'exp_name', 'exp_name_list', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_list', 'grader_run', 'grader_run_list', 'points', 'grade']}
116
+ # self.value = {k: '' for k in ['k', 'k_list', 'comb', 'comb_list', 'model', 'model_list', 'exp_name', 'exp_name_list', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_list', 'grader_run', 'grader_run_list', 'points', 'sent_limit_point', 'grade']}
117
+ self.value = self.update(self.value)
118
+ self.value = [self.value.copy() for _ in range(5)]
119
+ self.component = [{k: '' for k in ['k', 'comb', 'model', 'exp_name', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_run', 'points', 'grade']} for _ in range(5)]
120
+ # self.component = [{k: '' for k in ['k', 'comb', 'model', 'exp_name', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_run', 'points', 'sent_limit_point', 'grade']} for _ in range(5)]
121
+
122
+ def update(self, value):
123
+ cdf = self.df
124
+ k, comb, model, exp_name, grader, grader_run = value['k'], value['comb'], value['model'], value['exp_name'], value['grader'], value['grader_run']
125
+ k_list = sorted(list(cdf.k.unique()))
126
+ if k not in k_list:
127
+ k = k_list[0]
128
+ value['k'] = k
129
+ value['k_list'] = k_list
130
+ cdf = cdf[cdf.k==k]
131
+
132
+ comb_list = sorted(list(cdf['topic+skills'].unique()))
133
+ if comb not in comb_list:
134
+ comb = best_match(comb, comb_list)
135
+ value['comb'] = comb
136
+ value['comb_list'] = comb_list
137
+ cdf = cdf[cdf['topic+skills']==comb]
138
+
139
+ model_list = sorted(list(cdf['model'].unique()))
140
+ if model not in model_list:
141
+ model = model_list[0]
142
+ value['model'] = model
143
+ value['model_list'] = model_list
144
+ cdf = cdf[cdf.model==model]
145
+
146
+ exp_name_list = sorted(list(cdf['exp_name'].unique()))
147
+ if exp_name not in exp_name_list:
148
+ exp_name = exp_name_list[0]
149
+ value['exp_name'] = exp_name
150
+ value['exp_name_list'] = exp_name_list
151
+ cdf = cdf[cdf.exp_name==exp_name]
152
+
153
+ value['topic'] = "*Topic*: " + cdf['topic'].unique()[0]
154
+ value['skills'] = "*Skills*: \n\n" + cdf['skills'].unique()[0]
155
+ value['gen_prompt'] = "*Skill Definition and Example*:\n\n" + cdf['gen_prompt'].unique()[0]
156
+ value['gen'] = "*Model Answer*:\n\n" + cdf['gen'].unique()[0]
157
+
158
+ grader_list = sorted(list(cdf['grader'].unique()))
159
+ if grader not in grader_list:
160
+ grader = grader_list[0]
161
+ value['grader'] = grader
162
+ value['grader_list'] = grader_list
163
+ cdf = cdf[cdf.grader==grader]
164
+
165
+ grader_run_list = sorted(list(cdf['grade_run'].unique()))
166
+ if grader_run not in grader_run_list:
167
+ grader_run = grader_run_list[0]
168
+ value['grader_run'] = grader_run
169
+ value['grader_run_list'] = grader_run_list
170
+ cdf = cdf[cdf.grade_run==grader_run]
171
+
172
+ value['points'] = "Points: " + cdf['points'].unique()[0] + "\n\n(After deducting points for explicitly mentioning skill names: " + cdf['points_no_skill_name'].unique()[0] + ")"
173
+ # value['sent_limit_point'] = "within sentence limit? " + cdf['sent_limit_point'].unique()[0]
174
+ value['grade'] = cdf['grade'].unique()[0]
175
+
176
+ return value
177
+
178
+ def procedure(self, c):
179
+
180
+ input_list = []
181
+ output_list = []
182
+ fn_list = []
183
+ # binding = [['k', [0, 1, 3]],
184
+ # ['comb', [0, 1, 3]],
185
+ # ['model', [0, 1, 3]],
186
+ # ['exp_name', [0, 1, 3]],
187
+ # ['grader', [0, 1, 3]],
188
+ # ['grader_run', [0, 1, 3]],
189
+ # ]
190
+ binding = []
191
+ idx = -1
192
+ for i in range(5):
193
+ for k, v in self.component[i].items():
194
+ if v is c:
195
+ idx = i
196
+ key = k
197
+ break
198
+ if idx != -1:
199
+ break
200
+ assert(idx != -1)
201
+ # print(id(c), id(self.component[idx][key]), idx, key)
202
+ sync_list = []
203
+ for b in binding:
204
+ if (key == b[0]) and (idx in b[1]):
205
+ sync_list = [j for j in b[1] if j != idx]
206
+ sync_component = [self.component[j][key] for j in sync_list]
207
+ # print(c.label, key, idx, sync_list)
208
+
209
+ def sync(v, sync_list=[0]):
210
+ return [gr.Dropdown.update(value=v) for _ in range(len(sync_list))]
211
+
212
+ if len(sync_list) > 0:
213
+ input_list.append(c)
214
+ output_list.append(sync_component)
215
+ fn_list.append(partial(sync, sync_list=sync_list))
216
+
217
+ def update(k, comb, model, exp_name, grader, grader_run):
218
+ value = {
219
+ 'k': k,
220
+ 'k_list': '',
221
+ 'comb': comb,
222
+ 'comb_list': '',
223
+ 'model': model,
224
+ 'model_list': '',
225
+ 'exp_name': exp_name,
226
+ 'exp_name_list': '',
227
+ 'topic': '',
228
+ 'skills': '',
229
+ 'gen_prompt': '',
230
+ 'gen': '',
231
+ 'grader': grader,
232
+ 'grader_list': '',
233
+ 'grader_run': grader_run,
234
+ 'grader_run_list': '',
235
+ 'points': '',
236
+ 'sent_limit_point': '',
237
+ 'grade': ''
238
+ }
239
+ value = self.update(value)
240
+ return [gr.Dropdown.update(value=value['k'], choices=value['k_list']),
241
+ gr.Dropdown.update(value=value['comb'], choices=value['comb_list']),
242
+ gr.Dropdown.update(value=value['model'], choices=value['model_list']),
243
+ gr.Dropdown.update(value=value['exp_name'], choices=value['exp_name_list']),
244
+ value['topic'],
245
+ value['skills'],
246
+ value['gen_prompt'],
247
+ value['gen'],
248
+ gr.Dropdown.update(value=value['grader'], choices=value['grader_list']),
249
+ gr.Dropdown.update(value=value['grader_run'], choices=value['grader_run_list']),
250
+ value['points'],
251
+ # value['sent_limit_point'],
252
+ value['grade']
253
+ ]
254
+
255
+ sync_list += [idx]
256
+ update_list = []
257
+ for i in range(5):
258
+ for j in sync_list:
259
+ if self.component[j][key] is self.component[i][key]:
260
+ update_list.append(i)
261
+ break
262
+
263
+ for j in update_list:
264
+ input_list.append([self.component[j][k] for k in ['k', 'comb', 'model', 'exp_name', 'grader', 'grader_run']])
265
+ output_list.append([self.component[j][k] for k in ['k', 'comb', 'model', 'exp_name', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_run', 'points', 'grade']])
266
+ # output_list.append([self.component[j][k] for k in ['k', 'comb', 'model', 'exp_name', 'topic', 'skills', 'gen_prompt', 'gen', 'grader', 'grader_run', 'points', 'sent_limit_point', 'grade']])
267
+ fn_list.append(update)
268
+
269
+ return input_list, output_list, fn_list
270
+
271
+ def build_demo(df):
272
+
273
+ tracker = Tracker(df)
274
+
275
+ with gr.Blocks(
276
+ title="Skill Mix Browsing",
277
+ theme=gr.themes.Base(text_size=gr.themes.sizes.text_lg),
278
+ css=block_css,
279
+ ) as demo:
280
+ gr.Markdown(
281
+ """
282
+ # Skill Mix Browsing
283
+ """
284
+ )
285
+
286
+ with gr.Tab('Browse Single Generation'):
287
+ v = tracker.value[0]
288
+ with gr.Row():
289
+ k = gr.Dropdown(choices=v['k_list'], value=v['k'], label="k")
290
+ tracker.component[0]['k'] = k
291
+ comb = gr.Dropdown(choices=v['comb_list'], value=v['comb'], label="topic+skills")
292
+ tracker.component[0]['comb'] = comb
293
+ with gr.Row():
294
+ with gr.Column():
295
+ with gr.Row():
296
+ model = gr.Dropdown(choices=v['model_list'], value=v['model'], label="model")
297
+ tracker.component[0]['model'] = model
298
+ exp_name = gr.Dropdown(choices=v['exp_name_list'], value=v['exp_name'], label="exp_name")
299
+ tracker.component[0]['exp_name'] = exp_name
300
+ with gr.Row():
301
+ topic = gr.Markdown(value=v['topic'], elem_id='a')
302
+ tracker.component[0]['topic'] = topic
303
+ skills = gr.Markdown(value=v['skills'], elem_id='a')
304
+ tracker.component[0]['skills'] = skills
305
+ gen = gr.Markdown(value=v['gen'], elem_id='b')
306
+ tracker.component[0]['gen'] = gen
307
+ gen_prompt = gr.Markdown(value=v['gen_prompt'], elem_id='a')
308
+ tracker.component[0]['gen_prompt'] = gen_prompt
309
+ with gr.Column():
310
+ with gr.Row():
311
+ grader = gr.Dropdown(choices=v['grader_list'], value=v['grader'], label="grader")
312
+ tracker.component[0]['grader'] = grader
313
+ grader_run = gr.Dropdown(choices=v['grader_run_list'], value=v['grader_run'], label="grader_run")
314
+ tracker.component[0]['grader_run'] = grader_run
315
+ points = gr.Markdown(value=v['points'], elem_id='c')
316
+ tracker.component[0]['points'] = points
317
+ # sent_limit_point = gr.Markdown(value=v['sent_limit_point'], elem_id='c')
318
+ # tracker.component[0]['sent_limit_point'] = sent_limit_point
319
+ grade = gr.Markdown(value=v['grade'], elem_id='d')
320
+ tracker.component[0]['grade'] = grade
321
+ with gr.Tab('Compare Two Generations'):
322
+ v = tracker.value[1]
323
+ with gr.Row():
324
+ k = gr.Dropdown(choices=v['k_list'], value=v['k'], label="k")
325
+ tracker.component[1]['k'] = tracker.component[2]['k'] = k
326
+ comb = gr.Dropdown(choices=v['comb_list'], value=v['comb'], label="topic+skills")
327
+ tracker.component[1]['comb'] = tracker.component[2]['comb'] = comb
328
+ with gr.Row():
329
+ for col in range(1, 3):
330
+ v = tracker.value[col]
331
+ with gr.Column():
332
+ with gr.Row():
333
+ model = gr.Dropdown(choices=v['model_list'], value=v['model'], label="model")
334
+ tracker.component[col]['model'] = model
335
+ exp_name = gr.Dropdown(choices=v['exp_name_list'], value=v['exp_name'], label="exp_name")
336
+ tracker.component[col]['exp_name'] = exp_name
337
+ with gr.Row():
338
+ topic = gr.Markdown(value=v['topic'], elem_id='a')
339
+ tracker.component[col]['topic'] = topic
340
+ skills = gr.Markdown(value=v['skills'], elem_id='a')
341
+ tracker.component[col]['skills'] = skills
342
+ gen = gr.Markdown(value=v['gen'], elem_id='b')
343
+ tracker.component[col]['gen'] = gen
344
+ with gr.Row():
345
+ grader = gr.Dropdown(choices=v['grader_list'], value=v['grader'], label="grader")
346
+ tracker.component[col]['grader'] = grader
347
+ grader_run = gr.Dropdown(choices=v['grader_run_list'], value=v['grader_run'], label="grader_run")
348
+ tracker.component[col]['grader_run'] = grader_run
349
+ points = gr.Markdown(value=v['points'], elem_id='c')
350
+ tracker.component[col]['points'] = points
351
+ # sent_limit_point = gr.Markdown(value=v['sent_limit_point'], elem_id='c')
352
+ # tracker.component[col]['sent_limit_point'] = sent_limit_point
353
+ gen_prompt = gr.Markdown(value=v['gen_prompt'], elem_id='a')
354
+ tracker.component[col]['gen_prompt'] = gen_prompt
355
+
356
+ grade = gr.Markdown(value=v['grade'], elem_id='d')
357
+ tracker.component[col]['grade'] = grade
358
+
359
+ # with gr.Tab('One Generation Two Grading'):
360
+ # v = tracker.value[3]
361
+ # with gr.Row():
362
+ # k = gr.Dropdown(choices=v['k_list'], value=v['k'], label="k")
363
+ # tracker.component[3]['k'] = tracker.component[4]['k'] = k
364
+ # comb = gr.Dropdown(choices=v['comb_list'], value=v['comb'], label="topic+skills")
365
+ # tracker.component[3]['comb'] = tracker.component[4]['comb'] = comb
366
+ # with gr.Row():
367
+ # model = gr.Dropdown(choices=v['model_list'], value=v['model'], label="model")
368
+ # tracker.component[3]['model'] = tracker.component[4]['model'] = model
369
+ # exp_name = gr.Dropdown(choices=v['exp_name_list'], value=v['exp_name'], label="exp_name")
370
+ # tracker.component[3]['exp_name'] = tracker.component[4]['exp_name'] = exp_name
371
+ # with gr.Row():
372
+ # topic = gr.Markdown(value=v['topic'], elem_id='a')
373
+ # tracker.component[3]['topic'] = tracker.component[4]['topic'] = topic
374
+ # skills = gr.Markdown(value=v['skills'], elem_id='a')
375
+ # tracker.component[3]['skills'] = tracker.component[4]['skills'] = skills
376
+ # gen = gr.Markdown(value=v['gen'], elem_id='b')
377
+ # tracker.component[3]['gen'] = tracker.component[4]['gen'] = gen
378
+ # with gr.Row():
379
+ # for col in range(3, 5):
380
+ # v = tracker.value[col]
381
+ # with gr.Column():
382
+ # with gr.Row():
383
+ # grader = gr.Dropdown(choices=v['grader_list'], value=v['grader'], label="grader")
384
+ # tracker.component[col]['grader'] = grader
385
+ # grader_run = gr.Dropdown(choices=v['grader_run_list'], value=v['grader_run'], label="grader_run")
386
+ # tracker.component[col]['grader_run'] = grader_run
387
+ # points = gr.Markdown(value=v['points'], elem_id='c')
388
+ # tracker.component[col]['points'] = points
389
+ # sent_limit_point = gr.Markdown(value=v['sent_limit_point'], elem_id='c')
390
+ # tracker.component[col]['sent_limit_point'] = sent_limit_point
391
+ # grade = gr.Markdown(value=v['grade'], elem_id='d')
392
+ # tracker.component[col]['grade'] = grade
393
+
394
+ # gen_prompt = gr.Markdown(value=v['gen_prompt'], elem_id='a')
395
+ # tracker.component[3]['gen_prompt'] = tracker.component[4]['gen_prompt'] = gen_prompt
396
+
397
+
398
+
399
+
400
+ all_components = sum([list(tracker.component[i].values()) for i in range(5)], [])
401
+ all_components = [c for c in all_components if c != '']
402
+ all_components = list(set(all_components))
403
+ # print(all_components)
404
+ for c in all_components:
405
+ input_list, output_list, fn_list = tracker.procedure(c)
406
+ if len(fn_list) > 0:
407
+ if len(fn_list) == 1:
408
+ c.change(fn_list[0], input_list[0], output_list[0])
409
+ elif len(fn_list) == 2:
410
+ c.change(fn_list[0], input_list[0], output_list[0]).then(fn_list[1], input_list[1], output_list[1])
411
+ elif len(fn_list) == 3:
412
+ c.change(fn_list[0], input_list[0], output_list[0]).then(fn_list[1], input_list[1], output_list[1]).then(fn_list[2], input_list[2], output_list[2])
413
+ elif len(fn_list) == 4:
414
+ c.change(fn_list[0], input_list[0], output_list[0]).then(fn_list[1], input_list[1], output_list[1]).then(fn_list[2], input_list[2], output_list[2]).then(fn_list[3], input_list[3], output_list[3])
415
+ elif len(fn_list) == 5:
416
+ c.change(fn_list[0], input_list[0], output_list[0]).then(fn_list[1], input_list[1], output_list[1]).then(fn_list[2], input_list[2], output_list[2]).then(fn_list[3], input_list[3], output_list[3]).then(fn_list[4], input_list[4], output_list[4])
417
+ elif len(fn_list) == 6:
418
+ c.change(fn_list[0], input_list[0], output_list[0]).then(fn_list[1], input_list[1], output_list[1]).then(fn_list[2], input_list[2], output_list[2]).then(fn_list[3], input_list[3], output_list[3]).then(fn_list[4], input_list[4], output_list[4]).then(fn_list[5], input_list[5], output_list[5])
419
+ else:
420
+ raise NotImplementedError
421
+ return demo
422
+
423
+
424
+ if __name__ == '__main__':
425
+ # df = load_all_results(path="../../on_released_topics_and_skills")
426
+
427
+ # pickle.dump(df, open('on_released_topics_and_skills.pkl', 'wb'))
428
+ df = pickle.load(open('on_released_topics_and_skills.pkl', 'rb'))
429
+
430
+ demo = build_demo(df)
431
+
432
+ # demo.launch(share=True)
433
+ demo.launch()