asahi417 commited on
Commit
2160c60
β€’
1 Parent(s): 620ed38

model update

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. README.md +377 -51
  2. eval/{metric.first.answer.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.first.answer.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  3. eval/{metric.first.answer.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.first.answer.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  4. eval/{metric.first.answer.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.first.answer.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  5. eval/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  6. eval/{metric.first.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  7. eval/{metric.first.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  8. eval/{metric.last.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  9. eval/{metric.last.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  10. eval/{metric.last.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  11. eval/{metric.long.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  12. eval/{metric.long.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  13. eval/{metric.long.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  14. eval/{metric.middle.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  15. eval/{metric.middle.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  16. eval/{metric.middle.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  17. eval/{metric.short.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} +0 -0
  18. eval/{metric.short.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} +0 -0
  19. eval/{metric.short.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.sentence_answer.question.lmqg_qg_squad.default.json} +0 -0
  20. eval/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squad.default.txt} +0 -0
  21. eval/{samples.test.hyp.paragraph_sentence.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.paragraph_sentence.question.lmqg_qg_squad.default.txt} +0 -0
  22. eval/{samples.test.hyp.sentence_answer.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.sentence_answer.question.lmqg_qg_squad.default.txt} +0 -0
  23. eval/{samples.validation.hyp.paragraph_answer.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.paragraph_answer.question.lmqg_qg_squad.default.txt} +0 -0
  24. eval/{samples.validation.hyp.paragraph_sentence.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.paragraph_sentence.question.lmqg_qg_squad.default.txt} +0 -0
  25. eval/{samples.validation.hyp.sentence_answer.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.sentence_answer.question.lmqg_qg_squad.default.txt} +0 -0
  26. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.amazon.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json} +0 -0
  27. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.default.json} +0 -0
  28. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.new_wiki.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json} +0 -0
  29. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.nyt.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json} +0 -0
  30. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.reddit.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json} +0 -0
  31. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.books.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.books.json} +0 -0
  32. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.default.json} +0 -0
  33. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.electronics.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json} +0 -0
  34. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.grocery.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.grocery.json} +0 -0
  35. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.movies.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.movies.json} +0 -0
  36. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.restaurants.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json} +0 -0
  37. eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.tripadvisor.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.json} +0 -0
  38. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.amazon.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.amazon.txt} +0 -0
  39. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.default.txt} +0 -0
  40. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.new_wiki.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.txt} +0 -0
  41. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.nyt.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.nyt.txt} +0 -0
  42. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.reddit.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.reddit.txt} +0 -0
  43. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.books.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.books.txt} +0 -0
  44. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.default.txt} +0 -0
  45. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.electronics.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.electronics.txt} +0 -0
  46. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.grocery.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.grocery.txt} +0 -0
  47. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.movies.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.movies.txt} +0 -0
  48. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.restaurants.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.restaurants.txt} +0 -0
  49. eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.tripadvisor.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.txt} +0 -0
  50. eval_ood/{samples.validation.hyp.paragraph_answer.question.asahi417_qg_squadshifts.amazon.txt β†’ samples.validation.hyp.paragraph_answer.question.lmqg_qg_squadshifts.amazon.txt} +0 -0
README.md CHANGED
@@ -1,78 +1,404 @@
 
1
  ---
2
- language:
3
- - en
4
- tags:
5
- - question generation
6
- license: mit
7
- datasets:
8
- - asahi417/qg_squad
9
  metrics:
10
- - bleu
11
  - meteor
12
- - rouge
13
  - bertscore
14
  - moverscore
 
 
 
 
 
 
15
  widget:
16
- - text: "<hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records."
17
- example_title: "Example 1"
18
- - text: "Beyonce further expanded her acting career, starring as blues singer <hl> Etta James <hl> in the 2008 musical biopic, Cadillac Records."
19
- example_title: "Example 2"
20
- - text: "Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, <hl> Cadillac Records <hl> ."
21
- example_title: "Example 3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  ---
23
 
24
- # BART LARGE fine-tuned for English Question Generation
25
- BART LARGE Model fine-tuned on English question generation dataset (SQuAD) with an extensive hyper-parameter search.
26
- - [Online Demo](https://autoqg.net/)
27
- - [Project Repository](https://github.com/asahi417/lm-question-generation)
28
 
29
- ## Overview
30
 
31
- **Language model:** facebook/bart-large
32
- **Language:** English (en)
33
- **Downstream-task:** Question Generation
34
- **Training data:** SQuAD
35
- **Eval data:** SQuAD
36
- **Code:** See [our repository](https://github.com/asahi417/lm-question-generation)
 
37
 
38
- ## Usage
39
- ### In Transformers
40
  ```python
 
41
  from transformers import pipeline
42
 
43
- model_path = 'asahi417/lmqg-bart-large-squad'
44
  pipe = pipeline("text2text-generation", model_path)
45
 
46
- paragraph = 'Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records.'
47
- # highlight an answer in the paragraph to generate question
48
- answer = 'Etta James'
49
- highlight_token = '<hl>'
50
- input_text = paragraph.replace(answer, '{0} {1} {0}'.format(highlight_token, answer))
51
- input_text = 'generate question: {}'.format(input_text) # add task specific prefix
52
- generation = pipe(input_text)
53
- print(generation)
54
- >>> [{'generated_text': 'What is the name of the biopic that Beyonce starred in?'}]
55
  ```
56
 
57
- ## Evaluations
58
 
59
- Evaluation on the test set of [SQuAD QG dataset](https://huggingface.co/datasets/asahi417/qg_squad).
60
- The results are comparable with the [leaderboard](https://paperswithcode.com/sota/question-generation-on-squad11) and previous works.
61
- All evaluations were done using our [evaluation script](https://github.com/asahi417/lm-question-generation).
62
 
 
63
 
64
- | BLEU 4 | ROUGE L | METEOR | BERTScore | MoverScore |
65
- | ------ | -------- | ------ | --------- | ---------- |
66
- | 26.16 | 53.84 | 27.07 | 91.00 | 64.99 |
67
 
68
- - [metric file](https://huggingface.co/asahi417/lmqg-bart-large-squad/raw/main/eval/metric.first.sentence.paragraph_answer.question.asahi417_qg_squad.default.json)
69
 
70
 
71
- ## Fine-tuning Parameters
72
- We ran grid search to find the best hyper-parameters and continued fine-tuning until the validation metric decrease.
73
- The best hyper-parameters can be found [here](https://huggingface.co/asahi417/lmqg-bart-large-squad/raw/main/trainer_config.json), and fine-tuning script is released in [our repository](https://github.com/asahi417/lm-question-generation).
 
 
 
 
 
 
 
 
 
 
 
 
 
74
 
75
- ## Citation
76
- TBA
77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
 
 
 
 
 
1
+
2
  ---
3
+ license: cc-by-4.0
 
 
 
 
 
 
4
  metrics:
5
+ - bleu4
6
  - meteor
7
+ - rouge-l
8
  - bertscore
9
  - moverscore
10
+ language: en
11
+ datasets:
12
+ - lmqg/qg_squad
13
+ pipeline_tag: text2text-generation
14
+ tags:
15
+ - question generation
16
  widget:
17
+ - text: "generate question: <hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records."
18
+ example_title: "Question Generation Example 1"
19
+ - text: "generate question: Beyonce further expanded her acting career, starring as blues singer <hl> Etta James <hl> in the 2008 musical biopic, Cadillac Records."
20
+ example_title: "Question Generation Example 2"
21
+ - text: "generate question: Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, <hl> Cadillac Records <hl> ."
22
+ example_title: "Question Generation Example 3"
23
+ model-index:
24
+ - name: lmqg/bart-large-squad
25
+ results:
26
+ - task:
27
+ name: Text2text Generation
28
+ type: text2text-generation
29
+ dataset:
30
+ name: lmqg/qg_squad
31
+ type: default
32
+ args: default
33
+ metrics:
34
+ - name: BLEU4
35
+ type: bleu4
36
+ value: 0.26168385362299557
37
+ - name: ROUGE-L
38
+ type: rouge-l
39
+ value: 0.5384959163821219
40
+ - name: METEOR
41
+ type: meteor
42
+ value: 0.27073122286541956
43
+ - name: BERTScore
44
+ type: bertscore
45
+ value: 0.9100413219045603
46
+ - name: MoverScore
47
+ type: moverscore
48
+ value: 0.6499011626820898
49
+ - task:
50
+ name: Text2text Generation
51
+ type: text2text-generation
52
+ dataset:
53
+ name: lmqg/qg_squadshifts
54
+ type: reddit
55
+ args: reddit
56
+ metrics:
57
+ - name: BLEU4
58
+ type: bleu4
59
+ value: 0.059525104157825456
60
+ - name: ROUGE-L
61
+ type: rouge-l
62
+ value: 0.22365090580055863
63
+ - name: METEOR
64
+ type: meteor
65
+ value: 0.21499800504546457
66
+ - name: BERTScore
67
+ type: bertscore
68
+ value: 0.9095144685254328
69
+ - name: MoverScore
70
+ type: moverscore
71
+ value: 0.6059332247878408
72
+ - task:
73
+ name: Text2text Generation
74
+ type: text2text-generation
75
+ dataset:
76
+ name: lmqg/qg_squadshifts
77
+ type: new_wiki
78
+ args: new_wiki
79
+ metrics:
80
+ - name: BLEU4
81
+ type: bleu4
82
+ value: 0.11118273173452982
83
+ - name: ROUGE-L
84
+ type: rouge-l
85
+ value: 0.2967546690273089
86
+ - name: METEOR
87
+ type: meteor
88
+ value: 0.27315087810722966
89
+ - name: BERTScore
90
+ type: bertscore
91
+ value: 0.9322739617807421
92
+ - name: MoverScore
93
+ type: moverscore
94
+ value: 0.6623000084761579
95
+ - task:
96
+ name: Text2text Generation
97
+ type: text2text-generation
98
+ dataset:
99
+ name: lmqg/qg_subjqa
100
+ type: tripadvisor
101
+ args: tripadvisor
102
+ metrics:
103
+ - name: BLEU4
104
+ type: bleu4
105
+ value: 8.380171318718442e-07
106
+ - name: ROUGE-L
107
+ type: rouge-l
108
+ value: 0.1402922852924756
109
+ - name: METEOR
110
+ type: meteor
111
+ value: 0.1372146070365174
112
+ - name: BERTScore
113
+ type: bertscore
114
+ value: 0.8891002409937424
115
+ - name: MoverScore
116
+ type: moverscore
117
+ value: 0.5604572211470809
118
+ - task:
119
+ name: Text2text Generation
120
+ type: text2text-generation
121
+ dataset:
122
+ name: lmqg/qg_squadshifts
123
+ type: default
124
+ args: default
125
+ metrics:
126
+ - name: BLEU4
127
+ type: bleu4
128
+ value: 0.07839941048417529
129
+ - name: ROUGE-L
130
+ type: rouge-l
131
+ value: 0.25357667226247294
132
+ - name: METEOR
133
+ type: meteor
134
+ value: 0.24046838149047955
135
+ - name: BERTScore
136
+ type: bertscore
137
+ value: 0.9182198703598111
138
+ - name: MoverScore
139
+ type: moverscore
140
+ value: 0.6274693859765924
141
+ - task:
142
+ name: Text2text Generation
143
+ type: text2text-generation
144
+ dataset:
145
+ name: lmqg/qg_squadshifts
146
+ type: nyt
147
+ args: nyt
148
+ metrics:
149
+ - name: BLEU4
150
+ type: bleu4
151
+ value: 0.08117757543966063
152
+ - name: ROUGE-L
153
+ type: rouge-l
154
+ value: 0.25292097720734297
155
+ - name: METEOR
156
+ type: meteor
157
+ value: 0.25254205113198686
158
+ - name: BERTScore
159
+ type: bertscore
160
+ value: 0.9249009759439454
161
+ - name: MoverScore
162
+ type: moverscore
163
+ value: 0.6406329128556304
164
+ - task:
165
+ name: Text2text Generation
166
+ type: text2text-generation
167
+ dataset:
168
+ name: lmqg/qg_subjqa
169
+ type: restaurants
170
+ args: restaurants
171
+ metrics:
172
+ - name: BLEU4
173
+ type: bleu4
174
+ value: 1.1301750984972448e-06
175
+ - name: ROUGE-L
176
+ type: rouge-l
177
+ value: 0.13083168975354642
178
+ - name: METEOR
179
+ type: meteor
180
+ value: 0.12419733006916912
181
+ - name: BERTScore
182
+ type: bertscore
183
+ value: 0.8797711839570719
184
+ - name: MoverScore
185
+ type: moverscore
186
+ value: 0.5542757411268555
187
+ - task:
188
+ name: Text2text Generation
189
+ type: text2text-generation
190
+ dataset:
191
+ name: lmqg/qg_subjqa
192
+ type: electronics
193
+ args: electronics
194
+ metrics:
195
+ - name: BLEU4
196
+ type: bleu4
197
+ value: 0.00866799444965211
198
+ - name: ROUGE-L
199
+ type: rouge-l
200
+ value: 0.1601628874804186
201
+ - name: METEOR
202
+ type: meteor
203
+ value: 0.15348605312210778
204
+ - name: BERTScore
205
+ type: bertscore
206
+ value: 0.8783386920680519
207
+ - name: MoverScore
208
+ type: moverscore
209
+ value: 0.5634845371093992
210
+ - task:
211
+ name: Text2text Generation
212
+ type: text2text-generation
213
+ dataset:
214
+ name: lmqg/qg_subjqa
215
+ type: books
216
+ args: books
217
+ metrics:
218
+ - name: BLEU4
219
+ type: bleu4
220
+ value: 0.006278914808207679
221
+ - name: ROUGE-L
222
+ type: rouge-l
223
+ value: 0.12368226019088967
224
+ - name: METEOR
225
+ type: meteor
226
+ value: 0.11576293675813865
227
+ - name: BERTScore
228
+ type: bertscore
229
+ value: 0.8807110440044503
230
+ - name: MoverScore
231
+ type: moverscore
232
+ value: 0.5555905941686486
233
+ - task:
234
+ name: Text2text Generation
235
+ type: text2text-generation
236
+ dataset:
237
+ name: lmqg/qg_subjqa
238
+ type: movies
239
+ args: movies
240
+ metrics:
241
+ - name: BLEU4
242
+ type: bleu4
243
+ value: 1.0121579426501661e-06
244
+ - name: ROUGE-L
245
+ type: rouge-l
246
+ value: 0.12508697028506718
247
+ - name: METEOR
248
+ type: meteor
249
+ value: 0.11862284941640638
250
+ - name: BERTScore
251
+ type: bertscore
252
+ value: 0.8748829724726739
253
+ - name: MoverScore
254
+ type: moverscore
255
+ value: 0.5528899173535703
256
+ - task:
257
+ name: Text2text Generation
258
+ type: text2text-generation
259
+ dataset:
260
+ name: lmqg/qg_subjqa
261
+ type: grocery
262
+ args: grocery
263
+ metrics:
264
+ - name: BLEU4
265
+ type: bleu4
266
+ value: 0.00528043272450429
267
+ - name: ROUGE-L
268
+ type: rouge-l
269
+ value: 0.12343711316491492
270
+ - name: METEOR
271
+ type: meteor
272
+ value: 0.15133496445452477
273
+ - name: BERTScore
274
+ type: bertscore
275
+ value: 0.8778951253890991
276
+ - name: MoverScore
277
+ type: moverscore
278
+ value: 0.5701949938103265
279
+ - task:
280
+ name: Text2text Generation
281
+ type: text2text-generation
282
+ dataset:
283
+ name: lmqg/qg_squadshifts
284
+ type: amazon
285
+ args: amazon
286
+ metrics:
287
+ - name: BLEU4
288
+ type: bleu4
289
+ value: 0.06530369842068952
290
+ - name: ROUGE-L
291
+ type: rouge-l
292
+ value: 0.25030985091008146
293
+ - name: METEOR
294
+ type: meteor
295
+ value: 0.2229994442645732
296
+ - name: BERTScore
297
+ type: bertscore
298
+ value: 0.9092814804525936
299
+ - name: MoverScore
300
+ type: moverscore
301
+ value: 0.6086538514008419
302
+ - task:
303
+ name: Text2text Generation
304
+ type: text2text-generation
305
+ dataset:
306
+ name: lmqg/qg_subjqa
307
+ type: default
308
+ args: default
309
+ metrics:
310
+ - name: BLEU4
311
+ type: bleu4
312
+ value: 0.005121882223046874
313
+ - name: ROUGE-L
314
+ type: rouge-l
315
+ value: 0.1346485324169255
316
+ - name: METEOR
317
+ type: meteor
318
+ value: 0.13733272662214893
319
+ - name: BERTScore
320
+ type: bertscore
321
+ value: 0.8811488576438816
322
+ - name: MoverScore
323
+ type: moverscore
324
+ value: 0.5614233235005509
325
  ---
326
 
327
+ # Language Models Fine-tuning on Question Generation: `lmqg/bart-large-squad`
328
+ This model is fine-tuned version of [facebook/bart-large](https://huggingface.co/facebook/bart-large) for question generation task on the
329
+ [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) (dataset_name: default).
 
330
 
 
331
 
332
+ ### Overview
333
+ - **Language model:** [facebook/bart-large](https://huggingface.co/facebook/bart-large)
334
+ - **Language:** en
335
+ - **Training data:** [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) (default)
336
+ - **Online Demo:** [https://autoqg.net/](https://autoqg.net/)
337
+ - **Repository:** [https://github.com/asahi417/lm-question-generation](https://github.com/asahi417/lm-question-generation)
338
+ - **Paper:** [TBA](TBA)
339
 
340
+ ### Usage
 
341
  ```python
342
+
343
  from transformers import pipeline
344
 
345
+ model_path = 'lmqg/bart-large-squad'
346
  pipe = pipeline("text2text-generation", model_path)
347
 
348
+ # Question Generation
349
+ input_text = 'generate question: <hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records.'
350
+ question = pipe(input_text)
 
 
 
 
 
 
351
  ```
352
 
353
+ ## Evaluation Metrics
354
 
 
 
 
355
 
356
+ ### Metrics
357
 
358
+ | Dataset | Type | BLEU4 | ROUGE-L | METEOR | BERTScore | MoverScore | Link |
359
+ |:--------|:-----|------:|--------:|-------:|----------:|-----------:|-----:|
360
+ | [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) | default | 0.26168385362299557 | 0.5384959163821219 | 0.27073122286541956 | 0.9100413219045603 | 0.6499011626820898 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval/metric.first.sentence.paragraph_answer.question.lmqg_qg_squad.default.json) |
361
 
 
362
 
363
 
364
+ ### Out-of-domain Metrics
365
+
366
+ | Dataset | Type | BLEU4 | ROUGE-L | METEOR | BERTScore | MoverScore | Link |
367
+ |:--------|:-----|------:|--------:|-------:|----------:|-----------:|-----:|
368
+ | [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | reddit | 0.059525104157825456 | 0.22365090580055863 | 0.21499800504546457 | 0.9095144685254328 | 0.6059332247878408 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json) |
369
+ | [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | new_wiki | 0.11118273173452982 | 0.2967546690273089 | 0.27315087810722966 | 0.9322739617807421 | 0.6623000084761579 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json) |
370
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | tripadvisor | 8.380171318718442e-07 | 0.1402922852924756 | 0.1372146070365174 | 0.8891002409937424 | 0.5604572211470809 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.json) |
371
+ | [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | default | 0.07839941048417529 | 0.25357667226247294 | 0.24046838149047955 | 0.9182198703598111 | 0.6274693859765924 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.default.json) |
372
+ | [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | nyt | 0.08117757543966063 | 0.25292097720734297 | 0.25254205113198686 | 0.9249009759439454 | 0.6406329128556304 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json) |
373
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | restaurants | 1.1301750984972448e-06 | 0.13083168975354642 | 0.12419733006916912 | 0.8797711839570719 | 0.5542757411268555 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json) |
374
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | electronics | 0.00866799444965211 | 0.1601628874804186 | 0.15348605312210778 | 0.8783386920680519 | 0.5634845371093992 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json) |
375
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | books | 0.006278914808207679 | 0.12368226019088967 | 0.11576293675813865 | 0.8807110440044503 | 0.5555905941686486 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.books.json) |
376
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | movies | 1.0121579426501661e-06 | 0.12508697028506718 | 0.11862284941640638 | 0.8748829724726739 | 0.5528899173535703 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.movies.json) |
377
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | grocery | 0.00528043272450429 | 0.12343711316491492 | 0.15133496445452477 | 0.8778951253890991 | 0.5701949938103265 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.grocery.json) |
378
+ | [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | amazon | 0.06530369842068952 | 0.25030985091008146 | 0.2229994442645732 | 0.9092814804525936 | 0.6086538514008419 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json) |
379
+ | [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | default | 0.005121882223046874 | 0.1346485324169255 | 0.13733272662214893 | 0.8811488576438816 | 0.5614233235005509 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.default.json) |
380
 
 
 
381
 
382
+ ## Training hyperparameters
383
+
384
+ The following hyperparameters were used during fine-tuning:
385
+ - dataset_path: lmqg/qg_squad
386
+ - dataset_name: default
387
+ - input_types: ['paragraph_answer']
388
+ - output_types: ['question']
389
+ - prefix_types: None
390
+ - model: facebook/bart-large
391
+ - max_length: 512
392
+ - max_length_output: 32
393
+ - epoch: 4
394
+ - batch: 32
395
+ - lr: 5e-05
396
+ - fp16: False
397
+ - random_seed: 1
398
+ - gradient_accumulation_steps: 4
399
+ - label_smoothing: 0.15
400
 
401
+ The full configuration can be found at [fine-tuning config file](https://huggingface.co/lmqg/bart-large-squad/raw/main/trainer_config.json).
402
+
403
+ ## Citation
404
+ TBA
eval/{metric.first.answer.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.first.answer.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.first.answer.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.first.answer.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.first.answer.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.first.answer.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.first.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.first.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.first.sentence.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.last.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.last.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.last.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.last.sentence.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.long.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.long.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.long.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.long.sentence.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.middle.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.middle.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.middle.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.middle.sentence.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.short.sentence.paragraph_answer.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.paragraph_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.short.sentence.paragraph_sentence.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.paragraph_sentence.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{metric.short.sentence.sentence_answer.question.asahi417_qg_squad.default.json β†’ metric.short.sentence.sentence_answer.question.lmqg_qg_squad.default.json} RENAMED
File without changes
eval/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval/{samples.test.hyp.paragraph_sentence.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.paragraph_sentence.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval/{samples.test.hyp.sentence_answer.question.asahi417_qg_squad.default.txt β†’ samples.test.hyp.sentence_answer.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval/{samples.validation.hyp.paragraph_answer.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.paragraph_answer.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval/{samples.validation.hyp.paragraph_sentence.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.paragraph_sentence.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval/{samples.validation.hyp.sentence_answer.question.asahi417_qg_squad.default.txt β†’ samples.validation.hyp.sentence_answer.question.lmqg_qg_squad.default.txt} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.amazon.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.default.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.new_wiki.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.nyt.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_squadshifts.reddit.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.books.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.books.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.default.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.default.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.electronics.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.grocery.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.grocery.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.movies.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.movies.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.restaurants.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json} RENAMED
File without changes
eval_ood/{metric.first.sentence.paragraph_answer.question.asahi417_qg_subjqa.tripadvisor.json β†’ metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.json} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.amazon.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.amazon.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.default.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.new_wiki.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.nyt.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.nyt.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_squadshifts.reddit.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_squadshifts.reddit.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.books.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.books.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.default.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.default.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.electronics.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.electronics.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.grocery.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.grocery.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.movies.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.movies.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.restaurants.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.restaurants.txt} RENAMED
File without changes
eval_ood/{samples.test.hyp.paragraph_answer.question.asahi417_qg_subjqa.tripadvisor.txt β†’ samples.test.hyp.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.txt} RENAMED
File without changes
eval_ood/{samples.validation.hyp.paragraph_answer.question.asahi417_qg_squadshifts.amazon.txt β†’ samples.validation.hyp.paragraph_answer.question.lmqg_qg_squadshifts.amazon.txt} RENAMED
File without changes