Harsh Trivedi commited on
Commit
d0d2508
1 Parent(s): fcce6f7
README.md ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - question-answering, multi-step-reasoning, multi-hop-reasoning
4
+ thumbnail: https://raw.githubusercontent.com/StonyBrookNLP/teabreac/main/teabreac_icon.png
5
+ license: cc-by-4.0
6
+ ---
7
+
8
+ # What's this?
9
+
10
+ This is one of the models reported in the paper: ["Teaching Broad Reasoning Skills for Multi-Step QA by Generating Hard Contexts".](https://arxiv.org/abs/2205.12496).
11
+
12
+ This paper proposes a procedure to synthetically generate a QA dataset, TeaBReaC, for pretraining language models for robust multi-step reasoning. Pretraining plain LMs like Bart, T5 and numerate LMs like NT5, PReasM, POET on TeaBReaC leads to improvemed downstream performance on several multi-step QA datasets. Please checkout out the paper for the details.
13
+
14
+ We release the following models:
15
+
16
+ - **A:** Base Models finetuned on target datasets: `{base_model}-{target_dataset}`
17
+ - **B:** Base models pretrained on TeaBReaC: `teabreac-{base_model}`
18
+ - **C:** Base models pretrained on TeaBReaC and then finetuned on target datasets: `teabreac-{base_model}-{target_dataset}`
19
+
20
+ The `base_model` above can be from: `bart-large`, `t5-large`, `t5-3b`, `nt5-small`, `preasm-large`.
21
+ The `target_dataset` above can be from: `drop`, `tatqa`, `iirc-gold`, `iirc-retrieved`, `numglue`.
22
+
23
+ The **A** models are only released for completeness / reproducibility. In your end application you probably just want to use either **B** or **C**.
24
+
25
+ # How to use it?
26
+
27
+ Please checkout the details in our [github repository](https://github.com/stonybrooknlp/teabreac), but in a nutshell:
28
+
29
+ ```python
30
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
31
+ from digit_tokenization import enable_digit_tokenization # digit_tokenization.py from https://github.com/stonybrooknlp/teabreac
32
+
33
+ model_name = "teabreac-nt5-small-tatqa"
34
+ tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) # Fast doesn't work with digit tokenization
35
+ model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
36
+ enable_digit_tokenization(tokenizer)
37
+ input_texts = [
38
+ "answer_me: Who scored the first touchdown of the game?" +
39
+ "context: ... Oakland would get the early lead in the first quarter as quarterback JaMarcus Russell completed a 20-yard touchdown pass to rookie wide receiver Chaz Schilens..."
40
+ # Note: some models have slightly different qn/ctxt format. See the github repo.
41
+ ]
42
+ input_ids = tokenizer(
43
+ input_texts, return_tensors="pt",
44
+ truncation=True, max_length=800,
45
+ add_special_tokens=True, padding=True,
46
+ )
47
+ generated_ids = model.generate(input_ids, min_length=1, max_length=50)
48
+ generated_predictions = tokenizer.batch_decode(generated_ids, skip_special_tokens=False)
49
+ generated_predictions = [
50
+ tokenizer.fix_decoded_text(generated_prediction) for generated_prediction in generated_predictions
51
+ ]
52
+ # => ["Chaz Schilens"]
53
+ ```
added_tokens.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<ss>": 32100, "<sv>": 32101, "select": 32102, "boolean": 32103, "are_items_same": 32104, "are_items_different": 32105, "arg_minimum_number": 32106, "arg_maximum_number": 32107, "minimum_number": 32108, "maximum_number": 32109, "arithmetic_mean_single": 32110, "arithmetic_sum_single": 32111, "arithmetic_sum_multiple": 32112, "arithmetic_division": 32113, "arithmetic_subtraction": 32114, "arithmetic_multiplication": 32115, "compare_numbers": 32116, "arg_minimum_date": 32117, "arg_maximum_date": 32118, "minimum_date": 32119, "maximum_date": 32120, "date_subtraction": 32121, "compare_dates": 32122, "list_subtraction": 32123, "filter_a_where_b_is_max_date": 32124, "filter_a_where_b_is_min_date": 32125, "filter_a_where_b_is_compared_to_date": 32126, "filter_a_where_b_is_max": 32127, "filter_a_where_b_is_min": 32128, "filter_a_where_b_is_given_value": 32129, "filter_a_where_b_is_compared_to": 32130, "logical_or": 32131, "logical_and": 32132, "arg_bool": 32133, "arg_intersection": 32134, "intersection": 32135, "grouped_count": 32136, "grouped_mean": 32137, "grouped_sum": 32138, "kth_highest": 32139, "kth_lowest": 32140, "null": 32141, "[COLD]": 32142, "[ROWD]": 32143, "[DFB]": 32144, "[DFE]": 32145, "[DFD]": 32146, "[MDFD]": 32147, "[CND]": 32148, "[SOMD]": 32149, "[SIND]": 32150, "[CHPD]": 32151, "[CTANS]": 32152, "[CTOUINTA]": 32153, "[CTOUINSINA]": 32154, "[CTCNINTA]": 32155, "[CTCNINSINA]": 32156, "[SNS]": 32157, "[SNE]": 32158, "[SCD]": 32159, "[SCBP]": 32160, "[SCBN]": 32161, "[SCMP]": 32162, "[SCMN]": 32163, "[DAY]": 32164, "[MONTH]": 32165, "[YEAR]": 32166}
config.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "t5-small",
3
+ "architectures": [
4
+ "T5ForConditionalGeneration"
5
+ ],
6
+ "d_ff": 2048,
7
+ "d_kv": 64,
8
+ "d_model": 512,
9
+ "decoder_start_token_id": 0,
10
+ "dropout_rate": 0.1,
11
+ "eos_token_id": 1,
12
+ "feed_forward_proj": "relu",
13
+ "gradient_checkpointing": false,
14
+ "initializer_factor": 1.0,
15
+ "is_encoder_decoder": true,
16
+ "layer_norm_epsilon": 1e-06,
17
+ "model_type": "t5",
18
+ "n_positions": 512,
19
+ "num_decoder_layers": 6,
20
+ "num_heads": 8,
21
+ "num_layers": 6,
22
+ "output_past": true,
23
+ "pad_token_id": 0,
24
+ "relative_attention_num_buckets": 32,
25
+ "task_specific_params": {
26
+ "summarization": {
27
+ "early_stopping": true,
28
+ "length_penalty": 2.0,
29
+ "max_length": 200,
30
+ "min_length": 30,
31
+ "no_repeat_ngram_size": 3,
32
+ "num_beams": 4,
33
+ "prefix": "summarize: "
34
+ },
35
+ "translation_en_to_de": {
36
+ "early_stopping": true,
37
+ "max_length": 300,
38
+ "num_beams": 4,
39
+ "prefix": "translate English to German: "
40
+ },
41
+ "translation_en_to_fr": {
42
+ "early_stopping": true,
43
+ "max_length": 300,
44
+ "num_beams": 4,
45
+ "prefix": "translate English to French: "
46
+ },
47
+ "translation_en_to_ro": {
48
+ "early_stopping": true,
49
+ "max_length": 300,
50
+ "num_beams": 4,
51
+ "prefix": "translate English to Romanian: "
52
+ }
53
+ },
54
+ "torch_dtype": "float32",
55
+ "transformers_version": "4.10.0",
56
+ "use_cache": true,
57
+ "vocab_size": 32167
58
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8af8348983fcc79f87068c8030fa8297c7546e87f327dcaf3e2c0c4173219c1
3
+ size 242163643
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "<ss>", "<sv>"]}
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
3
+ size 791656
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "sp_model_kwargs": {}, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "nielsr/nt5-small-rc1", "add_special_tokens": false, "tokenizer_file": "/home/hjtrivedi/.cache/huggingface/transformers/48740ae5885e0d9d3a063548c3af14d6da9fd64f0599df5a5ae366da5ae8f827.f558102be59ecf48d8076cabd5bd4c68221dd51715467ad1b5e289f268dc8f0e", "tokenizer_class": "T5Tokenizer"}