Herve commited on
Commit
c397abe
1 Parent(s): 065ad37
config.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "distilbert-base-uncased",
3
+ "activation": "gelu",
4
+ "architectures": [
5
+ "DistilBertForMaskedLM"
6
+ ],
7
+ "attention_dropout": 0.1,
8
+ "dim": 768,
9
+ "dropout": 0.1,
10
+ "hidden_dim": 3072,
11
+ "initializer_range": 0.02,
12
+ "max_position_embeddings": 512,
13
+ "model_type": "distilbert",
14
+ "n_heads": 12,
15
+ "n_layers": 6,
16
+ "pad_token_id": 0,
17
+ "qa_dropout": 0.1,
18
+ "seq_classif_dropout": 0.2,
19
+ "sinusoidal_pos_embds": false,
20
+ "tie_weights_": true,
21
+ "transformers_version": "4.29.0",
22
+ "vocab_size": 30522
23
+ }
data_args.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "training_data_type": "pkl_dict",
3
+ "training_data_path": "/gfs-ssd/project/neuralsearch/new_scores/ensemble_scores_5050_rescored.pkl.gz",
4
+ "document_dir": "/nfs/data/neuralsearch/msmarco/documents/raw.tsv",
5
+ "query_dir": "/nfs/data/neuralsearch/msmarco/training_queries/raw.tsv",
6
+ "qrels_path": "/nfs/data/neuralsearch/msmarco/training_queries/qrels.json",
7
+ "n_negatives": 8,
8
+ "n_queries": -1
9
+ }
model_args.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name_or_path": "distilbert-base-uncased",
3
+ "max_length": 128,
4
+ "shared_weights": true,
5
+ "splade_doc": false,
6
+ "model_q": null,
7
+ "dense_pooling": "cls",
8
+ "dense": false,
9
+ "adapter_name": null,
10
+ "adapter_config": null,
11
+ "load_adapter": null
12
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec4c6e9c997b7c61297759ad529ef7ed4f056491aa9fb536c21e26ff7f11ca77
3
+ size 267984305
special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "[CLS]",
3
+ "mask_token": "[MASK]",
4
+ "pad_token": "[PAD]",
5
+ "sep_token": "[SEP]",
6
+ "unk_token": "[UNK]"
7
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "clean_up_tokenization_spaces": true,
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "mask_token": "[MASK]",
6
+ "model_max_length": 512,
7
+ "pad_token": "[PAD]",
8
+ "sep_token": "[SEP]",
9
+ "strip_accents": null,
10
+ "tokenize_chinese_chars": true,
11
+ "tokenizer_class": "DistilBertTokenizer",
12
+ "unk_token": "[UNK]"
13
+ }
training_args.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "experiments/hf/baseline_distilbert_mse/checkpoint",
3
+ "overwrite_output_dir": false,
4
+ "do_train": false,
5
+ "do_eval": false,
6
+ "do_predict": false,
7
+ "evaluation_strategy": "no",
8
+ "prediction_loss_only": false,
9
+ "per_device_train_batch_size": 20,
10
+ "per_device_eval_batch_size": 8,
11
+ "per_gpu_train_batch_size": null,
12
+ "per_gpu_eval_batch_size": null,
13
+ "gradient_accumulation_steps": 1,
14
+ "eval_accumulation_steps": null,
15
+ "eval_delay": 0,
16
+ "learning_rate": 2e-05,
17
+ "weight_decay": 0.0,
18
+ "adam_beta1": 0.9,
19
+ "adam_beta2": 0.999,
20
+ "adam_epsilon": 1e-08,
21
+ "max_grad_norm": 1.0,
22
+ "num_train_epochs": 5,
23
+ "max_steps": -1,
24
+ "lr_scheduler_type": "linear",
25
+ "warmup_ratio": 0.01,
26
+ "warmup_steps": 0,
27
+ "log_level": "passive",
28
+ "log_level_replica": "warning",
29
+ "log_on_each_node": true,
30
+ "logging_dir": null,
31
+ "logging_strategy": "steps",
32
+ "logging_first_step": false,
33
+ "logging_steps": 1000,
34
+ "logging_nan_inf_filter": true,
35
+ "save_strategy": "epoch",
36
+ "save_steps": 500,
37
+ "save_total_limit": null,
38
+ "save_safetensors": false,
39
+ "save_on_each_node": false,
40
+ "no_cuda": false,
41
+ "use_mps_device": false,
42
+ "seed": 123,
43
+ "data_seed": null,
44
+ "jit_mode_eval": false,
45
+ "use_ipex": false,
46
+ "bf16": false,
47
+ "fp16": true,
48
+ "fp16_opt_level": "O1",
49
+ "half_precision_backend": "cuda_amp",
50
+ "bf16_full_eval": false,
51
+ "fp16_full_eval": false,
52
+ "tf32": null,
53
+ "local_rank": 0,
54
+ "ddp_backend": null,
55
+ "tpu_num_cores": null,
56
+ "tpu_metrics_debug": false,
57
+ "debug": [],
58
+ "dataloader_drop_last": true,
59
+ "eval_steps": null,
60
+ "dataloader_num_workers": 0,
61
+ "past_index": -1,
62
+ "run_name": null,
63
+ "disable_tqdm": false,
64
+ "remove_unused_columns": false,
65
+ "label_names": null,
66
+ "load_best_model_at_end": false,
67
+ "metric_for_best_model": null,
68
+ "greater_is_better": null,
69
+ "ignore_data_skip": false,
70
+ "sharded_ddp": [],
71
+ "fsdp": [],
72
+ "fsdp_min_num_params": 0,
73
+ "fsdp_config": {
74
+ "fsdp_min_num_params": 0,
75
+ "xla": false,
76
+ "xla_fsdp_grad_ckpt": false
77
+ },
78
+ "fsdp_transformer_layer_cls_to_wrap": null,
79
+ "deepspeed": null,
80
+ "label_smoothing_factor": 0.0,
81
+ "optim": "adamw_hf",
82
+ "optim_args": null,
83
+ "adafactor": false,
84
+ "group_by_length": false,
85
+ "length_column_name": "length",
86
+ "report_to": [
87
+ "tensorboard"
88
+ ],
89
+ "ddp_find_unused_parameters": false,
90
+ "ddp_bucket_cap_mb": null,
91
+ "dataloader_pin_memory": true,
92
+ "skip_memory_metrics": true,
93
+ "use_legacy_prediction_loop": false,
94
+ "push_to_hub": false,
95
+ "resume_from_checkpoint": true,
96
+ "hub_model_id": null,
97
+ "hub_strategy": "every_save",
98
+ "hub_token": "<HUB_TOKEN>",
99
+ "hub_private_repo": false,
100
+ "gradient_checkpointing": false,
101
+ "include_inputs_for_metrics": false,
102
+ "fp16_backend": "auto",
103
+ "push_to_hub_model_id": null,
104
+ "push_to_hub_organization": null,
105
+ "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>",
106
+ "mp_parameters": "",
107
+ "auto_find_batch_size": false,
108
+ "full_determinism": false,
109
+ "torchdynamo": null,
110
+ "ray_scope": "last",
111
+ "ddp_timeout": 1800,
112
+ "torch_compile": false,
113
+ "torch_compile_backend": null,
114
+ "torch_compile_mode": null,
115
+ "xpu_backend": null,
116
+ "training_loss": "kldiv_mse_margin_with_weights",
117
+ "l0d": 0.02,
118
+ "l0q": 0.01,
119
+ "T_d": 30000,
120
+ "T_q": 30000,
121
+ "top_d": -1,
122
+ "top_q": -1,
123
+ "lexical_type": "none"
124
+ }
vocab.txt ADDED
The diff for this file is too large to render. See raw diff