MorenoLaQuatra commited on
Commit
42bd82d
1 Parent(s): 75856b2

Uploading model and tokenizer

Browse files
README.md CHANGED
@@ -1,3 +1,68 @@
1
  ---
 
2
  license: mit
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ language: "it"
3
  license: mit
4
+ datasets:
5
+ - gsarti/clean_mc4_it
6
+ tags:
7
+ - bart
8
+ - pytorch
9
+ pipeline:
10
+ - text2text-generation
11
  ---
12
+
13
+ # BART-IT: Italian pretraining for BART sequence to sequence model
14
+
15
+ BART-IT is a sequence-to-sequence model, based on the BART architecture that is specifically tailored to the Italian language. The model is pre-trained on a [large corpus of Italian text](https://huggingface.co/datasets/gsarti/clean_mc4_it), and can be fine-tuned on a variety of tasks.
16
+
17
+ ## Model description
18
+
19
+ The model is a `base-`sized BART model, with a vocabulary size of 52,000 tokens. It has 140M parameters and can be used for any task that requires a sequence-to-sequence model. It is trained from scratch on a large corpus of Italian text, and can be fine-tuned on a variety of tasks.
20
+
21
+
22
+ ## Pre-training
23
+
24
+ The code used to pre-train BART-IT together with additional information on model parameters can be found [here](https://github.com/MorenoLaQuatra/bart-it).
25
+
26
+ ## Fine-tuning
27
+
28
+ The model in this repository is a pre-trained model without any fine-tuning. In order to use the model for a specific task, you can fine-tune it on a specific dataset.
29
+
30
+ The model has been fine-tuned for the abstractive summarization task on 3 different Italian datasets:
31
+
32
+ - [FanPage](https://huggingface.co/datasets/ARTeLab/fanpage) - finetuned model [here](https://huggingface.co/MorenoLaQuatra/bart-it-fanpage)
33
+ - [IlPost](https://huggingface.co/datasets/ARTeLab/ilpost) - finetuned model [here](https://huggingface.co/MorenoLaQuatra/bart-it-ilpost)
34
+ - [WITS](https://huggingface.co/datasets/Silvia/WITS) - finetuned model [here](https://huggingface.co/MorenoLaQuatra/bart-it-WITS)
35
+
36
+ ## Usage
37
+
38
+ In order to use the model, you can use the following code:
39
+
40
+ ```python
41
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
42
+
43
+ tokenizer = AutoTokenizer.from_pretrained("morenolq/bart-it")
44
+ model = AutoModelForSeq2SeqLM.from_pretrained("morenolq/bart-it")
45
+
46
+ input_ids = tokenizer.encode("Il modello BART-IT è stato pre-addestrato su un corpus di testo italiano", return_tensors="pt")
47
+ outputs = model.generate(input_ids, max_length=40, num_beams=4, early_stopping=True)
48
+ print(tokenizer.decode(outputs[0], skip_special_tokens=True))
49
+ ```
50
+
51
+ # Citation
52
+
53
+ If you find this model useful for your research, please cite the following paper:
54
+
55
+ ```bibtex
56
+ @Article{BARTIT,
57
+ AUTHOR = {La Quatra, Moreno and Cagliero, Luca},
58
+ TITLE = {BART-IT: An Efficient Sequence-to-Sequence Model for Italian Text Summarization},
59
+ JOURNAL = {Future Internet},
60
+ VOLUME = {15},
61
+ YEAR = {2023},
62
+ NUMBER = {1},
63
+ ARTICLE-NUMBER = {15},
64
+ URL = {https://www.mdpi.com/1999-5903/15/1/15},
65
+ ISSN = {1999-5903},
66
+ DOI = {10.3390/fi15010015}
67
+ }
68
+ ```
config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_dropout": 0.0,
3
+ "activation_function": "gelu",
4
+ "architectures": [
5
+ "BartForConditionalGeneration"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 0,
9
+ "classifier_dropout": 0.0,
10
+ "d_model": 768,
11
+ "decoder_attention_heads": 12,
12
+ "decoder_ffn_dim": 3072,
13
+ "decoder_layerdrop": 0.0,
14
+ "decoder_layers": 6,
15
+ "decoder_start_token_id": 2,
16
+ "dropout": 0.1,
17
+ "encoder_attention_heads": 12,
18
+ "encoder_ffn_dim": 3072,
19
+ "encoder_layerdrop": 0.0,
20
+ "encoder_layers": 6,
21
+ "eos_token_id": 2,
22
+ "forced_eos_token_id": 2,
23
+ "id2label": {
24
+ "0": "LABEL_0",
25
+ "1": "LABEL_1",
26
+ "2": "LABEL_2"
27
+ },
28
+ "init_std": 0.02,
29
+ "is_encoder_decoder": true,
30
+ "label2id": {
31
+ "LABEL_0": 0,
32
+ "LABEL_1": 1,
33
+ "LABEL_2": 2
34
+ },
35
+ "max_position_embeddings": 1024,
36
+ "model_type": "bart",
37
+ "num_hidden_layers": 6,
38
+ "pad_token_id": 1,
39
+ "scale_embedding": false,
40
+ "torch_dtype": "float32",
41
+ "transformers_version": "4.22.1",
42
+ "use_cache": true,
43
+ "vocab_size": 52000
44
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5330a2ce3a0b805d4168188392194e351554bbb8920849196ce57f6c4e402e83
3
+ size 563305977
special_tokens_map.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "cls_token": {
10
+ "content": "<s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "eos_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "mask_token": {
24
+ "content": "<mask>",
25
+ "lstrip": true,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "pad_token": {
31
+ "content": "<pad>",
32
+ "lstrip": false,
33
+ "normalized": true,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ },
37
+ "sep_token": {
38
+ "content": "</s>",
39
+ "lstrip": false,
40
+ "normalized": true,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "unk_token": {
45
+ "content": "<unk>",
46
+ "lstrip": false,
47
+ "normalized": true,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ }
51
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "pad_token": {
37
+ "__type": "AddedToken",
38
+ "content": "<pad>",
39
+ "lstrip": false,
40
+ "normalized": true,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "sep_token": {
45
+ "__type": "AddedToken",
46
+ "content": "</s>",
47
+ "lstrip": false,
48
+ "normalized": true,
49
+ "rstrip": false,
50
+ "single_word": false
51
+ },
52
+ "tokenizer_class": "BartTokenizer",
53
+ "unk_token": {
54
+ "__type": "AddedToken",
55
+ "content": "<unk>",
56
+ "lstrip": false,
57
+ "normalized": true,
58
+ "rstrip": false,
59
+ "single_word": false
60
+ }
61
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a128215921a95e7afb04256a99f42f68c405b5883a0c6e05fa5e68d81828fd84
3
+ size 3311
vocab.json ADDED
The diff for this file is too large to render. See raw diff