feat: add initial trained model and scripts

* Pretrained a model;
* Add a training script with docker;
* Add a test script to try the model;

.dockerignore

@@ -0,0 +1 @@
+.venv

.gitignore

@@ -0,0 +1 @@
+.venv

README.md

@@ -0,0 +1,45 @@
+---
+language: en
+tags:
+- grammar
+- spell
+- correction
+---
+
+# Spellcheck Model
+
+## Installation
+1. Install Python 3.8 or higher and Docker;
+2. Clone the repo;
+3. Create a virtual environment: `python3 -m venv venv`;
+4. Activate the virtual environment: `source venv/bin/activate`;
+5. Install dependencies: `pip install -r requirements.txt`;
+
+#
+
+## Usage
+### Training
+There's a `data` directory with two files:
+1. `eval.csv`: evaluation dataset;
+2. `train.csv`: training dataset;
+Both are in the same format:
+```txt
+input,target
+"grammar: som sentense","some sentence"
+"grammar: anoder centence","another sentence"
+...
+```
+
+You can edit them to use your own dataset.
+
+Now if you want to fine-tune the existing model here, just run the following commands:
+```bash
+docker build -t spellcheck_train .
+docker run -v $(pwd):/app spellcheck_train
+```
+Also make sure you have docker running on your machine.
+### Testing
+If you want to test your model, just run the following command:
+```bash
+python test.py
+```

config.json

@@ -0,0 +1,61 @@
+{
+  "_name_or_path": ".",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 3072,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "relu",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "relu",
+  "gradient_checkpointing": false,
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "vocab_size": 32128
+}

data/eval.csv

@@ -0,0 +1,31 @@
+input,target
+"grammar: I luv goin for lng waks in the prk.","I love going for long walks in the park."
+"grammar: The quik brown fox jumps ovr the lzy dg.","The quick brown fox jumps over the lazy dog."
+"grammar: I enjoye playing the pino in my fre time.","I enjoy playing the piano in my free time."
+"grammar: The sun rises in the est and sets in the west.","The sun rises in the east and sets in the west."
+"grammar: I lik to reed boks in my spair time.","I like to read books in my spare time."
+"grammar: The cat is siting on the mat.","The cat is sitting on the mat."
+"grammar: I hav a pashun for fotografi.","I have a passion for photography."
+"grammar: The Erth revolvs around the Son.","The Earth revolves around the Sun."
+"grammar: I lik to watch moovies on weekendz.","I like to watch movies on weekends."
+"grammar: The rane is poring outside.","The rain is pouring outside."
+"grammar: I have a lot of work to do todai.","I have a lot of work to do today."
+"grammar: The mun shines brite in the nite sky.","The moon shines brightly in the night sky."
+"grammar: I enjoye spnding time with my famili.","I enjoy spending time with my family."
+"grammar: The coffe is brewing in the kichen.","The coffee is brewing in the kitchen."
+"grammar: I lik to go swiming in the sumer.","I like to go swimming in the summer."
+"grammar: The birds are chirpin in the trez.","The birds are chirping in the trees."
+"grammar: I am studyng compute science at universiti.","I am studying computer science at university."
+"grammar: The car is parkd in the garaje.","The car is parked in the garage."
+"grammar: I luv eating icecrem on hot dais.","I love eating ice cream on hot days."
+"grammar: The flowers are blooming in the gardn.","The flowers are blooming in the garden."
+"grammar: I enjoy hikng in the mountins.","I enjoy hiking in the mountains."
+"grammar: The air is fresh and cris in the morning.","The air is fresh and crisp in the morning."
+"grammar: I lik to trvl and explore new places.","I like to travel and explore new places."
+"grammar: The clok is tikin on the wal.","The clock is ticking on the wall."
+"grammar: I have a meting at 2 PM.","I have a meeting at 2 PM."
+"grammar: The children are playin in the playgrund.","The children are playing in the playground."
+"grammar: I enjoye coking and trying new recipz.","I enjoy cooking and trying new recipes."
+"grammar: The bus is ariving at the stacion.","The bus is arriving at the station."
+"grammar: I lik to go for a run in the evening.","I like to go for a run in the evening."
+"grammar: The wavs are crashin on the shore.","The waves are crashing on the shore."

data/train.csv

@@ -0,0 +1,42 @@
+input,target
+"grammar: they fought a deadly wear","they fought a deadly war"
+"grammar: thier house was father away from my place","their house was farther away from my place"
+"grammar: So I think we would not be live if our ancestors did not develop siences and tecnologies .","So I think we would not be alive if our ancestors did not develop sciences and technologies ."
+"grammar: Imagine yourself you are working in factory just to do one thing like put air a on car if they fire you you will be destroyed , becouse you do n't know more than to put air a in car .","Imagine yourself you are working in factory just to do one thing like put air a on car if they fire you you will be destroyed , because you do n't know more than to put air a in car ."
+"grammar: For example , they can play football whenever they want , but the olders can not .","For example , they can play football whenever they want , but the elders can not ."
+"grammar: While It is true that consumers preffer to buy products with lower prices , when international companies that are already certified begin to send their products to market , people will preffer to consume those goods because the difference in price will probbably not affect them too much .","While It is true that consumers prefer to buy products with lower prices , when international companies that are already certified begin to send their products to market , people will prefer to consume those goods because the difference in price will probably not affect them too much ."
+"grammar: And young people spend more time on ther lifestyles .","And young people spend more time on their lifestyles ."
+"grammar: Students can focus on only a few subjects they are intwerested in and they will become experts in those areas .","Students can focus on only a few subjects they are interested in and they will become experts in those areas ."
+"grammar: He thinks differently than others and he has succeded .","He thinks differently than others and he has succeeded ."
+"grammar: These activities make the community a better place to live and include these values in all the members .","These activities make the community a better place to live and include those values in all the members ."
+"grammar: all the broad knowledge helps they to understand their major in university classes , as well as help they to make a correct choice in a specialized area .","all the broad knowledge helps them to understand their major in university classes , as well as help them to make a correct choice in a specialized area ."
+"grammar: I could very easily understand why he would always brang bad marks home .","I could very easily understand why he would always bring bad marks home ."
+"grammar: Then , when we went to the Science Olympiad , she had diverse knowledge about the subects , and it was not new for her to think about new things .","Then , when we went to the Science Olympiad , she had diverse knowledge about the subjects , and it was not new for her to think about new things ."
+"grammar: If every person tries to learn and understand lots of scientifc subjects , no person will do it and , as a result of this , no science will be improved .","If every person tries to learn and understand lots of scientific subjects , no person will do it and , as a result of this , no science will be improved ."
+"grammar: they will run out very soon at the current rate of utilisation .","they will run out very soon at the current rate of utilization ."
+"grammar: and it will put your maind into non-stop learning .","and it will put your mind into non-stop learning ."
+"grammar: In today 's world Compuer skills are the first important life skill .","In today 's world Computer skills are the first important life skill ."
+"grammar: I have never stopped myself to think this , but this is a real possibility for the fucture .","I have never stopped myself to think this , but this is a real possibility for the future ."
+"grammar: If we conseder an idea as a totally autonomous concept within the individual process of defining reality , I think the ideas are useless , and not based on experience .","If we consider an idea as a totally autonomous concept within the individual process of defining reality , I think the ideas are useless , and not based on experience ."
+"grammar: If there are specialized docters that have done the operation often , he becomes talented in his job .","If there are specialized doctors that have done the operation often , he becomes talented in his job ."
+"grammar: It will be good situstion .","It will be good situation ."
+"grammar: the school teachers there are the ones who create the future of the younger genaration so we have to teach them better .","the school teachers there are the ones who create the future of the younger generation so we have to teach them better ."
+"grammar: some tour guide will want to set maximum security to make the tour difficult because you will only have a wonderful view throught the bus .","some tour guide will want to set maximum security to make the tour difficult because you will only have a wonderful view through the bus ."
+"grammar: this will effect exams .","this will affect exams ."
+"grammar: But on the other hand ther are also people that are often convinced by the interesting advertisements they see everywhere , because of this many times they have to face problems with the product that they bought and many times ther is no way to give back that product .","But on the other hand there are also people that are often convinced by the interesting advertisements they see everywhere , because of this many times they have to face problems with the product that they bought and many times there is no way to give back that product ."
+"grammar: Ever increasing competancy rates force them into frequent business model changes for a compatible transitional flexibility .","Ever increasing competency rates force them into frequent business model changes for a compatible transitional flexibility ."
+"grammar: However , this reading passage casts douts on the speaker 's mention .","However , this reading passage casts doubts on the speaker 's mention ."
+"grammar: For example , in the 2 0 0 6 world cup form Germany , many coaches on a work .","For example , in the 2 0 0 6 world cup form Germany , many coaches on a team ."
+"grammar: Many Scientists obtained clear results of investigations after the facts were on the table , but before they could even begin to theorise about them .","Many Scientists obtained clear results of investigations after the facts were on the table , but before they could even begin to theorize about them ."
+"grammar: They want to make people understend thet their product is the best and you can really trust it .","They want to make people understand that their product is the best and you can really trust it ."
+"grammar: Video is convenient , but if teachers are concerned about students , using texetbooks is a good ability for students .","Video is convenient , but if teachers are concerned about students , using textbooks is a good ability for students ."
+"grammar: In my opinion , it is dependend on a particular person .","In my opinion , it is dependent on a particular person ."
+"grammar: for example when we talk about speed they must anderstand why it is dangerous it is beteer than if they have an accident and learn after that .","for example when we talk about speed they must understand why it is dangerous it is better than if they have an accident and learn after that ."
+"grammar: Therefore peaple will be able to live with the automobile sosiety and nature for the future .","Therefore people will be able to live with the automobile society and nature for the future ."
+"grammar: finally , the third piece of evidence that birds use a type of internal compass is that birds have crystals of the mineral magetite embedded in their break .","finally , the third piece of evidence that birds use a type of internal compass is that birds have crystals of the mineral magnetite embedded in their break ."
+"grammar: The lecture says it is more important to provide enough fish to the people .","The lecturer says it is more important to provide enough fish to the people ."
+"grammar: Futhermore , a tour guide will also provide safety and security for travel , since they already know the dos and don'ts of the tour .","Furthermore , a tour guide will also provide safety and security for travel , since they already know the dos and don'ts of the tour ."
+"grammar: Which caused her situation to worse .","Which caused her situation to worsen ."
+"grammar: This arguement is not only true now , it has always been .","This argument is not only true now , it has always been ."
+"grammar: The Last thing they have to study is disease , which means they will be save .","The Last thing they have to study is disease , which means they will be safe ."
+"grammar: There are very successful politicians that have never tried somthing new .","There are very successful politicians that have never tried something new ."

dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.9
+WORKDIR /app
+COPY . .
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ["python", "train.py"]

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.30.2"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fbfd83d5242b7be46067a12c862a3c9345cff788fb38dfa329bc4620761f721
+size 891699345

requirements.txt

@@ -0,0 +1,2 @@
+accelerate==0.20.3
+happytransformer==2.4.1

special_tokens_map.json

@@ -0,0 +1,107 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

test.py

@@ -0,0 +1,12 @@
+from happytransformer import HappyTextToText, TTSettings
+
+input_text = input('Enter your mispelled text: ')
+happy_tt = HappyTextToText('T5', load_path='.')
+
+beam_settings = TTSettings(num_beams=5, min_length=1, max_length=100)
+
+output_text_1 = happy_tt.generate_text(
+    f'grammar: {input_text}',
+    args=beam_settings,
+)
+print(output_text_1.text)

tokenizer_config.json

@@ -0,0 +1,111 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

train.py

@@ -0,0 +1,11 @@
+from happytransformer import HappyTextToText, TTTrainArgs
+
+happy_tt = HappyTextToText("T5", load_path=".")
+
+args = TTTrainArgs(batch_size=8)
+happy_tt.train("data/train.csv", args=args)
+
+before_loss = happy_tt.eval("data/eval.csv")
+print("After loss: ", before_loss.loss)
+
+happy_tt.save('.')