Update README.md

README.md

@@ -1,3 +1,137 @@
 ---
 license: apache-2.0
+tags:
+- ESG
 ---
+## Main information
+We introduce the model for multilabel ESG risks classification. There is 47 classes methodology with granularial risk definition.   
+
+## Usage 
+```python
+
+from transformers import MPNetPreTrainedModel, MPNetModel
+import torch
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+        token_embeddings = model_output #First element of model_output contains all token embeddings
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+# Definition of ESGify class because of custom,sentence-transformers like, mean pooling function and classifier head
+class ESGify(MPNetPreTrainedModel):
+    """Model for Classification ESG risks from text."""
+
+    def __init__(self,config): #tuning only the head
+        """
+        """
+        super().__init__(config)
+        # Instantiate Parts of model
+        self.mpnet = MPNetModel(config,add_pooling_layer=False)
+        self.classifier = torch.nn.Sequential(OrderedDict([('norm',torch.nn.BatchNorm1d(768)),
+                                                ('linear',torch.nn.Linear(768,512)),
+                                                ('act',torch.nn.ReLU()),
+                                                ('batch_n',torch.nn.BatchNorm1d(512)),
+                                                ('drop_class', torch.nn.Dropout(0.2)),
+                                                ('class_l',torch.nn.Linear(512 ,47))]))
+
+
+    def forward(self, input_ids, attention_mask):
+
+
+         # Feed input to mpnet model
+        outputs = self.mpnet(input_ids=input_ids,
+                             attention_mask=attention_mask)
+         
+        # mean pooling dataset
+        logits = self.classifier( mean_pooling(outputs['last_hidden_state'],attention_mask))
+         # Feed input to classifier to compute logits
+ 
+        return logits
+
+model = ESGify.from_pretrained('ai-lab/ESGify')
+tokenizer = AutoTokenizer.from_pretrained('ai-lab/ESGify')
+texts = ['text1','text2']
+to_model = tokenizer.batch_encode_plus(
+                  texts,
+                  add_special_tokens=True,
+                  max_length=512,
+                  return_token_type_ids=False,
+                  padding="max_length",
+                  truncation=True,
+                  return_attention_mask=True,
+                  return_tensors='pt',
+                )
+results = model(**to_model)
+
+
+# We also recommend preprocess texts with using FLAIR model
+
+from flair.data import Sentence
+from flair.nn import Classifier
+from torch.utils.data import DataLoader
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+stop_words = set(stopwords.words('english'))
+tagger = Classifier.load('ner-ontonotes-large')
+tag_list = ['FAC','LOC','ORG','PERSON']
+texts_with_masks = []
+for example_sent in texts:
+
+    word_tokens = word_tokenize(example_sent)
+    # converts the words in word_tokens to lower case and then checks whether 
+    #they are present in stop_words or not
+    for w in word_tokens:
+        if w.lower() not in stop_words:
+            filtered_sentence.append(w)
+    # make a sentence
+    sentence = Sentence(' '.join(filtered_sentence))
+    # run NER over sentence
+    tagger.predict(sentence)
+    sent = ' '.join(filtered_sentence)
+    k = 0
+    new_string = ''
+    start_t = 0 
+    for i in sentence.get_labels():
+        info = i.to_dict()
+        val = info['value']
+        if info['confidence']>0.8 and val in tag_list : 
+
+            if i.data_point.start_position>start_t :
+                new_string+=sent[start_t:i.data_point.start_position]
+            start_t = i.data_point.end_position
+            new_string+= f'<{val}>'
+    new_string+=sent[start_t:-1]
+    texts_with_masks.append(new_string)
+
+to_model = tokenizer.batch_encode_plus(
+                  texts_with_masks,
+                  add_special_tokens=True,
+                  max_length=512,
+                  return_token_type_ids=False,
+                  padding="max_length",
+                  truncation=True,
+                  return_attention_mask=True,
+                  return_tensors='pt',
+                )
+results = model(**to_model)
+```
+
+------
+
+## Background
+
+The project aims to develop the ESG Risks classification model with a custom ESG risks definition methodology. 
+
+
+## Training procedure
+
+### Pre-training 
+
+We use the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model. 
+Next, we do the domain-adaptation procedure by Mask Language Modeling pertaining with using texts of ESG reports. 
+
+
+#### Training data
+
+We use the ESG news dataset of 2000 texts with manually annotation of ESG specialists.