Upload 10 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ESGify.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,112 @@
 ---
 license: apache-2.0
+tags:
+- ESG
+- finance
+language:
+- en
+
 ---
+![esgify](ESGify.png)
+# About ESGify
+<img src="ESGify_logo.jpeg" alt="image" width="20%" height="auto">
+**ESGify** is a model for multilabel news classification with respect to ESG risks. Our custom methodology includes 46 ESG classes and 1 non-relevant to ESG class, resulting in 47 classes in total:
+
+![esgify_classes](ESGify_classes.jpg)
+
+# Usage 
+
+ESGify is based on MPNet architecture but with a custom classification head. The ESGify class is defined is follows.
+
+```python
+from collections import OrderedDict
+from transformers import MPNetPreTrainedModel, MPNetModel, AutoTokenizer
+import torch
+
+# Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+        token_embeddings = model_output #First element of model_output contains all token embeddings
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+# Definition of ESGify class because of custom,sentence-transformers like, mean pooling function and classifier head
+class ESGify(MPNetPreTrainedModel):
+    """Model for Classification ESG risks from text."""
+
+    def __init__(self,config): #tuning only the head
+        """
+        """
+        super().__init__(config)
+        # Instantiate Parts of model
+        self.mpnet = MPNetModel(config,add_pooling_layer=False)
+        self.id2label =  config.id2label
+        self.label2id =  config.label2id
+        self.classifier = torch.nn.Sequential(OrderedDict([('norm',torch.nn.BatchNorm1d(768)),
+                                                ('linear',torch.nn.Linear(768,512)),
+                                                ('act',torch.nn.ReLU()),
+                                                ('batch_n',torch.nn.BatchNorm1d(512)),
+                                                ('drop_class', torch.nn.Dropout(0.2)),
+                                                ('class_l',torch.nn.Linear(512 ,47))]))
+
+
+    def forward(self, input_ids, attention_mask):
+         # Feed input to mpnet model
+        outputs = self.mpnet(input_ids=input_ids,
+                             attention_mask=attention_mask)
+         
+        # mean pooling dataset and eed input to classifier to compute logits
+        logits = self.classifier( mean_pooling(outputs['last_hidden_state'],attention_mask))
+         
+        # apply sigmoid
+        logits  = 1.0 / (1.0 + torch.exp(-logits))
+        return logits
+```
+
+After defining model class, we initialize ESGify and tokenizer with the pre-trained weights
+
+```python
+model = ESGify.from_pretrained('ai-lab/ESGify')
+tokenizer = AutoTokenizer.from_pretrained('ai-lab/ESGify')
+```
+
+Getting results from the model:
+
+```python
+texts = ['text1','text2']
+to_model = tokenizer.batch_encode_plus(
+                  texts,
+                  add_special_tokens=True,
+                  max_length=512,
+                  return_token_type_ids=False,
+                  padding="max_length",
+                  truncation=True,
+                  return_attention_mask=True,
+                  return_tensors='pt',
+                )
+results = model(**to_model)
+```
+
+To identify top-3 classes by relevance and their scores: 
+
+```python
+for i in torch.topk(results, k=3).indices.tolist()[0]:
+    print(f"{model.id2label[i]}: {np.round(results.flatten()[i].item(), 3)}")
+```
+
+For example, for the news "She faced employment rejection because of her gender", we get the following top-3 labels:
+```
+Discrimination: 0.944
+Strategy Implementation: 0.82
+Indigenous People: 0.499
+```
+
+Before training our model, we masked words related to Organisation, Date, Country, and Person to prevent false associations between these entities and risks. Hence, we recommend to process text with FLAIR NER model before inference.
+An example of such preprocessing is given in https://colab.research.google.com/drive/15YcTW9KPSWesZ6_L4BUayqW_omzars0l?usp=sharing.
+
+
+# Training procedure
+
+We use the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model.
+Next, we do the domain-adaptation procedure by Mask Language Modeling with using texts of ESG reports. 
+Finally, we fine-tune our model on 2000 texts with manually annotation of ESG specialists.
+

config.json

@@ -0,0 +1,129 @@
+{
+  "_name_or_path": "ai-forever/rubert-base",
+  "architectures": [
+    "ESGify_ru"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 120138,
+    "id2label": {"0": "Legal Proceedings & Law Violations",
+                 "1": "Biodiversity",
+                 "2": "Communities Health and Safety",
+                 "3": "Land Acquisition and Resettlement (S)",
+                 "4": "Emergencies (Social)",
+                 "5": "Corporate Governance",
+                 "6": "Responsible Investment & Greenwashing",
+                 "7": "Not Relevant to ESG",
+                 "8": "Economic Crime",
+                 "9": "Emergencies (Environmental)",
+                 "10": "Hazardous Materials Management",
+                 "11": "Environmental Management",
+                 "12": "Landscape Transformation",
+                 "13": "Human Rights",
+                 "14": "Climate Risks",
+                 "15": "Labor Relations Management",
+                 "16": "Freedom of Association and Right to Organise",
+                 "17": "Employee Health and Safety",
+                 "18": "Surface Water Pollution",
+                 "19": "Animal Welfare",
+                 "20": "Water Consumption",
+                 "21": "Disclosure",
+                 "22": "Product Safety and Quality",
+                 "23": "Greenhouse Gas Emissions",
+                 "24": "Indigenous People",
+                 "25": "Cultural Heritage",
+                 "26": "Air Pollution",
+                 "27": "Waste Management",
+                 "28": "Soil and Groundwater Impact",
+                 "29": "Forced Labour",
+                 "30": "Wastewater Management",
+                 "31": "Natural Resources",
+                 "32": "Physical Impacts",
+                 "33": "Values and Ethics",
+                 "34": "Risk Management and Internal Control",
+                 "35": "Supply Chain (Environmental)",
+                 "36": "Supply Chain (Social)",
+                 "37": "Discrimination",
+                 "38": "Minimum Age and Child Labour",
+                 "39": "Planning Limitations",
+                 "40": "Data Safety",
+                 "41": "Strategy Implementation",
+                 "42": "Energy Efficiency and Renewables",
+                 "43": "Land Acquisition and Resettlement (E)",
+                 "44": "Supply Chain (Economic / Governance)",
+                 "45": "Land Rehabilitation",
+                 "46": "Retrenchment",
+                 "47":"Positive"
+              },
+    "label2id":{"Legal Proceedings & Law Violations": "0",
+                 "Biodiversity": "1",
+                 "Communities Health and Safety": "2",
+                 "Land Acquisition and Resettlement (S)": "3",
+                 "Emergencies (Social)": "4",
+                 "Corporate Governance": "5",
+                 "Responsible Investment & Greenwashing": "6",
+                 "Not Relevant to ESG": "7",
+                 "Economic Crime": "8",
+                 "Emergencies (Environmental)": "9",
+                 "Hazardous Materials Management": "10",
+                 "Environmental Management": "11",
+                 "Landscape Transformation": "12",
+                 "Human Rights": "13",
+                 "Climate Risks": "14",
+                 "Labor Relations Management": "15",
+                 "Freedom of Association and Right to Organise": "16",
+                 "Employee Health and Safety": "17",
+                 "Surface Water Pollution": "18",
+                 "Animal Welfare": "19",
+                 "Water Consumption": "20",
+                 "Disclosure": "21",
+                 "Product Safety and Quality": "22",
+                 "Greenhouse Gas Emissions": "23",
+                 "Indigenous People": "24",
+                 "Cultural Heritage": "25",
+                 "Air Pollution": "26",
+                 "Waste Management": "27",
+                 "Soil and Groundwater Impact": "28",
+                 "Forced Labour": "29",
+                 "Wastewater Management": "30",
+                 "Natural Resources": "31",
+                 "Physical Impacts": "32",
+                 "Values and Ethics": "33",
+                 "Risk Management and Internal Control": "34",
+                 "Supply Chain (Environmental)": "35",
+                 "Supply Chain (Social)": "36",
+                 "Discrimination": "37",
+                 "Minimum Age and Child Labour": "38",
+                 "Planning Limitations": "39",
+                 "Data Safety": "40",
+                 "Strategy Implementation": "41",
+                 "Energy Efficiency and Renewables": "42",
+                 "Land Acquisition and Resettlement (E)": "43",
+                 "Supply Chain (Economic / Governance)": "44",
+                 "Land Rehabilitation": "45",
+                 "Retrenchment": "46",
+                 "Positive":"47"}
+    
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f80348b43849ada02addc53fa454eb8873b117f4ded7ec4c676d6c0051dd37
+size 712633885

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}