Update tokenizer based on new language model

added_tokens.json

@@ -0,0 +1 @@
+{"environmental": 50268, "reduced": 50422, "Eni": 50476, "assess": 50378, "decisions": 50412, "scenarios": 50330, "towards": 50414, "renewable": 50274, "reputation": 50327, "—": 50430, "strategy": 50286, "flooding": 50407, "regulations": 50321, "Governance": 50308, "supply": 50297, "sectors": 50312, "severity": 50468, "exposed": 50386, "resulting": 50361, "failure": 50499, "directly": 50427, "manage": 50326, "statements": 50453, "commitment": 50337, "Corporate": 50408, "technologies": 50352, "basis": 50473, "institutions": 50450, "engagement": 50397, "Agreement": 50344, "affect": 50320, "processes": 50354, "2030": 50339, "suppliers": 50402, "investments": 50282, "companies": 50271, "distribution": 50429, "portfolios": 50363, "finance": 50298, "2025": 50403, "manufacturing": 50425, "emerging": 50489, "metrics": 50442, "exposure": 50343, "mainly": 50466, "equity": 50434, "profitability": 50454, "managing": 50421, "uncertainty": 50406, "2050": 50401, "assessment": 50351, "materials": 50357, "insurance": 50296, "Sustainability": 50362, "conditions": 50376, "further": 50315, "challenges": 50400, "CO2": 50365, "emissions": 50266, "funds": 50398, "investors": 50356, "planning": 50435, "cement": 50492, "fuels": 50375, "regarding": 50413, "annual": 50342, "continued": 50443, "society": 50364, "activities": 50283, "reducing": 50359, "potential": 50281, "claims": 50470, "impacts": 50272, "objectives": 50387, "partners": 50482, "construction": 50462, "employees": 50332, "consumption": 50314, "bond": 50418, "additional": 50367, "plans": 50383, "sustainability": 50285, "scenario": 50290, "investing": 50392, "factors": 50301, "solutions": 50355, "addition": 50287, "financing": 50294, "provides": 50409, "infrastructure": 50310, "greater": 50444, "economy": 50304, "buildings": 50404, "vehicles": 50455, "requirements": 50335, "significantly": 50458, "lending": 50368, "bonds": 50452, "solar": 50340, "considered": 50479, "governance": 50345, "dioxide": 50471, "achieving": 50498, "regulatory": 50295, "identify": 50377, "Sustainable": 50341, "criteria": 50477, "sources": 50380, "–": 50317, "ESG": 50360, "BNP": 50370, "unable": 50464, "losses": 50346, "purchase": 50478, "developments": 50474, "practices": 50394, "indirect": 50445, "included": 50393, "initiative": 50424, "Committee": 50329, "prices": 50484, "particular": 50390, "’": 50267, "goals": 50348, "mitigate": 50428, "transition": 50273, "sustainable": 50288, "approach": 50311, "transport": 50419, "committed": 50334, "compared": 50467, "annually": 50488, "therefore": 50456, "resilience": 50433, "waste": 50331, "greenhouse": 50323, "Principles": 50494, "least": 50487, "increase": 50280, "developing": 50381, "€": 50303, "emission": 50388, "following": 50371, "‘": 50426, "operational": 50302, "floods": 50449, "commitments": 50465, "portfolio": 50277, "fossil": 50374, "stakeholders": 50384, "launched": 50440, "asset": 50325, "strategic": 50300, "overall": 50366, "decision": 50481, "supporting": 50490, "strategies": 50391, "contribute": 50405, "relating": 50461, "Investment": 50369, "plants": 50350, "businesses": 50309, "Greenhouse": 50463, "achieve": 50379, "areas": 50318, "BlackRock": 50480, "provide": 50313, "patterns": 50438, "returns": 50495, "equivalent": 50457, "caused": 50410, "deliver": 50437, "loan": 50396, "facilities": 50336, "transactions": 50451, "investment": 50269, "several": 50416, "possible": 50420, "clients": 50289, "risks": 50265, "“": 50373, "sands": 50441, "opportunities": 50278, "loans": 50417, "experience": 50448, "invested": 50439, "mortgage": 50459, "ensure": 50399, "increased": 50293, "communities": 50389, "longer": 50431, "Risk": 50319, "tonnes": 50447, "operating": 50372, "standards": 50349, "corporate": 50307, "reputational": 50469, "countries": 50328, "•": 50276, "across": 50299, "innovative": 50491, "temperature": 50423, "GHG": 50322, "others": 50472, "IFC": 50475, "costs": 50275, "operations": 50270, "various": 50382, "industry": 50292, "fiscal": 50436, "reduction": 50306, "legislation": 50485, "footprint": 50338, "electricity": 50291, "Paribas": 50385, "methodology": 50483, "reduce": 50284, "2021": 50347, "systems": 50333, "efforts": 50324, "customer": 50395, "initiatives": 50316, "already": 50496, "increases": 50446, "adversely": 50497, "Operational": 50486, "taken": 50460, "disasters": 50432, "policies": 50353, "Finance": 50493, "Carbon": 50358, "customers": 50279, "operate": 50415, "adverse": 50411, "targets": 50305}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "distilroberta-base", "tokenizer_class": "RobertaTokenizer"}
+{"unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "errors": "replace", "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 512, "name_or_path": "task2_model/tcfd/checkpoint-7", "special_tokens_map_file": "pre_model/30092021_roberta/special_tokens_map.json", "tokenizer_class": "RobertaTokenizer"}