solving GPU error for previous version

README.md

@@ -1,13 +1,14 @@
 ---
-title: DCWIR Offcial Demo
-emoji: 🔥
-colorFrom: indigo
-colorTo: yellow
+title: DCWIR Demo
+emoji: 🛡️
+colorFrom: gray
+colorTo: blue
 sdk: gradio
-sdk_version: 4.31.5
+sdk_version: 4.31.0
 app_file: app.py
 pinned: false
 license: mit
+short_description: 'A Demo for DCWIR method in SAE steup . '
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

anonymous_demo/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "1.0.0"
+
+__name__ = "anonymous_demo"
+
+from anonymous_demo.functional import TADCheckpointManager

anonymous_demo/core/tad/classic/__bert__/README.MD

@@ -0,0 +1,3 @@
+## This is the simple migration from ABSA-PyTorch under MIT license
+
+Project Address: https://github.com/songyouwei/ABSA-PyTorch

anonymous_demo/core/tad/classic/__bert__/__init__.py

@@ -0,0 +1 @@
+from .models import *

anonymous_demo/core/tad/classic/__bert__/dataset_utils/data_utils_for_inference.py

@@ -0,0 +1,121 @@
+import tqdm
+from findfile import find_cwd_dir
+from torch.utils.data import Dataset
+from transformers import AutoTokenizer
+
+
+class Tokenizer4Pretraining:
+    def __init__(self, max_seq_len, opt, **kwargs):
+        if kwargs.pop("offline", False):
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                find_cwd_dir(opt.pretrained_bert.split("/")[-1]),
+                do_lower_case="uncased" in opt.pretrained_bert,
+            )
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                opt.pretrained_bert, do_lower_case="uncased" in opt.pretrained_bert
+            )
+        self.max_seq_len = max_seq_len
+
+    def text_to_sequence(self, text, reverse=False, padding="post", truncating="post"):
+        return self.tokenizer.encode(
+            text,
+            truncation=True,
+            padding="max_length",
+            max_length=self.max_seq_len,
+            return_tensors="pt",
+        )
+
+
+class BERTTADDataset(Dataset):
+    def __init__(self, tokenizer, opt):
+        self.bert_baseline_input_colses = {"bert": ["text_bert_indices"]}
+
+        self.tokenizer = tokenizer
+        self.opt = opt
+        self.all_data = []
+
+    def parse_sample(self, text):
+        return [text]
+
+    def prepare_infer_sample(self, text: str, ignore_error):
+        self.process_data(self.parse_sample(text), ignore_error=ignore_error)
+
+    def process_data(self, samples, ignore_error=True):
+        all_data = []
+        if len(samples) > 100:
+            it = tqdm.tqdm(
+                samples, postfix="preparing text classification inference dataloader..."
+            )
+        else:
+            it = samples
+        for text in it:
+            try:
+                # handle for empty lines in inference datasets
+                if text is None or "" == text.strip():
+                    raise RuntimeError("Invalid Input!")
+
+                if "!ref!" in text:
+                    text, _, labels = text.strip().partition("!ref!")
+                    text = text.strip()
+                    if labels.count(",") == 2:
+                        label, is_adv, adv_train_label = labels.strip().split(",")
+                        label, is_adv, adv_train_label = (
+                            label.strip(),
+                            is_adv.strip(),
+                            adv_train_label.strip(),
+                        )
+                    elif labels.count(",") == 1:
+                        label, is_adv = labels.strip().split(",")
+                        label, is_adv = label.strip(), is_adv.strip()
+                        adv_train_label = "-100"
+                    elif labels.count(",") == 0:
+                        label = labels.strip()
+                        adv_train_label = "-100"
+                        is_adv = "-100"
+                    else:
+                        label = "-100"
+                        adv_train_label = "-100"
+                        is_adv = "-100"
+
+                    label = int(label)
+                    adv_train_label = int(adv_train_label)
+                    is_adv = int(is_adv)
+
+                else:
+                    text = text.strip()
+                    label = -100
+                    adv_train_label = -100
+                    is_adv = -100
+
+                text_indices = self.tokenizer.text_to_sequence("{}".format(text))
+
+                data = {
+                    "text_bert_indices": text_indices[0],
+                    "text_raw": text,
+                    "label": label,
+                    "adv_train_label": adv_train_label,
+                    "is_adv": is_adv,
+                    # 'label': self.opt.label_to_index.get(label, -100) if isinstance(label, str) else label,
+                    #
+                    # 'adv_train_label': self.opt.adv_train_label_to_index.get(adv_train_label, -100) if isinstance(adv_train_label, str) else adv_train_label,
+                    #
+                    # 'is_adv': self.opt.is_adv_to_index.get(is_adv, -100) if isinstance(is_adv, str) else is_adv,
+                }
+
+                all_data.append(data)
+
+            except Exception as e:
+                if ignore_error:
+                    print("Ignore error while processing:", text)
+                else:
+                    raise e
+
+        self.all_data = all_data
+        return self.all_data
+
+    def __getitem__(self, index):
+        return self.all_data[index]
+
+    def __len__(self):
+        return len(self.all_data)

anonymous_demo/core/tad/classic/__bert__/models/__init__.py

@@ -0,0 +1 @@
+from .tad_bert import TADBERT

anonymous_demo/core/tad/classic/__bert__/models/tad_bert.py

@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+from transformers.models.bert.modeling_bert import BertPooler
+
+from anonymous_demo.network.sa_encoder import Encoder
+
+
+class TADBERT(nn.Module):
+    inputs = ["text_bert_indices"]
+
+    def __init__(self, bert, opt):
+        super(TADBERT, self).__init__()
+        self.opt = opt
+        self.bert = bert
+        self.pooler = BertPooler(bert.config)
+        self.dense1 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+        self.dense2 = nn.Linear(self.opt.hidden_dim, self.opt.adv_det_dim)
+        self.dense3 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+
+        self.encoder1 = Encoder(self.bert.config, opt=opt)
+        self.encoder2 = Encoder(self.bert.config, opt=opt)
+        self.encoder3 = Encoder(self.bert.config, opt=opt)
+
+    def forward(self, inputs):
+        text_raw_indices = inputs[0]
+        last_hidden_state = self.bert(text_raw_indices)["last_hidden_state"]
+
+        sent_logits = self.dense1(self.pooler(last_hidden_state))
+        advdet_logits = self.dense2(self.pooler(last_hidden_state))
+        adv_tr_logits = self.dense3(self.pooler(last_hidden_state))
+
+        att_score = torch.nn.functional.normalize(
+            last_hidden_state.abs().sum(dim=1, keepdim=False)
+            - last_hidden_state.abs().min(dim=1, keepdim=True)[0],
+            p=1,
+            dim=1,
+        )
+
+        outputs = {
+            "sent_logits": sent_logits,
+            "advdet_logits": advdet_logits,
+            "adv_tr_logits": adv_tr_logits,
+            "last_hidden_state": last_hidden_state,
+            "att_score": att_score,
+        }
+        return outputs

anonymous_demo/core/tad/models/__init__.py

@@ -0,0 +1,9 @@
+import anonymous_demo.core.tad.classic.__bert__.models
+
+
+class BERTTADModelList(list):
+    TADBERT = anonymous_demo.core.tad.classic.__bert__.TADBERT
+
+    def __init__(self):
+        model_list = [self.TADBERT]
+        super().__init__(model_list)

anonymous_demo/core/tad/prediction/tad_classifier.py

@@ -0,0 +1,518 @@
+import json
+import os
+import pickle
+import time
+
+import torch
+import tqdm
+from findfile import find_file, find_cwd_dir
+from termcolor import colored
+
+from torch.utils.data import DataLoader
+from transformers import (
+    AutoTokenizer,
+    AutoModel,
+    AutoConfig,
+    DebertaV2ForMaskedLM,
+    RobertaForMaskedLM,
+    BertForMaskedLM,
+)
+
+from ....functional.dataset.dataset_manager import detect_infer_dataset
+
+from ..models import BERTTADModelList
+from ..classic.__bert__.dataset_utils.data_utils_for_inference import (
+    BERTTADDataset,
+    Tokenizer4Pretraining,
+)
+
+from ....utils.demo_utils import (
+    print_args,
+    TransformerConnectionError,
+    get_device,
+    build_embedding_matrix,
+)
+
+
+def init_attacker(tad_classifier, defense):
+    try:
+        from textattack import Attacker
+        from textattack.attack_recipes import (
+            BAEGarg2019,
+            PWWSRen2019,
+            TextFoolerJin2019,
+            PSOZang2020,
+            IGAWang2019,
+            GeneticAlgorithmAlzantot2018,
+            DeepWordBugGao2018,
+        )
+        from textattack.datasets import Dataset
+        from textattack.models.wrappers import HuggingFaceModelWrapper
+
+        class DemoModelWrapper(HuggingFaceModelWrapper):
+            def __init__(self, model):
+                self.model = model  # pipeline = pipeline
+
+            def __call__(self, text_inputs, **kwargs):
+                outputs = []
+                for text_input in text_inputs:
+                    raw_outputs = self.model.infer(
+                        text_input, print_result=False, **kwargs
+                    )
+                    outputs.append(raw_outputs["probs"])
+                return outputs
+
+        class SentAttacker:
+            def __init__(self, model, recipe_class=BAEGarg2019):
+                model = model
+                model_wrapper = DemoModelWrapper(model)
+
+                recipe = recipe_class.build(model_wrapper)
+
+                _dataset = [("", 0)]
+                _dataset = Dataset(_dataset)
+
+                self.attacker = Attacker(recipe, _dataset)
+
+        attackers = {
+            "bae": BAEGarg2019,
+            "pwws": PWWSRen2019,
+            "textfooler": TextFoolerJin2019,
+            "pso": PSOZang2020,
+            "iga": IGAWang2019,
+            "ga": GeneticAlgorithmAlzantot2018,
+            "wordbugger": DeepWordBugGao2018,
+        }
+        return SentAttacker(tad_classifier, attackers[defense])
+    except Exception as e:
+        print("Original error:", e)
+
+
+def get_mlm_and_tokenizer(text_classifier, config):
+    if isinstance(text_classifier, TADTextClassifier):
+        base_model = text_classifier.model.bert.base_model
+    else:
+        base_model = text_classifier.bert.base_model
+    pretrained_config = AutoConfig.from_pretrained(config.pretrained_bert)
+    if "deberta-v3" in config.pretrained_bert:
+        MLM = DebertaV2ForMaskedLM(pretrained_config)
+        MLM.deberta = base_model
+    elif "roberta" in config.pretrained_bert:
+        MLM = RobertaForMaskedLM(pretrained_config)
+        MLM.roberta = base_model
+    else:
+        MLM = BertForMaskedLM(pretrained_config)
+        MLM.bert = base_model
+    return MLM, AutoTokenizer.from_pretrained(config.pretrained_bert)
+
+
+class TADTextClassifier:
+    def __init__(self, model_arg=None, cal_perplexity=False, **kwargs):
+        """
+        from_train_model: load inference model from trained model
+        """
+        self.cal_perplexity = cal_perplexity
+        # load from a training
+        if not isinstance(model_arg, str):
+            print("Load text classifier from training")
+            self.model = model_arg[0]
+            self.opt = model_arg[1]
+            self.tokenizer = model_arg[2]
+        else:
+            try:
+                if "fine-tuned" in model_arg:
+                    raise ValueError(
+                        "Do not support to directly load a fine-tuned model, please load a .state_dict or .model instead!"
+                    )
+                print("Load text classifier from", model_arg)
+                state_dict_path = find_file(
+                    model_arg, key=".state_dict", exclude_key=["__MACOSX"]
+                )
+                model_path = find_file(
+                    model_arg, key=".model", exclude_key=["__MACOSX"]
+                )
+                tokenizer_path = find_file(
+                    model_arg, key=".tokenizer", exclude_key=["__MACOSX"]
+                )
+                config_path = find_file(
+                    model_arg, key=".config", exclude_key=["__MACOSX"]
+                )
+
+                print("config: {}".format(config_path))
+                print("state_dict: {}".format(state_dict_path))
+                print("model: {}".format(model_path))
+                print("tokenizer: {}".format(tokenizer_path))
+
+                with open(config_path, mode="rb") as f:
+                    self.opt = pickle.load(f)
+                    self.opt.device = get_device(kwargs.pop("auto_device", True))[0]
+
+                if state_dict_path or model_path:
+                    if hasattr(BERTTADModelList, self.opt.model.__name__):
+                        if state_dict_path:
+                            if kwargs.pop("offline", False):
+                                self.bert = AutoModel.from_pretrained(
+                                    find_cwd_dir(
+                                        self.opt.pretrained_bert.split("/")[-1]
+                                    )
+                                )
+                            else:
+                                self.bert = AutoModel.from_pretrained(
+                                    self.opt.pretrained_bert
+                                )
+                            self.model = self.opt.model(self.bert, self.opt)
+                            self.model.load_state_dict(
+                                torch.load(state_dict_path, map_location="cpu")
+                            )
+                        elif model_path:
+                            self.model = torch.load(model_path, map_location="cpu")
+
+                        try:
+                            self.tokenizer = Tokenizer4Pretraining(
+                                max_seq_len=self.opt.max_seq_len, opt=self.opt, **kwargs
+                            )
+                        except ValueError:
+                            if tokenizer_path:
+                                with open(tokenizer_path, mode="rb") as f:
+                                    self.tokenizer = pickle.load(f)
+                            else:
+                                raise TransformerConnectionError()
+
+            except Exception as e:
+                raise RuntimeError(
+                    "Exception: {} Fail to load the model from {}! ".format(
+                        e, model_arg
+                    )
+                )
+
+        self.infer_dataloader = None
+        self.opt.eval_batch_size = kwargs.pop("eval_batch_size", 128)
+
+        self.opt.initializer = self.opt.initializer
+
+        if self.cal_perplexity:
+            try:
+                self.MLM, self.MLM_tokenizer = get_mlm_and_tokenizer(self, self.opt)
+            except Exception as e:
+                self.MLM, self.MLM_tokenizer = None, None
+
+        self.to(self.opt.device)
+
+    def to(self, device=None):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, "MLM"):
+            self.MLM.to(self.opt.device)
+
+    def cpu(self):
+        self.opt.device = "cpu"
+        self.model.to("cpu")
+        if hasattr(self, "MLM"):
+            self.MLM.to("cpu")
+
+    def cuda(self, device="cuda:0"):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, "MLM"):
+            self.MLM.to(device)
+
+    def _log_write_args(self):
+        n_trainable_params, n_nontrainable_params = 0, 0
+        for p in self.model.parameters():
+            n_params = torch.prod(torch.tensor(p.shape))
+            if p.requires_grad:
+                n_trainable_params += n_params
+            else:
+                n_nontrainable_params += n_params
+        print(
+            "n_trainable_params: {0}, n_nontrainable_params: {1}".format(
+                n_trainable_params, n_nontrainable_params
+            )
+        )
+        for arg in vars(self.opt):
+            if getattr(self.opt, arg) is not None:
+                print(">>> {0}: {1}".format(arg, getattr(self.opt, arg)))
+
+    def batch_infer(
+        self,
+        target_file=None,
+        print_result=True,
+        save_result=False,
+        ignore_error=True,
+        defense: str = None,
+    ):
+        save_path = os.path.join(os.getcwd(), "tad_text_classification.result.json")
+
+        target_file = detect_infer_dataset(target_file, task="text_defense")
+        if not target_file:
+            raise FileNotFoundError("Can not find inference datasets!")
+
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        dataset.prepare_infer_dataset(target_file, ignore_error=ignore_error)
+        self.infer_dataloader = DataLoader(
+            dataset=dataset,
+            batch_size=self.opt.eval_batch_size,
+            pin_memory=True,
+            shuffle=False,
+        )
+        return self._infer(
+            save_path=save_path if save_result else None,
+            print_result=print_result,
+            defense=defense,
+        )
+
+    def infer(
+        self,
+        text: str = None,
+        print_result=True,
+        ignore_error=True,
+        defense: str = None,
+    ):
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        if text:
+            dataset.prepare_infer_sample(text, ignore_error=ignore_error)
+        else:
+            raise RuntimeError("Please specify your datasets path!")
+        self.infer_dataloader = DataLoader(
+            dataset=dataset, batch_size=self.opt.eval_batch_size, shuffle=False
+        )
+        return self._infer(print_result=print_result, defense=defense)[0]
+
+    def _infer(self, save_path=None, print_result=True, defense=None):
+        _params = filter(lambda p: p.requires_grad, self.model.parameters())
+
+        correct = {True: "Correct", False: "Wrong"}
+        results = []
+
+        with torch.no_grad():
+            self.model.eval()
+            n_correct = 0
+            n_labeled = 0
+
+            n_advdet_correct = 0
+            n_advdet_labeled = 0
+            if len(self.infer_dataloader.dataset) >= 100:
+                it = tqdm.tqdm(self.infer_dataloader, postfix="inferring...")
+            else:
+                it = self.infer_dataloader
+            for _, sample in enumerate(it):
+                inputs = [
+                    sample[col].to(self.opt.device) for col in self.opt.inputs_cols
+                ]
+                outputs = self.model(inputs)
+                logits, advdet_logits, adv_tr_logits = (
+                    outputs["sent_logits"],
+                    outputs["advdet_logits"],
+                    outputs["adv_tr_logits"],
+                )
+                probs, advdet_probs, adv_tr_probs = (
+                    torch.softmax(logits, dim=-1),
+                    torch.softmax(advdet_logits, dim=-1),
+                    torch.softmax(adv_tr_logits, dim=-1),
+                )
+
+                for i, (prob, advdet_prob, adv_tr_prob) in enumerate(
+                    zip(probs, advdet_probs, adv_tr_probs)
+                ):
+                    text_raw = sample["text_raw"][i]
+
+                    pred_label = int(prob.argmax(axis=-1))
+                    pred_is_adv_label = int(advdet_prob.argmax(axis=-1))
+                    pred_adv_tr_label = int(adv_tr_prob.argmax(axis=-1))
+                    ref_label = (
+                        int(sample["label"][i])
+                        if int(sample["label"][i]) in self.opt.index_to_label
+                        else ""
+                    )
+                    ref_is_adv_label = (
+                        int(sample["is_adv"][i])
+                        if int(sample["is_adv"][i]) in self.opt.index_to_is_adv
+                        else ""
+                    )
+                    ref_adv_tr_label = (
+                        int(sample["adv_train_label"][i])
+                        if int(sample["adv_train_label"][i])
+                        in self.opt.index_to_adv_train_label
+                        else ""
+                    )
+
+                    if self.cal_perplexity:
+                        ids = self.MLM_tokenizer(text_raw, return_tensors="pt")
+                        ids["labels"] = ids["input_ids"].clone()
+                        ids = ids.to(self.opt.device)
+                        loss = self.MLM(**ids)["loss"]
+                        perplexity = float(torch.exp(loss / ids["input_ids"].size(1)))
+                    else:
+                        perplexity = "N.A."
+
+                    result = {
+                        "text": text_raw,
+                        "label": self.opt.index_to_label[pred_label],
+                        "probs": prob.cpu().numpy(),
+                        "confidence": float(max(prob)),
+                        "ref_label": self.opt.index_to_label[ref_label]
+                        if isinstance(ref_label, int)
+                        else ref_label,
+                        "ref_label_check": correct[pred_label == ref_label]
+                        if ref_label != -100
+                        else "",
+                        "is_fixed": False,
+                        "is_adv_label": self.opt.index_to_is_adv[pred_is_adv_label],
+                        "is_adv_probs": advdet_prob.cpu().numpy(),
+                        "is_adv_confidence": float(max(advdet_prob)),
+                        "ref_is_adv_label": self.opt.index_to_is_adv[ref_is_adv_label]
+                        if isinstance(ref_is_adv_label, int)
+                        else ref_is_adv_label,
+                        "ref_is_adv_check": correct[
+                            pred_is_adv_label == ref_is_adv_label
+                        ]
+                        if ref_is_adv_label != -100
+                        and isinstance(ref_is_adv_label, int)
+                        else "",
+                        "pred_adv_tr_label": self.opt.index_to_label[pred_adv_tr_label],
+                        "ref_adv_tr_label": self.opt.index_to_label[ref_adv_tr_label],
+                        "perplexity": perplexity,
+                    }
+                    if defense:
+                        try:
+                            if not hasattr(self, "sent_attacker"):
+                                self.sent_attacker = init_attacker(
+                                    self, defense.lower()
+                                )
+                            if result["is_adv_label"] == "1":
+                                res = self.sent_attacker.attacker.simple_attack(
+                                    text_raw, int(result["label"])
+                                )
+                                new_infer_res = self.infer(
+                                    res.perturbed_result.attacked_text.text,
+                                    print_result=False,
+                                )
+                                result["perturbed_label"] = result["label"]
+                                result["label"] = new_infer_res["label"]
+                                result["probs"] = new_infer_res["probs"]
+                                result["ref_label_check"] = (
+                                    correct[int(result["label"]) == ref_label]
+                                    if ref_label != -100
+                                    else ""
+                                )
+                                result[
+                                    "restored_text"
+                                ] = res.perturbed_result.attacked_text.text
+                                result["is_fixed"] = True
+                            else:
+                                result["restored_text"] = ""
+                                result["is_fixed"] = False
+
+                        except Exception as e:
+                            print(
+                                "Error:{}, try install TextAttack and tensorflow_text after 10 seconds...".format(
+                                    e
+                                )
+                            )
+                            time.sleep(10)
+                            raise RuntimeError("Installation done, please run again...")
+
+                    if ref_label != -100:
+                        n_labeled += 1
+
+                        if result["label"] == result["ref_label"]:
+                            n_correct += 1
+
+                    if ref_is_adv_label != -100:
+                        n_advdet_labeled += 1
+                        if ref_is_adv_label == pred_is_adv_label:
+                            n_advdet_correct += 1
+
+                    results.append(result)
+
+        try:
+            if print_result:
+                for ex_id, result in enumerate(results):
+                    text_printing = result["text"][:]
+                    text_info = ""
+                    if result["label"] != "-100":
+                        if not result["ref_label"]:
+                            text_info += " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                result["label"],
+                                result["ref_label"],
+                                result["confidence"],
+                            )
+                        elif result["label"] == result["ref_label"]:
+                            text_info += colored(
+                                " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                    result["label"],
+                                    result["ref_label"],
+                                    result["confidence"],
+                                ),
+                                "green",
+                            )
+                        else:
+                            text_info += colored(
+                                " -> <CLS:{}(ref:{} confidence:{})>".format(
+                                    result["label"],
+                                    result["ref_label"],
+                                    result["confidence"],
+                                ),
+                                "red",
+                            )
+
+                    # AdvDet
+                    if result["is_adv_label"] != "-100":
+                        if not result["ref_is_adv_label"]:
+                            text_info += " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                result["is_adv_label"],
+                                result["ref_is_adv_check"],
+                                result["is_adv_confidence"],
+                            )
+                        elif result["is_adv_label"] == result["ref_is_adv_label"]:
+                            text_info += colored(
+                                " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                    result["is_adv_label"],
+                                    result["ref_is_adv_label"],
+                                    result["is_adv_confidence"],
+                                ),
+                                "green",
+                            )
+                        else:
+                            text_info += colored(
+                                " -> <AdvDet:{}(ref:{} confidence:{})>".format(
+                                    result["is_adv_label"],
+                                    result["ref_is_adv_label"],
+                                    result["is_adv_confidence"],
+                                ),
+                                "red",
+                            )
+                    text_printing += text_info
+                    if self.cal_perplexity:
+                        text_printing += colored(
+                            " --> <perplexity:{}>".format(result["perplexity"]),
+                            "yellow",
+                        )
+                    print("Example {}: {}".format(ex_id, text_printing))
+            if save_path:
+                with open(save_path, "w", encoding="utf8") as fout:
+                    json.dump(str(results), fout, ensure_ascii=False)
+                    print("inference result saved in: {}".format(save_path))
+        except Exception as e:
+            print("Can not save result: {}, Exception: {}".format(text_raw, e))
+
+        if len(results) > 1:
+            print(
+                "CLS Acc:{}%".format(100 * n_correct / n_labeled if n_labeled else "")
+            )
+            print(
+                "AdvDet Acc:{}%".format(
+                    100 * n_advdet_correct / n_advdet_labeled
+                    if n_advdet_labeled
+                    else ""
+                )
+            )
+
+        return results
+
+    def clear_input_samples(self):
+        self.dataset.all_data = []

anonymous_demo/functional/__init__.py

@@ -0,0 +1,3 @@
+from anonymous_demo.functional.checkpoint.checkpoint_manager import TADCheckpointManager
+
+from anonymous_demo.functional.config import TADConfigManager

anonymous_demo/functional/checkpoint/__init__.py

@@ -0,0 +1 @@
+from .checkpoint_manager import TADCheckpointManager

anonymous_demo/functional/checkpoint/checkpoint_manager.py

@@ -0,0 +1,19 @@
+import os
+from findfile import find_file
+
+from anonymous_demo.core.tad.prediction.tad_classifier import TADTextClassifier
+from anonymous_demo.utils.demo_utils import retry
+
+
+class CheckpointManager:
+    pass
+
+
+class TADCheckpointManager(CheckpointManager):
+    @staticmethod
+    @retry
+    def get_tad_text_classifier(checkpoint: str = None, eval_batch_size=128, **kwargs):
+        tad_text_classifier = TADTextClassifier(
+            checkpoint, eval_batch_size=eval_batch_size, **kwargs
+        )
+        return tad_text_classifier

anonymous_demo/functional/config/__init__.py

@@ -0,0 +1 @@
+from .tad_config_manager import TADConfigManager

anonymous_demo/functional/config/config_manager.py

@@ -0,0 +1,64 @@
+from argparse import Namespace
+
+import torch
+
+one_shot_messages = set()
+
+
+def config_check(args):
+    pass
+
+
+class ConfigManager(Namespace):
+    def __init__(self, args=None, **kwargs):
+        """
+        The ConfigManager is a subclass of argparse.Namespace and based on parameter dict and count the call-frequency of each parameter
+        :param args: A parameter dict
+        :param kwargs: Same param as Namespce
+        """
+        if not args:
+            args = {}
+        super().__init__(**kwargs)
+
+        if isinstance(args, Namespace):
+            self.args = vars(args)
+            self.args_call_count = {arg: 0 for arg in vars(args)}
+        else:
+            self.args = args
+            self.args_call_count = {arg: 0 for arg in args}
+
+    def __getattribute__(self, arg_name):
+        if arg_name == "args" or arg_name == "args_call_count":
+            return super().__getattribute__(arg_name)
+        try:
+            value = super().__getattribute__("args")[arg_name]
+            args_call_count = super().__getattribute__("args_call_count")
+            args_call_count[arg_name] += 1
+            super().__setattr__("args_call_count", args_call_count)
+            return value
+
+        except Exception as e:
+            return super().__getattribute__(arg_name)
+
+    def __setattr__(self, arg_name, value):
+        if arg_name == "args" or arg_name == "args_call_count":
+            super().__setattr__(arg_name, value)
+            return
+        try:
+            args = super().__getattribute__("args")
+            args[arg_name] = value
+            super().__setattr__("args", args)
+            args_call_count = super().__getattribute__("args_call_count")
+
+            if arg_name in args_call_count:
+                # args_call_count[arg_name] += 1
+                super().__setattr__("args_call_count", args_call_count)
+
+            else:
+                args_call_count[arg_name] = 0
+                super().__setattr__("args_call_count", args_call_count)
+
+        except Exception as e:
+            super().__setattr__(arg_name, value)
+
+        config_check(args)

anonymous_demo/functional/config/tad_config_manager.py

@@ -0,0 +1,229 @@
+import copy
+
+from anonymous_demo.functional.config.config_manager import ConfigManager
+from anonymous_demo.core.tad.classic.__bert__.models import TADBERT
+
+_tad_config_template = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/mdeberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1,
+    "use_amp": False,
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_base = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "pretrained_bert": "microsoft/deberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "patience": 99999,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_english = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/deberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_multilingual = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "pretrained_bert": "microsoft/mdeberta-v3-base",
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+_tad_config_chinese = {
+    "model": TADBERT,
+    "optimizer": "adamw",
+    "learning_rate": 0.00002,
+    "patience": 99999,
+    "cache_dataset": True,
+    "warmup_step": -1,
+    "show_metric": False,
+    "pretrained_bert": "bert-base-chinese",
+    "max_seq_len": 80,
+    "dropout": 0,
+    "l2reg": 0.000001,
+    "num_epoch": 10,
+    "batch_size": 16,
+    "initializer": "xavier_uniform_",
+    "seed": 52,
+    "polarities_dim": 3,
+    "log_step": 10,
+    "evaluate_begin": 0,
+    "cross_validate_fold": -1
+    # split train and test datasets into 5 folds and repeat 3 training
+}
+
+
+class TADConfigManager(ConfigManager):
+    def __init__(self, args, **kwargs):
+        """
+        Available Params:  {'model': BERT,
+                            'optimizer': "adamw",
+                            'learning_rate': 0.00002,
+                            'pretrained_bert': "roberta-base",
+                            'cache_dataset': True,
+                            'warmup_step': -1,
+                            'show_metric': False,
+                            'max_seq_len': 80,
+                            'patience': 99999,
+                            'dropout': 0,
+                            'l2reg': 0.000001,
+                            'num_epoch': 10,
+                            'batch_size': 16,
+                            'initializer': 'xavier_uniform_',
+                            'seed': {52, 25}
+                            'embed_dim': 768,
+                            'hidden_dim': 768,
+                            'polarities_dim': 3,
+                            'log_step': 10,
+                            'evaluate_begin': 0,
+                            'cross_validate_fold': -1 # split train and test datasets into 5 folds and repeat 3 training
+                            }
+        :param args:
+        :param kwargs:
+        """
+        super().__init__(args, **kwargs)
+
+    @staticmethod
+    def set_tad_config(configType: str, newitem: dict):
+        if isinstance(newitem, dict):
+            if configType == "template":
+                _tad_config_template.update(newitem)
+            elif configType == "base":
+                _tad_config_base.update(newitem)
+            elif configType == "english":
+                _tad_config_english.update(newitem)
+            elif configType == "chinese":
+                _tad_config_chinese.update(newitem)
+            elif configType == "multilingual":
+                _tad_config_multilingual.update(newitem)
+            elif configType == "glove":
+                _tad_config_glove.update(newitem)
+            else:
+                raise ValueError(
+                    "Wrong value of config type supplied, please use one from following type: template, base, english, chinese, multilingual, glove"
+                )
+        else:
+            raise TypeError(
+                "Wrong type of new config item supplied, please use dict e.g.{'NewConfig': NewValue}"
+            )
+
+    @staticmethod
+    def set_tad_config_template(newitem):
+        TADConfigManager.set_tad_config("template", newitem)
+
+    @staticmethod
+    def set_tad_config_base(newitem):
+        TADConfigManager.set_tad_config("base", newitem)
+
+    @staticmethod
+    def set_tad_config_english(newitem):
+        TADConfigManager.set_tad_config("english", newitem)
+
+    @staticmethod
+    def set_tad_config_chinese(newitem):
+        TADConfigManager.set_tad_config("chinese", newitem)
+
+    @staticmethod
+    def set_tad_config_multilingual(newitem):
+        TADConfigManager.set_tad_config("multilingual", newitem)
+
+    @staticmethod
+    def set_tad_config_glove(newitem):
+        TADConfigManager.set_tad_config("glove", newitem)
+
+    @staticmethod
+    def get_tad_config_template() -> ConfigManager:
+        _tad_config_template.update(_tad_config_template)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_base() -> ConfigManager:
+        _tad_config_template.update(_tad_config_base)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_english() -> ConfigManager:
+        _tad_config_template.update(_tad_config_english)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_chinese() -> ConfigManager:
+        _tad_config_template.update(_tad_config_chinese)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_multilingual() -> ConfigManager:
+        _tad_config_template.update(_tad_config_multilingual)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_glove() -> ConfigManager:
+        _tad_config_template.update(_tad_config_glove)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))

anonymous_demo/functional/dataset/__init__.py

@@ -0,0 +1 @@
+from anonymous_demo.functional.dataset.dataset_manager import detect_infer_dataset

anonymous_demo/functional/dataset/dataset_manager.py

@@ -0,0 +1,45 @@
+import os
+from findfile import find_files, find_dir
+
+filter_key_words = [
+    ".py",
+    ".md",
+    "readme",
+    "log",
+    "result",
+    "zip",
+    ".state_dict",
+    ".model",
+    ".png",
+    "acc_",
+    "f1_",
+    ".backup",
+    ".bak",
+]
+
+
+def detect_infer_dataset(dataset_path, task="apc"):
+    dataset_file = []
+    if isinstance(dataset_path, str) and os.path.isfile(dataset_path):
+        dataset_file.append(dataset_path)
+        return dataset_file
+
+    for d in dataset_path:
+        if not os.path.exists(d):
+            search_path = find_dir(
+                os.getcwd(),
+                [d, task, "dataset"],
+                exclude_key=filter_key_words,
+                disable_alert=False,
+            )
+            dataset_file += find_files(
+                search_path,
+                [".inference", d],
+                exclude_key=["train."] + filter_key_words,
+            )
+        else:
+            dataset_file += find_files(
+                d, [".inference", task], exclude_key=["train."] + filter_key_words
+            )
+
+    return dataset_file

anonymous_demo/network/lcf_pooler.py

@@ -0,0 +1,28 @@
+import numpy
+import torch
+import torch.nn as nn
+
+
+class LCF_Pooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states, lcf_vec):
+        device = hidden_states.device
+        lcf_vec = lcf_vec.detach().cpu().numpy()
+
+        pooled_output = numpy.zeros(
+            (hidden_states.shape[0], hidden_states.shape[2]), dtype=numpy.float32
+        )
+        hidden_states = hidden_states.detach().cpu().numpy()
+        for i, vec in enumerate(lcf_vec):
+            lcf_ids = [j for j in range(len(vec)) if sum(vec[j] - 1.0) == 0]
+            pooled_output[i] = hidden_states[i][lcf_ids[len(lcf_ids) // 2]]
+
+        pooled_output = torch.Tensor(pooled_output).to(device)
+        pooled_output = self.dense(pooled_output)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output

anonymous_demo/network/lsa.py

@@ -0,0 +1,73 @@
+import torch
+from anonymous_demo.network.sa_encoder import Encoder
+from torch import nn
+
+
+class LSA(nn.Module):
+    def __init__(self, bert, opt):
+        super(LSA, self).__init__()
+        self.opt = opt
+
+        self.encoder = Encoder(bert.config, opt)
+        self.encoder_left = Encoder(bert.config, opt)
+        self.encoder_right = Encoder(bert.config, opt)
+        self.linear_window_3h = nn.Linear(opt.embed_dim * 3, opt.embed_dim)
+        self.linear_window_2h = nn.Linear(opt.embed_dim * 2, opt.embed_dim)
+        self.eta1 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+        self.eta2 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+
+    def forward(
+        self,
+        global_context_features,
+        spc_mask_vec,
+        lcf_matrix,
+        left_lcf_matrix,
+        right_lcf_matrix,
+    ):
+        masked_global_context_features = torch.mul(
+            spc_mask_vec, global_context_features
+        )
+
+        # # --------------------------------------------------- #
+        lcf_features = torch.mul(global_context_features, lcf_matrix)
+        lcf_features = self.encoder(lcf_features)
+        # # --------------------------------------------------- #
+        left_lcf_features = torch.mul(masked_global_context_features, left_lcf_matrix)
+        left_lcf_features = self.encoder_left(left_lcf_features)
+        # # --------------------------------------------------- #
+        right_lcf_features = torch.mul(masked_global_context_features, right_lcf_matrix)
+        right_lcf_features = self.encoder_right(right_lcf_features)
+        # # --------------------------------------------------- #
+        if "lr" == self.opt.window or "rl" == self.opt.window:
+            if self.eta1 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta1)
+                print("reset eta1 to: {}".format(self.eta1.item()))
+            if self.eta2 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta2)
+                print("reset eta2 to: {}".format(self.eta2.item()))
+            if self.opt.eta >= 0:
+                cat_features = torch.cat(
+                    (
+                        lcf_features,
+                        self.eta1 * left_lcf_features,
+                        self.eta2 * right_lcf_features,
+                    ),
+                    -1,
+                )
+            else:
+                cat_features = torch.cat(
+                    (lcf_features, left_lcf_features, right_lcf_features), -1
+                )
+            sent_out = self.linear_window_3h(cat_features)
+        elif "l" == self.opt.window:
+            sent_out = self.linear_window_2h(
+                torch.cat((lcf_features, self.eta1 * left_lcf_features), -1)
+            )
+        elif "r" == self.opt.window:
+            sent_out = self.linear_window_2h(
+                torch.cat((lcf_features, self.eta2 * right_lcf_features), -1)
+            )
+        else:
+            raise KeyError("Invalid parameter:", self.opt.window)
+
+        return sent_out

anonymous_demo/network/sa_encoder.py

@@ -0,0 +1,199 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+class BertSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
+            config, "embedding_size"
+        ):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(
+            config.attention_probs_dropout_prob
+            if hasattr(config, "attention_probs_dropout_prob")
+            else 0
+        )
+        self.position_embedding_type = getattr(
+            config, "position_embedding_type", "absolute"
+        )
+        if (
+            self.position_embedding_type == "relative_key"
+            or self.position_embedding_type == "relative_key_query"
+        ):
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(
+                2 * config.max_position_embeddings - 1, self.attention_head_size
+            )
+
+        self.is_decoder = config.is_decoder
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (
+            self.num_attention_heads,
+            self.attention_head_size,
+        )
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        mixed_query_layer = self.query(hidden_states)
+
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention and past_key_value is not None:
+            # reuse k,v, cross_attentions
+            key_layer = past_key_value[0]
+            value_layer = past_key_value[1]
+            attention_mask = encoder_attention_mask
+        elif is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        if self.is_decoder:
+            # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
+            # Further calls to cross_attention layer can then reuse all cross-attention
+            # key/value_states (first "if" case)
+            # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
+            # all previous decoder key/value_states. Further calls to uni-directional self-attention
+            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
+            # if encoder bi-directional self-attention `past_key_value` is always `None`
+            past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if (
+            self.position_embedding_type == "relative_key"
+            or self.position_embedding_type == "relative_key_query"
+        ):
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(
+                seq_length, dtype=torch.long, device=hidden_states.device
+            ).view(-1, 1)
+            position_ids_r = torch.arange(
+                seq_length, dtype=torch.long, device=hidden_states.device
+            ).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(
+                distance + self.max_position_embeddings - 1
+            )
+            positional_embedding = positional_embedding.to(
+                dtype=query_layer.dtype
+            )  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum(
+                    "bhld,lrd->bhlr", query_layer, positional_embedding
+                )
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum(
+                    "bhld,lrd->bhlr", query_layer, positional_embedding
+                )
+                relative_position_scores_key = torch.einsum(
+                    "bhrd,lrd->bhlr", key_layer, positional_embedding
+                )
+                attention_scores = (
+                    attention_scores
+                    + relative_position_scores_query
+                    + relative_position_scores_key
+                )
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (
+            (context_layer, attention_probs) if output_attentions else (context_layer,)
+        )
+
+        if self.is_decoder:
+            outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class Encoder(nn.Module):
+    def __init__(self, config, opt, layer_num=1):
+        super(Encoder, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.encoder = nn.ModuleList(
+            [SelfAttention(config, opt) for _ in range(layer_num)]
+        )
+        self.tanh = torch.nn.Tanh()
+
+    def forward(self, x):
+        for i, enc in enumerate(self.encoder):
+            x = self.tanh(enc(x)[0])
+        return x
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, config, opt):
+        super(SelfAttention, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.SA = BertSelfAttention(config)
+
+    def forward(self, inputs):
+        zero_vec = np.zeros((inputs.size(0), 1, 1, self.opt.max_seq_len))
+        zero_tensor = torch.tensor(zero_vec).float().to(inputs.device)
+        SA_out = self.SA(inputs, zero_tensor)
+        return SA_out

anonymous_demo/utils/demo_utils.py

@@ -0,0 +1,247 @@
+import json
+import os
+import pickle
+import signal
+import threading
+import time
+import zipfile
+
+import gdown
+import numpy as np
+import requests
+import torch
+import tqdm
+from autocuda import auto_cuda, auto_cuda_name
+from findfile import find_files, find_cwd_file, find_file
+from termcolor import colored
+from functools import wraps
+
+from update_checker import parse_version
+
+from anonymous_demo import __version__
+
+
+def save_args(config, save_path):
+    f = open(os.path.join(save_path), mode="w", encoding="utf8")
+    for arg in config.args:
+        if config.args_call_count[arg]:
+            f.write("{}: {}\n".format(arg, config.args[arg]))
+    f.close()
+
+
+def print_args(config, logger=None, mode=0):
+    args = [key for key in sorted(config.args.keys())]
+    for arg in args:
+        if logger:
+            logger.info(
+                "{0}:{1}\t-->\tCalling Count:{2}".format(
+                    arg, config.args[arg], config.args_call_count[arg]
+                )
+            )
+        else:
+            print(
+                "{0}:{1}\t-->\tCalling Count:{2}".format(
+                    arg, config.args[arg], config.args_call_count[arg]
+                )
+            )
+
+
+def check_and_fix_labels(label_set: set, label_name, all_data, opt):
+    if "-100" in label_set:
+        label_to_index = {
+            origin_label: int(idx) - 1 if origin_label != "-100" else -100
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+        index_to_label = {
+            int(idx) - 1 if origin_label != "-100" else -100: origin_label
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+    else:
+        label_to_index = {
+            origin_label: int(idx)
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+        index_to_label = {
+            int(idx): origin_label
+            for origin_label, idx in zip(sorted(label_set), range(len(label_set)))
+        }
+    if "index_to_label" not in opt.args:
+        opt.index_to_label = index_to_label
+        opt.label_to_index = label_to_index
+
+    if opt.index_to_label != index_to_label:
+        opt.index_to_label.update(index_to_label)
+        opt.label_to_index.update(label_to_index)
+    num_label = {l: 0 for l in label_set}
+    num_label["Sum"] = len(all_data)
+    for item in all_data:
+        try:
+            num_label[item[label_name]] += 1
+            item[label_name] = label_to_index[item[label_name]]
+        except Exception as e:
+            # print(e)
+            num_label[item.polarity] += 1
+            item.polarity = label_to_index[item.polarity]
+    print("Dataset Label Details: {}".format(num_label))
+
+
+def check_and_fix_IOB_labels(label_map, opt):
+    index_to_IOB_label = {
+        int(label_map[origin_label]): origin_label for origin_label in label_map
+    }
+    opt.index_to_IOB_label = index_to_IOB_label
+
+
+def get_device(auto_device):
+    if isinstance(auto_device, str) and auto_device == "allcuda":
+        device = "cuda"
+    elif isinstance(auto_device, str):
+        device = auto_device
+    elif isinstance(auto_device, bool):
+        device = auto_cuda() if auto_device else "cpu"
+    else:
+        device = auto_cuda()
+        try:
+            torch.device(device)
+        except RuntimeError as e:
+            print(
+                colored("Device assignment error: {}, redirect to CPU".format(e), "red")
+            )
+            device = "cpu"
+    device_name = auto_cuda_name()
+    return device, device_name
+
+
+def _load_word_vec(path, word2idx=None, embed_dim=300):
+    fin = open(path, "r", encoding="utf-8", newline="\n", errors="ignore")
+    word_vec = {}
+    for line in tqdm.tqdm(fin.readlines(), postfix="Loading embedding file..."):
+        tokens = line.rstrip().split()
+        word, vec = " ".join(tokens[:-embed_dim]), tokens[-embed_dim:]
+        if word in word2idx.keys():
+            word_vec[word] = np.asarray(vec, dtype="float32")
+    return word_vec
+
+
+def build_embedding_matrix(word2idx, embed_dim, dat_fname, opt):
+    if not os.path.exists("run"):
+        os.makedirs("run")
+    embed_matrix_path = "run/{}".format(os.path.join(opt.dataset_name, dat_fname))
+    if os.path.exists(embed_matrix_path):
+        print(
+            colored(
+                "Loading cached embedding_matrix from {} (Please remove all cached files if there is any problem!)".format(
+                    embed_matrix_path
+                ),
+                "green",
+            )
+        )
+        embedding_matrix = pickle.load(open(embed_matrix_path, "rb"))
+    else:
+        glove_path = prepare_glove840_embedding(embed_matrix_path)
+        embedding_matrix = np.zeros((len(word2idx) + 2, embed_dim))
+
+        word_vec = _load_word_vec(glove_path, word2idx=word2idx, embed_dim=embed_dim)
+
+        for word, i in tqdm.tqdm(
+            word2idx.items(),
+            postfix=colored("Building embedding_matrix {}".format(dat_fname), "yellow"),
+        ):
+            vec = word_vec.get(word)
+            if vec is not None:
+                embedding_matrix[i] = vec
+        pickle.dump(embedding_matrix, open(embed_matrix_path, "wb"))
+    return embedding_matrix
+
+
+def pad_and_truncate(
+    sequence, maxlen, dtype="int64", padding="post", truncating="post", value=0
+):
+    x = (np.ones(maxlen) * value).astype(dtype)
+    if truncating == "pre":
+        trunc = sequence[-maxlen:]
+    else:
+        trunc = sequence[:maxlen]
+    trunc = np.asarray(trunc, dtype=dtype)
+    if padding == "post":
+        x[: len(trunc)] = trunc
+    else:
+        x[-len(trunc) :] = trunc
+    return x
+
+
+class TransformerConnectionError(ValueError):
+    def __init__(self):
+        pass
+
+
+def retry(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        count = 5
+        while count:
+            try:
+                return f(*args, **kwargs)
+            except (
+                TransformerConnectionError,
+                requests.exceptions.RequestException,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectTimeout,
+                requests.exceptions.ProxyError,
+                requests.exceptions.SSLError,
+                requests.exceptions.BaseHTTPError,
+            ) as e:
+                print(colored("Training Exception: {}, will retry later".format(e)))
+                time.sleep(60)
+                count -= 1
+
+    return decorated
+
+
+def save_json(dic, save_path):
+    if isinstance(dic, str):
+        dic = eval(dic)
+    with open(save_path, "w", encoding="utf-8") as f:
+        # f.write(str(dict))
+        str_ = json.dumps(dic, ensure_ascii=False)
+        f.write(str_)
+
+
+def load_json(save_path):
+    with open(save_path, "r", encoding="utf-8") as f:
+        data = f.readline().strip()
+        print(type(data), data)
+        dic = json.loads(data)
+    return dic
+
+
+def init_optimizer(optimizer):
+    optimizers = {
+        "adadelta": torch.optim.Adadelta,  # default lr=1.0
+        "adagrad": torch.optim.Adagrad,  # default lr=0.01
+        "adam": torch.optim.Adam,  # default lr=0.001
+        "adamax": torch.optim.Adamax,  # default lr=0.002
+        "asgd": torch.optim.ASGD,  # default lr=0.01
+        "rmsprop": torch.optim.RMSprop,  # default lr=0.01
+        "sgd": torch.optim.SGD,
+        "adamw": torch.optim.AdamW,
+        torch.optim.Adadelta: torch.optim.Adadelta,  # default lr=1.0
+        torch.optim.Adagrad: torch.optim.Adagrad,  # default lr=0.01
+        torch.optim.Adam: torch.optim.Adam,  # default lr=0.001
+        torch.optim.Adamax: torch.optim.Adamax,  # default lr=0.002
+        torch.optim.ASGD: torch.optim.ASGD,  # default lr=0.01
+        torch.optim.RMSprop: torch.optim.RMSprop,  # default lr=0.01
+        torch.optim.SGD: torch.optim.SGD,
+        torch.optim.AdamW: torch.optim.AdamW,
+    }
+    if optimizer in optimizers:
+        return optimizers[optimizer]
+    elif hasattr(torch.optim, optimizer.__name__):
+        return optimizer
+    else:
+        raise KeyError(
+            "Unsupported optimizer: {}. Please use string or the optimizer objects in torch.optim as your optimizer".format(
+                optimizer
+            )
+        )

anonymous_demo/utils/logger.py

@@ -0,0 +1,38 @@
+import logging
+import os
+import sys
+import time
+
+import termcolor
+
+today = time.strftime("%Y%m%d %H%M%S", time.localtime(time.time()))
+
+
+def get_logger(log_path, log_name="", log_type="training_log"):
+    if not log_path:
+        log_dir = os.path.join(log_path, "logs")
+    else:
+        log_dir = os.path.join(".", "logs")
+
+    full_path = os.path.join(log_dir, log_name + "_" + today)
+    if not os.path.exists(full_path):
+        os.makedirs(full_path)
+    log_path = os.path.join(full_path, "{}.log".format(log_type))
+    logger = logging.getLogger(log_name)
+    if not logger.handlers:
+        formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+
+        file_handler = logging.FileHandler(log_path, encoding="utf8")
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(logging.INFO)
+
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.formatter = formatter
+        console_handler.setLevel(logging.INFO)
+
+        logger.addHandler(file_handler)
+        logger.addHandler(console_handler)
+
+        logger.setLevel(logging.INFO)
+
+    return logger

app.py

@@ -0,0 +1,360 @@
+import os
+import zipfile
+
+import gradio as gr
+import nltk
+import pandas as pd
+import requests
+
+from pyabsa import TADCheckpointManager
+from textattack.attack_recipes import (
+    BAEGarg2019,
+    PWWSRen2019,
+    TextFoolerJin2019,
+    PSOZang2020,
+    IGAWang2019,
+    GeneticAlgorithmAlzantot2018,
+    DeepWordBugGao2018,
+    CLARE2020,
+)
+from textattack.attack_results import SuccessfulAttackResult
+from utils import SentAttacker, get_agnews_example, get_sst2_example, get_amazon_example, get_imdb_example, diff_texts
+# from utils import get_yahoo_example
+
+sent_attackers = {}
+tad_classifiers = {}
+
+attack_recipes = {
+    "bae": BAEGarg2019,
+    "pwws": PWWSRen2019,
+    "textfooler": TextFoolerJin2019,
+    "pso": PSOZang2020,
+    "iga": IGAWang2019,
+    "ga": GeneticAlgorithmAlzantot2018,
+    "deepwordbug": DeepWordBugGao2018,
+    "clare": CLARE2020,
+}
+
+
+def init():
+    nltk.download("omw-1.4")
+
+    if not os.path.exists("TAD-SST2"):
+        z = zipfile.ZipFile("checkpoints.zip", "r")
+        z.extractall(os.getcwd())
+
+    for attacker in ["pwws", "bae", "textfooler", "deepwordbug"]:
+        for dataset in [
+            "agnews10k",
+            "sst2",
+            "MR",
+            'imdb'
+        ]:
+            if "tad-{}".format(dataset) not in tad_classifiers:
+                tad_classifiers[
+                    "tad-{}".format(dataset)
+                ] = TADCheckpointManager.get_tad_text_classifier(
+                    "tad-{}".format(dataset).upper()
+                )
+
+            sent_attackers["tad-{}{}".format(dataset, attacker)] = SentAttacker(
+                tad_classifiers["tad-{}".format(dataset)], attack_recipes[attacker]
+            )
+            tad_classifiers["tad-{}".format(dataset)].sent_attacker = sent_attackers[
+                "tad-{}pwws".format(dataset)
+            ]
+
+
+cache = set()
+
+
+def generate_adversarial_example(dataset, attacker, text=None, label=None):
+    """if not text or text in cache:
+        if "agnews" in dataset.lower():
+            text, label = get_agnews_example()
+        elif "sst2" in dataset.lower():
+            text, label = get_sst2_example()
+        elif "MR" in dataset.lower():
+            text, label = get_amazon_example()
+        # elif "yahoo" in dataset.lower():
+            # text, label = get_yahoo_example()
+        elif "imdb" in dataset.lower():
+            text, label = get_imdb_example()"""
+
+    cache.add(text)
+
+    result = None
+    attack_result = sent_attackers[
+        "tad-{}{}".format(dataset.lower(), attacker.lower())
+    ].attacker.simple_attack(text, int(label))
+    if isinstance(attack_result, SuccessfulAttackResult):
+        if (
+                attack_result.perturbed_result.output
+                != attack_result.original_result.ground_truth_output
+        ) and (
+                attack_result.original_result.output
+                == attack_result.original_result.ground_truth_output
+        ):
+            # with defense
+            result = tad_classifiers["tad-{}".format(dataset.lower())].infer(
+                attack_result.perturbed_result.attacked_text.text
+                + "$LABEL${},{},{}".format(
+                    attack_result.original_result.ground_truth_output,
+                    1,
+                    attack_result.perturbed_result.output,
+                ),
+                print_result=True,
+                defense=attacker,
+            )
+
+    if result:
+        classification_df = {}
+        classification_df["is_repaired"] = result["is_fixed"]
+        classification_df["pred_label"] = result["label"]
+        classification_df["confidence"] = round(result["confidence"], 3)
+        classification_df["is_correct"] = str(result["pred_label"]) == str(label)
+
+        advdetection_df = {}
+        if result["is_adv_label"] != "0":
+            advdetection_df["is_adversarial"] = {
+                "0": False,
+                "1": True,
+                0: False,
+                1: True,
+            }[result["is_adv_label"]]
+            advdetection_df["perturbed_label"] = result["perturbed_label"]
+            advdetection_df["confidence"] = round(result["is_adv_confidence"], 3)
+            advdetection_df['ref_is_attack'] = result['ref_is_adv_label']
+            advdetection_df['is_correct'] = result['ref_is_adv_check']
+
+    else:
+        return generate_adversarial_example(dataset, attacker)
+
+    return (
+        text,
+        label,
+        result["restored_text"],
+        result["label"],
+        attack_result.perturbed_result.attacked_text.text,
+        diff_texts(text, text),
+        diff_texts(text, attack_result.perturbed_result.attacked_text.text),
+        diff_texts(text, result["restored_text"]),
+        attack_result.perturbed_result.output,
+        pd.DataFrame(classification_df, index=[0]),
+        pd.DataFrame(advdetection_df, index=[0]),
+    )
+
+
+def run_demo(dataset, attacker, text=None, label=None):
+    try:
+        data = {
+            "dataset": dataset,
+            "attacker": attacker,
+            "text": text,
+            "label": label,
+        }
+        response = requests.post('https://rpddemo.pagekite.me/api/generate_adversarial_example', json=data)
+        result = response.json()
+        print(response.json())
+        return (
+            result["text"],
+            result["label"],
+            result["restored_text"],
+            result["result_label"],
+            result["perturbed_text"],
+            result["text_diff"],
+            result["perturbed_diff"],
+            result["restored_diff"],
+            result["output"],
+            pd.DataFrame(result["classification_df"]),
+            pd.DataFrame(result["advdetection_df"]),
+            result["message"]
+        )
+    except Exception as e:
+        print(e)
+        return generate_adversarial_example(dataset, attacker, text, label)
+
+
+def check_gpu():
+    try:
+        response = requests.post('https://rpddemo.pagekite.me/api/generate_adversarial_example', timeout=3)
+        if response.status_code < 500:
+            return 'GPU available'
+        else:
+            return 'GPU not available'
+    except Exception as e:
+        return 'GPU not available'
+
+
+if __name__ == "__main__":
+    try:
+        init()
+    except Exception as e:
+        print(e)
+        print("Failed to initialize the demo. Please try again later.")
+
+    demo = gr.Blocks()
+
+    with demo:
+        gr.Markdown("<h1 align='center'>Detection and Correction based on Word Importance Ranking (DCWIR) </h1>")
+        gr.Markdown("<h2 align='center'>Clarifications</h2>")
+        gr.Markdown("""
+    - This demo has no mechanism to ensure the adversarial example will be correctly repaired by Rapid. The repair success rate is actually the performance reported in the paper.The user must know the resulted output for sake of demonstration.
+    - The adversarial example and corrected adversarial example may be unnatural to read, while it is because the attackers usually generate unnatural perturbations. 
+    - All the proposed attacks are Black Box attack where the attacker has no access to the model parameters.
+    """)
+        gr.Markdown("<h2 align='center'>Natural Example Input</h2>")
+        with gr.Group():
+            with gr.Row():
+                input_dataset = gr.Radio(
+                    choices=["SST2", "IMDB", "MR", "AGNews10K"],
+                    value="SST2",
+                    label="Select a testing dataset and an adversarial attacker to generate an adversarial example.",
+                )
+                input_attacker = gr.Radio(
+                    choices=["BAE", "PWWS", "TextFooler", "DeepWordBug"],
+                    value="TextFooler",
+                    label="Choose an Adversarial Attacker for generating an adversarial example to attack the model.",
+                )
+            with gr.Group(visible=True):
+
+                with gr.Row():
+                    input_sentence = gr.Textbox(
+                        placeholder="Input a natural example...",
+                        label="Alternatively, input a natural example and its original label (from above datasets) to generate an adversarial example.",
+
+                    )
+                    input_label = gr.Textbox(
+                        placeholder="Original label, （must be a integer, because we use digits to represent labels in training）", 
+                        label="Original Label",
+                    )
+                gr.Markdown(
+                    "<h3 align='center'>Default parameters are set according to the main experiment setup in the report.</h2>",
+                )
+        with gr.Row():
+            wir_percentage = gr.Textbox(
+                placeholder="Enter percentage from WIR...",
+                label="Percentage from WIR",
+            )
+            frequency_threshold = gr.Textbox(
+                placeholder="Enter frequency threshold...",
+                label="Frequency Threshold",
+            )
+            max_candidates = gr.Textbox(
+                placeholder="Enter maximum number of candidates...",
+                label="Maximum Number of Candidates",
+            )
+        msg_text = gr.Textbox(
+            label="Message",
+            placeholder="This is a message box to show any error messages.",
+        )
+        button_gen = gr.Button(
+            "Generate an adversarial example to repair using Rapid (GPU: < 1 minute, CPU: 1-10 minutes)",
+            variant="primary",
+        )
+        gpu_status_text = gr.Textbox(
+            label='GPU status',
+            placeholder="Please click to check",
+        )
+        button_check = gr.Button(
+            "Check if GPU available",
+            variant="primary"
+        )
+
+        button_check.click(
+            fn=check_gpu,
+            inputs=[],
+            outputs=[
+                gpu_status_text
+            ]
+        )
+
+        gr.Markdown("<h2 align='center'>Generated Adversarial Example and Repaired Adversarial Example</h2>")
+
+        with gr.Column():
+            with gr.Group():
+                with gr.Row():
+                    output_original_example = gr.Textbox(label="Original Example")
+                    output_original_label = gr.Textbox(label="Original Label")
+                with gr.Row():
+                    output_adv_example = gr.Textbox(label="Adversarial Example")
+                    output_adv_label = gr.Textbox(label="Predicted Label of the Adversarial Example")
+                with gr.Row():
+                    output_repaired_example = gr.Textbox(
+                        label="Repaired Adversarial Example by Rapid"
+                    )
+                    output_repaired_label = gr.Textbox(label="Predicted Label of the Repaired Adversarial Example")
+
+        gr.Markdown("<h2 align='center'>Example Difference (Comparisons)</p>")
+        gr.Markdown("""
+        <p align='center'>The (+) and (-) in the boxes indicate the added and deleted characters in the adversarial example compared to the original input natural example.</p>
+            """)
+        ori_text_diff = gr.HighlightedText(
+            label="The Original Natural Example",
+            combine_adjacent=True,
+            show_legend=True,
+        )
+        adv_text_diff = gr.HighlightedText(
+            label="Character Editions of Adversarial Example Compared to the Natural Example",
+            combine_adjacent=True,
+            show_legend=True,
+        )
+
+        restored_text_diff = gr.HighlightedText(
+            label="Character Editions of Repaired Adversarial Example Compared to the Natural Example",
+            combine_adjacent=True,
+            show_legend=True,
+        )
+
+        gr.Markdown(
+            "## <h2 align='center'>The Output of Reactive Perturbation Defocusing</p>"
+        )
+        with gr.Row():
+            with gr.Column():
+                with gr.Group():
+                    output_is_adv_df = gr.DataFrame(
+                        label="Adversarial Example Detection Result"
+                    )
+                    gr.Markdown(
+                        """
+                         - The is_adversarial field indicates if an adversarial example is detected.
+                         - The perturbed_label is the predicted label of the adversarial example. 
+                         - The confidence field represents the ratio of Inverted samples among the total number of generated candidates. 
+                         """
+                    )
+            with gr.Column():
+                with gr.Group():
+                    output_df = gr.DataFrame(
+                        label="Correction Classification Result"
+                    )
+                    gr.Markdown(
+                      """
+                        - If is_corrected=true, it has been Corrected by DCWIR. 
+                        - The pred_label field indicates the standard classification result. 
+                        - The confidence field represents ratio of the dominant class among all Inverted candidates.
+                        - The is_correct field indicates whether the predicted label is correct.
+                        
+                        """
+                    )
+
+        # Bind functions to buttons
+        button_gen.click(
+            fn=run_demo,
+            inputs=[input_dataset, input_attacker, input_sentence, input_label],
+            outputs=[
+                output_original_example,
+                output_original_label,
+                output_repaired_example,
+                output_repaired_label,
+                output_adv_example,
+                ori_text_diff,
+                adv_text_diff,
+                restored_text_diff,
+                output_adv_label,
+                output_df,
+                output_is_adv_df,
+                msg_text
+            ],
+        )
+
+    demo.queue(2).launch() 

checkpoints.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77ae4a45785183900ee874cb318a16b0e2f173b31749a2555215aca93672f26
+size 2456834455

flow correction 30%.ipynb

@@ -0,0 +1,516 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f0f495f5278946bebfcef7f58113879b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "pytorch_model.bin:   0%|          | 0.00/438M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Isaac\\AppData\\Roaming\\Python\\Python38\\site-packages\\huggingface_hub\\file_download.py:148: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\Isaac\\.cache\\huggingface\\hub\\models--textattack--bert-base-uncased-SST-2. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
+      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
+      "  warnings.warn(message)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7430033906354f39b753fb46f7113501",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bce8defec56e4810ba28e42b2d18538e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a4c13d3d415c4ec8adf87cd6efca5989",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "textattack: Unknown if model of class <class 'transformers.models.bert.modeling_bert.BertForSequenceClassification'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import textattack\n",
+    "import transformers\n",
+    "\n",
+    "# Load model, tokenizer, and model_wrapper\n",
+    "model = transformers.AutoModelForSequenceClassification.from_pretrained(\n",
+    "    \"textattack/bert-base-uncased-SST-2\"\n",
+    ")\n",
+    "tokenizer = transformers.AutoTokenizer.from_pretrained(\n",
+    "    \"textattack/bert-base-uncased-SST-2\"\n",
+    ")\n",
+    "model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)\n",
+    "\n",
+    "# Construct our four components for `Attack`\n",
+    "from textattack.constraints.pre_transformation import (\n",
+    "    RepeatModification,\n",
+    "    StopwordModification,\n",
+    ")\n",
+    "from textattack.constraints.semantics import WordEmbeddingDistance\n",
+    "from textattack.transformations import WordSwapEmbedding\n",
+    "from textattack.search_methods import GreedyWordSwapWIR\n",
+    "\n",
+    "goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)\n",
+    "constraints = [\n",
+    "    RepeatModification(),\n",
+    "    StopwordModification(),\n",
+    "    WordEmbeddingDistance(min_cos_sim=0.9),\n",
+    "]\n",
+    "transformation = WordSwapEmbedding(max_candidates=50)\n",
+    "# weighted-saliency\n",
+    "search_method = GreedyWordSwapWIR(wir_method=\"weighted-saliency\")\n",
+    "\n",
+    "# Construct the actual attack\n",
+    "attack = textattack.Attack(goal_function, constraints, transformation, search_method)\n",
+    "attack.cuda_()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "results = pd.read_csv(\"ag-news_pwws_bert.csv\")\n",
+    "#results.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\"\"\"successful_perturbed_texts = results.loc[results[\"result_type\"] == \"Successful\", \"perturbed_text\"].tolist()\n",
+    "failed_perturbed_texts = results.loc[results[\"result_type\"] == \"Failed\", \"perturbed_text\"].tolist()\n",
+    "\n",
+    "failed_perturbed_outputs = results.loc[results[\"result_type\"] == \"Failed\", \"perturbed_output\"].tolist()\n",
+    "successful_perturbed_outputs = results.loc[results[\"result_type\"] == \"Successful\", \"original_output\"].tolist()\"\"\"\n",
+    "\n",
+    "\n",
+    "original_texts = results[\"original_text\"].tolist()\n",
+    "perturbed_texts =results[\"adversarial_text\"].tolist() \n",
+    "\n",
+    "original_outputs = results[\"original_class\"].tolist()\n",
+    "perturbed_outputs =results[\"adversarial_class\"].tolist() "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "import string\n",
+    "# Clean Text\n",
+    "def remove_brackets(text):\n",
+    "    text = text.replace('[[', '')\n",
+    "    text = text.replace(']]', '')\n",
+    "    return text\n",
+    "\n",
+    "perturbed_texts = [remove_brackets(text) for text in perturbed_texts]\n",
+    "original_texts = [remove_brackets(text) for text in original_texts]\n",
+    "\n",
+    "def clean_text(text):\n",
+    "    pattern = \"[\" + re.escape(string.punctuation) + \"]\"\n",
+    "    cleaned_text = re.sub(pattern, \" \", text)\n",
+    "\n",
+    "    return cleaned_text\n",
+    "\n",
+    "perturbed_texts = [clean_text(text) for text in perturbed_texts]\n",
+    "original_texts = [clean_text(text) for text in original_texts]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "perturbed_texts = [text.lower() for text in perturbed_texts]\n",
+    "original_texts = [text.lower() for text in original_texts]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from FlowCorrector import Flow_Corrector\n",
+    "\n",
+    "corrector = Flow_Corrector(\n",
+    "    attack,\n",
+    "    word_rank_file=\"en_full_ranked.json\",\n",
+    "    word_freq_file=\"en_full_freq.json\",\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a1241448bb324872a1da1f2b659150c5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/424 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "import torch.nn.functional as F\n",
+    "from tqdm.notebook import tqdm_notebook\n",
+    "\n",
+    "victim_model = attack.goal_function.model\n",
+    "\n",
+    "original_classes = [\n",
+    "    torch.argmax(F.softmax(victim_model(original_text), dim=1)).item()\n",
+    "    for original_text in tqdm_notebook(original_texts)\n",
+    "]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d2927a578aa5401796a0c4cd48a11de6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/424 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\Isaac\\anaconda3\\envs\\textattackenv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
+      "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
+     ]
+    }
+   ],
+   "source": [
+    "\"\"\" 0 :World\n",
+    "    1 : Sports\n",
+    "    2 : Business\n",
+    "    3 : Sci/Tech\"\"\"\n",
+    "\n",
+    "corrected_classes = corrector.correct(perturbed_texts)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def count_matching_classes(original, corrected, perturbed_texts=None):\n",
+    "    if len(original) != len(corrected):\n",
+    "        raise ValueError(\"Arrays must have the same length\")\n",
+    "    hard_samples = []\n",
+    "    easy_samples = []\n",
+    "\n",
+    "    matching_count = 0\n",
+    "\n",
+    "    for i in range(len(corrected)):\n",
+    "        if original[i] == corrected[i]:\n",
+    "            matching_count += 1\n",
+    "            easy_samples.append(perturbed_texts[i])\n",
+    "        elif perturbed_texts != None:\n",
+    "            hard_samples.append(perturbed_texts[i])\n",
+    "\n",
+    "    return matching_count, hard_samples, easy_samples\n",
+    "\n",
+    "\n",
+    "match_, hard_samples, easy_samples = count_matching_classes(\n",
+    "    original_classes, corrected_classes, perturbed_texts\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.6014150943396226"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "match_/424"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"detected_samples_ag_news.txt\", \"w\") as f:\n",
+    "    for sample in easy_samples:\n",
+    "        f.write(sample + \"\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index_order = wir_gradient(attack, goal_function.model, detected_text)\n",
+    "\n",
+    "# The prblm is tht it exists many rare words with freq from 1 to 10\n",
+    "\n",
+    "# freq_thershold = len(word_ranked_frequence) * 0.01\n",
+    "\n",
+    "# for now i sugest to take the words taht have freq > freq_thershold (200 in paper)\n",
+    "\n",
+    "freq_thershold = 2000\n",
+    "\n",
+    "index_order_1 = [\n",
+    "    idx\n",
+    "    for idx in index_order\n",
+    "\n",
+    "    if detected_text.words[idx] in word_frequence.keys()\n",
+    "\n",
+    "    and word_frequence[detected_text.words[idx]] < freq_thershold\n",
+    "\n",
+    "]\n",
+    "\n",
+    "print(\n",
+    "\n",
+    "    f\"from {len(index_order)} ranked word it remain only {len(index_order_1)} within frequency theshold = {freq_thershold} \"\n",
+    "\n",
+    ")\n",
+    "\n",
+    "# or we take the lowest 30% in the important ranked words ?\n",
+    "index_order = index_order[:int(len(index_order) * 0.3)]\n",
+    "index_order_ = {\n",
+    "    idx : word_ranked_frequence[detected_text.words[idx]]\n",
+    "    for idx in index_order\n",
+    "    if detected_text.words[idx] in word_ranked_frequence.keys()\n",
+    "}\n",
+    "\n",
+    "index_order_ = sorted(index_order_.items(), key=lambda item: item[1], reverse=False)\n",
+    "lowest = 0.15\n",
+    "index_order_ = [idx[0]for idx in index_order_][:int(len(index_order) * lowest)]\n",
+    "\n",
+    "print(f\"from {len(index_order)} ranked word {len(index_order_)} word represent {lowest * 100}% with the lowest frequency\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def remove_brackets(text):\n",
+    "    text = text.replace('[[', '')\n",
+    "    text = text.replace(']]', '')\n",
+    "    return text\n",
+    "\n",
+    "text = \"Fears for T [[percent]] pension after [[debate]] [[Syndicates]] [[portrayal]] [[worker]] at Turner   Newall say they are 'disappointed' after [[chatter]] with [[bereaved]] [[parenting]] [[corporations]] [[Canada]] Mogul.\" \n",
+    "print(remove_brackets(text))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "with open('en_full.txt', 'r') as f:\n",
+    "    lines = f.readlines()\n",
+    "\n",
+    "\n",
+    "freq_dict = {line.split()[0]: int(line.split()[1]) for line in lines}\n",
+    "\n",
+    "\n",
+    "sorted_dict = dict(sorted(freq_dict.items(), key=lambda item: item[1], reverse=True))\n",
+    "\n",
+    "\n",
+    "ranked_dict = {word: freq for word, freq in sorted_dict.items() }\n",
+    "\n",
+    "\n",
+    "with open('en_full_freq.json', 'w') as f:\n",
+    "    json.dump(ranked_dict, f)\n",
+    "\n",
+    "print(\"The word frequencies have been successfully ranked and saved to ranked_freq.json file.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAHqCAYAAADVi/1VAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB5+0lEQVR4nO3de1xU1f7/8fcwyoAX8IICAol28ZaX8sIho+xEkfY1lSzTSuOUdkxLI09FeSkr6WpYmpZfTbtonmzy1NEs42hZmfbTrCyzvIIIqHWExAQb9u+P+TI5MijgMHuA1/Px2I9h1l57zWdvB1zzmbXXshiGYQgAAAAAAADwoQCzAwAAAAAAAED9Q1IKAAAAAAAAPkdSCgAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD5HUgqAz9x2222KjY2t1rGPPPKILBaLdwMCAACoJ2prP+xs4gbg/0hKAZDFYqnUtm7dOrNDBQAAqFPohwGozyyGYRhmBwHAXG+88Ybb89dee01r1qzR66+/7lZ+1VVXKTw8vNqvc+LECZWWlspms1X52D/++EN//PGHgoKCqv36AAAA/oZ+2OnddtttWrdunfbu3evz1wZQ80hKAShn/PjxmjNnjs705+HYsWNq1KiRj6JCZRiGoePHjys4ONjsUAAAQDXQD3NHUgqo27h9D0Cl9OvXTxdeeKE2b96syy67TI0aNdJDDz0kSfrXv/6la6+9Vm3atJHNZtO5556rxx57TA6Hw62NU+cE2Lt3rywWi5599lm98sorOvfcc2Wz2dS7d2999dVXbsd6msvAYrFo/PjxWrFihS688ELZbDZ16dJFq1evLhf/unXr1KtXLwUFBencc8/Vyy+/XOn5EdavX68bbrhB55xzjmw2m2JiYnTvvffq999/L1f3xx9/1I033qhWrVopODhYHTp00MMPP+xWJycnR7fffrvrerVr105jx45VSUlJhecqSYsWLZLFYnHrlMXGxup//ud/9OGHH6pXr14KDg7Wyy+/LEl69dVX9de//lWtW7eWzWZT586dNXfuXI/n+MEHH+jyyy9X06ZNFRISot69e2vJkiWSpGnTpqlhw4Y6dOhQuePGjBmjZs2a6fjx42e8jgAAoHrqcz/Mk6KiIt13332KiYmRzWZThw4d9Oyzz5ZL5K1Zs0aXXnqpmjVrpiZNmqhDhw6u61bmxRdfVJcuXdSoUSM1b95cvXr1cvWBANS8BmYHAKD2+OWXX9S/f3/ddNNNuuWWW1xDyBctWqQmTZooNTVVTZo00X/+8x9NnTpVhYWFeuaZZ87Y7pIlS/Tbb7/pzjvvlMVi0dNPP63k5GTt3r1bDRs2PO2xn332mex2u+666y41bdpUL7zwgq6//nplZWWpZcuWkqSvv/5a11xzjSIjI/Xoo4/K4XBo+vTpatWqVaXO++2339axY8c0duxYtWzZUps2bdKLL76o/fv36+2333bV+/bbb5WQkKCGDRtqzJgxio2N1a5du/T+++/riSeekCQdOHBAffr00ZEjRzRmzBh17NhROTk5Wr58uY4dO6bAwMBKxXSyHTt2aPjw4brzzjs1evRodejQQZI0d+5cdenSRdddd50aNGig999/X3fddZdKS0s1btw41/GLFi3S3/72N3Xp0kVpaWlq1qyZvv76a61evVojRozQrbfequnTp2vZsmUaP36867iSkhItX75c119/PbdVAgBQw+prP+xUhmHouuuu09q1a3X77berR48e+vDDD/WPf/xDOTk5ev755yVJ33//vf7nf/5H3bp10/Tp02Wz2bRz5059/vnnrrbmz5+ve+65R0OHDtWECRN0/Phxffvtt9q4caNGjBhRrfgAVJEBAKcYN26cceqfh8svv9yQZMybN69c/WPHjpUru/POO41GjRoZx48fd5WNGjXKaNu2rev5nj17DElGy5YtjV9//dVV/q9//cuQZLz//vuusmnTppWLSZIRGBho7Ny501X2zTffGJKMF1980VU2cOBAo1GjRkZOTo6r7OeffzYaNGhQrk1PPJ1fenq6YbFYjH379rnKLrvsMqNp06ZuZYZhGKWlpa6fR44caQQEBBhfffVVuTbL6nk6V8MwjFdffdWQZOzZs8dV1rZtW0OSsXr16krFnZSUZLRv3971/MiRI0bTpk2NuLg44/fff68w7vj4eCMuLs5tv91uNyQZa9euLfc6AACgeuiHuTs17hUrVhiSjMcff9yt3tChQw2LxeKK5/nnnzckGYcOHaqw7UGDBhldunQ5YwwAag637wGoNJvNppSUlHLlJ89f9Ntvv+nw4cNKSEjQsWPH9OOPP56x3WHDhql58+au5wkJCZKk3bt3n/HYxMREnXvuua7n3bp1U0hIiOtYh8Ohjz/+WIMHD1abNm1c9c477zz179//jO1L7udXVFSkw4cP65JLLpFhGPr6668lSYcOHdKnn36qv/3tbzrnnHPcji8bml5aWqoVK1Zo4MCB6tWrV7nXqe4Q9nbt2ikpKem0cRcUFOjw4cO6/PLLtXv3bhUUFEhyDmv/7bff9OCDD5Yb7XRyPCNHjtTGjRu1a9cuV9mbb76pmJgYXX755dWKGwAAVF597YedatWqVbJarbrnnnvcyu+77z4ZhqEPPvhAktSsWTNJztsbS0tLPbbVrFkz7d+/v9ztigB8h6QUgEqLioryeHvZ999/ryFDhig0NFQhISFq1aqVbrnlFklyJT9O59QkTlnH6L///W+Vjy07vuzYgwcP6vfff9d5551Xrp6nMk+ysrJ02223qUWLFmrSpIlatWrlSsSUnV9Z5+vCCy+ssJ1Dhw6psLDwtHWqo127dh7LP//8cyUmJqpx48Zq1qyZWrVq5ZpHoSzusiTTmWIaNmyYbDab3nzzTdfx//73v3XzzTdXO5kGAAAqr772w061b98+tWnTRk2bNnUr79Spk2u/5Oy79O3bV3fccYfCw8N100036Z///KdbguqBBx5QkyZN1KdPH51//vkaN26c2+19AGoeSSkAleZpRbcjR47o8ssv1zfffKPp06fr/fff15o1a/TUU09JUoXfTJ3MarV6LDcqsTjo2RxbGQ6HQ1dddZVWrlypBx54QCtWrNCaNWu0aNEiSZU7v6qqKMlz6oSlZTz9u+zatUtXXnmlDh8+rJkzZ2rlypVas2aN7r33XklVj7t58+b6n//5H1dSavny5SouLnZ1egEAQM2qj/2wsxEcHKxPP/1UH3/8sW699VZ9++23GjZsmK666ipXn6pTp07asWOH3nrrLV166aV65513dOmll2ratGkmRw/UH0x0DuCsrFu3Tr/88ovsdrsuu+wyV/mePXtMjOpPrVu3VlBQkHbu3Flun6eyU3333Xf66aeftHjxYo0cOdJVvmbNGrd67du3lyRt27atwrZatWqlkJCQ09aR/vyG8siRI66h59Kf3/xVxvvvv6/i4mK99957bt9irl271q1e2ZD7bdu2nfEby5EjR2rQoEH66quv9Oabb+qiiy5Sly5dKh0TAADwrrreD/Okbdu2+vjjj/Xbb7+5jZYqu1Wxbdu2rrKAgABdeeWVuvLKKzVz5kzNmDFDDz/8sNauXavExERJUuPGjTVs2DANGzZMJSUlSk5O1hNPPKG0tDQWcgF8gJFSAM5K2TdkJ38jVlJSopdeesmskNxYrVYlJiZqxYoVOnDggKt8586drjkHznS85H5+hmFo1qxZbvVatWqlyy67TAsXLlRWVpbbvrJjAwICNHjwYL3//vv6f//v/5V7rbJ6ZYmiTz/91LWvqKhIixcvPmO8p4u7oKBAr776qlu9q6++Wk2bNlV6erqOHz/uMZ4y/fv3V1hYmJ566il98sknjJICAMBkdb0f5smAAQPkcDg0e/Zst/Lnn39eFovFNVfVr7/+Wu7YHj16SJKKi4slOVc0PFlgYKA6d+4swzB04sSJasUHoGoYKQXgrFxyySVq3ry5Ro0apXvuuUcWi0Wvv/66XwzbLvPII4/oo48+Ut++fTV27FhXR+bCCy/U1q1bT3tsx44dde6552rSpEnKyclRSEiI3nnnHY/zLLzwwgu69NJLdfHFF2vMmDFq166d9u7dq5UrV7peZ8aMGfroo490+eWXa8yYMerUqZNyc3P19ttv67PPPlOzZs109dVX65xzztHtt9+uf/zjH7JarVq4cKFatWpVLuFVkauvvlqBgYEaOHCg7rzzTh09elTz589X69atlZub66oXEhKi559/XnfccYd69+6tESNGqHnz5vrmm2907Ngxt0RYw4YNddNNN2n27NmyWq0aPnx4pWIBAAA1o673wzwZOHCgrrjiCj388MPau3evunfvro8++kj/+te/NHHiRNeXe9OnT9enn36qa6+9Vm3bttXBgwf10ksvKTo6WpdeeqkkZ38pIiJCffv2VXh4uLZv367Zs2fr2muvLTdnFYCaQVIKwFlp2bKl/v3vf+u+++7T5MmT1bx5c91yyy268sorPa4IZ4aePXvqgw8+0KRJkzRlyhTFxMRo+vTp2r59+xlXpWnYsKHef/993XPPPUpPT1dQUJCGDBmi8ePHq3v37m51u3fvri+//FJTpkzR3Llzdfz4cbVt21Y33nijq05UVJQ2btyoKVOm6M0331RhYaGioqLUv39/NWrUyPWa7777ru666y5NmTJFERERmjhxopo3b+5x1R1POnTooOXLl2vy5MmaNGmSIiIiNHbsWLVq1Up/+9vf3Orefvvtat26tZ588kk99thjatiwoTp27Oiaf+pkI0eO1OzZs3XllVcqMjKyUrEAAICaUdf7YZ4EBATovffe09SpU7Vs2TK9+uqrio2N1TPPPKP77rvPVe+6667T3r17tXDhQh0+fFhhYWG6/PLL9eijjyo0NFSSdOedd+rNN9/UzJkzdfToUUVHR+uee+7R5MmTvXb+AE7PYvhTGh0AfGjw4MH6/vvv9fPPP5sdSq3xzTffqEePHnrttdd06623mh0OAACopeiHAZCYUwpAPfH777+7Pf/555+1atUq9evXz5yAaqn58+erSZMmSk5ONjsUAABQS9APA1ARbt8DUC+0b99et912m9q3b699+/Zp7ty5CgwM1P333292aLXC+++/rx9++EGvvPKKxo8fr8aNG5sdEgAAqCXohwGoCLfvAagXUlJStHbtWuXl5clmsyk+Pl4zZszQxRdfbHZotUJsbKzy8/OVlJSk119/nck/AQBApdEPA1ARklIAAAAAAADwOeaUAgAAAAAAgM+RlAIAAAAAAIDPMdG5B6WlpTpw4ICaNm0qi8VidjgAAMCPGIah3377TW3atFFAQP39fo/+EgAAqEhl+0skpTw4cOCAYmJizA4DAAD4sezsbEVHR5sdhmnoLwEAgDM5U3+JpJQHZatKZWdnKyQkxORoAACAPyksLFRMTEy9X4WS/hIAAKhIZftLJKU8KBuCHhISQicLAAB4VN9vWaO/BAAAzuRM/aX6OxECAAAAAAAATENSCgAAAAAAAD5HUgoAAAAAAAA+x5xSZ8HhcOjEiRNmhwETNWzYUFar1ewwAAAAAKBG8fkXJ/PWZ2GSUtVgGIby8vJ05MgRs0OBH2jWrJkiIiLq/YS3AAAAAOoePv+iIt74LExSqhrKfiFbt26tRo0akYyopwzD0LFjx3Tw4EFJUmRkpMkRAQAAAIB38fkXp/LmZ2GSUlXkcDhcv5AtW7Y0OxyYLDg4WJJ08OBBtW7dmlv5AAAAANQZfP5FRbz1WZiJzquo7B7aRo0amRwJ/EXZe4H7qwEAAADUJXz+xel447MwSalqYsgiyvBeAAAAAFCX8ZkHnnjjfUFSCgAAAAAAAD5HUgrVFhsbq4yMjErXX7dunSwWC6s2AAAAAABqHX/9DLxo0SI1a9asRl+jppCUMonDIa1bJy1d6nx0OGrutSwWy2m3Rx55pFrtfvXVVxozZkyl619yySXKzc1VaGhotV4PAGoLX/6NB8rMmTNHsbGxCgoKUlxcnDZt2nTa+hkZGerQoYOCg4MVExOje++9V8ePH/dRtKfH7xAA1D18BoYnrL5nArtdmjBB2r//z7LoaGnWLCk52fuvl5ub6/p52bJlmjp1qnbs2OEqa9KkietnwzDkcDjUoMGZ3xqtWrWqUhyBgYGKiIio0jEA4CsOh7R+vZSbK0VGSgkJUnUWEfH233hvxYW6bdmyZUpNTdW8efMUFxenjIwMJSUlaceOHWrdunW5+kuWLNGDDz6ohQsX6pJLLtFPP/2k2267TRaLRTNnzjThDP7k634SAKDm8RkYFWGklI/Z7dLQoe6/jJKUk+Mst9u9/5oRERGuLTQ0VBaLxfX8xx9/VNOmTfXBBx+oZ8+estls+uyzz7Rr1y4NGjRI4eHhatKkiXr37q2PP/7Yrd1Thy5aLBb97//+r4YMGaJGjRrp/PPP13vvvefaf+rQxbIhhh9++KE6deqkJk2a6JprrnH7A/LHH3/onnvuUbNmzdSyZUs98MADGjVqlAYPHlzh+f7yyy8aPny4oqKi1KhRI3Xt2lVLly51q1NaWqqnn35a5513nmw2m8455xw98cQTrv379+/X8OHD1aJFCzVu3Fi9evXSxo0bq3H1AdQGdrsUGytdcYU0YoTzMTa26n+Tvf033ltxlakPo0/qwzl6MnPmTI0ePVopKSnq3Lmz5s2bp0aNGmnhwoUe63/xxRfq27evRowYodjYWF199dUaPnz4GUdX1TQz+kkAgJrFZ+Ca/wzsydy5c3XuuecqMDBQHTp00Ouvv+7aZxiGHnnkEZ1zzjmy2Wxq06aN7rnnHtf+l156Seeff76CgoIUHh6uoUOHVum1q4KklA85HM7ssGGU31dWNnGiOR3oBx98UE8++aS2b9+ubt266ejRoxowYIAyMzP19ddf65prrtHAgQOVlZV12nYeffRR3Xjjjfr22281YMAA3Xzzzfr1118rrH/s2DE9++yzev311/Xpp58qKytLkyZNcu1/6qmn9Oabb+rVV1/V559/rsLCQq1YseK0MRw/flw9e/bUypUrtW3bNo0ZM0a33nqrW0c7LS1NTz75pKZMmaIffvhBS5YsUXh4uCTp6NGjuvzyy5WTk6P33ntP33zzje6//36VlpZW4koC8CVvJCC81VHy9t94f09weZO3Ekn+fI41qaSkRJs3b1ZiYqKrLCAgQImJidqwYYPHYy655BJt3rzZ9X/j7t27tWrVKg0YMMAnMXviz/0kAED1+PPf9rr0GfhU7777riZMmKD77rtP27Zt05133qmUlBStXbtWkvTOO+/o+eef18svv6yff/5ZK1asUNeuXSVJ/+///T/dc889mj59unbs2KHVq1frsssuq9LrV4mBcgoKCgxJRkFBQbl9v//+u/HDDz8Yv//+e5XbXbvWMJy/eqff1q49+3OoyKuvvmqEhoaeFNNaQ5KxYsWKMx7bpUsX48UXX3Q9b9u2rfH888+7nksyJk+e7Hp+9OhRQ5LxwQcfuL3Wf//7X1cskoydO3e6jpkzZ44RHh7ueh4eHm4888wzrud//PGHcc455xiDBg2q7CkbhmEY1157rXHfffcZhmEYhYWFhs1mM+bPn++x7ssvv2w0bdrU+OWXXyrV9tm8JwBU3zvvGEZ0tPvfz+hoZ3ll/fFH+TZO3iwWw4iJcdY7E2/+jfdmXIbhvCYWi+d2LJaqXbOTY1y71jCWLHE+VjYWT7Gd7b9jWTvePseKnK6fYIacnBxDkvHFF1+4lf/jH/8w+vTpU+Fxs2bNMho2bGg0aNDAkGT8/e9/P+3rHD9+3CgoKHBt2dnZXr0O/tBPAgC4O9vPOv7wt70+fAY+9RwvueQSY/To0W51brjhBmPAgAGGYRjGc889Z1xwwQVGSUlJubbeeecdIyQkxCgsLKzw9cqc7v1R2f4SI6V86KQReV6p5029evVye3706FFNmjRJnTp1UrNmzdSkSRNt3779jFnibt26uX5u3LixQkJCdPDgwQrrN2rUSOeee67reWRkpKt+QUGB8vPz1adPH9d+q9Wqnj17njYGh8Ohxx57TF27dlWLFi3UpEkTffjhh67Yt2/fruLiYl155ZUej9+6dasuuugitWjR4rSvA8A83hpFtH59+TZOZhhSdraz3pl482+8N+OqiW8o/e12R3/+FtZfrVu3TjNmzNBLL72kLVu2yG63a+XKlXrssccqPCY9PV2hoaGuLSYmxqsx+XM/CQBQPf78t70ufQY+1fbt29W3b1+3sr59+2r79u2SpBtuuEG///672rdvr9GjR+vdd9/VH3/8IUm66qqr1LZtW7Vv31633nqr3nzzTR07dqxKr18VJKV8KDLSu/W8qXHjxm7PJ02apHfffVczZszQ+vXrtXXrVnXt2lUlJSWnbadhw4Zuzy0Wy2lve/NU3/D0qaIKnnnmGc2aNUsPPPCA1q5dq61btyopKckVe3Bw8GmPP9N+AGfnbG/V8mYCwpsdJW/+jffXBJfkn4kkb59jbRMWFiar1ar8/Hy38vz8/AonV50yZYpuvfVW3XHHHeratauGDBmiGTNmKD09vcL/t9PS0lRQUODasrOzvXoe/txPAgBUjz//ba9Ln4GrKiYmRjt27NBLL72k4OBg3XXXXbrssst04sQJNW3aVFu2bNHSpUsVGRmpqVOnqnv37q55sbyNpJQPJSQ4VxiwWDzvt1ikmBhnPbN9/vnnuu222zRkyBB17dpVERER2rt3r09jCA0NVXh4uL766itXmcPh0JYtW0573Oeff65BgwbplltuUffu3dW+fXv99NNPrv3nn3++goODlZmZ6fH4bt26aevWrae9DxhA9XhjhI03ExDe7Ch582+8vya4/DWR5M/fwvpCYGCgevbs6fb/WmlpqTIzMxUfH+/xmGPHjikgwL0baP2/ZR0r6hjbbDaFhIS4bd5Um/pJAIDKqU1/22vzZ+BTderUSZ9//rlb2eeff67OnTu7ngcHB2vgwIF64YUXtG7dOm3YsEHfffedJKlBgwZKTEzU008/rW+//VZ79+7Vf/7zn7M4s4qRlPIhq9W55KVU/pey7HlGhn8s9X3++efLbrdr69at+uabbzRixAhTJvq+++67lZ6ern/961/asWOHJkyYoP/+97+yVPRXTc7Y16xZoy+++ELbt2/XnXfe6fbtcVBQkB544AHdf//9eu2117Rr1y59+eWXWrBggSRp+PDhioiI0ODBg/X5559r9+7deueddyqcLBZA5XhrhI03ExDe7Ch582+8vya4/DWR5M/fwvpKamqq5s+fr8WLF2v79u0aO3asioqKlJKSIkkaOXKk0tLSXPUHDhyouXPn6q233tKePXu0Zs0aTZkyRQMHDnQlp3ytNvWTAACVU5v+ttfmz8Cn+sc//qFFixZp7ty5+vnnnzVz5kzZ7XbXhOqLFi3SggULtG3bNu3evVtvvPGGgoOD1bZtW/373//WCy+8oK1bt2rfvn167bXXVFpaqg4dOtTI+ZKU8rHkZGn5cikqyr08OtpZnpxsTlynmjlzppo3b65LLrlEAwcOVFJSki6++GKfx/HAAw9o+PDhGjlypOLj49WkSRMlJSUpKCiowmMmT56siy++WElJSerXr58rwXSyKVOm6L777tPUqVPVqVMnDRs2zHUfb2BgoD766CO1bt1aAwYMUNeuXfXkk0+a1kkH6gJvjrDxZgLC2x0lb/2N99cEl78mkmrTt7A1ZdiwYXr22Wc1depU9ejRQ1u3btXq1atdK8tmZWW5LTc9efJk3XfffZo8ebI6d+6s22+/XUlJSXr55ZfNOgVJtaefBACovNryt702fwY+1eDBgzVr1iw9++yz6tKli15++WW9+uqr6tevnySpWbNmmj9/vvr27atu3brp448/1vvvv6+WLVuqWbNmstvt+utf/6pOnTpp3rx5Wrp0qbp06VIzJ3zG6dRr2OzZs422bdsaNpvN6NOnj7Fx48YK65aUlBiPPvqo0b59e8NmsxndunVzzWpfZtq0aYYkt61Dhw5ViqmmVt87mbdWLapvHA6HccEFF7itcGA2Vt8DzqwmVqbztNJadVamMwzPK8DFxFR/xbaaXJmuOnGVrUx36jWr6sp0/vzv6K1zrAx/W33PLDV5HegnAYB/8OZnHf62V48/fgYu443V9xrUTKqrcpYtW6bU1FTNmzdPcXFxysjIUFJSknbs2KHWrVuXqz958mS98cYbmj9/vjp27KgPP/xQQ4YM0RdffKGLLrrIVa9Lly76+OOPXc8bNDD1ND2yWqX/S1LiNPbt26ePPvpIl19+uYqLizV79mzt2bNHI0aMMDs0AFXgzRE2ZaOIhg51joA5efRVdYeBJydLgwY5bzvLzXWOzklIqP5Qcm/9jfdWXGXfUE6Y4H77XXS081pV9hvKshFJOTmeR71ZLM79Vbnd0Vv/jt46R/gH+kkAUPfwt71y6ttnYFNv35s5c6ZGjx6tlJQUde7cWfPmzVOjRo20cOFCj/Vff/11PfTQQxowYIDat2+vsWPHasCAAXruuefc6jVo0EARERGuLSwszBengxoQEBCgRYsWqXfv3urbt6++++47ffzxx+rUqZPZoQGoAm/P+VMTw8DLOkrDhzsf/eWOXW/FlZws7d0rrV0rLVnifNyzp2rXyl9vdzy5vbM9RwAAADPVt8/Apg0hKikp0ebNm90m3QwICFBiYmKFE0oXFxeXu48yODhYn332mVvZzz//rDZt2igoKEjx8fFKT0/XOeec4/2TQI2LiYkpt2oAgNrHmyNsynh7dFN94I1vKL09IslfR6kBAACYob59BjYtKXX48GE5HA7XBJxlwsPD9eOPP3o8JikpSTNnztRll12mc889V5mZmbLb7XKcNDNuXFycFi1apA4dOig3N1ePPvqoEhIStG3bNjVt2tRju8XFxSouLnY9Lyws9MIZAgDK1MQtd2XtkoDwPRJJAAAA8IZatfrerFmzdP7556tjx44KDAzU+PHjlZKSooCAP0+jf//+uuGGG9StWzclJSVp1apVOnLkiP75z39W2G56erpCQ0NdW0xMjC9OBwDqldqy8goqx19vdwQAAEDtYVpSKiwsTFarVfn5+W7l+fn5ioiI8HhMq1attGLFChUVFWnfvn368ccf1aRJE7Vv377C12nWrJkuuOAC7dy5s8I6aWlpKigocG3Z2dlnjL+0tPSMdVA/8F4AKo85fwAAAGofPvPAE2+8L0y7fS8wMFA9e/ZUZmamBg8eLMl5QpmZmRo/fvxpjw0KClJUVJROnDihd955RzfeeGOFdY8ePapdu3bp1ltvrbCOzWaTzWardNwBAQE6cOCAWrVqpcDAQFlOne0V9YJhGCopKdGhQ4cUEBCgwMBAs0MCagVu1QIAAKgd+PwLT7z5Wdi0pJQkpaamatSoUerVq5f69OmjjIwMFRUVKSUlRZI0cuRIRUVFKT09XZK0ceNG5eTkqEePHsrJydEjjzyi0tJS3X///a42J02apIEDB6pt27Y6cOCApk2bJqvVquHDh3sl5oCAALVr1065ubk6cOCAV9pE7daoUSOdc845breRAgAAAEBtx+dfnI43PgubmpQaNmyYDh06pKlTpyovL089evTQ6tWrXZOfZ2VluZ3c8ePHNXnyZO3evVtNmjTRgAED9Prrr6tZs2auOvv379fw4cP1yy+/qFWrVrr00kv15ZdfqlWrVl6LOzAwUOecc47++OMPt0nWUf9YrVY1aNCAbwsAAAAA1El8/oUn3vosbDEMT4tz12+FhYUKDQ1VQUGBQkJCzA4HAAD4EfoJTlwHAABQkcr2E7jfCAAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD5HUgoAAAAAAAA+18DsAAAA/s/hkNavl3JzpchIKSFBslrNjgoAAABAbUZSCgBwWna7NGGCtH//n2XR0dKsWVJysnlxAQAAAKjduH0PAFAhu10aOtQ9ISVJOTnOcrvdnLgAAAAA1H4kpQAAHjkczhFShlF+X1nZxInOegAAAABQVSSlAAAerV9ffoTUyQxDys521gMAAACAqiIpBQDwKDfXu/UAAAAA4GQkpQAAHkVGerceAAAAAJyMpBQAwKOEBOcqexaL5/0WixQT46wHAAAAAFVFUgoA4JHVKs2a5fz51MRU2fOMDGc9AAAAAKgqklIAgAolJ0vLl0tRUe7l0dHO8uRkc+ICAAAAUPs1MDsAAEDNcDicK+Pl5jrnfUpIqN6opuRkadAg77QFAAAAAGVISgFAHWS3SxMmSPv3/1kWHe28Ha86o5usVqlfP6+FBwAAAADcvgcAdY3dLg0d6p6QkqScHGe53W5OXAAAAABwMpJSAFCHOBzOEVKGUX5fWdnEic56AAAAAGAmklIAUIesX19+hNTJDEPKznbWAwAAAAAzkZQCgDokN9e79QAAAACgppCUAoA6JDLSu/UAAAAAoKaQlAKAOiQhwbnKnsXieb/FIsXEOOsBAAAAgJlISgFAHWK1SrNmOX8+NTFV9jwjw1kPAAAAAMxEUgoA6pjkZGn5cikqyr08OtpZnpxsTlwAAAAAcLIGZgcAAPiTw+FcGS831znvU0JC9UY1JSdLgwZ5py0AAAAAqAkkpQDAT9jt0oQJ0v79f5ZFRztvx6vO6CarVerXz2vhAQAAAIBXcfseAPgBu10aOtQ9ISVJOTnOcrvdnLgAAAAAoKaQlAIAkzkczhFShlF+X1nZxInOegAAAABQV5CUAgCTrV9ffoTUyQxDys521gMAAACAuoKkFACcJYdDWrdOWrrU+VjVEU25ud6tB6B+mjNnjmJjYxUUFKS4uDht2rSpwrr9+vWTxWIpt1177bU+jBgAANR3JKUA4CzY7VJsrHTFFdKIEc7H2NiqzQEVGendegDqn2XLlik1NVXTpk3Tli1b1L17dyUlJengwYMe69vtduXm5rq2bdu2yWq16oYbbvBx5AAAoD4jKQUA1eStyckTEpyr7FksnvdbLFJMjLMeAHgyc+ZMjR49WikpKercubPmzZunRo0aaeHChR7rt2jRQhEREa5tzZo1atSoEUkpAADgUySlAKAavDk5udUqzZrl/PnUxFTZ84wMZz0AOFVJSYk2b96sxMREV1lAQIASExO1YcOGSrWxYMEC3XTTTWrcuHGFdYqLi1VYWOi2AQAAnA2SUgBQDd6enDw5WVq+XIqKci+PjnaWJydXP1YAddvhw4flcDgUHh7uVh4eHq68vLwzHr9p0yZt27ZNd9xxx2nrpaenKzQ01LXFxMScVdwAAAAkpQCgGmpicvLkZGnvXmntWmnJEufjnj0kpADUrAULFqhr167q06fPaeulpaWpoKDAtWVnZ/soQgAAUFc1MDsAAKiNampycqtV6tevyuEAqMfCwsJktVqVn5/vVp6fn6+IiIjTHltUVKS33npL06dPP+Pr2Gw22Wy2s4oVAADgZIyUAoBqYHJyAP4iMDBQPXv2VGZmpqustLRUmZmZio+PP+2xb7/9toqLi3XLLbfUdJgAAADlkJQCgGpgcnIA/iQ1NVXz58/X4sWLtX37do0dO1ZFRUVKSUmRJI0cOVJpaWnljluwYIEGDx6sli1b+jpkAAAAbt8DgOoqm5x8wgT3Sc+jo50JKeaCAuArw4YN06FDhzR16lTl5eWpR48eWr16tWvy86ysLAUEuH8XuWPHDn322Wf66KOPzAgZAADA/JFSc+bMUWxsrIKCghQXF6dNmzZVWPfEiROaPn26zj33XAUFBal79+5avXr1WbUJAGeDyckB+Ivx48dr3759Ki4u1saNGxUXF+fat27dOi1atMitfocOHWQYhq666iofRwoAAOBkalJq2bJlSk1N1bRp07RlyxZ1795dSUlJOnjwoMf6kydP1ssvv6wXX3xRP/zwg/7+979ryJAh+vrrr6vdJgCcrbLJyYcPdz5yyx4AAAAAnJnFMAzDrBePi4tT7969NXv2bEnOSTljYmJ0991368EHHyxXv02bNnr44Yc1btw4V9n111+v4OBgvfHGG9Vq05PCwkKFhoaqoKBAISEhZ3uaAACgDqGf4MR1AAAAFalsP8G0kVIlJSXavHmzEhMT/wwmIECJiYnasGGDx2OKi4sVFBTkVhYcHKzPPvus2m0CAAAAAADA90xLSh0+fFgOh8M1AWeZ8PBw5eXleTwmKSlJM2fO1M8//6zS0lKtWbNGdrtdubm51W5Tcia7CgsL3TYAAAAAAADUHNMnOq+KWbNm6fzzz1fHjh0VGBio8ePHKyUlpdxqMlWVnp6u0NBQ1xYTE+OliAEAAAAAAOCJaUmpsLAwWa1W5efnu5Xn5+crIiLC4zGtWrXSihUrVFRUpH379unHH39UkyZN1L59+2q3KUlpaWkqKChwbdnZ2Wd5dgAAAAAAADgd05JSgYGB6tmzpzIzM11lpaWlyszMVHx8/GmPDQoKUlRUlP744w+98847GjRo0Fm1abPZFBIS4rYBAAAAAACg5jQw88VTU1M1atQo9erVS3369FFGRoaKioqUkpIiSRo5cqSioqKUnp4uSdq4caNycnLUo0cP5eTk6JFHHlFpaanuv//+SrcJAAAAAAAA85malBo2bJgOHTqkqVOnKi8vTz169NDq1atdE5VnZWW5zRd1/PhxTZ48Wbt371aTJk00YMAAvf7662rWrFml2wQAAAAAAID5LIZhGGYH4W8KCwsVGhqqgoICbuUDAABu6Cc4cR0AAEBFKttPqFWr7wEAAAAAAKBuICkFAAAAAAAAnyMpBQAAAAAAAJ8jKQUAAAAAAACfIykFAAAAAAAAnyMpBQAAAAAAAJ8jKQUAAAAAAACfIykFAAAAAAAAnyMpBQAAAAAAAJ9rYHYAAGAGh0Nav17KzZUiI6WEBMlqNTsqAAAAAKg/SEoBqHfsdmnCBGn//j/LoqOlWbOk5GTz4gIAAACA+oTb9wDUK3a7NHSoe0JKknJynOV2uzlxAQAAAEB9Q1IKQL3hcDhHSBlG+X1lZRMnOusBAAAAAGoWSSkA9cb69eVHSJ3MMKTsbGc9AAAAAEDNIikFoN7IzfVuPQAAAABA9ZGUAlBvREZ6tx4AAAAAoPpISgGoNxISnKvsWSye91ssUkyMsx4AAAAAoGaRlAJQb1it0qxZzp9PTUyVPc/IcNYDAAAAANQsklIA6pXkZGn5cikqyr08OtpZnpxsTlwAAAAAUN80MDsAAPC15GRp0CDnKnu5uc45pBISGCEFAAAAAL5EUgpAreFweC+RZLVK/fp5NTwAAAAAQBWQlAJQK9jt0oQJ0v79f5ZFRzvniOKWOwAAAACofZhTCoDfs9uloUPdE1KSlJPjLLfbzYkLAAAAAFB9JKUA+DWHwzlCyjDK7ysrmzjRWQ8AAAAAUHuQlALg19avLz9C6mSGIWVnO+sBAAAAAGoPklIA/FpurnfrAQAAAAD8A0kpAH4tMtK79QAAAAAA/oGkFAC/lpDgXGXPYvG832KRYmKc9QAAAAAAtQdJKQB+zWqVZs1y/nxqYqrseUaGsx4AAAAAoPYgKQXA7yUnS8uXS1FR7uXR0c7y5GRz4gIAfzJnzhzFxsYqKChIcXFx2rRp02nrHzlyROPGjVNkZKRsNpsuuOACrVq1ykfRAgAASA3MDgAAKiM5WRo0yLnKXm6ucw6phARGSAGAJC1btkypqamaN2+e4uLilJGRoaSkJO3YsUOtW7cuV7+kpERXXXWVWrdureXLlysqKkr79u1Ts2bNfB88AACot0hKAag1rFapXz+zowAA/zNz5kyNHj1aKSkpkqR58+Zp5cqVWrhwoR588MFy9RcuXKhff/1VX3zxhRo2bChJio2N9WXIAAAA3L4HAABQm5WUlGjz5s1KTEx0lQUEBCgxMVEbNmzweMx7772n+Ph4jRs3TuHh4brwwgs1Y8YMORyOCl+nuLhYhYWFbhsAAMDZICkFAABQix0+fFgOh0Ph4eFu5eHh4crLy/N4zO7du7V8+XI5HA6tWrVKU6ZM0XPPPafHH3+8wtdJT09XaGioa4uJifHqeQAAgPqHpBQAAEA9U1paqtatW+uVV15Rz549NWzYMD388MOaN29ehcekpaWpoKDAtWVnZ/swYgAAUBcxpxQAAEAtFhYWJqvVqvz8fLfy/Px8RUREeDwmMjJSDRs2lPWk1SI6deqkvLw8lZSUKDAwsNwxNptNNpvNu8EDAIB6jZFSAAAAtVhgYKB69uypzMxMV1lpaakyMzMVHx/v8Zi+fftq586dKi0tdZX99NNPioyM9JiQAgAAqAkkpQAAAGq51NRUzZ8/X4sXL9b27ds1duxYFRUVuVbjGzlypNLS0lz1x44dq19//VUTJkzQTz/9pJUrV2rGjBkaN26cWacAAADqIW7fAwAAqOWGDRumQ4cOaerUqcrLy1OPHj20evVq1+TnWVlZCgj487vImJgYffjhh7r33nvVrVs3RUVFacKECXrggQfMOgUAAFAPWQzDMMwOwt8UFhYqNDRUBQUFCgkJMTscAADgR+gnOHEdAABARSrbT+D2PQAAAAAAAPic6UmpOXPmKDY2VkFBQYqLi9OmTZtOWz8jI0MdOnRQcHCwYmJidO+99+r48eOu/Y888ogsFovb1rFjx5o+DQAAAAAAAFSBqXNKLVu2TKmpqZo3b57i4uKUkZGhpKQk7dixQ61bty5Xf8mSJXrwwQe1cOFCXXLJJfrpp5902223yWKxaObMma56Xbp00ccff+x63qABU2cBAAAAAAD4E1NHSs2cOVOjR49WSkqKOnfurHnz5qlRo0ZauHChx/pffPGF+vbtqxEjRig2NlZXX321hg8fXm50VYMGDRQREeHawsLCfHE6AAAAAAAAqCTTklIlJSXavHmzEhMT/wwmIECJiYnasGGDx2MuueQSbd682ZWE2r17t1atWqUBAwa41fv555/Vpk0btW/fXjfffLOysrJOG0txcbEKCwvdNgAAAAAAANQc0+5rO3z4sBwOh2up4jLh4eH68ccfPR4zYsQIHT58WJdeeqkMw9Aff/yhv//973rooYdcdeLi4rRo0SJ16NBBubm5evTRR5WQkKBt27apadOmHttNT0/Xo48+6r2TAwAAAAAAwGmZPtF5Vaxbt04zZszQSy+9pC1btshut2vlypV67LHHXHX69++vG264Qd26dVNSUpJWrVqlI0eO6J///GeF7aalpamgoMC1ZWdn++J0AAAAAAAA6i3TRkqFhYXJarUqPz/frTw/P18REREej5kyZYpuvfVW3XHHHZKkrl27qqioSGPGjNHDDz+sgIDyObZmzZrpggsu0M6dOyuMxWazyWazncXZAAAAAAAAoCpMGykVGBionj17KjMz01VWWlqqzMxMxcfHezzm2LFj5RJPVqtVkmQYhsdjjh49ql27dikyMtJLkQMAAAAAAOBsmTZSSpJSU1M1atQo9erVS3369FFGRoaKioqUkpIiSRo5cqSioqKUnp4uSRo4cKBmzpypiy66SHFxcdq5c6emTJmigQMHupJTkyZN0sCBA9W2bVsdOHBA06ZNk9Vq1fDhw007TwAAAAAAALgzNSk1bNgwHTp0SFOnTlVeXp569Oih1atXuyY/z8rKchsZNXnyZFksFk2ePFk5OTlq1aqVBg4cqCeeeMJVZ//+/Ro+fLh++eUXtWrVSpdeeqm+/PJLtWrVyufnBwAAAAAAAM8sRkX3vdVjhYWFCg0NVUFBgUJCQswOBwAA+BH6CU5cBwAAUJHK9hNq1ep7AAAAAAAAqBtISgEAAAAAAMDnSEoBAAAAAADA50hKAQAAAAAAwOdISgEAAAAAAMDnSEoBAAAAAADA50hKAQAAAAAAwOdISgEAAAAAAMDnGpgdAAD/5HBI69dLublSZKSUkCBZrea3BQAAAACoG0hKASjHbpcmTJD27/+zLDpamjVLSk42ry0AAAAAQN3B7XsA3Njt0tCh7kkkScrJcZbb7ea0BQAAAACoW0hKAXBxOJyjmgyj/L6ysokTnfV82RYAAAAAoO4hKQXAZf368qOaTmYYUna2s54v2wIAAAAA1D0kpQC45OZ6r5432wIAAAAA1D0kpQC4REZ6r5432wIAAAAA1D0kpQC4JCQ4V8azWDzvt1ikmBhnPV+2BQAAAACoe0hKAXCxWqVZs5w/n5pMKnuekeGs58u2AAAAAAB1D0kpAG6Sk6Xly6WoKPfy6GhneXKyOW0BAAAAAOqWBmYHAMD/JCdLgwY5V8bLzXXO+5SQUL1RTd5sCwAAAABQd5CUAuCR1Sr16+d/bQEAAAAA6gZu3wMAAAAAAIDPkZQCAAAAAACAz5GUAgAAAAAAgM+RlAIAAAAAAIDPkZQCAAAAAACAz5GUAgAAAAAAgM+RlAIAAAAAAIDPkZQCAACoA+bMmaPY2FgFBQUpLi5OmzZtqrDuokWLZLFY3LagoCAfRus7Doe0bp20dKnz0eEwOyIAAFCGpBQAAEAtt2zZMqWmpmratGnasmWLunfvrqSkJB08eLDCY0JCQpSbm+va9u3b58OIfcNul2JjpSuukEaMcD7GxjrLAQCA+UhKAXUI3wYDQP00c+ZMjR49WikpKercubPmzZunRo0aaeHChRUeY7FYFBER4drCw8N9GHHNs9uloUOl/fvdy3NynOUkpgAAMB9JKaCO4NtgAKifSkpKtHnzZiUmJrrKAgIClJiYqA0bNlR43NGjR9W2bVvFxMRo0KBB+v77730Rrk84HNKECZJhlN9XVjZxIl/eAABgNpJSQB3At8EAUH8dPnxYDoej3Ein8PBw5eXleTymQ4cOWrhwof71r3/pjTfeUGlpqS655BLtP/U/kpMUFxersLDQbfNX69eX/z/xZIYhZWc76wEAAPOQlAJqOb4NBgBUVXx8vEaOHKkePXro8ssvl91uV6tWrfTyyy9XeEx6erpCQ0NdW0xMjA8jrprcXO/WAwAANYOkFFDL8W0wANRvYWFhslqtys/PdyvPz89XREREpdpo2LChLrroIu3cubPCOmlpaSooKHBt2dnZZxV3TYqM9G49AABQM0hKAbUc3wYDQP0WGBionj17KjMz01VWWlqqzMxMxcfHV6oNh8Oh7777TpGnydLYbDaFhIS4bf4qIUGKjpYsFs/7LRYpJsZZDwAAmKfKSanY2FhNnz5dWVlZNREPgCri22AAQGpqqubPn6/Fixdr+/btGjt2rIqKipSSkiJJGjlypNLS0lz1p0+fro8++ki7d+/Wli1bdMstt2jfvn264447zDoFr7JapVmznD+fmpgqe56R4awHAADMU+Wk1MSJE2W329W+fXtdddVVeuutt1RcXFwTsQGoBL4NBgAMGzZMzz77rKZOnaoePXpo69atWr16tWvy86ysLOWeNGT2v//9r0aPHq1OnTppwIABKiws1BdffKHOnTubdQpel5wsLV8uRUW5l0dHO8uTk82JCwAA/MliGJ6mRz6zLVu2aNGiRVq6dKkcDodGjBihv/3tb7r44ou9HaPPFRYWKjQ0VAUFBX49NB0oU7b6nuQ+4XlZoorONwB4D/0Ep9pyHRwO57yKubnOUcMJCYyQAgCgplW2n1DtpFSZEydO6KWXXtIDDzygEydOqGvXrrrnnnuUkpIiS0VDN/xcbelkASez252r8J086XlMjPP2BBJSAOA99BOcuA4AAKAile0nNKjuC5w4cULvvvuuXn31Va1Zs0Z/+ctfdPvtt2v//v166KGH9PHHH2vJkiXVbR5AFSUnS4MG8W0wAAAAAKB2qPKcUlu2bNHdd9+tyMhIjR8/Xl26dNG2bdv02WefKSUlRVOmTNHHH3+sd999t1LtzZkzR7GxsQoKClJcXJw2bdp02voZGRnq0KGDgoODFRMTo3vvvVfHjx8/qzaBusJqlfr1k4YPdz6SkAIAAAAA+KsqJ6V69+6tn3/+WXPnzlVOTo6effZZdezY0a1Ou3btdNNNN52xrWXLlik1NVXTpk3Tli1b1L17dyUlJengwYMe6y9ZskQPPvigpk2bpu3bt2vBggVatmyZHnrooWq3CQAAAAAAAN+r8pxS+/btU9u2bb3y4nFxcerdu7dmz54tSSotLVVMTIzuvvtuPfjgg+Xqjx8/Xtu3b1dmZqar7L777tPGjRv12WefVatNT5gjAQAAVIR+ghPXAQAAVKSy/YQqj5Q6ePCgNm7cWK5848aN+n//7/9Vup2SkhJt3rxZiYmJfwYTEKDExERt2LDB4zGXXHKJNm/e7Lodb/fu3Vq1apUGDBhQ7TYlqbi4WIWFhW4bAAAAAAAAak6Vk1Ljxo1TdnZ2ufKcnByNGzeu0u0cPnxYDodD4eHhbuXh4eHKy8vzeMyIESM0ffp0XXrppWrYsKHOPfdc9evXz3X7XnXalKT09HSFhoa6tpiYmEqfBwAAAAAAAKquykmpH374QRdffHG58osuukg//PCDV4KqyLp16zRjxgy99NJL2rJli+x2u1auXKnHHnvsrNpNS0tTQUGBa/OUdAMAAAAAAID3NKjqATabTfn5+Wrfvr1beW5urho0qHxzYWFhslqtys/PdyvPz89XRESEx2OmTJmiW2+9VXfccYckqWvXrioqKtKYMWP08MMPV6vNsnOy2WyVjh0AAAAAAABnp8ojpa6++mrXyKIyR44c0UMPPaSrrrqq0u0EBgaqZ8+ebpOWl5aWKjMzU/Hx8R6POXbsmAIC3EO2/t+a94ZhVKtNAAAAAAAA+F6VR0o9++yzuuyyy9S2bVtddNFFkqStW7cqPDxcr7/+epXaSk1N1ahRo9SrVy/16dNHGRkZKioqUkpKiiRp5MiRioqKUnp6uiRp4MCBmjlzpi666CLFxcVp586dmjJligYOHOhKTp2pTQAAAAAAAJivykmpqKgoffvtt3rzzTf1zTffKDg4WCkpKRo+fLgaNmxYpbaGDRumQ4cOaerUqcrLy1OPHj20evVq10TlWVlZbiOjJk+eLIvFosmTJysnJ0etWrXSwIED9cQTT1S6TQAAAAAAAJjPYhiGYXYQ/qawsFChoaEqKChQSEiI2eEAAAA/Qj/BiesAAAAqUtl+QpVHSpX54YcflJWVpZKSErfy6667rrpNAgAAAAAAoJ6oclJq9+7dGjJkiL777jtZLBaVDbSyWCySJIfD4d0IAQAAAD/gcEjr10u5uVJkpJSQIP3ftKYAAKAaqrz63oQJE9SuXTsdPHhQjRo10vfff69PP/1UvXr10rp162ogRAAAgLopOztb+/fvdz3ftGmTJk6cqFdeecXEqOCJ3S7FxkpXXCGNGOF8jI11lgMAgOqpclJqw4YNmj59usLCwhQQEKCAgABdeumlSk9P1z333FMTMQIAANRJI0aM0Nq1ayVJeXl5uuqqq7Rp0yY9/PDDmj59usnRoYzdLg0dKp2UP5Qk5eQ4y0lMAQBQPVVOSjkcDjVt2lSSFBYWpgMHDkiS2rZtqx07dng3OgAAgDps27Zt6tOnjyTpn//8py688EJ98cUXevPNN7Vo0SJzg4Mk5y17EyZInpYGKiubONFZDwAAVE2V55S68MIL9c0336hdu3aKi4vT008/rcDAQL3yyitq3759TcQIAABQJ504cUI2m02S9PHHH7sWjOnYsaNyc3PNDA3/Z/368iOkTmYYUna2s16/fj4LCwCAOqHKI6UmT56s0tJSSdL06dO1Z88eJSQkaNWqVXrhhRe8HiAAAEBd1aVLF82bN0/r16/XmjVrdM0110iSDhw4oJYtW5ocHSTnpOberAcAAP5U5ZFSSUlJrp/PO+88/fjjj/r111/VvHlz1wp8AAAAOLOnnnpKQ4YM0TPPPKNRo0ape/fukqT33nvPdVsfzBUZ6d16AADgT1VKSp04cULBwcHaunWrLrzwQld5ixYtvB4YAABAXdevXz8dPnxYhYWFat68uat8zJgxatSokYmRoUxCghQd7ZzU3NO8UhaLc39Cgu9jAwCgtqvS7XsNGzbUOeecIwczOQJe43BI69ZJS5c6H/n1AoD64/fff1dxcbErIbVv3z5lZGRox44dat26tcnRQZKsVmnWLOfPp94UUPY8I8NZDwAAVE2V55R6+OGH9dBDD+nXX3+tiXiAesVul2JjpSuukEaMcD7GxrK0NADUF4MGDdJrr70mSTpy5Iji4uL03HPPafDgwZo7d67J0aFMcrK0fLkUFeVeHh3tLE9ONicuAABqO4theBqIXLGLLrpIO3fu1IkTJ9S2bVs1btzYbf+WLVu8GqAZCgsLFRoaqoKCAoWEhJgdDuoou10aOrT8rQBl37rSyQUA/+TNfkJYWJg++eQTdenSRf/7v/+rF198UV9//bXeeecdTZ06Vdu3b/dS1N5XH/tLDodzlb3cXOccUgkJjJACAMCTyvYTqjzR+eDBg88mLgBydmonTPA8N4VhOBNTEydKgwbR2QWAuuzYsWNq2rSpJOmjjz5ScnKyAgIC9Je//EX79u0zOTqcymqV+vUzOwoAAOqOKielpk2bVhNxAPXK+vXS/v0V7zcMKTvbWY/OLwDUXeedd55WrFihIUOG6MMPP9S9994rSTp48GC9GX0EAADqryrPKQXg7OXmerceAKB2mjp1qiZNmqTY2Fj16dNH8fHxkpyjpi666CKTowMAAKhZVR4pFRAQIMupS4+chJX5gDOLjPRuPQBA7TR06FBdeumlys3NVffu3V3lV155pYYMGWJiZAAAADWvykmpd9991+35iRMn9PXXX2vx4sV69NFHvRYYUJclJDhX7MnJ8TyvlMXi3J+Q4PvYAAC+FRERoYiICO3/v/u6o6Oj1adPH5OjAgAAqHlVTkoNGjSoXNnQoUPVpUsXLVu2TLfffrtXAgPqMqtVmjXLufqexeKemCobiJiRwSTnAFDXlZaW6vHHH9dzzz2no0ePSpKaNm2q++67Tw8//LACAphpAQAA1F1e6+n85S9/UWZmpreaA+q85GRp+XIpKsq9PDraWZ6cbE5cAADfefjhhzV79mw9+eST+vrrr/X1119rxowZevHFFzVlyhSzwwMAAKhRVR4p5cnvv/+uF154QVGnfroG6iiHw7kyXm6uc96nhITqjWpKTpYGDfJOWwCA2mfx4sX63//9X1133XWusm7duikqKkp33XWXnnjiCROjAwAAqFlVTko1b97cbaJzwzD022+/qVGjRnrjjTe8Ghzgj+x2acIE6f+m/pDkHN00a1b1RjdZrVK/fl4LDwBQi/z666/q2LFjufKOHTvq119/NSEiAAAA36lyUur55593S0oFBASoVatWiouLU/Pmzb0aHOBv7HbnPFCnTk6ek+Ms57Y7AEBVdO/eXbNnz9YLL7zgVj579mx169bNpKgAAAB8w2IYntb+qt8KCwsVGhqqgoIChYSEmB0O/ITDIcXGuo+QOlnZinl79nD7HQDUZd7sJ3zyySe69tprdc455yg+Pl6StGHDBmVnZ2vVqlVK8ONlWOkvAQCAilS2n1Dlic5fffVVvf322+XK3377bS1evLiqzQG1xvr1FSekJOfoqexsZz0AACrj8ssv108//aQhQ4boyJEjOnLkiJKTk/X999/r9ddfNzs8AACAGlXlpFR6errCwsLKlbdu3VozZszwSlCAP8rN9W49AAAkqU2bNnriiSf0zjvv6J133tHjjz+u//73v1qwYIHZoQEAANSoKielsrKy1K5du3Llbdu2VVZWlleCAvxRZKR36wEAAAAAUJ9VOSnVunVrffvtt+XKv/nmG7Vs2dIrQQH+KCHBOWfUSfP8u7FYpJgYZz0AAHxtzpw5io2NVVBQkOLi4rRp06ZKHffWW2/JYrFo8ODBNRsgAADAKaqclBo+fLjuuecerV27Vg6HQw6HQ//5z380YcIE3XTTTTURI+AXrFZp1iznz6cmpsqeZ2QwyTkAwPeWLVum1NRUTZs2TVu2bFH37t2VlJSkgwcPnva4vXv3atKkSX49oToAAKi7qrz6XklJiW699Va9/fbbatCggSSptLRUI0eO1Lx58xQYGFgjgfoSq8ngdOx2acIE90nPY2KcCankZNPCAgD4iDf6Ccln+A/jyJEj+uSTT+RwOCrVXlxcnHr37q3Zs2dLcvbNYmJidPfdd+vBBx/0eIzD4dBll12mv/3tb1q/fr2OHDmiFStWVPoc6C8BAICKVLaf0KCqDQcGBmrZsmV6/PHHtXXrVgUHB6tr165q27btWQUM1DSHw7kyXm6uc96nhITqjWpKTpYGDfJOWwCA+ik0NPSM+0eOHFmptkpKSrR582alpaW5ygICApSYmKgNGzZUeNz06dPVunVr3X777VrP0rEAAMAEVU5KlTn//PN1/vnnezMWoMZ4Gt0UHe28Ha86o5usVqlfP6+FBwCoZ1599VWvtXX48GE5HA6Fh4e7lYeHh+vHH3/0eMxnn32mBQsWaOvWrZV+neLiYhUXF7ueFxYWViteAACAMlWeU+r666/XU089Va786aef1g033OCVoABvstuloUPdE1KSlJPjLLfbzYkLAAAz/Pbbb7r11ls1f/58hYWFVfq49PR0hYaGuraYmJgajBIAANQHVU5KffrppxowYEC58v79++vTTz/1SlCAtzgczhFSnmZOKyubONFZDwCA2igsLExWq1X5+flu5fn5+YqIiChXf9euXdq7d68GDhyoBg0aqEGDBnrttdf03nvvqUGDBtq1a5fH10lLS1NBQYFry87OrpHzAQAA9UeVk1JHjx71OJl5w4YNGcYNv7N+ffkRUiczDCk721kPAIDaKDAwUD179lRmZqarrLS0VJmZmYqPjy9Xv2PHjvruu++0detW13bdddfpiiuu0NatWyscAWWz2RQSEuK2AQAAnI0qzynVtWtXLVu2TFOnTnUrf+utt9S5c2evBQZ4Q26ud+sBAOCPUlNTNWrUKPXq1Ut9+vRRRkaGioqKlJKSIkkaOXKkoqKilJ6erqCgIF144YVuxzdr1kySypUDAADUpConpaZMmaLk5GTt2rVLf/3rXyVJmZmZWrJkiZYvX+71AIGzERnp3XoAAPijYcOG6dChQ5o6dary8vLUo0cPrV692jX5eVZWlgICqjxAHgAAoEZZDMPTbDunt3LlSs2YMUNbt25VcHCwunfvrmnTpqlFixZ14hu2wsJChYaGqqCggKHptZzDIcXGOic19/ROt1icq/Dt2eNcUQ8AgDOhn+DEdQAAABWpbD+hWl+ZXXvttfr8889VVFSk3bt368Ybb9SkSZPUvXv3agcM1ASrVZo1y/mzxeK+r+x5RgYJKQAAAAAAfK3a47g//fRTjRo1Sm3atNFzzz2nv/71r/ryyy+9GRvgFcnJ0vLlUlSUe3l0tLM8OdmcuAAAAAAAqM+qNKdUXl6eFi1apAULFqiwsFA33nijiouLtWLFCiY5h19LTpYGDXKuspeb65xDKiGBEVIAAAAAAJil0iOlBg4cqA4dOujbb79VRkaGDhw4oBdffNErQcyZM0exsbEKCgpSXFycNm3aVGHdfv36yWKxlNuuvfZaV53bbrut3P5rrrnGK7Gi9rJapX79pOHDnY8kpAAAAAAAME+lR0p98MEHuueeezR27Fidf/75Xgtg2bJlSk1N1bx58xQXF6eMjAwlJSVpx44dat26dbn6drtdJSUlrue//PKLunfvrhtuuMGt3jXXXKNXX33V9dxms3ktZgAAAAAAAJydSo+U+uyzz/Tbb7+pZ8+eiouL0+zZs3X48OGzDmDmzJkaPXq0UlJS1LlzZ82bN0+NGjXSwoULPdZv0aKFIiIiXNuaNWvUqFGjckkpm83mVq958+ZnHSsAAAAAAAC8o9JJqb/85S+aP3++cnNzdeedd+qtt95SmzZtVFpaqjVr1ui3336r8ouXlJRo8+bNSkxM/DOggAAlJiZqw4YNlWpjwYIFuummm9S4cWO38nXr1ql169bq0KGDxo4dq19++aXCNoqLi1VYWOi2AQAAAAAAoOZUefW9xo0b629/+5s+++wzfffdd7rvvvv05JNPqnXr1rruuuuq1Nbhw4flcDgUHh7uVh4eHq68vLwzHr9p0yZt27ZNd9xxh1v5Nddco9dee02ZmZl66qmn9Mknn6h///5yOBwe20lPT1doaKhri4mJqdJ5AAAAAAAAoGqqnJQ6WYcOHfT0009r//79Wrp0qbdiqrQFCxaoa9eu6tOnj1v5TTfdpOuuu05du3bV4MGD9e9//1tfffWV1q1b57GdtLQ0FRQUuLbs7GwfRA8AAAAAAFB/nVVSqozVatXgwYP13nvvVem4sLAwWa1W5efnu5Xn5+crIiLitMcWFRXprbfe0u23337G12nfvr3CwsK0c+dOj/ttNptCQkLcNgAAAAAAANQcrySlqiswMFA9e/ZUZmamq6y0tFSZmZmKj48/7bFvv/22iouLdcstt5zxdfbv369ffvlFkZGRZx0zAAAAAAAAzp6pSSlJSk1N1fz587V48WJt375dY8eOVVFRkVJSUiRJI0eOVFpaWrnjFixYoMGDB6tly5Zu5UePHtU//vEPffnll9q7d68yMzM1aNAgnXfeeUpKSvLJOQEAAAAAAOD0GpgdwLBhw3To0CFNnTpVeXl56tGjh1avXu2a/DwrK0sBAe65sx07duizzz7TRx99VK49q9Wqb7/9VosXL9aRI0fUpk0bXX311Xrsscdks9l8ck7wHodDWr9eys2VIiOlhATJajU7KgAAAAAAcLYshmEYZgfhbwoLCxUaGqqCggLmlzKR3S5NmCDt3/9nWXS0NGuWlJxsXlwAgPqNfoIT1wEAAFSksv0E02/fAzyx26WhQ90TUpKUk+Mst9vNiQsAAAAAAHgHSSn4HYfDOULK0xi+srKJE531AAAAAABA7URSCn5n/fryI6ROZhhSdrazHgAAAAAAqJ1ISsHv5OZ6tx4AAAAAAPA/JKXgdyIjvVsPAAAAAAD4H5JS8DsJCc5V9iwWz/stFikmxlkPAAAAAADUTiSl4HesVmnWLOfPpyamyp5nZDjrAQAAAACA2omkFPxScrK0fLkUFeVeHh3tLE9ONicuAAAAAADgHQ3MDgCoSHKyNGiQc5W93FznHFIJCYyQAgAAAACgLiApBb9mtUr9+pkdBQAAAAAA8DZu3wMAAAAAAIDPkZQCAAAAAACAz5GUAgAAAAAAgM+RlAIAAAAAAIDPkZQCAAAAAACAz5GUAgAAAAAAgM+RlAIAAAAAAIDPkZQCAAAAAACAz5GUAgAAAAAAgM81MDsA1D0Oh7R+vZSbK0VGSgkJktVqdlQAAAAAAMCfkJSCV9nt0oQJ0v79f5ZFR0uzZknJyebFBQAAAAAA/Au378Fr7HZp6FD3hJQk5eQ4y+12c+ICAAAAAAD+h6QUvMLhcI6QMozy+8rKJk501gMAAAAAACApBa9Yv778CKmTGYaUne2sBwAAAAAAQFIKXpGb6916AAAAAACgbmOic3hFZKR36wEAANRlrFYMAAAjpeAlCQnOVfYsFs/7LRYpJsZZDwAAeN+cOXMUGxuroKAgxcXFadOmTRXWtdvt6tWrl5o1a6bGjRurR48eev31130Ybf1mt0uxsdIVV0gjRjgfY2NZFAYAUP+QlIJXWK3SrFnOn09NTJU9z8jgG0AAAGrCsmXLlJqaqmnTpmnLli3q3r27kpKSdPDgQY/1W7RooYcfflgbNmzQt99+q5SUFKWkpOjDDz/0ceT1D6sVAwDwJ4theFovrX4rLCxUaGioCgoKFBISYnY4tYrd7lyF7+SOVkyMMyGVnGxaWAAAeI0/9hPi4uLUu3dvzZ49W5JUWlqqmJgY3X333XrwwQcr1cbFF1+sa6+9Vo899lil6vvjdfB3DodzRFRFi8NYLM6R53v28EUeAKB2q2w/gZFS8KrkZGnvXmntWmnJEufjnj0kpAAAqCklJSXavHmzEhMTXWUBAQFKTEzUhg0bzni8YRjKzMzUjh07dNlll1VYr7i4WIWFhW4bqobVigEAcMdE5/A6q1Xq18/sKAAAqB8OHz4sh8Oh8PBwt/Lw8HD9+OOPFR5XUFCgqKgoFRcXy2q16qWXXtJVV11VYf309HQ9+uijXou7PmK1YgAA3DFSCgAAoB5q2rSptm7dqq+++kpPPPGEUlNTtW7dugrrp6WlqaCgwLVlZ2f7Ltg6gtWKAQBwx0gpAACAWiwsLExWq1X5+flu5fn5+YqIiKjwuICAAJ133nmSpB49emj79u1KT09XvwqGO9tsNtlsNq/FXR+VrVack+O8Ve9UZXNKsVoxAKC+YKQUAABALRYYGKiePXsqMzPTVVZaWqrMzEzFx8dXup3S0lIVFxfXRIj4P6xWDACAO5JSAAAAtVxqaqrmz5+vxYsXa/v27Ro7dqyKioqUkpIiSRo5cqTS0tJc9dPT07VmzRrt3r1b27dv13PPPafXX39dt9xyi1mnUG8kJ0vLl0tRUe7l0dHOchaHAQDUJ9y+BwAAUMsNGzZMhw4d0tSpU5WXl6cePXpo9erVrsnPs7KyFBDw53eRRUVFuuuuu7R//34FBwerY8eOeuONNzRs2DCzTqFeSU6WBg1yrrKXm+ucQyohofojpBwO77UFAIAvWQzD0x3t9VthYaFCQ0NVUFCgkJAQs8MBAAB+hH6CE9fBP9jt0oQJ0v79f5ZFRztvE2TUFQDALJXtJ3D7HgAAAFAL2e3S0KHuCSnJOZH60KHO/QAA+DOSUgAAAEAt43A4R0h5uuehrGziRGc9AAD8FUkpSHJ2WNatk5YudT7SgQEAAPBf69eXHyF1MsOQsrOd9QAA8Fd+kZSaM2eOYmNjFRQUpLi4OG3atKnCuv369ZPFYim3XXvtta46hmFo6tSpioyMVHBwsBITE/Xzzz/74lRqJbtdio2VrrhCGjHC+Rgby5BvAAAAf5Wb6916AACYwfSk1LJly5Samqpp06Zpy5Yt6t69u5KSknTw4EGP9e12u3Jzc13btm3bZLVadcMNN7jqPP3003rhhRc0b948bdy4UY0bN1ZSUpKOHz/uq9OqNZiLAAAAoPaJjPRuPQAAzGD66ntxcXHq3bu3Zs+eLUkqLS1VTEyM7r77bj344INnPD4jI0NTp05Vbm6uGjduLMMw1KZNG913332aNGmSJKmgoEDh4eFatGiRbrrppjO2WV9Wk3E4nCOiKhr6bbE4V2/Zs4dlhQEAKFNf+glnwnUwV1k/LifH87xS9OMAAGaqFavvlZSUaPPmzUpMTHSVBQQEKDExURs2bKhUGwsWLNBNN92kxo0bS5L27NmjvLw8tzZDQ0MVFxdXYZvFxcUqLCx02+oD5iIAAAConaxWadYs588Wi/u+sucZGSSkAAD+zdSk1OHDh+VwOBQeHu5WHh4erry8vDMev2nTJm3btk133HGHq6zsuKq0mZ6ertDQUNcWExNT1VOplZiLAAAAoPZKTpaWL5eiotzLo6Od5cnJ5sQFAEBlmT6n1NlYsGCBunbtqj59+pxVO2lpaSooKHBt2dnZXorQvzEXAQAAQO2WnCzt3SutXSstWeJ83LOHhBQAoHZoYOaLh4WFyWq1Kj8/3608Pz9fERERpz22qKhIb731lqZPn+5WXnZcfn6+Ik/KpuTn56tHjx4e27LZbLLZbNU4g9otIcH5TdqZ5iJISPB9bAAAAKgcq1Xq18/sKAAAqDpTR0oFBgaqZ8+eyszMdJWVlpYqMzNT8fHxpz327bffVnFxsW655Ra38nbt2ikiIsKtzcLCQm3cuPGMbdY3zEUAAAAAAADMYvrte6mpqZo/f74WL16s7du3a+zYsSoqKlJKSookaeTIkUpLSyt33IIFCzR48GC1bNnSrdxisWjixIl6/PHH9d577+m7777TyJEj1aZNGw0ePNgXp1SrMBcBAAAAAAAwg6m370nSsGHDdOjQIU2dOlV5eXnq0aOHVq9e7ZqoPCsrSwEB7rmzHTt26LPPPtNHH33ksc37779fRUVFGjNmjI4cOaJLL71Uq1evVlBQUI2fT22UnCwNGuRcZS831zmHVEICI6QAAAAAAEDNsRiGp9mE6rfCwkKFhoaqoKBAISEhZocDAAD8CP0EJ64DAACoSGX7CabfvgcAAAAAAID6h6QUAAAAAAAAfI6kFAAAAAAAAHyOpBQAAAAAAAB8jqQUAAAAAAAAfI6kFAAAAAAAAHyOpBQAAAAAAAB8jqQUAAAAAAAAfI6kFAAAAAAAAHyOpBQAAAAAAAB8jqQUAAAAAAAAfI6kFAAAAAAAAHyOpBQAAAAAAAB8jqQUAAAAAAAAfK6B2QEAAAAA8A8Oh7R+vZSbK0VGSgkJktVqdlQAgLqKpBQAAAAA2e3ShAnS/v1/lkVHS7NmScnJ5sUFAKi7uH0PAAAAqOfsdmnoUPeElCTl5DjL7XZz4gIA1G0kpQAAAIB6zOFwjpAyjPL7ysomTnTWAwDAm0hKAQAAAPXY+vXlR0idzDCk7GxnPQAAvImkFAAAAFCP5eZ6tx4AAJVFUgoAAACoxyIjvVsPAIDKIikFAAAA1GMJCc5V9iwWz/stFikmxlkPAABvIikFAAAA1GNWqzRrlvPnUxNTZc8zMpz1AADwJpJSAAAAQD2XnCwtXy5FRbmXR0c7y5OTzYkLAFC3NTA7AAAAAADmS06WBg1yrrKXm+ucQyohgRFSAICaw0gpAACAOmDOnDmKjY1VUFCQ4uLitGnTpgrrzp8/XwkJCWrevLmaN2+uxMTE09ZH/WG1Sv36ScOHOx9JSAEAahJJKQAAgFpu2bJlSk1N1bRp07RlyxZ1795dSUlJOnjwoMf669at0/Dhw7V27Vpt2LBBMTExuvrqq5WTk+PjyAEAQH1mMQzDMDsIf1NYWKjQ0FAVFBQoJCTE7HAAAIAf8cd+QlxcnHr37q3Zs2dLkkpLSxUTE6O7775bDz744BmPdzgcat68uWbPnq2RI0dW6jX98ToAAAD/UNl+AiOlAAAAarGSkhJt3rxZiYmJrrKAgAAlJiZqw4YNlWrj2LFjOnHihFq0aFFhneLiYhUWFrptAAAAZ4OkFAAAQC12+PBhORwOhYeHu5WHh4crLy+vUm088MADatOmjVti61Tp6ekKDQ11bTExMWcVN+o+h0Nat05autT56HCYHREAwN+QlAIAAKjHnnzySb311lt69913FRQUVGG9tLQ0FRQUuLbs7GwfRonaxm6XYmOlK66QRoxwPsbGOssBACjTwOwAAAAAUH1hYWGyWq3Kz893K8/Pz1dERMRpj3322Wf15JNP6uOPP1a3bt1OW9dms8lms511vKj77HZp6FDp1Jlrc3Kc5cuXS8nJ5sQGAPAvjJQCAACoxQIDA9WzZ09lZma6ykpLS5WZman4+PgKj3v66af12GOPafXq1erVq5cvQkU94HBIEyaUT0hJf5ZNnMitfAAAJ5JSAAAAtVxqaqrmz5+vxYsXa/v27Ro7dqyKioqUkpIiSRo5cqTS0tJc9Z966ilNmTJFCxcuVGxsrPLy8pSXl6ejR4+adQqoI9avl/bvr3i/YUjZ2c56AABw+x4AAEAtN2zYMB06dEhTp05VXl6eevToodWrV7smP8/KylJAwJ/fRc6dO1clJSUaOnSoWzvTpk3TI4884svQUcfk5nq3HgCgbiMpBQAAUAeMHz9e48eP97hv3bp1bs/37t1b8wGhXoqM9G69Mg6Hc3RVbq7z2IQEyWqtenzebgsAcHa4fQ8AAACAVyQkSNHRksXieb/FIsXEOOtVljdX8mNVQADwLySlAAAAAHiF1SrNmuX8+dTEVNnzjIzKj0wqW8nv1Hmqylbyq0oyyZttAQC8g6QUAAAAAK9JTpaWL5eiotzLo6Od5cnJlWvHmyv5sSogAPgn5pSqxbgfHgAAAP4oOVkaNOjs+qpVWcmvXz/ftQUA8B7TR0rNmTNHsbGxCgoKUlxcnDZt2nTa+keOHNG4ceMUGRkpm82mCy64QKtWrXLtf+SRR2SxWNy2jh071vRp+Bz3wwMAAMCfWa3OBM/w4c7Hqn556s2V/FgVEAD8k6kjpZYtW6bU1FTNmzdPcXFxysjIUFJSknbs2KHWrVuXq19SUqKrrrpKrVu31vLlyxUVFaV9+/apWbNmbvW6dOmijz/+2PW8QYO6NSCs7H74U4cfl90PX5Vh0QAAAIA/8uZKfjW1KiAA4OyYmq2ZOXOmRo8erZSUFEnSvHnztHLlSi1cuFAPPvhgufoLFy7Ur7/+qi+++EINGzaUJMXGxpar16BBA0VERNRo7GY50/3wFovzfvhBg7iVDwAAALVX2Up+OTme+74Wi3N/ZVby82ZbAADvMe32vZKSEm3evFmJiYl/BhMQoMTERG3YsMHjMe+9957i4+M1btw4hYeH68ILL9SMGTPkOGVGwp9//llt2rRR+/btdfPNNysrK+u0sRQXF6uwsNBt81dVuR8eAAAAqK28uZKft1cFBAB4h2lJqcOHD8vhcCg8PNytPDw8XHl5eR6P2b17t5YvXy6Hw6FVq1ZpypQpeu655/T444+76sTFxWnRokVavXq15s6dqz179ighIUG//fZbhbGkp6crNDTUtcXExHjnJGsA98MDAACgvvDWSn7ebgsA4B21arKl0tJStW7dWq+88oqsVqt69uypnJwcPfPMM5o2bZokqX///q763bp1U1xcnNq2bat//vOfuv322z22m5aWptTUVNfzwsJCv01McT88AAAA6hNvrORXE20BAM6eaUmpsLAwWa1W5efnu5Xn5+dXOB9UZGSkGjZsKOtJ/2t06tRJeXl5KikpUWBgYLljmjVrpgsuuEA7d+6sMBabzSabzVbNM/Et7ocHAABAfVO2kp+/tQUAODum3b4XGBionj17KjMz01VWWlqqzMxMxcfHezymb9++2rlzp0pLS11lP/30kyIjIz0mpCTp6NGj2rVrlyLryNAh7ocHAAAAAAB1gWlJKUlKTU3V/PnztXjxYm3fvl1jx45VUVGRazW+kSNHKi0tzVV/7Nix+vXXXzVhwgT99NNPWrlypWbMmKFx48a56kyaNEmffPKJ9u7dqy+++EJDhgyR1WrV8OHDfX5+NYX74QEAAAAAQG1n6pxSw4YN06FDhzR16lTl5eWpR48eWr16tWvy86ysLAUE/Jk3i4mJ0Ycffqh7771X3bp1U1RUlCZMmKAHHnjAVWf//v0aPny4fvnlF7Vq1UqXXnqpvvzyS7Vq1crn51eTuB8eAAAAAADUZhbD8DQzUf1WWFio0NBQFRQUKCQkxOxwAACAH6Gf4MR1ACSHw3tfEvtrWwBQHZXtJ9Sq1fcAAAAAwB/Y7dKECdL+/X+WRUc753+t6nQa/toWANQ0U+eUAgAAAIDaxm6Xhg51T/xIzhWyhw517q/tbQGAL5CUAgAAAIBKcjicI5E8TYJSVjZxorNebW0LAHyFpBQAAAAAVNL69eVHIp3MMKTsbGe92toWAPgKSSkAAAAAqKTcXO/V89e2AMBXSEoBAAAAQCVFRnqvnr+2BQC+QlIKAAAAACopIcG5mp3F4nm/xSLFxDjr1da2AMBXSEoBAAAAQCVZrdKsWc6fT00AlT3PyHDWq61tAYCvkJQCAAAAgCpITpaWL5eiotzLo6Od5cnJtb8tAPAFi2F4WjS0fissLFRoaKgKCgoUEhJidjgAAMCP0E9w4joAksPhXM0uN9c5V1NCQvVHIvlrW97kr3EB8L7K9hMa+DAmAAAAAKgzrFapX7+63Za32O3ShAnS/v1/lkVHO285ZAQXUH9x+x4AAAAAoMbY7dLQoe4JKUnKyXGW2+3mxAXAfCSlAAAAAAA1wuFwjpDyNGlMWdnEic56AOofklIAAAAAgBqxfn35EVInMwwpO9tZD0D9w5xSAAAAAACPznZy8txc79YDULeQlAIAAAAAlOONyckjI71bD0Ddwu17AAAAAAA33pqcPCHBmciyWDzvt1ikmBhnPQD1D0kpAAAAAICLNycnt1qdI6uk8ompsucZGVW7JRBA3UFSCgAAAADg4u3JyZOTpeXLpago9/LoaGd5ZW8FBFD3MKcUAAAAAMClJiYnT06WBg06u0nTT3a2E7AD8A8kpQAAAAAALjU1ObnVKvXrV+VwyvHGBOwA/AO37wEAAAAAXPx5cnJvTcAOwD+QlAIAAAAAuPjr5OTenIAdgH8gKQUAAAAAcOOPk5N7ewJ2AOYjKQUAAFAHzJkzR7GxsQoKClJcXJw2bdpUYd3vv/9e119/vWJjY2WxWJSRkeG7QAHUGsnJ0t690tq10pIlzsc9e8ybt6kmJmAHYC6SUgAAALXcsmXLlJqaqmnTpmnLli3q3r27kpKSdPDgQY/1jx07pvbt2+vJJ59URESEj6MFUJuUTU4+fLjz0cwV7mpqAnYA5iEpBQAAUMvNnDlTo0ePVkpKijp37qx58+apUaNGWrhwocf6vXv31jPPPKObbrpJNpvNx9ECQPX48wTsAKqHpBQAAEAtVlJSos2bNysxMdFVFhAQoMTERG3YsMHEyADAu/x1AnYA1UdSCgAAoBY7fPiwHA6HwsPD3crDw8OVl5fntdcpLi5WYWGh2wYAvuaPE7ADqL4GZgcAAAAA/5eenq5HH33U7DAAQMnJ0qBBzlX2cnOdc0glJFR/hJTD4b22AFQNSSkAAIBaLCwsTFarVfn5+W7l+fn5Xp3EPC0tTampqa7nhYWFiomJ8Vr7AFAVZROwny27XZowQdq//8+y6GjnbYKMugJqHrfvAQAA1GKBgYHq2bOnMjMzXWWlpaXKzMxUfHy8117HZrMpJCTEbQOA2sxul4YOdU9ISVJOjrPcbq9aew6HtG6dtHSp89Hh8FakQN3FSCkAAIBaLjU1VaNGjVKvXr3Up08fZWRkqKioSCkpKZKkkSNHKioqSunp6ZKck6P/8MMPrp9zcnK0detWNWnSROedd55p5wEAvuJwOEdIGUb5fYbhnDh94kTnbYKVuZXP2yOuuKUQ9QVJKQAAgFpu2LBhOnTokKZOnaq8vDz16NFDq1evdk1+npWVpYCAPwfIHzhwQBdddJHr+bPPPqtnn31Wl19+udatW+fr8AHA59avLz9C6mSGIWVnO+ud6TbBshFXpya4ykZcVXUCdhJcqE8shuEpN1y/FRYWKjQ0VAUFBQxNBwAAbugnOHEdANRmS5dKI0acud6SJdLw4RXvdzik2NiKE1wWizOhtGdP5UdceUpwWSzOR7MTXEBlVbafwJxSAAAAAIB6JTLSO/WqMuLqTM50S6HkvKWwsnNVeXvOrLIYmTcL3kRSCgAAAABQryQkOEcMlY1AOpXFIsXEOOudTm5u5V6vMvX8OcElOZNYsbHSFVc4R5ldcYXzeXWSW2UxkuACSSkAAAAAQL1itTpvYZPKJ6bKnmdknPmWO2+NuJL8N8EleX/UlT8nuEiW+RZJKQAAAABAvZOc7JyjKSrKvTw6uvJzN3lrxJXkvwkuf7+t0JsJLn9OltVVJKV8jDclAAAAAPiH5GRp715p7VrnpOZr1zonJa/sJODeGnEl+W+Cy59vK/Rmgsufk2VS3R0NZnpSas6cOYqNjVVQUJDi4uK0adOm09Y/cuSIxo0bp8jISNlsNl1wwQVatWrVWbXpK95+UwIAAAAAzo7VKvXr51xlr1+/yiWQTuaNEVdlcfhjgstfbyv0ZoLLn5NlZe3562iws2VqUmrZsmVKTU3VtGnTtGXLFnXv3l1JSUk6ePCgx/olJSW66qqrtHfvXi1fvlw7duzQ/PnzFXXSb39V2/SVmlj5AAAAAABgvrMdcXVyO/6W4PLX2wq9meDy12SZ5N+jwbzBYhieLpVvxMXFqXfv3po9e7YkqbS0VDExMbr77rv14IMPlqs/b948PfPMM/rxxx/VsGFDr7TpSWFhoUJDQ1VQUKCQkJBqnt2fHA5n5rGiN7nF4vwjs2dP1bPyAADAt7zdT6ituA4AUHMcDmcCJDfXmexJSKjeZ0W73ZkgOfmzaEyMMyFV2QRX2efZnBzPiZaqfJ5dt845MudM1q51jlo7naVLnSN9zmTJEucoOF+15c1z9GYuwdd5icr2E0wbKVVSUqLNmzcrMTHxz2ACApSYmKgNGzZ4POa9995TfHy8xo0bp/DwcF144YWaMWOGHP+XYqxOm5JUXFyswsJCt82bvL3yAQAAAACg7jrbWwrLeGMEl7/eVujNEVyMBjMvL2FaUurw4cNyOBwKDw93Kw8PD1deXp7HY3bv3q3ly5fL4XBo1apVmjJlip577jk9/vjj1W5TktLT0xUaGuraYmJizvLs3HnzTQkAAAAAQGV5I8Hlj7cVejPB5a/JMm/mEvw1L2H6ROdVUVpaqtatW+uVV15Rz549NWzYMD388MOaN2/eWbWblpamgoIC15adne2liJ28+aYEAAAAAMDX6vK8Wf6aLPPX0WDeZFpSKiwsTFarVfn5+W7l+fn5ioiI8HhMZGSkLrjgAllPeid06tRJeXl5KikpqVabkmSz2RQSEuK2eZM335QAAAAAAJjBn24rLGvHGwkub7blrwkuf81LmJaUCgwMVM+ePZWZmekqKy0tVWZmpuLj4z0e07dvX+3cuVOlpaWusp9++kmRkZEKDAysVpu+4M03JQAAAAAAtZ2/Jbi82ZY/Jrj8NS9h6u17qampmj9/vhYvXqzt27dr7NixKioqUkpKiiRp5MiRSktLc9UfO3asfv31V02YMEE//fSTVq5cqRkzZmjcuHGVbtMs3szgAgAAAAAAJ28luLzZlr8luLzdlrc08P1L/mnYsGE6dOiQpk6dqry8PPXo0UOrV692TVSelZWlgIA/82YxMTH68MMPde+996pbt26KiorShAkT9MADD1S6TTMlJ0uDBnlnaU8AAAAAAOC/yhJcZ8ubuQR/y0tYDMMwzHlp/1VYWKjQ0FAVFBR4fX4pAABQu9FPcOI6AACAilS2n1CrVt8DAAAAAABA3UBSCgAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD5HUgoAAAAAAAA+R1IKAAAAAAAAPkdSCgAAAAAAAD7XwOwA/JFhGJKkwsJCkyMBAAD+pqx/UNZfqK/oLwEAgIpUtr9EUsqD3377TZIUExNjciQAAMBf/fbbbwoNDTU7DNPQXwIAAGdypv6SxajvX/N5UFpaqgMHDqhp06ayWCySnFm+mJgYZWdnKyQkxOQI6xeuvXm49ubgupuHa2+e2nTtDcPQb7/9pjZt2iggoP7OhEB/yb9w7c3DtTcP194cXHfz1KZrX9n+EiOlPAgICFB0dLTHfSEhIX7/j19Xce3Nw7U3B9fdPFx789SWa1+fR0iVob/kn7j25uHam4drbw6uu3lqy7WvTH+p/n69BwAAAAAAANOQlAIAAAAAAIDPkZSqJJvNpmnTpslms5kdSr3DtTcP194cXHfzcO3Nw7WvG/h3NA/X3jxce/Nw7c3BdTdPXbz2THQOAAAAAAAAn2OkFAAAAAAAAHyOpBQAAAAAAAB8jqQUAAAAAAAAfI6kVCXMmTNHsbGxCgoKUlxcnDZt2mR2SHXeI488IovF4rZ17NjR7LDqpE8//VQDBw5UmzZtZLFYtGLFCrf9hmFo6tSpioyMVHBwsBITE/Xzzz+bE2wdc6Zrf9ttt5X7PbjmmmvMCbaOSU9PV+/evdW0aVO1bt1agwcP1o4dO9zqHD9+XOPGjVPLli3VpEkTXX/99crPzzcp4rqhMte9X79+5d73f//7302KGFVBf8n36C/5Dv0l89BfMg/9JfPUpz4TSakzWLZsmVJTUzVt2jRt2bJF3bt3V1JSkg4ePGh2aHVely5dlJub69o+++wzs0Oqk4qKitS9e3fNmTPH4/6nn35aL7zwgubNm6eNGzeqcePGSkpK0vHjx30cad1zpmsvSddcc43b78HSpUt9GGHd9cknn2jcuHH68ssvtWbNGp04cUJXX321ioqKXHXuvfdevf/++3r77bf1ySef6MCBA0pOTjYx6tqvMtddkkaPHu32vn/66adNihiVRX/JPPSXfIP+knnoL5mH/pJ56lWfycBp9enTxxg3bpzrucPhMNq0aWOkp6ebGFXdN23aNKN79+5mh1HvSDLeffdd1/PS0lIjIiLCeOaZZ1xlR44cMWw2m7F06VITIqy7Tr32hmEYo0aNMgYNGmRKPPXNwYMHDUnGJ598YhiG833esGFD4+2333bV2b59uyHJ2LBhg1lh1jmnXnfDMIzLL7/cmDBhgnlBoVroL5mD/pI56C+Zh/6Suegvmacu95kYKXUaJSUl2rx5sxITE11lAQEBSkxM1IYNG0yMrH74+eef1aZNG7Vv314333yzsrKyzA6p3tmzZ4/y8vLcfgdCQ0MVFxfH74CPrFu3Tq1bt1aHDh00duxY/fLLL2aHVCcVFBRIklq0aCFJ2rx5s06cOOH23u/YsaPOOecc3vtedOp1L/Pmm28qLCxMF154odLS0nTs2DEzwkMl0V8yF/0l89FfMh/9Jd+gv2SeutxnamB2AP7s8OHDcjgcCg8PdysPDw/Xjz/+aFJU9UNcXJwWLVqkDh06KDc3V48++qgSEhK0bds2NW3a1Ozw6o28vDxJ8vg7ULYPNeeaa65RcnKy2rVrp127dumhhx5S//79tWHDBlmtVrPDqzNKS0s1ceJE9e3bVxdeeKEk53s/MDBQzZo1c6vLe997PF13SRoxYoTatm2rNm3a6Ntvv9UDDzygHTt2yG63mxgtTof+knnoL/kH+kvmor/kG/SXzFPX+0wkpeCX+vfv7/q5W7duiouLU9u2bfXPf/5Tt99+u4mRAb5z0003uX7u2rWrunXrpnPPPVfr1q3TlVdeaWJkdcu4ceO0bds25mHxsYqu+5gxY1w/d+3aVZGRkbryyiu1a9cunXvuub4OE/Br9JcA+ku+Qn/JPHW9z8Tte6cRFhYmq9VabvWA/Px8RUREmBRV/dSsWTNdcMEF2rlzp9mh1Ctl73N+B/xD+/btFRYWxu+BF40fP17//ve/tXbtWkVHR7vKIyIiVFJSoiNHjrjV573vHRVdd0/i4uIkife9H6O/5D/oL5mD/pJ/ob/kffSXzFMf+kwkpU4jMDBQPXv2VGZmpqustLRUmZmZio+PNzGy+ufo0aPatWuXIiMjzQ6lXmnXrp0iIiLcfgcKCwu1ceNGfgdMsH//fv3yyy/8HniBYRgaP3683n33Xf3nP/9Ru3bt3Pb37NlTDRs2dHvv79ixQ1lZWbz3z8KZrrsnW7dulSTe936M/pL/oL9kDvpL/oX+kvfQXzJPfeozcfveGaSmpmrUqFHq1auX+vTpo4yMDBUVFSklJcXs0Oq0SZMmaeDAgWrbtq0OHDigadOmyWq1avjw4WaHVuccPXrULZu+Z88ebd26VS1atNA555yjiRMn6vHHH9f555+vdu3aacqUKWrTpo0GDx5sXtB1xOmufYsWLfToo4/q+uuvV0REhHbt2qX7779f5513npKSkkyMum4YN26clixZon/9619q2rSpa96D0NBQBQcHKzQ0VLfffrtSU1PVokULhYSE6O6771Z8fLz+8pe/mBx97XWm675r1y4tWbJEAwYMUMuWLfXtt9/q3nvv1WWXXaZu3bqZHD1Oh/6SOegv+Q79JfPQXzIP/SXz1Ks+k7mL/9UOL774onHOOecYgYGBRp8+fYwvv/zS7JDqvGHDhhmRkZFGYGCgERUVZQwbNszYuXOn2WHVSWvXrjUkldtGjRplGIZzmeMpU6YY4eHhhs1mM6688kpjx44d5gZdR5zu2h87dsy4+uqrjVatWhkNGzY02rZta4wePdrIy8szO+w6wdN1l2S8+uqrrjq///67cddddxnNmzc3GjVqZAwZMsTIzc01L+g64EzXPSsry7jsssuMFi1aGDabzTjvvPOMf/zjH0ZBQYG5gaNS6C/5Hv0l36G/ZB76S+ahv2Se+tRnshiGYdRMugsAAAAAAADwjDmlAAAAAAAA4HMkpQAAAAAAAOBzJKUAAAAAAADgcySlAAAAAAAA4HMkpQAAAAAAAOBzJKUAAAAAAADgcySlAAAAAAAA4HMkpQAAAAAAAOBzJKUAwIssFotWrFhhdhgAAAB+i/4SgDIkpQDUGbfddpssFku57ZprrjE7NAAAAL9AfwmAP2lgdgAA4E3XXHONXn31Vbcym81mUjQAAAD+h/4SAH/BSCkAdYrNZlNERITb1rx5c0nOoeJz585V//79FRwcrPbt22v58uVux3/33Xf661//quDgYLVs2VJjxozR0aNH3eosXLhQXbp0kc1mU2RkpMaPH++2//DhwxoyZIgaNWqk888/X++9955r33//+1/dfPPNatWqlYKDg3X++eeX6xQCAADUJPpLAPwFSSkA9cqUKVN0/fXX65tvvtHNN9+sm266Sdu3b5ckFRUVKSkpSc2bN9dXX32lt99+Wx9//LFbJ2ru3LkaN26cxowZo++++07vvfeezjvvPLfXePTRR3XjjTfq22+/1YABA3TzzTfr119/db3+Dz/8oA8++EDbt2/X3LlzFRYW5rsLAAAAcAb0lwD4jAEAdcSoUaMMq9VqNG7c2G174oknDMMwDEnG3//+d7dj4uLijLFjxxqGYRivvPKK0bx5c+Po0aOu/StXrjQCAgKMvLw8wzAMo02bNsbDDz9cYQySjMmTJ7ueHz161JBkfPDBB4ZhGMbAgQONlJQU75wwAABAFdFfAuBPmFMKQJ1yxRVXaO7cuW5lLVq0cP0cHx/vti8+Pl5bt26VJG3fvl3du3dX48aNXfv79u2r0tJS7dixQxaLRQcOHNCVV1552hi6devm+rlx48YKCQnRwYMHJUljx47V9ddfry1btujqq6/W4MGDdckll1TrXAEAAKqD/hIAf0FSCkCd0rhx43LDw70lODi4UvUaNmzo9txisai0tFSS1L9/f+3bt0+rVq3SmjVrdOWVV2rcuHF69tlnvR4vAACAJ/SXAPgL5pQCUK98+eWX5Z536tRJktSpUyd98803Kioqcu3//PPPFRAQoA4dOqhp06aKjY1VZmbmWcXQqlUrjRo1Sm+88YYyMjL0yiuvnFV7AAAA3kR/CYCvMFIKQJ1SXFysvLw8t7IGDRq4Jsd8++231atXL1166aV68803tWnTJi1YsECSdPPNN2vatGkaNWqUHnnkER06dEh33323br31VoWHh0uSHnnkEf39739X69at1b9/f/3222/6/PPPdffdd1cqvqlTp6pnz57q0qWLiouL9e9//9vVyQMAAPAF+ksA/AVJKQB1yurVqxUZGelW1qFDB/3444+SnCu9vPXWW7rrrrsUGRmppUuXqnPnzpKkRo0a6cMPP9SECRPUu3dvNWrUSNdff71mzpzpamvUqFE6fvy4nn/+eU2aNElhYWEaOnRopeMLDAxUWlqa9u7dq+DgYCUkJOitt97ywpkDAABUDv0lAP7CYhiGYXYQAOALFotF7777rgYPHmx2KAAAAH6J/hIAX2JOKQAAAAAAAPgcSSkAAAAAAAD4HLfvAQAAAAAAwOcYKQUAAAAAAACfIykFAAAAAAAAnyMpBQAAAAAAAJ8jKQUAAAAAAACfIykFAAAAAAAAnyMpBQAAAAAAAJ8jKQUAAAAAAACfIykFAAAAAAAAnyMpBQAAAAAAAJ/7/wq794z1ii/7AAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 1200x500 with 2 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<Figure size 640x480 with 0 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "# Assuming these are your accuracy and loss values\n",
+    "accuracy = [0.6, 0.65, 0.7, 0.72, 0.74, 0.76, 0.77, 0.78, 0.81, 0.83, 0.83, 0.87,0.88, 0.91, 0.915, 0.924, 0.934, 0.954, 0.957, 0.959, 0.96, 0.959, 0.958, 0.956, 0.957, 0.958]\n",
+    "loss = [0.8, 0.5, 0.45, 0.30, 0.28, 0.22, 0.19, 0.18, 0.18, 0.15, 0.15, 0.15,  0.12, 0.13, 0.11, 0.09, 0.086, 0.083, 0.082, 0.077, 0.076, 0.074, 0.073, 0.072, 0.070, 0.069]\n",
+    "\n",
+    "epochs = range(1, len(accuracy) + 1)\n",
+    "\n",
+    "plt.figure(figsize=(12, 5))\n",
+    "\n",
+    "# Plotting accuracy\n",
+    "plt.subplot(1, 2, 1)\n",
+    "plt.plot(epochs, accuracy, 'bo', label='Training acc')\n",
+    "plt.title('Training accuracy')\n",
+    "plt.xlabel('Epochs')\n",
+    "plt.ylabel('Accuracy')\n",
+    "plt.legend()\n",
+    "\n",
+    "# Plotting loss\n",
+    "plt.subplot(1, 2, 2)\n",
+    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
+    "plt.title('Training loss')\n",
+    "plt.xlabel('Epochs')\n",
+    "plt.ylabel('Loss')\n",
+    "plt.legend()\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
+    "\n",
+    "plt.savefig(\"accuracy loss.pdf\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textattackenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

flow_correction_ag_news.py

@@ -0,0 +1,388 @@
+import textattack
+import transformers
+import pandas as pd
+import csv
+import string
+import pickle
+# Construct our four components for `Attack`
+from textattack.constraints.pre_transformation import (
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics import WordEmbeddingDistance
+from textattack.transformations import WordSwapEmbedding
+from textattack.search_methods import GreedyWordSwapWIR
+
+import numpy as np
+import json
+import random
+import re
+import textattack.shared.attacked_text as atk
+import torch.nn.functional as F
+import torch
+
+
+class InvertedText:
+
+    def __init__(
+        self,
+        swapped_indexes,
+        score,
+        attacked_text,
+        new_class,
+    ):
+        self.attacked_text = attacked_text
+        self.swapped_indexes = (
+            swapped_indexes  # dict of swapped indexes with their synonym
+        )
+        self.score = score  # value of original class
+        self.new_class = new_class  # class after inversion
+
+    def __repr__(self):
+        return f"InvertedText:\n attacked_text='{self.attacked_text}', \n swapped_indexes={self.swapped_indexes},\n score={self.score}"
+
+
+def count_matching_classes(original, corrected, perturbed_texts=None):
+    if len(original) != len(corrected):
+        raise ValueError("Arrays must have the same length")
+    hard_samples = []
+    easy_samples = []
+
+    matching_count = 0
+
+    for i in range(len(corrected)):
+        if original[i] == corrected[i]:
+            matching_count += 1
+            easy_samples.append(perturbed_texts[i])
+        elif perturbed_texts != None:
+            hard_samples.append(perturbed_texts[i])
+
+    return matching_count, hard_samples, easy_samples
+
+
+class Flow_Corrector:
+    def __init__(
+        self,
+        attack,
+        word_rank_file="en_full_ranked.json",
+        word_freq_file="en_full_freq.json",
+        wir_threshold=0.3,
+    ):
+        self.attack = attack
+        self.attack.cuda_()
+        self.wir_threshold = wir_threshold
+        with open(word_rank_file, "r") as f:
+            self.word_ranked_frequence = json.load(f)
+        with open(word_freq_file, "r") as f:
+            self.word_frequence = json.load(f)
+        self.victim_model = attack.goal_function.model
+
+    def wir_gradient(
+        self,
+        attack,
+        victim_model,
+        detected_text,
+    ):
+        _, indices_to_order = attack.get_indices_to_order(detected_text)
+
+        index_scores = np.zeros(len(indices_to_order))
+        grad_output = victim_model.get_grad(detected_text.tokenizer_input)
+        gradient = grad_output["gradient"]
+        word2token_mapping = detected_text.align_with_model_tokens(victim_model)
+        for i, index in enumerate(indices_to_order):
+            matched_tokens = word2token_mapping[index]
+            if not matched_tokens:
+                index_scores[i] = 0.0
+            else:
+                agg_grad = np.mean(gradient[matched_tokens], axis=0)
+                index_scores[i] = np.linalg.norm(agg_grad, ord=1)
+        index_order = np.array(indices_to_order)[(-index_scores).argsort()]
+        return index_order
+
+    def get_syn_freq_dict(
+        self,
+        index_order,
+        detected_text,
+    ):
+        most_frequent_syn_dict = {}
+
+        no_syn = []
+        freq_thershold = len(self.word_ranked_frequence) / 10
+
+        for idx in index_order:
+            # get the synonyms of a specific index
+
+            try:
+                synonyms = [
+                    attacked_text.words[idx]
+                    for attacked_text in self.attack.get_transformations(
+                        detected_text, detected_text, indices_to_modify=[idx]
+                    )
+                ]
+                # getting synonyms that exists in dataset with thiere frequency rank
+                ranked_synonyms = {
+                    syn: self.word_ranked_frequence[syn]
+                    for syn in synonyms
+                    if syn in self.word_ranked_frequence.keys()
+                    and self.word_ranked_frequence[syn] < freq_thershold
+                    and self.word_ranked_frequence[detected_text.words[idx]]
+                    > self.word_ranked_frequence[syn]
+                }
+                # selecting the M most frequent synonym
+
+                if list(ranked_synonyms.keys()) != []:
+                    most_frequent_syn_dict[idx] = list(ranked_synonyms.keys())
+            except:
+                # no synonyms avaialble in the dataset
+                no_syn.append(idx)
+
+        return most_frequent_syn_dict
+
+    def build_candidates(
+        self, detected_text, most_frequent_syn_dict: dict, max_attempt: int
+    ):
+        candidates = {}
+        for _ in range(max_attempt):
+            syn_dict = {}
+            current_text = detected_text
+            for index in most_frequent_syn_dict.keys():
+                syn = random.choice(most_frequent_syn_dict[index])
+                syn_dict[index] = syn
+                current_text = current_text.replace_word_at_index(index, syn)
+
+            candidates[current_text] = syn_dict
+        return candidates
+
+    def find_dominant_class(self, inverted_texts):
+        class_counts = {}  # Dictionary to store the count of each new class
+
+        for text in inverted_texts:
+            new_class = text.new_class
+            class_counts[new_class] = class_counts.get(new_class, 0) + 1
+
+        # Find the most dominant class
+        most_dominant_class = max(class_counts, key=class_counts.get)
+
+        return most_dominant_class
+
+    def correct(self, detected_texts):
+        corrected_classes = []
+        for detected_text in detected_texts:
+
+            # convert to Attacked texts
+            detected_text = atk.AttackedText(detected_text)
+
+            # getting 30% most important indexes
+            index_order = self.wir_gradient(
+                self.attack, self.victim_model, detected_text
+            )
+            index_order = index_order[: int(len(index_order) * self.wir_threshold)]
+
+            # getting synonyms according to frequency conditiontions
+            most_frequent_syn_dict = self.get_syn_freq_dict(index_order, detected_text)
+
+            # generate M candidates
+            candidates = self.build_candidates(
+                detected_text, most_frequent_syn_dict, max_attempt=100
+            )
+
+            original_probs = F.softmax(self.victim_model(detected_text.text), dim=1)
+            original_class = torch.argmax(original_probs).item()
+            original_golden_prob = float(original_probs[0][original_class])
+
+            nbr_inverted = 0
+            inverted_texts = []  # a dictionary of inverted texts with
+            bad, impr = 0, 0
+            dict_deltas = {}
+
+            batch_inputs = [candidate.text for candidate in candidates.keys()]
+
+            batch_outputs = self.victim_model(batch_inputs)
+
+            probabilities = F.softmax(batch_outputs, dim=1)
+            for i, (candidate, syn_dict) in enumerate(candidates.items()):
+
+                corrected_class = torch.argmax(probabilities[i]).item()
+                new_golden_probability = float(probabilities[i][corrected_class])
+                if corrected_class != original_class:
+                    nbr_inverted += 1
+                    inverted_texts.append(
+                        InvertedText(
+                            syn_dict, new_golden_probability, candidate, corrected_class
+                        )
+                    )
+                else:
+                    delta = new_golden_probability - original_golden_prob
+                    if delta <= 0:
+                        bad += 1
+                    else:
+                        impr += 1
+                        dict_deltas[candidate] = delta
+
+            if len(original_probs[0]) > 2 and len(inverted_texts) >= len(candidates) / (
+                len(original_probs[0])
+            ):
+                # selecting the most dominant class
+                dominant_class = self.find_dominant_class(inverted_texts)
+            elif len(inverted_texts) >= len(candidates) / 2:
+                dominant_class = corrected_class
+            else:
+                dominant_class = original_class
+
+            corrected_classes.append(dominant_class)
+
+        return corrected_classes
+
+
+def remove_brackets(text):
+    text = text.replace("[[", "")
+    text = text.replace("]]", "")
+    return text
+
+
+def clean_text(text):
+    pattern = "[" + re.escape(string.punctuation) + "]"
+    cleaned_text = re.sub(pattern, " ", text)
+
+    return cleaned_text
+
+
+# Load model, tokenizer, and model_wrapper
+model = transformers.AutoModelForSequenceClassification.from_pretrained(
+    "textattack/bert-base-uncased-ag-news"
+)
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    "textattack/bert-base-uncased-ag-news"
+)
+model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)
+
+
+goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)
+constraints = [
+    RepeatModification(),
+    StopwordModification(),
+    WordEmbeddingDistance(min_cos_sim=0.9),
+]
+transformation = WordSwapEmbedding(max_candidates=50)
+search_method = GreedyWordSwapWIR(wir_method="gradient")
+
+# Construct the actual attack
+attack = textattack.Attack(goal_function, constraints, transformation, search_method)
+attack.cuda_()
+
+
+results = pd.read_csv("ag_news_results.csv")
+perturbed_texts = [
+    results["perturbed_text"][i]
+    for i in range(len(results))
+    if results["result_type"][i] == "Successful"
+]
+original_texts = [
+    results["original_text"][i]
+    for i in range(len(results))
+    if results["result_type"][i] == "Successful"
+]
+
+perturbed_texts = [remove_brackets(text) for text in perturbed_texts]
+original_texts = [remove_brackets(text) for text in original_texts]
+
+perturbed_texts = [clean_text(text) for text in perturbed_texts]
+original_texts = [clean_text(text) for text in original_texts]
+
+
+victim_model = attack.goal_function.model
+
+print("Getting corrected classes")
+print("This may take a while ...")
+# we can use directly resultds in csv file
+original_classes = [
+    torch.argmax(F.softmax(victim_model(original_text), dim=1)).item()
+    for original_text in original_texts
+]
+
+batch_size = 1000
+num_batches = (len(perturbed_texts) + batch_size - 1) // batch_size
+batched_perturbed_texts = []
+batched_original_texts = []
+batched_original_classes = []
+
+for i in range(num_batches):
+    start = i * batch_size
+    end = min(start + batch_size, len(perturbed_texts))
+    batched_perturbed_texts.append(perturbed_texts[start:end])
+    batched_original_texts.append(original_texts[start:end])
+    batched_original_classes.append(original_classes[start:end])
+print(batched_original_classes)
+hard_samples_list = []
+easy_samples_list = []
+
+
+# Open a CSV file for writing
+csv_filename = "flow_correction_results_ag_news.csv"
+with open(csv_filename, "w", newline="") as csvfile:
+    fieldnames = ["freq_threshold", "batch_num", "match_perturbed", "match_original"]
+    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+    # Write the header row
+    writer.writeheader()
+
+    # Iterate over batched lists
+    batch_num = 0
+    for perturbed, original, classes in zip(
+        batched_perturbed_texts, batched_original_texts, batched_original_classes
+    ):
+        batch_num += 1
+        print(f"Processing batch number: {batch_num}")
+
+        for i in range(2):
+            wir_threshold = 0.1 * (i + 1)
+            print(f"Setting Word threshold to: {wir_threshold}")
+
+            corrector = Flow_Corrector(
+                attack,
+                word_rank_file="en_full_ranked.json",
+                word_freq_file="en_full_freq.json",
+                wir_threshold=wir_threshold,
+            )
+
+            # Correct perturbed texts
+            print("Correcting perturbed texts...")
+            corrected_perturbed_classes = corrector.correct(perturbed)
+
+            match_perturbed, hard_samples, easy_samples = count_matching_classes(
+                classes, corrected_perturbed_classes, perturbed
+            )
+            hard_samples_list.extend(hard_samples)
+            easy_samples_list.extend(easy_samples)
+
+
+            print(f"Number of matching classes (perturbed): {match_perturbed}")
+
+            # Correct original texts
+            print("Correcting original texts...")
+            corrected_original_classes = corrector.correct(original)
+            match_original, hard_samples, easy_samples = count_matching_classes(
+                classes, corrected_original_classes, perturbed
+            )
+            print(f"Number of matching classes (original): {match_original}")
+
+            # Write results to CSV file
+            print("Writing results to CSV file...")
+            writer.writerow(
+                {
+                    "freq_threshold": wir_threshold,
+                    "batch_num": batch_num,
+                    "match_perturbed": match_perturbed/len(perturbed),
+                    "match_original": match_original/len(perturbed),
+                }
+            )
+            print("-" * 20)
+
+print("savig samples for more statistics studies")
+
+# Save hard_samples_list and easy_samples_list to files
+with open('hard_samples.pkl', 'wb') as f:
+    pickle.dump(hard_samples_list, f)
+
+with open('easy_samples.pkl', 'wb') as f:
+    pickle.dump(easy_samples_list, f)

flow_correction_imdb.py

@@ -0,0 +1,388 @@
+import textattack
+import transformers
+import pandas as pd
+import csv
+import string
+import pickle
+# Construct our four components for `Attack`
+from textattack.constraints.pre_transformation import (
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics import WordEmbeddingDistance
+from textattack.transformations import WordSwapEmbedding
+from textattack.search_methods import GreedyWordSwapWIR
+
+import numpy as np
+import json
+import random
+import re
+import textattack.shared.attacked_text as atk
+import torch.nn.functional as F
+import torch
+
+
+class InvertedText:
+
+    def __init__(
+        self,
+        swapped_indexes,
+        score,
+        attacked_text,
+        new_class,
+    ):
+        self.attacked_text = attacked_text
+        self.swapped_indexes = (
+            swapped_indexes  # dict of swapped indexes with their synonym
+        )
+        self.score = score  # value of original class
+        self.new_class = new_class  # class after inversion
+
+    def __repr__(self):
+        return f"InvertedText:\n attacked_text='{self.attacked_text}', \n swapped_indexes={self.swapped_indexes},\n score={self.score}"
+
+
+def count_matching_classes(original, corrected, perturbed_texts=None):
+    if len(original) != len(corrected):
+        raise ValueError("Arrays must have the same length")
+    hard_samples = []
+    easy_samples = []
+
+    matching_count = 0
+
+    for i in range(len(corrected)):
+        if original[i] == corrected[i]:
+            matching_count += 1
+            easy_samples.append(perturbed_texts[i])
+        elif perturbed_texts != None:
+            hard_samples.append(perturbed_texts[i])
+
+    return matching_count, hard_samples, easy_samples
+
+
+class Flow_Corrector:
+    def __init__(
+        self,
+        attack,
+        word_rank_file="en_full_ranked.json",
+        word_freq_file="en_full_freq.json",
+        wir_threshold=0.3,
+    ):
+        self.attack = attack
+        self.attack.cuda_()
+        self.wir_threshold = wir_threshold
+        with open(word_rank_file, "r") as f:
+            self.word_ranked_frequence = json.load(f)
+        with open(word_freq_file, "r") as f:
+            self.word_frequence = json.load(f)
+        self.victim_model = attack.goal_function.model
+
+    def wir_gradient(
+        self,
+        attack,
+        victim_model,
+        detected_text,
+    ):
+        _, indices_to_order = attack.get_indices_to_order(detected_text)
+
+        index_scores = np.zeros(len(indices_to_order))
+        grad_output = victim_model.get_grad(detected_text.tokenizer_input)
+        gradient = grad_output["gradient"]
+        word2token_mapping = detected_text.align_with_model_tokens(victim_model)
+        for i, index in enumerate(indices_to_order):
+            matched_tokens = word2token_mapping[index]
+            if not matched_tokens:
+                index_scores[i] = 0.0
+            else:
+                agg_grad = np.mean(gradient[matched_tokens], axis=0)
+                index_scores[i] = np.linalg.norm(agg_grad, ord=1)
+        index_order = np.array(indices_to_order)[(-index_scores).argsort()]
+        return index_order
+
+    def get_syn_freq_dict(
+        self,
+        index_order,
+        detected_text,
+    ):
+        most_frequent_syn_dict = {}
+
+        no_syn = []
+        freq_thershold = len(self.word_ranked_frequence) / 10
+
+        for idx in index_order:
+            # get the synonyms of a specific index
+
+            try:
+                synonyms = [
+                    attacked_text.words[idx]
+                    for attacked_text in self.attack.get_transformations(
+                        detected_text, detected_text, indices_to_modify=[idx]
+                    )
+                ]
+                # getting synonyms that exists in dataset with thiere frequency rank
+                ranked_synonyms = {
+                    syn: self.word_ranked_frequence[syn]
+                    for syn in synonyms
+                    if syn in self.word_ranked_frequence.keys()
+                    and self.word_ranked_frequence[syn] < freq_thershold
+                    and self.word_ranked_frequence[detected_text.words[idx]]
+                    > self.word_ranked_frequence[syn]
+                }
+                # selecting the M most frequent synonym
+
+                if list(ranked_synonyms.keys()) != []:
+                    most_frequent_syn_dict[idx] = list(ranked_synonyms.keys())
+            except:
+                # no synonyms avaialble in the dataset
+                no_syn.append(idx)
+
+        return most_frequent_syn_dict
+
+    def build_candidates(
+        self, detected_text, most_frequent_syn_dict: dict, max_attempt: int
+    ):
+        candidates = {}
+        for _ in range(max_attempt):
+            syn_dict = {}
+            current_text = detected_text
+            for index in most_frequent_syn_dict.keys():
+                syn = random.choice(most_frequent_syn_dict[index])
+                syn_dict[index] = syn
+                current_text = current_text.replace_word_at_index(index, syn)
+
+            candidates[current_text] = syn_dict
+        return candidates
+
+    def find_dominant_class(self, inverted_texts):
+        class_counts = {}  # Dictionary to store the count of each new class
+
+        for text in inverted_texts:
+            new_class = text.new_class
+            class_counts[new_class] = class_counts.get(new_class, 0) + 1
+
+        # Find the most dominant class
+        most_dominant_class = max(class_counts, key=class_counts.get)
+
+        return most_dominant_class
+
+    def correct(self, detected_texts):
+        corrected_classes = []
+        for detected_text in detected_texts:
+
+            # convert to Attacked texts
+            detected_text = atk.AttackedText(detected_text)
+
+            # getting 30% most important indexes
+            index_order = self.wir_gradient(
+                self.attack, self.victim_model, detected_text
+            )
+            index_order = index_order[: int(len(index_order) * self.wir_threshold)]
+
+            # getting synonyms according to frequency conditiontions
+            most_frequent_syn_dict = self.get_syn_freq_dict(index_order, detected_text)
+
+            # generate M candidates
+            candidates = self.build_candidates(
+                detected_text, most_frequent_syn_dict, max_attempt=100
+            )
+
+            original_probs = F.softmax(self.victim_model(detected_text.text), dim=1)
+            original_class = torch.argmax(original_probs).item()
+            original_golden_prob = float(original_probs[0][original_class])
+
+            nbr_inverted = 0
+            inverted_texts = []  # a dictionary of inverted texts with
+            bad, impr = 0, 0
+            dict_deltas = {}
+
+            batch_inputs = [candidate.text for candidate in candidates.keys()]
+
+            batch_outputs = self.victim_model(batch_inputs)
+
+            probabilities = F.softmax(batch_outputs, dim=1)
+            for i, (candidate, syn_dict) in enumerate(candidates.items()):
+
+                corrected_class = torch.argmax(probabilities[i]).item()
+                new_golden_probability = float(probabilities[i][corrected_class])
+                if corrected_class != original_class:
+                    nbr_inverted += 1
+                    inverted_texts.append(
+                        InvertedText(
+                            syn_dict, new_golden_probability, candidate, corrected_class
+                        )
+                    )
+                else:
+                    delta = new_golden_probability - original_golden_prob
+                    if delta <= 0:
+                        bad += 1
+                    else:
+                        impr += 1
+                        dict_deltas[candidate] = delta
+
+            if len(original_probs[0]) > 2 and len(inverted_texts) >= len(candidates) / (
+                len(original_probs[0])
+            ):
+                # selecting the most dominant class
+                dominant_class = self.find_dominant_class(inverted_texts)
+            elif len(inverted_texts) >= len(candidates) / 2:
+                dominant_class = corrected_class
+            else:
+                dominant_class = original_class
+
+            corrected_classes.append(dominant_class)
+
+        return corrected_classes
+
+
+def remove_brackets(text):
+    text = text.replace("[[", "")
+    text = text.replace("]]", "")
+    return text
+
+
+def clean_text(text):
+    pattern = "[" + re.escape(string.punctuation) + "]"
+    cleaned_text = re.sub(pattern, " ", text)
+
+    return cleaned_text
+
+
+# Load model, tokenizer, and model_wrapper
+model = transformers.AutoModelForSequenceClassification.from_pretrained(
+    "textattack/bert-base-uncased-imdb"
+)
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    "textattack/bert-base-uncased-imdb"
+)
+model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)
+
+
+goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)
+constraints = [
+    RepeatModification(),
+    StopwordModification(),
+    WordEmbeddingDistance(min_cos_sim=0.9),
+]
+transformation = WordSwapEmbedding(max_candidates=50)
+search_method = GreedyWordSwapWIR(wir_method="gradient")
+
+# Construct the actual attack
+attack = textattack.Attack(goal_function, constraints, transformation, search_method)
+attack.cuda_()
+
+
+results = pd.read_csv("IMDB_results.csv")
+perturbed_texts = [
+    results["perturbed_text"][i]
+    for i in range(len(results))
+    if results["result_type"][i] == "Successful"
+]
+original_texts = [
+    results["original_text"][i]
+    for i in range(len(results))
+    if results["result_type"][i] == "Successful"
+]
+
+perturbed_texts = [remove_brackets(text) for text in perturbed_texts]
+original_texts = [remove_brackets(text) for text in original_texts]
+
+perturbed_texts = [clean_text(text) for text in perturbed_texts]
+original_texts = [clean_text(text) for text in original_texts]
+
+
+victim_model = attack.goal_function.model
+
+print("Getting corrected classes")
+print("This may take a while ...")
+# we can use directly resultds in csv file
+original_classes = [
+    torch.argmax(F.softmax(victim_model(original_text), dim=1)).item()
+    for original_text in original_texts
+]
+
+batch_size = 1000
+num_batches = (len(perturbed_texts) + batch_size - 1) // batch_size
+batched_perturbed_texts = []
+batched_original_texts = []
+batched_original_classes = []
+
+for i in range(num_batches):
+    start = i * batch_size
+    end = min(start + batch_size, len(perturbed_texts))
+    batched_perturbed_texts.append(perturbed_texts[start:end])
+    batched_original_texts.append(original_texts[start:end])
+    batched_original_classes.append(original_classes[start:end])
+print(batched_original_classes)
+hard_samples_list = []
+easy_samples_list = []
+
+
+# Open a CSV file for writing
+csv_filename = "flow_correction_results_imdb.csv"
+with open(csv_filename, "w", newline="") as csvfile:
+    fieldnames = ["freq_threshold", "batch_num", "match_perturbed", "match_original"]
+    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+    # Write the header row
+    writer.writeheader()
+
+    # Iterate over batched lists
+    batch_num = 0
+    for perturbed, original, classes in zip(
+        batched_perturbed_texts, batched_original_texts, batched_original_classes
+    ):
+        batch_num += 1
+        print(f"Processing batch number: {batch_num}")
+
+        for i in range(2):
+            wir_threshold = 0.1 * (i + 1)
+            print(f"Setting Word threshold to: {wir_threshold}")
+
+            corrector = Flow_Corrector(
+                attack,
+                word_rank_file="en_full_ranked.json",
+                word_freq_file="en_full_freq.json",
+                wir_threshold=wir_threshold,
+            )
+
+            # Correct perturbed texts
+            print("Correcting perturbed texts...")
+            corrected_perturbed_classes = corrector.correct(perturbed)
+
+            match_perturbed, hard_samples, easy_samples = count_matching_classes(
+                classes, corrected_perturbed_classes, perturbed
+            )
+            hard_samples_list.extend(hard_samples)
+            easy_samples_list.extend(easy_samples)
+
+
+            print(f"Number of matching classes (perturbed): {match_perturbed}")
+
+            # Correct original texts
+            print("Correcting original texts...")
+            corrected_original_classes = corrector.correct(original)
+            match_original, hard_samples, easy_samples = count_matching_classes(
+                classes, corrected_original_classes, perturbed
+            )
+            print(f"Number of matching classes (original): {match_original}")
+
+            # Write results to CSV file
+            print("Writing results to CSV file...")
+            writer.writerow(
+                {
+                    "freq_threshold": wir_threshold,
+                    "batch_num": batch_num,
+                    "match_perturbed": match_perturbed/len(perturbed),
+                    "match_original": match_original/len(perturbed),
+                }
+            )
+            print("-" * 20)
+
+print("savig samples for more statistics studies")
+
+# Save hard_samples_list and easy_samples_list to files
+with open('hard_samples.pkl', 'wb') as f:
+    pickle.dump(hard_samples_list, f)
+
+with open('easy_samples.pkl', 'wb') as f:
+    pickle.dump(easy_samples_list, f)

gitignore

@@ -0,0 +1,143 @@
+# dev files
+*.cache
+*.dev.py
+state_dict/
+TAD*/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+tests/
+*.result.json
+.idea/
+
+# Embedding
+glove.840B.300d.txt
+glove.42B.300d.txt
+glove.twitter.27B.txt
+
+# project main files
+release_note.json
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer training_logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+.DS_Store
+examples/.DS_Store

main_correction.py

@@ -0,0 +1,89 @@
+import textattack
+import transformers
+from FlowCorrector import Flow_Corrector
+import torch
+import torch.nn.functional as F
+
+def count_matching_classes(original, corrected):
+    if len(original) != len(corrected):
+        raise ValueError("Arrays must have the same length")
+
+    matching_count = 0
+
+    for i in range(len(corrected)):
+        if original[i] == corrected[i]:
+            matching_count += 1
+
+    return matching_count
+
+if __name__ == "main" :
+
+    # Load model, tokenizer, and model_wrapper
+    model = transformers.AutoModelForSequenceClassification.from_pretrained(
+        "textattack/bert-base-uncased-ag-news"
+    )
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        "textattack/bert-base-uncased-ag-news"
+    )
+    model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)
+
+    # Construct our four components for `Attack`
+    from textattack.constraints.pre_transformation import (
+        RepeatModification,
+        StopwordModification,
+    )
+    from textattack.constraints.semantics import WordEmbeddingDistance
+    from textattack.transformations import WordSwapEmbedding
+    from textattack.search_methods import GreedyWordSwapWIR
+
+    goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)
+    constraints = [
+        RepeatModification(),
+        StopwordModification(),
+        WordEmbeddingDistance(min_cos_sim=0.9),
+    ]
+    transformation = WordSwapEmbedding(max_candidates=50)
+    search_method = GreedyWordSwapWIR(wir_method="weighted-saliency")
+
+    # Construct the actual attack
+    attack = textattack.Attack(goal_function, constraints, transformation, search_method)
+    attack.cuda_()
+
+    # intialisation de coreecteur
+    corrector = Flow_Corrector(
+        attack,
+        word_rank_file="en_full_ranked.json",
+        word_freq_file="en_full_freq.json",
+    )
+
+    # All these texts are adverserial ones
+
+    with open('perturbed_texts_ag_news.txt', 'r') as f:
+        detected_texts = [line.strip() for line in f]
+
+
+    #These are orginal texts in same order of adverserial ones 
+
+    with open("original_texts_ag_news.txt", "r") as f:
+        original_texts = [line.strip() for line in f]
+
+    victim_model = attack.goal_function.model
+
+    # getting original labels for benchmarking later
+    original_classes = [
+        torch.argmax(F.softmax(victim_model(original_text), dim=1)).item()
+        for original_text in original_texts
+    ]
+
+    """ 0 :World
+        1 : Sports
+        2 : Business
+        3 : Sci/Tech"""
+
+    corrected_classes = corrector.correct(original_texts)
+    print(f"match {count_matching_classes()}")
+
+
+
+
+

requirements.txt

@@ -0,0 +1,28 @@
+findfile>=1.7.9.8
+autocuda>=0.11
+metric-visualizer>=0.5.5
+boostaug>=2.2.3
+spacy
+networkx
+seqeval
+update-checker
+typing_extensions
+tqdm
+pytorch_warmup
+termcolor
+gitpython
+gdown>=4.4.0
+transformers>4.20.0
+torch>1.0.0
+sentencepiece
+tensorflow_text
+tensorflow_hub
+tensorflow>=2.6.0
+datasets
+textattack
+jieba
+pycld2
+OpenHowNet
+pinyin
+flask
+

textattack/__init__.py

@@ -0,0 +1,39 @@
+"""Welcome to the API references for TextAttack!
+
+What is TextAttack?
+
+`TextAttack <https://github.com/QData/TextAttack>`__ is a Python framework for adversarial attacks, adversarial training, and data augmentation in NLP.
+
+TextAttack makes experimenting with the robustness of NLP models seamless, fast, and easy. It's also useful for NLP model training, adversarial training, and data augmentation.
+
+TextAttack provides components for common NLP tasks like sentence encoding, grammar-checking, and word replacement that can be used on their own.
+"""
+from .attack_args import AttackArgs, CommandLineAttackArgs
+from .augment_args import AugmenterArgs
+from .dataset_args import DatasetArgs
+from .model_args import ModelArgs
+from .training_args import TrainingArgs, CommandLineTrainingArgs
+from .attack import Attack
+from .attacker import Attacker
+from .trainer import Trainer
+from .metrics import Metric
+
+from . import (
+    attack_recipes,
+    attack_results,
+    augmentation,
+    commands,
+    constraints,
+    datasets,
+    goal_function_results,
+    goal_functions,
+    loggers,
+    metrics,
+    models,
+    search_methods,
+    shared,
+    transformations,
+)
+
+
+name = "textattack"

textattack/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+if __name__ == "__main__":
+    import textattack
+
+    textattack.commands.textattack_cli.main()