init space

.gitattributes

@@ -32,3 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.hdf filter=lfs diff=lfs merge=lfs -text
+*.ttf filter=lfs diff=lfs merge=lfs -text

README.ipynb

@@ -0,0 +1,265 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Welcome!\n",
+    "to the repo for\n",
+    "\n",
+    "*Learning the Legibility of Visual Text Perturbations* (EACL 2023)\n",
+    "\n",
+    "by Dev Seth, Rickard Stureborg, Danish Pruthi and Bhuwan Dhingra"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### A `LEGIT` Introduction\n",
+    "This notebook provides a helpful starting point to interact with the datasets and models presented in the Learning Legibility paper.\n",
+    "\n",
+    "All assets are hosted on the HuggingFace Hub and can be used with the `transformers` and `datasets` libraries: \n",
+    "  - TrOCR-MT Model: https://huggingface.co/dvsth/LEGIT-TrOCR-MT \n",
+    "  - LEGIT Dataset: https://huggingface.co/datasets/dvsth/LEGIT\n",
+    "  - Perturbed Jigsaw Dataset: https://huggingface.co/datasets/dvsth/LEGIT-VIPER-Jigsaw-Toxic-Comment-Perturbed"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**For an interactive preview of the perturbation process and legibility assessment model, run `demo.py` using the command `python demo.py` (will open a browser-based interface). The demo allows you to perturb a word with your chosen attack parameters, then see the model's legibility estimate for the generated perturbations.**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# external imports -- use pip or conda to install these packages\n",
+    "import torch\n",
+    "from transformers import TrOCRProcessor, AutoModel, TrainingArguments\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# local imports\n",
+    "from classes.LegibilityModel import LegibilityModel\n",
+    "from classes.Trainer import MultiTaskTrainer\n",
+    "from classes.Metrics import binary_classification_metric, ranking_metric"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Loading the Model and Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load the model schema and pretrained weights\n",
+    "# (this may take some time to download)\n",
+    "model = AutoModel.from_pretrained(\"dvsth/LEGIT-TrOCR-MT\", revision='main', trust_remote_code=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Interactive dataset preview available [here](https://huggingface.co/datasets/dvsth/LEGIT/viewer/dvsth--LEGIT/test)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration dvsth--LEGIT-d84a4d72774d3652\n",
+      "Found cached dataset parquet (/Users/dvsth/.cache/huggingface/datasets/dvsth___parquet/dvsth--LEGIT-d84a4d72774d3652/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "22f8a468229a4760bd2829ef894a5472",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = load_dataset('dvsth/LEGIT').with_format('torch')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Training/Eval Loop"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Trainer setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Could not find image processor class in the image processor config or the model config. Loading based on pattern matching with the model's feature extractor configuration.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# preprocessor provides image normalization and resizing\n",
+    "preprocessor = TrOCRProcessor.from_pretrained(\n",
+    "    \"microsoft/trocr-base-handwritten\")\n",
+    "\n",
+    "# apply preprocessing batch-wise\n",
+    "def collate_fn(data):\n",
+    "    return {\n",
+    "        'choice': torch.tensor([d['choice'].item() for d in data]),\n",
+    "        'img0': preprocessor([d['img0'] for d in data], return_tensors='pt')['pixel_values'],\n",
+    "        'img1': preprocessor([d['img1'] for d in data], return_tensors='pt')['pixel_values']\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "train_args = TrainingArguments(\n",
+    "    output_dir=f'runs',             # change this to a unique path for each run, e.g. f'runs/{run_id}'\n",
+    "    overwrite_output_dir=True,\n",
+    "    num_train_epochs=5,             # we found 3 epochs to be sufficient for convergence on the base models\n",
+    "    per_device_train_batch_size=26, # fits on 1 x NVIDIA A6000, 48GB VRAM\n",
+    "    per_device_eval_batch_size=26,  # can be increased to 32\n",
+    "    gradient_accumulation_steps=2,  # increase this to fit on a smaller GPU\n",
+    "    warmup_steps=0,             \n",
+    "    weight_decay=0.0,\n",
+    "    learning_rate=1e-5,             # we found this to be the best initial learning rate for the base models\n",
+    "    save_strategy=\"steps\",\n",
+    "    save_steps=200,\n",
+    "    eval_steps=200,\n",
+    "    evaluation_strategy=\"steps\",\n",
+    "    logging_strategy='steps',\n",
+    "    logging_steps=50,\n",
+    "    fp16=False,                     \n",
+    "    load_best_model_at_end=True,    # load the best model at the end of training based on validation F1\n",
+    "    metric_for_best_model='f1_score')\n",
+    "\n",
+    "trainer = MultiTaskTrainer(\n",
+    "    model=model,\n",
+    "    compute_metrics=binary_classification_metric, # check out metrics.py for a list of metrics\n",
+    "    args=train_args,\n",
+    "    data_collator=collate_fn,\n",
+    "    train_dataset=dataset['train'],\n",
+    "    eval_dataset=dataset['valid'])\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Generate predictions and compute metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Parameter 'indices'=range(0, 100) of the transform datasets.arrow_dataset.Dataset.select couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n",
+      "The following columns in the test set don't have a corresponding argument in `LegibilityModel.forward` and have been ignored: k, word1, word0, n, model1, word, n1, k1, model0. If k, word1, word0, n, model1, word, n1, k1, model0 are not expected by `LegibilityModel.forward`,  you can safely ignore this message.\n",
+      "***** Running Prediction *****\n",
+      "  Num examples = 100\n",
+      "  Batch size = 26\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1e9f7a59e7624c129d2dae1c85c2d11a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'test_loss': 0.5344929695129395, 'test_precision': 0.9479166567925349, 'test_recall': 0.8921568539984622, 'test_accuracy': 0.8787878721303949, 'test_f1_score': 0.9191914103665608, 'test_runtime': 47.6671, 'test_samples_per_second': 2.098, 'test_steps_per_second': 0.084}\n"
+     ]
+    }
+   ],
+   "source": [
+    "predictions = trainer.predict(dataset['test'].select(range(100))) # takes ~1-2 minutes on a laptop CPU\n",
+    "print(predictions.metrics)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.12 ('base')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "a9a33fd02dcd74fd53701f10c0433ded41be0a0f53c9699722a73f690e69c2bc"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

README.md

@@ -1,12 +1,9 @@
----
-title: Learning Legibility 2023
-emoji: 🌍
-colorFrom: green
-colorTo: purple
-sdk: gradio
-sdk_version: 3.20.1
-app_file: app.py
-pinned: false
----
+# learning-legibility-2023
+This is the repo for *Learning the Legibility of Visual Text Perturbations* (EACL 2023)
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Dev Seth, Rickard Stureborg, Danish Pruthi and Bhuwan Dhingra
+
+Dataset available [here](https://huggingface.co/datasets/dvsth/LEGIT).
+Model available [here](https://huggingface.co/dvsth/LEGIT-TrOCR-MT).
+
+Run `README.ipynb` for a quick start.

classes/LegibilityModel.py

@@ -0,0 +1,27 @@
+import torch.nn as nn
+from transformers import VisionEncoderDecoderModel, PreTrainedModel, AutoConfig
+
+class LegibilityModel(PreTrainedModel):
+    def __init__(self, config):
+        config = AutoConfig.from_pretrained("microsoft/trocr-base-handwritten")
+        super(LegibilityModel, self).__init__(config=config)
+
+        # base model architecture
+        self.model = VisionEncoderDecoderModel(config).encoder
+        
+        # change dropout during training
+        self.stack = nn.Sequential(
+            nn.Dropout(0),
+            nn.Linear(768, 768),
+            nn.ReLU(),
+            nn.Dropout(0),
+            nn.Linear(768, 1)
+        )
+
+    # choice, img0, img1 are not used by the model, but are passed by the trainer
+    def forward(self, img_batch, choice=None, img0=None, img1=None):
+        output = self.model(img_batch)
+        # average the output of the last hidden layer
+        output = output.last_hidden_state.mean(dim=1)
+        scores = self.stack(output)
+        return scores.squeeze()

classes/LegibilityPlot.py

@@ -0,0 +1,73 @@
+import matplotlib.font_manager as fm
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import numpy as np
+
+class LegibilityPlot:
+    def __init__(self):
+        fe = fm.FontEntry(fname='unifont.ttf', name='unifont')
+        fm.fontManager.ttflist.append(fe)
+
+    @staticmethod
+    def gradient_crim_to_darkg(mix):
+        c1 = '#3B724B'
+        c2 = '#9A2C44'
+        # increase saturation
+        c1=np.array(mpl.colors.to_rgb(c1))
+        c2=np.array(mpl.colors.to_rgb(c2))
+        return mpl.colors.to_hex((mix)*c1 + (1-mix)*c2)
+
+    def plot(self, scores, perturbations):
+        # convert the raw scores to probabilities
+        scores = [1 / (1 + np.exp(-x)) for x in scores]
+        fig, ax = plt.subplots(1, 1, figsize=(7, 10), dpi=600)
+
+        # add a horizontal bar for each probability, showing the difference between the probability and 0.5
+        # center the bar at 0.5
+        # make the bar 0.25 high
+        # make the bar color a gradient from crimson to darkgreen
+        # with opacity proportional to the probability
+        ax.barh(range(len(scores)), [x-0.5 for x in scores], height=0.1, color=[self.gradient_crim_to_darkg(x) for x in scores], left=0.5, alpha=0.9)
+
+        ax.scatter(scores, range(len(scores)), s=40, color='white', marker='v', alpha=1.0, zorder=10000, edgecolors='black', linewidths=0.5)
+        # place the image under the point
+        for i in range(len(perturbations)):
+            # place the image to the right of the point
+            ax.text(scores[i], i-0.25, perturbations[i], horizontalalignment='center', verticalalignment='top', fontsize=26, fontfamily='unifont', bbox=dict(facecolor=self.gradient_crim_to_darkg(scores[i]), edgecolor='white', alpha=0.3, boxstyle='round,pad=0.3'))
+            
+        ax.set_xlim(0, 1)
+        # make ticks from 0 to 1 spaced by 0.1
+        ax.set_xticks(np.arange(0, 1.1, 0.20))
+        ax.set_ylim(-1, len(scores)-0.5)
+        ax.set_yticks([])
+        # show a line at 0.5
+        ax.axvline(0.5, color='black', linestyle=':', linewidth=1, alpha=0.5)
+        # disable grid lines
+        ax.grid(False)
+        # make the background white
+        ax.set_facecolor('white')
+        # show the x axis line
+        ax.spines['bottom'].set_visible(True)
+        # make the x axis line black with 0.5 alpha
+        ax.spines['bottom'].set_color('black')
+        ax.spines['bottom'].set_alpha(0.5)
+        # add x axis label
+        ax.set_xlabel('Legibility Score', fontsize=22)
+        # axis ticklabel font size
+        ax.tick_params(axis='x', labelsize=18)
+        # aspect ratio
+        ax.set_aspect(0.25)
+        # x tick marks
+        ax.tick_params(axis='x', which='both', bottom=True, top=False, labelbottom=True)
+        # embolden the x axis label
+        ax.xaxis.label.set_fontweight('bold')
+        # add some padding above the x axis label
+        # ax.xaxis.labelpad = 15
+        
+        # remove the left and right spines
+        ax.spines['left'].set_visible(False)
+        ax.spines['right'].set_visible(False)
+        # remove the top spine
+        ax.spines['top'].set_visible(False)
+        
+        return fig

classes/Metrics.py

@@ -0,0 +1,87 @@
+import numpy as np
+
+def ranking_metric(evalpred):
+    scores0 = evalpred[0][0]
+    scores1 = evalpred[0][1]
+    labels = evalpred[1]
+
+    # labels:
+    # 0 or 1: word 0 or 1 is more legible, other unknown
+    # 2: both words are equally legible
+    # 3: neither word is legible
+
+    pairs_evaluated = 0
+    pairs_correct = 0
+    scores0 = 1 / (1 + np.exp(-scores0))
+    scores1 = 1 / (1 + np.exp(-scores1))
+    for i in range(scores0.shape[0]):
+        if labels[i] < 2:
+            pairs_evaluated += 1
+            if labels[i] == 0:
+                if scores0[i] >= scores1[i]:
+                    pairs_correct += 1
+            elif labels[i] == 1:
+                if scores1[i] >= scores0[i]:
+                    pairs_correct += 1
+
+        accuracy = pairs_correct / pairs_evaluated
+    return {'accuracy': accuracy}
+
+
+def binary_classification_metric(evalpred):
+    scores0 = evalpred[0][0]
+    scores1 = evalpred[0][1]
+    labels = evalpred[1]
+
+    # labels:
+    # 0 or 1: word 0 or 1 is more legible, other unknown
+    # 2: both words are equally legible
+    # 3: neither word is legible
+
+    words_evaluated = 0
+    true_positives = 0
+    false_positives = 0
+    false_negatives = 0
+    true_negatives = 0
+    scores0 = 1 / (1 + np.exp(-scores0))
+    scores1 = 1 / (1 + np.exp(-scores1))
+    for i in range(scores0.shape[0]):
+        if labels[i] < 2:
+            words_evaluated += 1
+        else:
+            words_evaluated += 2
+        if labels[i] == 0:
+            if scores0[i] > 0.5:
+                true_positives += 1
+            else:
+                false_negatives += 1
+        elif labels[i] == 1:
+            if scores1[i] > 0.5:
+                true_positives += 1
+            else:
+                false_negatives += 1
+        elif labels[i] == 2:
+            if scores0[i] > 0.5:
+                true_positives += 1
+            else:
+                false_negatives += 1
+            if scores1[i] > 0.5:
+                true_positives += 1
+            else:
+                false_negatives += 1
+        elif labels[i] == 3:
+            if scores0[i] < 0.5:
+                true_negatives += 1
+            else:
+                false_positives += 1
+            if scores1[i] < 0.5:
+                true_negatives += 1
+            else:
+                false_positives += 1
+
+        # calculate precision, recall, accuracy and f1 score
+        precision = true_positives / (true_positives + false_positives + 1e-6)
+        recall = true_positives / (true_positives + false_negatives + 1e-6)
+        accuracy = (true_positives + true_negatives) / (words_evaluated + 1e-6)
+        f1_score = 2 * precision * recall / (precision + recall + 1e-6)
+    return {'precision': precision, 'recall': recall, 'accuracy': accuracy, 'f1_score': f1_score}

classes/Perturber.py

@@ -0,0 +1,42 @@
+from classes.Similarity import SimHelper
+import numpy as np
+
+class Perturber:
+    def __init__(self, model: str = 'trocr', max_k: int = 30):
+        try:
+            self.sim_space = SimHelper.create_sim_space(
+                model, 'features/trocr.hdf', num_nearest=max_k)
+            self.model = model
+            self.max_k = max_k
+        except:
+            raise Exception(
+                f"Could not load similarity space for model {model}. Make sure {model} is a valid model name. Valid model names: imgdot, trocr, detr, beit, clip")
+
+    def perturb_word(self, word: str, k: int = 10, n: float = 0.5):
+        if k > self.max_k:
+            raise Exception(
+                f"Cannot use k={k} for model {self.model}. Maximum k for this model is {self.max_k}.")
+        if n > 1 or n < 0:
+            raise Exception(
+                f"Cannot use n={n} for model {self.model}. n must be between 0 and 1.")
+
+        metadata = {}
+        metadata['original'] = word
+        metadata['model'] = self.model
+        metadata['k'] = k
+        metadata['substitutions'] = []
+
+        word = list(word)
+        l = len(word)
+        idx_to_replace = np.random.choice(
+            np.arange(l), size=int(l * n), replace=False)
+        for i in idx_to_replace:
+            rand_k = np.random.randint(1, k+1)
+            neighbor = self.sim_space.topk_neighbors(ord(word[i]), rand_k)[-1]
+            neighbor = chr(int(neighbor))
+            metadata['substitutions'].append({'from': word[i], 'to': neighbor, 'k': rand_k})
+            word[i] = neighbor
+            
+        perturbation = ''.join(word)
+        
+        return perturbation, metadata

classes/Renderer.py

@@ -0,0 +1,18 @@
+from PIL import Image, ImageDraw, ImageFont
+
+class Renderer:
+    def __init__(self, fontpath='unifont.ttf'):
+        # load the font
+        self.font = ImageFont.truetype(fontpath, 32)
+
+    def render_image(self, corrupted, original):
+        # create a new image with height slightly larger than the font size
+        text_length_px = self.font.getsize(corrupted + '  ' + original)[0]
+        img = Image.new('RGB', (text_length_px + 20, 40), color='white')
+        # create a drawing context
+        draw = ImageDraw.Draw(img)
+        # draw the text
+        draw.text((10, 0), corrupted + '  ' +
+                  original, font=self.font, fill='black')
+        # return the image
+        return img

classes/Similarity.py

@@ -0,0 +1,73 @@
+import numpy as np
+import pandas as pd
+import torch
+import pickle
+
+class SimilaritySpace:
+    '''
+    PyTorch tensor implementation of a similarity space.
+    Much faster than sklearn's NearestNeighbors.
+    '''
+
+    def __init__(self, desc: str, feature_vectors: pd.DataFrame, num_nearest=10) -> None:
+        self.desc = desc
+        self.device = torch.device(
+            'cuda' if torch.cuda.is_available() else 'cpu')
+        print("using device", self.device)
+        self.idx_to_codepoint = np.array(
+            feature_vectors.codepoint, dtype=np.int64)
+        self.codepoint_to_idx = {codepoint: idx for idx,
+                                 codepoint in enumerate(self.idx_to_codepoint)}
+        feature_vectors = torch.tensor(
+            np.vstack(feature_vectors.features), dtype=torch.float32, device=self.device)
+        # create a pairwise distance matrix
+        distance_matrix = self.matrix_cosine_distance(
+            feature_vectors)
+        # calculate the num_nearest nearest neighbors for each codepoint
+        distances, indices = torch.topk(
+            distance_matrix, k=num_nearest, dim=1, largest=False)
+        self.distances = distances.cpu().numpy()
+        self.indices = indices.cpu().numpy().astype(np.int64)
+        for row in self.indices:
+            # replace every element of row of indices with the corresponding codepoint
+            row[:] = self.idx_to_codepoint[row]
+
+    @staticmethod
+    def cosine_distance(x, y) -> float:
+        return 1 - np.dot(x, y) / ((np.linalg.norm(x) * np.linalg.norm(y)) + 1e-6)
+
+    @staticmethod
+    def matrix_cosine_distance(X: torch.TensorType) -> torch.TensorType:
+        '''
+        Compute the pairwise cosine distance between all rows of X.
+        X is a tensor of shape (n_samples, n_features)
+        '''
+        norm = torch.norm(X, dim=1, keepdim=True)
+        return 1 - (X @ X.T) / (norm @ norm.T)
+
+    def topk_neighbors(self, codepoint: int, k: int):
+        return self.indices[self.codepoint_to_idx[codepoint]][:k+1]
+
+    def topk_distances(self, codepoint: int, k: int):
+        return self.distances[self.codepoint_to_idx[codepoint]][:k+1]
+
+    def set_desc(self, desc: str) -> None:
+        self.desc = desc
+
+
+class SimHelper:
+    @staticmethod
+    def create_sim_space(desc: str, path: str, key: str = 'df', num_nearest: int = 10) -> SimilaritySpace:
+        '''
+        Creates a similarity space from a feature vector HDF file stored at `path` with key `key`.
+        '''
+        df = pd.read_hdf(path, key)
+        return SimilaritySpace(desc=desc, feature_vectors=df, num_nearest=num_nearest)
+
+    @staticmethod
+    def load_sim_space(name: str):
+        return pickle.load(open(name + '.pkl', 'rb'))
+
+    @staticmethod
+    def save_sim_space(sim_space: SimilaritySpace, name: str) -> None:
+        pickle.dump(sim_space, open(name + '.pkl', 'wb'))

classes/Trainer.py

@@ -0,0 +1,64 @@
+import torch
+from transformers import Trainer
+from classes import LegibilityModel
+
+class MultiTaskTrainer(Trainer):
+    '''
+    Overrides the default HuggingFace Trainer class
+    '''
+    
+    def training_step(self, model: LegibilityModel.LegibilityModel, inputs):
+        # get the input images and target label from the data dictionary
+        imgs0, imgs1, labels = inputs['img0'], inputs['img1'], inputs['choice']
+        # run the model
+        scores0 = model(imgs0)
+        scores1 = model(imgs1)
+        # compute the loss
+        loss = self.compute_loss(model, scores0, scores1, labels)
+        loss.backward()
+
+        return loss.detach()
+
+    def eval_step(self, model: LegibilityModel.LegibilityModel, inputs):
+        with torch.no_grad():
+            # get the input images and target label from the data dictionary
+            imgs0, imgs1, labels = inputs['img0'], inputs['img1'], inputs['choice']
+            # run the model
+            scores0 = model(imgs0)
+            scores1 = model(imgs1)
+            # compute the loss
+            loss = self.compute_loss(model, scores0, scores1, labels)
+            return loss
+
+    def prediction_step(self, model: LegibilityModel.LegibilityModel, inputs, prediction_loss_only=True, ignore_keys=None):
+       with torch.no_grad():
+            # get the input images and target label from the data dictionary
+            imgs0, imgs1, labels = inputs['img0'], inputs['img1'], inputs['choice']
+            # run the model
+            scores0 = model(imgs0)
+            scores1 = model(imgs1)
+            # compute the loss
+            loss = self.compute_loss(model, scores0, scores1, labels)
+            return loss, [scores0, scores1], labels
+
+    def compute_loss(self, model: LegibilityModel.LegibilityModel, scores0: torch.Tensor, scores1: torch.Tensor, labels: torch.Tensor, return_outputs=False):
+        # labels:
+        # 0 or 1: word 0 or 1 is more legible, other unknown
+        # 2: both words are equally legible
+        # 3: neither word is legible
+
+        contrastive_term = torch.binary_cross_entropy_with_logits(
+            scores0 - scores1, (labels == 0).type(torch.float))
+        word0_term = torch.binary_cross_entropy_with_logits(
+            scores0, torch.logical_or(labels == 0, labels == 2).type(torch.float))
+        word1_term = torch.binary_cross_entropy_with_logits(
+            scores1, torch.logical_or(labels == 1, labels == 2).type(torch.float))
+
+        # mask out terms which are not relevant for the loss
+        mask_c = labels < 2
+        mask_0 = torch.logical_or(torch.logical_or(labels == 0, labels == 2), labels == 3)
+        mask_1 = labels > 0
+
+        # compute the loss
+        loss = mask_c * contrastive_term + mask_0 * word0_term + mask_1 * word1_term
+        return loss.mean()

demo.py

@@ -0,0 +1,60 @@
+import gradio as gr
+from classes.Perturber import Perturber
+from classes.Renderer import Renderer
+from classes.LegibilityPlot import LegibilityPlot
+
+from transformers import TrOCRProcessor, AutoModel
+
+# preprocessor provides image normalization and resizing
+preprocessor = TrOCRProcessor.from_pretrained(
+    "microsoft/trocr-base-handwritten")
+
+# load the model schema and pretrained weights
+# (this may take some time to download)
+model = AutoModel.from_pretrained("dvsth/LEGIT-TrOCR-MT", revision='main', trust_remote_code=True)
+
+perturber = Perturber('trocr', 50)
+renderer = Renderer('unifont.ttf')
+plotter = LegibilityPlot()
+
+def demo(word_to_perturb, k, n):
+    if ' ' in word_to_perturb:
+        return 'Please enter a single word.'
+
+    perturbations, metadatas, images, scores = [], [], [], []
+    for i in range(10):
+        perturbation, metadata = perturber.perturb_word(word_to_perturb, k, n)
+        inputimg = renderer.render_image(perturbation, word_to_perturb)
+        score = model(preprocessor(inputimg, return_tensors='pt').pixel_values).item()
+        
+        metadata['score'] = score
+        outputimg = renderer.render_image(perturbation, '')
+
+        perturbations.append(perturbation)
+        images.append(outputimg)
+        metadatas.append(metadata)
+        scores.append(score)
+
+    # sort perturbations by score
+    perturbations = [perturbation for perturbation, score in sorted(zip(perturbations, scores), key=lambda x: x[1])]
+    scores = sorted(scores)
+    images = [image for image, score in sorted(zip(images, scores), key=lambda x: x[1])]
+    metadatas = [metadata for metadata, score in sorted(zip(metadatas, scores), key=lambda x: x[1])]
+
+    # return as a single string in the format
+    # perturbation1 (score1)
+    # perturbation2 (score2)
+    # ...
+    # perturbationN (scoreN)
+    # with all scores rounded to 2 decimal places
+    ret_str = ''
+    for i in range(len(perturbations)):
+        ret_str += f'{perturbations[i]} ({round(scores[i], 2)}) -- ' + ("legible" if scores[i] > 0 else "not legible") + '\n'
+        
+    # plot the perturbations and scores
+    fig = plotter.plot(scores, perturbations)
+    return ret_str, fig
+
+interface = gr.Interface(fn=demo, inputs=["text", gr.Slider(1, 50, 20, step=1), gr.Slider(0., 1., 0.5)], outputs=["text", "plot"], allow_flagging='never')
+
+interface.launch(inbrowser=True)

features/beit.hdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:363347be388b8731dd54bbf537f1c2e459dcbec06e86a32e4ba7d24906200864
+size 35072936

features/clip.hdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94ed9b51b0b3a25e94939f7cf08b4a4b1a68edeba4e33ed45a5b904742aee24
+size 35072936

features/detr.hdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:260b70c631fcefade5ef9d521c1619c3e4298adc5d8e298cecffb47d00faa3de
+size 12761192

features/imgdot.hdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:930ebdf7588aa49dc70fd6c4c4beded7aba31e6d103abc7c25841e0ef16933f0
+size 12761192

features/trocr.hdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f17b863064d15f02807549e654d4088966d2422a0f39d7bbf1100440afa58d58
+size 35072936

requirements.txt

@@ -0,0 +1,197 @@
+absl-py @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_e3n1cffagz/croot/absl-py_1666362938899/work
+-e git+ssh://git@github.com/acl-org/aclpubcheck.git@1050a4fbca1602f39150e07f303627d36a452938#egg=aclpubcheck
+aiofiles==23.1.0
+aiohttp @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_4c_8pz93lf/croot/aiohttp_1670009562783/work
+aiosignal @ file:///tmp/build/80754af9/aiosignal_1637843061372/work
+altair==4.2.2
+anyio @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-t_zs64wy/anyio_1644482593257/work/dist
+appnope @ file:///Users/ktietz/demo/mc3/conda-bld/appnope_1629146036738/work
+argon2-cffi @ file:///opt/conda/conda-bld/argon2-cffi_1645000214183/work
+argon2-cffi-bindings @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-wbf5edig/argon2-cffi-bindings_1644845754377/work
+asttokens @ file:///opt/conda/conda-bld/asttokens_1646925590279/work
+async-timeout @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_19fv27ehgp/croots/recipe/async-timeout_1664876371666/work
+attrs @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_978y9aqcd7/croot/attrs_1668696180911/work
+autopep8 @ file:///opt/conda/conda-bld/autopep8_1650463822033/work
+Babel @ file:///tmp/build/80754af9/babel_1620871417480/work
+backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
+beautifulsoup4 @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-15cbtalq/beautifulsoup4_1650462161715/work
+bibtexparser==1.4.0
+bleach @ file:///opt/conda/conda-bld/bleach_1641577558959/work
+blinker==1.4
+Bottleneck @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_07078715-3ab7-4562-8d3d-d56b0eaa0f7dp504n_ny/croots/recipe/bottleneck_1657175566567/work
+brotlipy==0.7.0
+cachetools @ file:///tmp/build/80754af9/cachetools_1619597386817/work
+certifi @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5d968ni_yn/croot/certifi_1671487774636/work/certifi
+cffi @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-jgj0vmyy/cffi_1642701117808/work
+charset-normalizer @ file:///tmp/build/80754af9/charset-normalizer_1630003229654/work
+click @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-quwchbn8/click_1646123324461/work
+conda==23.1.0
+conda-content-trust @ file:///tmp/build/80754af9/conda-content-trust_1617045594566/work
+conda-package-handling @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-4sc96bd_/conda-package-handling_1649105290173/work
+cryptography @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5871d1ea-0250-4cd7-ac89-4b1e60514f5daqk8t0ow/croots/recipe/cryptography_1652101128666/work
+cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
+datasets @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_f0bj9iam3j/croot/datasets_1668181516280/work
+debugpy @ file:///Users/builder/miniconda3/envs/prefect/conda-bld/debugpy_1637092214173/work
+decorator @ file:///opt/conda/conda-bld/decorator_1643638310831/work
+defusedxml @ file:///tmp/build/80754af9/defusedxml_1615228127516/work
+dill @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_7eaenha9af/croot/dill_1667919539340/work
+entrypoints @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-jb01gaox/entrypoints_1650293758411/work
+et-xmlfile==1.1.0
+executing @ file:///opt/conda/conda-bld/executing_1646925071911/work
+fastapi==0.94.0
+fastjsonschema @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_43a0jaiddu/croots/recipe/python-fastjsonschema_1661368628129/work
+ffmpy==0.3.0
+filelock @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_20tv0lrp8x/croot/filelock_1672387134240/work
+flit_core @ file:///opt/conda/conda-bld/flit-core_1644941570762/work/source/flit_core
+fonttools==4.25.0
+frozenlist @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_38d0xkqt7k/croot/frozenlist_1670004509129/work
+fsspec @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_c8b8kofgq9/croot/fsspec_1670336600001/work
+google-auth @ file:///opt/conda/conda-bld/google-auth_1646735974934/work
+google-auth-oauthlib==0.4.1
+gradio==3.20.1
+grpcio @ file:///Users/builder/miniconda3/envs/prefect/conda-bld/grpcio_1637592348692/work
+h11==0.14.0
+httpcore==0.16.3
+httpx==0.23.3
+huggingface-hub==0.11.1
+idna @ file:///tmp/build/80754af9/idna_1637925883363/work
+importlib-metadata @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-5pqd2z6f/importlib-metadata_1648710902288/work
+ipykernel @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_7boejo_bxp/croots/recipe/ipykernel_1662361806503/work
+ipython @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_813bgy7kg6/croot/ipython_1668088123202/work
+ipython-genutils @ file:///tmp/build/80754af9/ipython_genutils_1606773439826/work
+ipywidgets @ file:///tmp/build/80754af9/ipywidgets_1634143127070/work
+jedi @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-f1t6hma6/jedi_1644315882177/work
+Jinja2 @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_9fjgzv9ant/croot/jinja2_1666908141308/work
+json5 @ file:///tmp/build/80754af9/json5_1624432770122/work
+jsonschema @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_e5_l6coyjh/croots/recipe/jsonschema_1663375475589/work
+jupyter @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_c96hs6nzjt/croots/recipe/jupyter_1659349054648/work
+jupyter-console @ file:///opt/conda/conda-bld/jupyter_console_1647002188872/work
+jupyter-server @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_20576569-56d2-46ec-9455-d266af658edbzbqhavnh/croots/recipe/jupyter_server_1658754492228/work
+jupyter_client @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_3bl8w897rj/croot/jupyter_client_1669040279222/work
+jupyter_core @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_9bflj8byfz/croot/jupyter_core_1668084444072/work
+jupyterlab @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_ebr25akxuh/croot/jupyterlab_1669368478002/work
+jupyterlab-pygments @ file:///tmp/build/80754af9/jupyterlab_pygments_1601490720602/work
+jupyterlab-widgets @ file:///tmp/build/80754af9/jupyterlab_widgets_1609884341231/work
+jupyterlab_server @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_bfh24ty__4/croot/jupyterlab_server_1669363632509/work
+kiwisolver @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_69de275f-e01d-4033-8ea6-9caf70c31072d5uirqak/croots/recipe/kiwisolver_1653292042660/work
+latexcodec==2.0.1
+linkify-it-py==2.0.0
+lxml @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_71bcfe2b-fe7b-414a-9d7e-4f32bdd95f6d2vxca0jd/croots/recipe/lxml_1657545136492/work
+Markdown @ file:///Users/ktietz/demo/mc3/conda-bld/markdown_1629712599908/work
+markdown-it-py==2.2.0
+MarkupSafe @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_12c133f5-0720-4727-9c18-599a3af825723lzwham3/croots/recipe/markupsafe_1654597866058/work
+matplotlib @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5fi9l6u9e6/croots/recipe/matplotlib-suite_1660167931049/work
+matplotlib-inline @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_f6fdc0hldi/croots/recipe/matplotlib-inline_1662014472341/work
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mistune @ file:///Users/ktietz/demo/mc3/conda-bld/mistune_1629356075445/work
+mock @ file:///tmp/build/80754af9/mock_1607622725907/work
+multidict @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_bd3x0mcjpv/croot/multidict_1665674237951/work
+multiprocess @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_c71oabygq3/croot/multiprocess_1668006436256/work
+munkres==1.1.4
+nbclassic @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_3btejl1i1y/croot/nbclassic_1668174982725/work
+nbclient @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-08wgx75f/nbclient_1650373566605/work
+nbconvert @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_36au3u9s44/croot/nbconvert_1668450648628/work
+nbformat @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_edm2jl54tz/croots/recipe/nbformat_1663744962951/work
+nest-asyncio @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-xymukih3/nest-asyncio_1649931465456/work
+notebook @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_92n14lq88x/croot/notebook_1668179891126/work
+notebook_shim @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_f8mr1gjfb7/croot/notebook-shim_1668160580414/work
+numexpr @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_08e1f2b1-735c-4635-9755-5afc6d3eb18ew8ilpa0t/croots/recipe/numexpr_1656940301129/work
+numpy @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_9bff6b9b-6adc-40a2-b632-076fce306026zq15ylrw/croots/recipe/numpy_and_numpy_base_1653915521605/work
+oauthlib @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_4aroovdrpz/croot/oauthlib_1665490908808/work
+openpyxl==3.0.10
+orjson==3.8.7
+packaging @ file:///tmp/build/80754af9/packaging_1637314298585/work
+pandas==1.4.4
+pandocfilters @ file:///opt/conda/conda-bld/pandocfilters_1643405455980/work
+parso @ file:///opt/conda/conda-bld/parso_1641458642106/work
+pdfminer.six==20221105
+pdfplumber==0.7.6
+pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
+pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
+Pillow==9.2.0
+pluggy @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_croot-w6jyveby/pluggy_1648109277227/work
+ply==3.11
+prometheus-client @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_25sgeyk0j5/croots/recipe/prometheus_client_1659455103277/work
+prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1633440160888/work
+protobuf==3.20.1
+psutil @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_1310b568-21f4-4cb0-b0e3-2f3d31e39728k9coaga5/croots/recipe/psutil_1656431280844/work
+ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+pure-eval @ file:///opt/conda/conda-bld/pure_eval_1646925070566/work
+pyarrow==8.0.0
+pyasn1 @ file:///Users/ktietz/demo/mc3/conda-bld/pyasn1_1629708007385/work
+pyasn1-modules==0.2.8
+pybtex==0.24.0
+pycodestyle @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5b2mq44vl0/croot/pycodestyle_1674267228581/work
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
+pycryptodome==3.17
+pydantic==1.10.6
+pydub==0.25.1
+Pygments @ file:///opt/conda/conda-bld/pygments_1644249106324/work
+PyJWT @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_83311740-c3bc-48a8-9688-62d7e2d625f5s6jo920w/croots/recipe/pyjwt_1657544592579/work
+pylatexenc==2.10
+pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
+pyparsing @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_3b_3vxnd07/croots/recipe/pyparsing_1661452540919/work
+PyQt5-sip==12.11.0
+pyrsistent @ file:///Users/ktietz/demo/mc3/conda-bld/pyrsistent_1628941062930/work
+PySocks @ file:///Users/ktietz/Code/oss/ci_pkgs/pysocks_1626781349491/work
+python-dateutil @ file:///tmp/build/80754af9/python-dateutil_1626374649649/work
+python-multipart==0.0.6
+pytz @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_830e8540-4c59-42fe-9515-a6f79c24ff6dh653xa67/croots/recipe/pytz_1654762630628/work
+PyYAML @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_8dd_9u21zz/croot/pyyaml_1670514759576/work
+pyzmq @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_8599562e-e9e5-443b-91db-7f7c0ba6aad3mrdoyvz4/croots/recipe/pyzmq_1657724196154/work
+qtconsole @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_213etb6fb4/croots/recipe/qtconsole_1662018260717/work
+QtPy @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_90hl8ymlpx/croots/recipe/qtpy_1662014534092/work
+rebiber==1.1.3
+regex==2022.10.31
+requests @ file:///opt/conda/conda-bld/requests_1641824580448/work
+requests-oauthlib==1.3.0
+responses @ file:///tmp/build/80754af9/responses_1619800270522/work
+rfc3986==1.5.0
+rsa @ file:///tmp/build/80754af9/rsa_1614366226499/work
+ruamel-yaml-conda @ file:///Users/ktietz/demo/mc3/conda-bld/ruamel_yaml_1629464769899/work
+ruamel.yaml @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_dby9tad7ud/croots/recipe/ruamel.yaml_1664988526357/work
+ruamel.yaml.clib @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_f64xdg2rww/croot/ruamel.yaml.clib_1666302244208/work
+seaborn @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_bfw6gzh1ye/croot/seaborn_1669625743104/work
+Send2Trash @ file:///tmp/build/80754af9/send2trash_1632406701022/work
+sip @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_fbqiv4bzwo/croots/recipe/sip_1659012372184/work
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+sniffio @ file:///Users/ktietz/demo/mc3/conda-bld/sniffio_1629145892482/work
+soupsieve @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_d2jpk7eoyp/croot/soupsieve_1666296398381/work
+stack-data @ file:///opt/conda/conda-bld/stack_data_1646927590127/work
+starlette==0.26.0.post1
+tables @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_19ckz1b_eh/croot/pytables_1673967665080/work
+tensorboard @ file:///tmp/build/80754af9/tensorboard_1633093581375/work/tensorboard-2.6.0-py3-none-any.whl
+tensorboard-data-server @ file:///var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_91lm1y0ip6/croot/tensorboard-data-server_1670853587549/work/tensorboard_data_server-0.6.1-py3-none-macosx_11_0_arm64.whl
+tensorboard-plugin-wit==1.6.0
+termcolor==2.2.0
+terminado @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-n1puqh63/terminado_1644395131327/work
+timm==0.6.12
+tinycss2 @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_fcw5_i306t/croot/tinycss2_1668168825117/work
+tokenizers==0.13.2
+toml @ file:///tmp/build/80754af9/toml_1616166611790/work
+tomli @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_d0e5ffbf-5cf1-45be-8693-c5dff8108a2awhthtjlq/croots/recipe/tomli_1657175508477/work
+toolz @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_362wyqvvgy/croot/toolz_1667464079070/work
+torch==1.13.1
+torchvision==0.14.1
+tornado @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_a61b4xoie9/croots/recipe/tornado_1662061692951/work
+tqdm @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_d8374dd1-2388-4771-b0ef-a205a18076f0p40h0nwh/croots/recipe/tqdm_1650891080348/work
+traitlets @ file:///tmp/build/80754af9/traitlets_1636710298902/work
+transformers==4.25.1
+tsv==1.2
+typing_extensions @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_10a8_u2ijw/croot/typing_extensions_1669923788997/work
+uc-micro-py==1.0.1
+Unidecode==1.3.6
+urllib3 @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_dc7bd08d-3aa9-4e45-9124-1c432b2bee1dmp2r0ell/croots/recipe/urllib3_1650640006820/work
+uvicorn==0.21.0
+Wand==0.6.11
+wcwidth @ file:///Users/ktietz/demo/mc3/conda-bld/wcwidth_1629357192024/work
+webencodings==0.5.1
+websocket-client @ file:///Users/ktietz/demo/mc3/conda-bld/websocket-client_1629357070578/work
+websockets==10.4
+Werkzeug @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_54t7yp6vfo/croot/werkzeug_1671215998207/work
+widgetsnbextension @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/croot-ppn8qoma/widgetsnbextension_1645010005457/work
+xxhash @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5dctqdmbn8/croot/python-xxhash_1667919512545/work
+yarl @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_a5sx8bz_hq/croots/recipe/yarl_1661437083380/work
+zipp @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_66c2c5f2-5dd5-4946-a16a-72af650ebd6cnmz4ou0f/croots/recipe/zipp_1652343960956/work

unifont.ttf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c632666c659ccdfdb841f151aa6cc48cb987e093b90806f5af3d5a4bea7c54a5
+size 12273956