Re-enable LOME

docker_commands.sh

@@ -1,3 +1,7 @@
+cd spanfinder/
+python -m sociolome.lome_webserver &
+cd ..
+
 rm -rfv /.cache/sfdata/.git
 mv -v /.cache/sfdata/* /app/
 du -h -d 2

requirements.txt

@@ -25,12 +25,12 @@ concrete==4.15.1
 jinja2==3.0.3  # added 2022-03-28 because of (new?) error
 itsdangerous==2.0.1  # idem
 
-# # LOME partpip install pyopenssl
-# allennlp==2.8.0
-# allennlp-models==2.8.0
-# transformers==4.12.5
-# numpy
-# torch>=1.7.0,<1.8.0
-# tqdm
-# overrides
-# scipy
+# LOME
+allennlp==2.8.0
+allennlp-models==2.8.0
+transformers==4.12.5
+numpy
+torch>=1.7.0,<1.8.0
+tqdm
+overrides
+scipy

spanfinder/.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Module",
+            "type": "python",
+            "request": "launch",
+            "module": "sociolome.lome_webserver",
+            "cwd": "${workspaceFolder}",
+            "justMyCode": true
+        }
+    ]
+}

spanfinder/README.md

@@ -0,0 +1,33 @@
+# Span Finder (v0.0.2)
+
+## Installation
+
+Environment:
+   - python >= 3.7
+   - pip
+
+To install the dependencies, execute
+
+``` shell script
+pip install -r requirements.txt
+pip uninstall -y dataclasses
+```
+
+Then install SFTP (Span Finding - Transductive Parsing) package:
+
+``` shell script
+python setup.py install
+```
+
+## Prediction
+
+If you use SpanFinder only for inference, please read [this example](scripts/predict_span.py).
+
+## Demo
+
+A demo (combined with Patrick's coref model) is [here](https://nlp.jhu.edu/demos/lome).
+
+## Pre-Trained Models
+
+Some parameters trained on FrameNet can be found at CLSP grid: `/home/gqin2/public/release/sftp/0.0.2`.
+The file `model.tar.gz` is pointed to the best model so for, and file named after dates will not be updated anymore.

spanfinder/config/ace/ace.jsonnet

@@ -0,0 +1,131 @@
+local env = import "../env.jsonnet";
+
+local dataset_path = env.str("DATA_PATH", "data/ace/events");
+local ontology_path = "data/ace/ontology.tsv";
+
+local debug = false;
+
+# embedding
+local label_dim = 64;
+local pretrained_model = env.str("ENCODER", "roberta-large");
+
+# module
+local dropout = 0.2;
+local bio_dim = 512;
+local bio_layers = 2;
+local span_typing_dims = [256, 256];
+local event_smoothing_factor = env.json("SMOOTHING", "0.0");
+local arg_smoothing_factor = env.json("SMOOTHING", "0.0");
+local layer_fix = 0;
+
+# training
+local typing_loss_factor = 8.0;
+local grad_acc = env.json("GRAD_ACC", "1");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+local lr = env.json("LR", "1e-3");
+local cuda_devices = env.json("CUDA_DEVICES", "[0]");
+
+{
+    dataset_reader: {
+        type: "concrete",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: event_smoothing_factor,
+        arg_smoothing_factor: event_smoothing_factor,
+    },
+    train_data_path: dataset_path + "/train.tar.gz",
+    validation_data_path: dataset_path + "/dev.tar.gz",
+    test_data_path: dataset_path + "/test.tar.gz",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        ontology_path: ontology_path,
+        typing_loss_factor: typing_loss_factor,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: null,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true,
+}

spanfinder/config/ace/ft.jsonnet

@@ -0,0 +1,51 @@
+local env = import "../env.jsonnet";
+local base = import "ace.jsonnet";
+
+local pretrained_path = env.str("PRETRAINED_PATH", "cache/ace/best");
+local lr = env.json("FT_LR", 5e-5);
+
+# training
+local cuda_devices = base.cuda_devices;
+
+{
+    dataset_reader: base.dataset_reader,
+    train_data_path: base.train_data_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+    datasets_for_vocab_creation: ["train"],
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: {
+        type: "from_archive",
+        archive_file: pretrained_path
+    },
+    vocabulary: {
+        type: "from_files",
+        directory: pretrained_path + "/vocabulary"
+    },
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/ace/pt.jsonnet

@@ -0,0 +1,69 @@
+local env = import "../env.jsonnet";
+local base = import "ace.jsonnet";
+
+local fn_path = "data/framenet/full/full.jsonl";
+local mapping_path = "data/ace/framenet2ace/";
+
+local debug = false;
+
+# training
+local lr = env.json("PT_LR", "5e-5");
+local cuda_devices = base.cuda_devices;
+
+# mapping
+local min_weight = env.json("MIN_WEIGHT", '0.0');
+local max_weight = env.json("MAX_WEIGHT", '5.0');
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: base.dataset_reader.pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: base.dataset_reader.event_smoothing_factor,
+        arg_smoothing_factor: base.dataset_reader.arg_smoothing_factor,
+        ontology_mapping_path: mapping_path + '/ontology_mapping.json',
+        min_weight: min_weight,
+        max_weight: max_weight,
+    },
+    validation_dataset_reader: base.dataset_reader,
+    train_data_path: fn_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+    vocabulary: {
+        type: "extend",
+        directory: mapping_path + "/vocabulary"
+    },
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: base.model,
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/ace/rt.jsonnet

@@ -0,0 +1,89 @@
+local env = import "../env.jsonnet";
+local base = import "ace.jsonnet";
+
+local dataset_path = env.str("DATA_PATH", "data/ace/events");
+
+local debug = false;
+
+# re-train
+local pretrained_path = env.str("PRETRAINED_PATH", "cache/fn/best");
+local rt_lr = env.json("RT_LR", 5e-5);
+
+# module
+local cuda_devices = base.cuda_devices;
+
+{
+    dataset_reader: base.dataset_reader,
+    train_data_path: base.train_data_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: {
+        type: "span",
+        word_embedding: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "word_embedding",
+                "freeze": false,
+            }
+        },
+        span_extractor: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "_span_extractor",
+                "freeze": false,
+            }
+        },
+        span_finder: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "_span_finder",
+                "freeze": false,
+            }
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: base.model.span_typing.hidden_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: base.model.typing_loss_factor,
+        label_dim: base.model.label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: base.trainer.optimizer.base.lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+            parameter_groups: [
+            [['_span_finder.*'], {'lr': rt_lr}],
+            [['_span_extractor.*'], {'lr': rt_lr}],
+            ]
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/basic/basic.jsonnet

@@ -0,0 +1,132 @@
+local env = import "../env.jsonnet";
+
+local dataset_path = "data/better/basic/sent/";
+local ontology_path = "data/better/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local patience = env.json("PATIENCE", "null");
+
+{
+    dataset_reader: {
+        type: "better",
+        eval_type: "basic",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+    },
+    train_data_path: dataset_path + "/basic.eng-provided-72.0pct.train-70.0pct.d.bp.json",
+    validation_data_path: dataset_path + "/basic.eng-provided-72.0pct.analysis-15.0pct.ref.d.bp.json",
+    test_data_path: dataset_path + "/basic.eng-provided-72.0pct.devtest-15.0pct.ref.d.bp.json",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: ontology_path,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/basic/ft.jsonnet

@@ -0,0 +1,51 @@
+local env = import "../env.jsonnet";
+local base = import "basic.jsonnet";
+
+local pretrained_path = env.str("PRETRAINED_PATH", "cache/basic/best");
+local lr = env.json("FT_LR", 5e-5);
+
+# training
+local cuda_devices = base.cuda_devices;
+
+{
+    dataset_reader: base.dataset_reader,
+    train_data_path: base.train_data_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+    datasets_for_vocab_creation: ["train"],
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: {
+        type: "from_archive",
+        archive_file: pretrained_path
+    },
+    vocabulary: {
+        type: "from_files",
+        directory: pretrained_path + "/vocabulary"
+    },
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/basic/pt.jsonnet

@@ -0,0 +1,67 @@
+local env = import "../env.jsonnet";
+local base = import "basic.jsonnet";
+
+local fn_path = "data/framenet/full/full.jsonl";
+local mapping_path = "data/basic/framenet2better/";
+
+local debug = false;
+
+# training
+local lr = env.json("PT_LR", "5e-5");
+local cuda_devices = base.cuda_devices;
+
+# mapping
+local min_weight = env.json("MIN_WEIGHT", '0.0');
+local max_weight = env.json("MAX_WEIGHT", '5.0');
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: base.dataset_reader.pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        ontology_mapping_path: mapping_path + '/ontology_mapping.json',
+        min_weight: min_weight,
+        max_weight: max_weight,
+    },
+    validation_dataset_reader: base.dataset_reader,
+    train_data_path: fn_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+    vocabulary: {
+        type: "extend",
+        directory: mapping_path + "/vocabulary"
+    },
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: base.model,
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/basic/rt.jsonnet

@@ -0,0 +1,87 @@
+local env = import "../env.jsonnet";
+local base = import "basic.jsonnet";
+
+local debug = false;
+
+# re-train
+local pretrained_path = env.str("PRETRAINED_PATH", "cache/fn/best");
+local rt_lr = env.json("RT_LR", 5e-5);
+
+# module
+local cuda_devices = base.cuda_devices;
+
+{
+    dataset_reader: base.dataset_reader,
+    train_data_path: base.train_data_path,
+    validation_data_path: base.validation_data_path,
+    test_data_path: base.test_data_path,
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: base.data_loader,
+    validation_data_loader: base.validation_data_loader,
+
+    model: {
+        type: "span",
+        word_embedding: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "word_embedding",
+                "freeze": false,
+            }
+        },
+        span_extractor: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "_span_extractor",
+                "freeze": false,
+            }
+        },
+        span_finder: {
+            "_pretrained": {
+                "archive_file": pretrained_path,
+                "module_path": "_span_finder",
+                "freeze": false,
+            }
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: base.model.span_typing.hidden_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: base.model.typing_loss_factor,
+        label_dim: base.model.label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: base.trainer.num_epochs,
+        patience: base.trainer.patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+arg-c_f",
+        num_gradient_accumulation_steps: base.trainer.num_gradient_accumulation_steps,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: base.trainer.optimizer.base.lr,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: base.trainer.optimizer.layer_fix,
+            parameter_groups: [
+            [['_span_finder.*'], {'lr': rt_lr}],
+            [['_span_extractor.*'], {'lr': rt_lr}],
+            ]
+        }
+    },
+
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/env.jsonnet

@@ -0,0 +1,4 @@
+{
+    json: function(name, default) if std.extVar("LOGNAME")=="tuning" then std.parseJson(std.extVar(name)) else std.parseJson(default),
+    str: function(name, default) if std.extVar("LOGNAME")=="tuning" then std.extVar(name) else default
+}

spanfinder/config/fn-evalita/evalita.framenet_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/evalita_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = "/data/p289731/cloned/lome-models/models/xlm-roberta-framenet/";
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/evalita_train.jsonl",
+    validation_data_path: dataset_path + "/evalita_dev.jsonl",
+    test_data_path: dataset_path + "/evalita_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-evalita/evalita.it_mono.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/evalita_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "Musixmatch/umberto-commoncrawl-cased-v1");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/evalita_train.jsonl",
+    validation_data_path: dataset_path + "/evalita_dev.jsonl",
+    test_data_path: dataset_path + "/evalita_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-evalita/evalita.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/evalita_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/evalita_train.jsonl",
+    validation_data_path: dataset_path + "/evalita_dev.jsonl",
+    test_data_path: dataset_path + "/evalita_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-evalita/evalita_plus_fn.vanilla_xlmr.freeze.jsonnet

@@ -0,0 +1,142 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/evalita_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/evalita_plus_fn_train.jsonl",
+    validation_data_path: dataset_path + "/evalita_dev.jsonl",
+    test_data_path: dataset_path + "/evalita_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                    train_parameters: false
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-evalita/evalita_plus_fn.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/evalita_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/evalita_plus_fn_train.jsonl",
+    validation_data_path: dataset_path + "/evalita_dev.jsonl",
+    test_data_path: dataset_path + "/evalita_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-kicktionary/kicktionary.concat_clipped.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/kicktionary_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/kicktionary_exemplars_train.concat_clipped.jsonl",
+    validation_data_path: dataset_path + "/kicktionary_exemplars_dev.jsonl",
+    test_data_path: dataset_path + "/kicktionary_exemplars_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-kicktionary/kicktionary.football_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/kicktionary_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "/data/p289731/cloned/lome-models/models/xlm-roberta-football/");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/kicktionary_exemplars_train.jsonl",
+    validation_data_path: dataset_path + "/kicktionary_exemplars_dev.jsonl",
+    test_data_path: dataset_path + "/kicktionary_exemplars_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-kicktionary/kicktionary.framenet_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/kicktionary_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = "/data/p289731/cloned/lome-models/models/xlm-roberta-framenet/";
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/kicktionary_exemplars_train.jsonl",
+    validation_data_path: dataset_path + "/kicktionary_exemplars_dev.jsonl",
+    test_data_path: dataset_path + "/kicktionary_exemplars_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-kicktionary/kicktionary.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/kicktionary_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/kicktionary_exemplars_train.jsonl",
+    validation_data_path: dataset_path + "/kicktionary_exemplars_dev.jsonl",
+    test_data_path: dataset_path + "/kicktionary_exemplars_test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a1.framenet_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = "/data/p289731/cloned/lome-models/models/xlm-roberta-framenet/";
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/dutch-sonar-train-A1.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A1.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A1.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a1.sonar_plus_fn.vanilla_xlmr.jsonnet

@@ -0,0 +1,142 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+
+    train_data_path: dataset_path + "/dutch-sonar-train-A1.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A1.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A1.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a1.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/dutch-sonar-train-A1.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A1.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A1.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a2.framenet_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = "/data/p289731/cloned/lome-models/models/xlm-roberta-framenet/";
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/dutch-sonar-train-A2.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A2.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A2.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a2.sonar_plus_fn.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/dutch-sonar-train-A2.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A2.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A2.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn-sonar/sonar-a2.vanilla_xlmr.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/sonar_jsonl";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/dutch-sonar-train-A2.jsonl",
+    validation_data_path: dataset_path + "/dutch-sonar-dev-A2.jsonl",
+    test_data_path: dataset_path + "/dutch-sonar-test-A2.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': 1.0,
+                'full text': 0.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn/fn.orig.jsonnet

@@ -0,0 +1,139 @@
+local env = import "../env.jsonnet";
+
+local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local patience = env.json("PATIENCE", "null");
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/train.jsonl",
+    validation_data_path: dataset_path + "/dev.jsonl",
+    test_data_path: dataset_path + "/test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': exemplar_ratio,
+                'full text': 1.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: ontology_path,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn/fn.train-football.jsonnet

@@ -0,0 +1,142 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/framenet_jsonl/full";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+#local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local pretrained_model = env.str("ENCODER", "/data/p289731/cloned/lome-models/models/xlm-roberta-football/");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: "xlm-roberta-large",
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/train.jsonl",
+    validation_data_path: dataset_path + "/dev.jsonl",
+    test_data_path: dataset_path + "/test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': exemplar_ratio,
+                'full text': 1.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/config/fn/fn.train3.jsonnet

@@ -0,0 +1,141 @@
+local env = import "../env.jsonnet";
+
+#local dataset_path = env.str("DATA_PATH", "data/framenet/full");
+local dataset_path = "/home/p289731/cloned/lome/preproc/framenet_jsonl/full";
+local ontology_path = "data/framenet/ontology.tsv";
+
+local debug = false;
+
+# reader
+local pretrained_model = env.str("ENCODER", "xlm-roberta-large");
+local smoothing_factor = env.json("SMOOTHING", "0.1");
+
+# model
+local label_dim = env.json("LABEL_DIM", "64");
+local dropout = env.json("DROPOUT", "0.2");
+local bio_dim = env.json("BIO_DIM", "512");
+local bio_layers = env.json("BIO_LAYER", "2");
+local span_typing_dims = env.json("TYPING_DIMS", "[256, 256]");
+local typing_loss_factor = env.json("LOSS_FACTOR", "8.0");
+
+# loader
+local exemplar_ratio = env.json("EXEMPLAR_RATIO", "0.05");
+local max_training_tokens = 512;
+local max_inference_tokens = 1024;
+
+# training
+local layer_fix = env.json("LAYER_FIX", "0");
+local grad_acc = env.json("GRAD_ACC", "1");
+#local cuda_devices  = env.json("CUDA_DEVICES", "[-1]");
+local cuda_devices = [0];
+local patience = 32;
+
+{
+    dataset_reader: {
+        type: "semantic_role_labeling",
+        debug: debug,
+        pretrained_model: pretrained_model,
+        ignore_label: false,
+        [ if debug then "max_instances" ]: 128,
+        event_smoothing_factor: smoothing_factor,
+        arg_smoothing_factor: smoothing_factor,
+    },
+    train_data_path: dataset_path + "/train.jsonl",
+    validation_data_path: dataset_path + "/dev.jsonl",
+    test_data_path: dataset_path + "/test.jsonl",
+
+    datasets_for_vocab_creation: ["train"],
+
+    data_loader: {
+        batch_sampler: {
+            type: "mix_sampler",
+            max_tokens: max_training_tokens,
+            sorting_keys: ['tokens'],
+            sampling_ratios: {
+                'exemplar': exemplar_ratio,
+                'full text': 1.0,
+            }
+        }
+    },
+
+    validation_data_loader: {
+        batch_sampler: {
+            type: "max_tokens_sampler",
+            max_tokens: max_inference_tokens,
+            sorting_keys: ['tokens']
+        }
+    },
+
+    model: {
+        type: "span",
+        word_embedding: {
+            token_embedders: {
+                "pieces": {
+                    type: "pretrained_transformer",
+                    model_name: pretrained_model,
+                }
+            },
+        },
+        span_extractor: {
+            type: 'combo',
+            sub_extractors: [
+                {
+                    type: 'self_attentive',
+                },
+                {
+                    type: 'bidirectional_endpoint',
+                }
+            ]
+        },
+        span_finder: {
+            type: "bio",
+            bio_encoder: {
+                type: "lstm",
+                hidden_size: bio_dim,
+                num_layers: bio_layers,
+                bidirectional: true,
+                dropout: dropout,
+            },
+            no_label: false,
+        },
+        span_typing: {
+            type: 'mlp',
+            hidden_dims: span_typing_dims,
+        },
+        metrics: [{type: "srl"}],
+
+        typing_loss_factor: typing_loss_factor,
+        ontology_path: null,
+        label_dim: label_dim,
+        max_decoding_spans: 128,
+        max_recursion_depth: 2,
+        debug: debug,
+    },
+
+    trainer: {
+        num_epochs: 128,
+        patience: patience,
+        [if std.length(cuda_devices) == 1 then "cuda_device"]: cuda_devices[0],
+        validation_metric: "+em_f",
+        grad_norm: 10,
+        grad_clipping: 10,
+        num_gradient_accumulation_steps: grad_acc,
+        optimizer: {
+            type: "transformer",
+            base: {
+                type: "adam",
+                lr: 1e-3,
+            },
+            embeddings_lr: 0.0,
+            encoder_lr: 1e-5,
+            pooler_lr: 1e-5,
+            layer_fix: layer_fix,
+        }
+    },
+
+    cuda_devices:: cuda_devices,
+    [if std.length(cuda_devices) > 1 then "distributed"]: {
+        "cuda_devices": cuda_devices
+    },
+    [if std.length(cuda_devices) == 1 then "evaluate_on_test"]: true
+}

spanfinder/docs/data.md

@@ -0,0 +1,68 @@
+# Data Format
+
+You can pass SpanFinder any formats of data, as long as you implement a dataset reader inherited from SpanReader. We also provide a Concrete dataset reader. Besides them, SpanFinder comes with its own JSON data format, which enables richer features for training and modeling.
+
+The minimal example of the JSON is
+
+```JSON
+{
+  "meta": {
+    "fully_annotated": true
+  },
+  "tokens": ["Bob", "attacks", "the", "building", "."],
+  "annotations": [
+    {
+      "span": [1, 1],
+      "label": "Attack",
+      "children": [
+        {
+          "span": [0, 0],
+          "label": "Assailant",
+          "children": []
+        },
+        {
+          "span": [2, 3],
+          "label": "Victim",
+          "children": []
+        }
+      ]
+    },
+    {
+      "span": [3, 3],
+      "label": "Buildings",
+      "children": [
+        {
+          "span": [3, 3],
+          "label": "Building",
+          "children": []
+        }
+      ]
+    }
+  ]
+}
+```
+
+You can have nested spans with unlimited depth.
+
+## Meta-info for Semantic Role Labeling (SRL)
+
+```JSON
+{
+  "ontology": {
+    "event": ["Violence-Attack"],
+    "argument": ["Agent", "Patient"],
+    "link": [[0, 0], [0, 1]]
+  },
+  "ontology_mapping": {
+    "event": {
+      "Attack": ["Violence-Attack", 0.8]
+    },
+    "argument": {
+      "Assault": ["Agent", 0.95],
+      "Victim": ["patient", 0.9]
+    }
+  }
+}
+```
+
+TODO: Guanghui needs to doc this.

spanfinder/docs/mapping.md

@@ -0,0 +1,17 @@
+## Mapping
+
+If a file is passed to the predictor,
+the predicted spans will be converted into a new ontology.
+The file format should be
+
+`<original parent label>\t<original label>\t<new label>`
+
+If the predicted span is labeled as `<original label>`,
+and its parent is labeled as `<orignal parent label>`,
+it will be re-labeled as `<new label>`.
+If no rules match, the span and all of its descendents will be ignored.
+
+The `<original parent label>` is optional.
+If the parent label is `@@VIRTUAL_ROOT@@`, then this rule matches the first layer of spans.
+In semantic parsing, it matches events.
+If the parent label is `*`, it means it can match anything.

spanfinder/docs/training.md

@@ -0,0 +1,65 @@
+# Training Span Finder
+
+## Metrics explanation
+
+By default, the following metrics will be used
+
+- em: (includes emp, emr, emf) Exact matching metric. A span is exactly matched iff its parent, boundaries, and label are all correctly predicted. Note that if a parent is not correctly predicted, all its children will be treated as false negative. In another word, errors are propagated.
+- sm: (includes smp, smr, smf) Span matching metric. Similar to EM but will not check the labels. If you observe high EM but low SM, then the typing system is not properly working.
+- finder: (includes finder-p, finder-r, finder-f) A metric to measure how well the model can find spans. Different from SM, in this metric, gold parent will be provided, so the errors will not be propagated.
+- typing_acc: Span typing accuracy with gold parent and gold span boundaries.
+
+
+Optional metrics that might be useful for SRL-style tasks. Put the following line
+
+`metrics: [{type: "srl", check_type: true}],` 
+
+to the span model in the config file to turn on this feature. You will see the following two metrics:
+
+- trigger: (include trigger-p, trigger-r, trigger-f) It measures how well the system can find the event triggers (or frames in FrameNet). If `check_type` is True, it also checks the event label.
+- role: (include role-p, role-r, role-f) It measures how well the system can find roles. Note if the event/trigger is not found, all its children will be treated as false negative. If `check_type` is True, it also checks the role label.
+
+## Ontology Constraint
+
+In some cases, certain spans can also be attached to specific spans.
+E.g., in SRL tasks, event can only be attached to the VirtualRoot, and arguments can only be attached to the events.
+The constraints of FrameNet is harsher, where each frame have some specific frame elements.
+
+These constraints can be abstracted as a boolean square matrix whose columns and rows are span labels including VIRTUAL_ROOT. 
+Say it's `M`, label2 can be label1's child iff `M[label1, label2]` if True.
+
+You can specify ontology constraint for SpanFinder with the `ontology_path` argument in the SpanModel class.
+The format of this file is simple. Each line is one row of the `M` matrix:
+
+```parent_label child_label_1 child_label_2```
+
+which means child1 and child2 can be attached to the parent. 
+Both `parent_label` and `child_label` are strings, and the space between them should be `\t` not ` `.
+If a parent_label is missing from the file, by default all children be attachable.
+If this file is not provided, all labels can be attached to all labels.
+
+An example of this file can be found at CLSP grid:
+
+```/home/gqin2/data/framenet/ontology.tsv```
+
+## Typing loss factor
+
+(This section might be updated soon -- Guanghui)
+
+The loss comes from two sources: SpanFinding and SpanTyping modules.
+SpanFinder uses CRF and use probability as loss, but SpanTyping uses cross entropy.
+They're of different scale so we have to re-scale them.
+The formula is:
+
+`loss = finding_loss + typing_loss_factor * typing_loss`
+
+Empirically Guanghui finds the optimal `typing_loss_factor` for FrameNet system is 750.
+
+In theory, we should put the two losses to the same space. Guanghui is looking into this, and this might be solved in SpanFinder 0.0.2.
+
+## Optimizer
+
+A custom optimizer `transformer` is used for span finder. 
+It allows you to specify special learning rate for transformer encoder and fix the parameters of certain modules.
+Empirically, fix embedding (so only fine-tune the encoder and pooler) and train with lr=1e-5 yields best results for FrameNet.
+For usage and more details, see its class doc.

spanfinder/model.kicktionary.mod.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44922d9261fa39f226ffc36e0c7b29e7c17aed7b9f516ab32e69b7d2eeedfd11
+size 1785047888

spanfinder/model.mod.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f5be5aeef50b2f4840317b8196c51186f9f138a853dc1eb2da980b1947ceb23
+size 1795605184

spanfinder/requirements.txt

@@ -0,0 +1,15 @@
+allennlp>=2.0.0
+allennlp-models>=2.0.0
+transformers>=4.0.0 # Why is huggingface so unstable?
+numpy
+torch>=1.7.0,<1.8.0
+tqdm
+nltk
+overrides
+concrete
+flask
+scipy
+https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.0.0/it_core_news_md-3.0.0-py3-none-any.whl
+https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.0.0/en_core_web_md-3.0.0-py3-none-any.whl
+https://github.com/explosion/spacy-models/releases/download/nl_core_news_md-3.0.0/nl_core_news_md-3.0.0-py3-none-any.whl
+https://github.com/explosion/spacy-models/releases/download/xx_sent_ud_sm-3.0.0/xx_sent_ud_sm-3.0.0-py3-none-any.whl

spanfinder/scripts/aida_experiment/predict_aida.py

@@ -0,0 +1,42 @@
+import json
+import os
+import copy
+from collections import defaultdict
+from argparse import ArgumentParser
+from tqdm import tqdm
+import random
+from tqdm import tqdm
+from scripts.predict_concrete import read_kairos
+
+from sftp import SpanPredictor
+
+
+parser = ArgumentParser()
+parser.add_argument('aida', type=str)
+parser.add_argument('model', type=str)
+parser.add_argument('dst', type=str)
+parser.add_argument('--topk', type=int, default=10)
+parser.add_argument('--device', type=int, default=0)
+args = parser.parse_args()
+
+k = args.topk
+corpus = json.load(open(args.aida))
+predictor = SpanPredictor.from_path(args.model, cuda_device=args.device)
+idx2fn = predictor._model.vocab.get_index_to_token_vocabulary('span_label')
+random.seed(42)
+random.shuffle(corpus)
+
+
+output_fp = open(args.dst, 'a')
+for line in tqdm(corpus):
+    tokens, ann = line['tokens'], line['annotation']
+    start, end, kairos_label = ann['start_idx'], ann['end_idx'], ann['label']
+    prob_dist = predictor.force_decode(tokens, [(start, end)])[0]
+    topk_indices = prob_dist.argsort(descending=True)[:k]
+    prob = prob_dist[topk_indices].tolist()
+    frames = [(idx2fn[int(idx)], p) for idx, p in zip(topk_indices, prob)]
+    output_fp.write(json.dumps({
+        'tokens': tokens,
+        'frames': frames,
+        'kairos': kairos_label
+    }) + '\n')

spanfinder/scripts/aida_experiment/read_aida.py

@@ -0,0 +1,107 @@
+import json
+import os
+import copy
+from collections import defaultdict
+from argparse import ArgumentParser
+from tqdm import tqdm
+
+
+def extract_sentences(raw_doc):
+    sentence_tokens = list()  # [(start, end), list_tokens, event_list]
+    for sent_boundary in raw_doc['_views']['_InitialView']['Sentence']:
+        start, end = sent_boundary.get('begin', 0), sent_boundary.get('end')
+        sentence_tokens.append([(start, end), list(), list()])
+    begin2sentence, end2sentence = dict(), dict()
+    for token in raw_doc['_views']['_InitialView']['Token']:
+        start, end = token.get('begin', 0), token.get('end')
+        added = False
+        for sent_idx, (bound, tl, _) in enumerate(sentence_tokens):
+            if start in range(*bound) and (end - 1) in range(*bound):
+                assert not added
+                begin2sentence[start] = (sent_idx, len(tl))
+                end2sentence[end] = (sent_idx, len(tl))
+                tl.append((start, end))
+                added = True
+        assert added
+    return sentence_tokens, begin2sentence, end2sentence
+
+
+def read_aida2kairos(mapping_path):
+    mapping = dict()
+    for line in open(mapping_path).readlines():
+        kairos, aida_list = line.replace('\n', '').replace(',', '').split('\t')
+        for aida in aida_list.split():
+            if aida in 'x?':
+                continue
+            if aida in mapping:
+                print('warning:', aida, 'already in the mapping, repeated.')
+            mapping[aida] = kairos
+    return mapping
+
+
+def read_aida(corpus_path, mapping_path):
+    print('reading aida data')
+    n_negative, n_span_mismatch, n_diff = 0, 0, 0
+    outputs = list()
+    mapping = read_aida2kairos(mapping_path)
+    for event_fn in tqdm(os.listdir(corpus_path)):
+        event_name = event_fn.split('-')[0]
+        if event_name not in mapping:
+            print('warning:', event_name, 'not in the mapping.')
+            continue
+        event_name = mapping[event_name]
+
+        for doc_name in os.listdir(os.path.join(corpus_path, event_fn)):
+            if not doc_name.endswith('json'):
+                continue
+            raw_doc = json.load(open(os.path.join(corpus_path, event_fn, doc_name)))
+            sentences, begin2sentence, end2sentence = extract_sentences(raw_doc)
+            for fss_no, fss in raw_doc['_referenced_fss'].items():
+                if fss_no == '1':
+                    continue
+                begin, end, is_negative = fss['begin'], fss['end'], fss['negative_example']
+                if is_negative:
+                    n_negative += 1
+                    continue
+                if begin not in begin2sentence or end not in end2sentence:
+                    n_span_mismatch += 1
+                    continue
+                (b_idx_sent, b_idx_token), (e_idx_sent, e_idx_token) = begin2sentence[begin], end2sentence[end]
+                if b_idx_sent != e_idx_sent:
+                    n_diff += 1
+                    continue
+                sentences[b_idx_sent][2].append([b_idx_token, e_idx_token])
+
+            text = raw_doc['_referenced_fss']['1']['sofaString']
+
+            for _, tokens, events in sentences:
+                tokens = [text[start:end] for start, end in tokens]
+                for (start, end) in events:
+                    outputs.append({
+                        'tokens': copy.deepcopy(tokens),
+                        'annotation': {
+                            'start_idx': start,
+                            'end_idx': end,
+                            'label': event_name,
+                        }
+                    })
+
+    print(f'Loaded {len(outputs)} annotations.')
+    print(f'{n_negative} negative annotations are ignored.')
+    print(f'{n_span_mismatch} mismatched annotations are ignored.')
+    print(f'{n_diff} annotations across sentences are ignored.')
+
+    return outputs
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('aida', type=str)
+    parser.add_argument('aida2kairos', type=str)
+    parser.add_argument('dst', type=str)
+    args = parser.parse_args()
+
+    aida = read_aida(args.aida, args.aida2kairos)
+
+    json.dump(aida, open(args.dst, 'w'))
+

spanfinder/scripts/aida_experiment/test_mapping.py

@@ -0,0 +1,59 @@
+import json
+import os
+import copy
+from collections import defaultdict
+from argparse import ArgumentParser
+from tqdm import tqdm
+import random
+from tqdm import tqdm
+from scripts.predict_concrete import read_kairos
+
+from sftp import SpanPredictor
+
+
+parser = ArgumentParser()
+parser.add_argument('aida', type=str)
+parser.add_argument('model', type=str)
+parser.add_argument('fn2kairos', type=str, default=None)
+parser.add_argument('--device', type=int, default=3)
+args = parser.parse_args()
+
+corpus = json.load(open(args.aida))
+mapping = read_kairos(args.fn2kairos)
+predictor = SpanPredictor.from_path(args.model, cuda_device=args.device)
+random.seed(42)
+random.shuffle(corpus)
+batch_size = 128
+
+
+def batchify(a_list):
+    cur = list()
+    for item in a_list:
+        cur.append(item)
+        if len(cur) == batch_size:
+            yield cur
+            cur = list()
+    if len(cur) > 0:
+        yield cur
+
+
+batches = list(batchify(corpus))
+
+
+n_total = n_pos = n_span_match = 0
+for idx, lines in tqdm(enumerate(batches)):
+    n_total += batch_size
+    prediction_lines = predictor.predict_batch_sentences(
+        [line['tokens'] for line in lines], max_tokens=1024, ontology_mapping=mapping
+    )
+    for preds, ann in zip(prediction_lines, lines):
+        ann = ann['annotation']
+        preds = preds['prediction']
+        for pred in preds:
+            if pred['start_idx'] == ann['start_idx'] and pred['end_idx'] == ann['end_idx']:
+                n_span_match += 1
+                if pred['label'] == ann['label']:
+                    n_pos += 1
+
+    print(f'exact match precision: {n_pos * 100 / n_total:.3f}')
+    print(f'span only precision: {n_span_match * 100 / n_total:.3f}')

spanfinder/scripts/archive/eval_tie.py

@@ -0,0 +1,50 @@
+import os
+import sys
+import json
+from pprint import pprint
+from collections import defaultdict
+
+from sftp.metrics.exact_match import ExactMatch
+
+
+def evaluate():
+    em = ExactMatch(True)
+    sm = ExactMatch(False)
+    gold_file, pred_file = sys.argv[1:]
+    test_sentences = {json.loads(line)['meta']['sentence ID']: json.loads(line) for line in open(gold_file).readlines()}
+    pred_sentences = defaultdict(list)
+    for line in open(pred_file).readlines():
+        one_pred = json.loads(line)
+        pred_sentences[one_pred['meta']['sentence ID']].append(one_pred)
+    for sent_id, gold_sent in test_sentences.items():
+        pred_sent = pred_sentences.get(sent_id, [])
+        pred_frames, pred_fes = [], []
+        for fr_idx, fr in enumerate(pred_sent):
+            pred_frames.append({key: fr[key] for key in ["start_idx", "end_idx", "label"]})
+            pred_frames[-1]['parent'] = 0
+            for fe in fr['children']:
+                pred_fes.append({key: fe[key] for key in ["start_idx", "end_idx", "label"]})
+                pred_fes[-1]['parent'] = fr_idx+1
+        pred_to_eval = pred_frames + pred_fes
+
+        gold_frames, gold_fes = [], []
+        for fr_idx, fr in enumerate(gold_sent['frame']):
+            gold_frames.append({
+                'start_idx': fr['target'][0], 'end_idx': fr['target'][-1], "label": fr['name'], 'parent': 0
+            })
+            for start_idx, end_idx, fe_name in fr['fe']:
+                gold_fes.append({
+                    "start_idx": start_idx, "end_idx": end_idx, "label": fe_name, "parent": fr_idx+1
+                })
+        gold_to_eval = gold_frames + gold_fes
+        em(pred_to_eval, gold_to_eval)
+        sm(pred_to_eval, gold_to_eval)
+
+    print('EM')
+    pprint(em.get_metric(True))
+    print('SM')
+    pprint(sm.get_metric(True))
+
+
+if __name__ == '__main__':
+    evaluate()

spanfinder/scripts/archive/frame_similarity.py

@@ -0,0 +1,143 @@
+from argparse import ArgumentParser
+from collections import defaultdict
+
+from torch import nn
+from copy import deepcopy
+import torch
+import os
+import json
+
+from sftp import SpanPredictor
+import nltk
+
+
+def shift_grid_cos_sim(mat: torch.Tensor):
+    mat1 = mat.unsqueeze(0).expand(mat.shape[0], -1, -1)
+    mat2 = mat.unsqueeze(1).expand(-1, mat.shape[0], -1)
+    cos = nn.CosineSimilarity(2)
+    sim = (cos(mat1, mat2) + 1) / 2
+    return sim
+
+
+def all_frames():
+    nltk.download('framenet_v17')
+    fn = nltk.corpus.framenet
+    return fn.frames()
+
+
+def extract_relations(fr):
+    ret = list()
+    added = {fr.name}
+    for rel in fr.frameRelations:
+        for key in ['subFrameName', 'superFrameName']:
+            rel_fr_name = rel[key]
+            if rel_fr_name in added:
+                continue
+            ret.append((rel_fr_name, key[:-4]))
+    return ret
+
+
+def run():
+    parser = ArgumentParser()
+    parser.add_argument('archive', metavar='ARCHIVE_PATH', type=str)
+    parser.add_argument('dst', metavar='DESTINATION', type=str)
+    parser.add_argument('kairos', metavar='KAIROS', type=str)
+    parser.add_argument('--topk', metavar='TOPK', type=int, default=10)
+    args = parser.parse_args()
+
+    predictor = SpanPredictor.from_path(args.archive, cuda_device=-1)
+    kairos_gold_mapping = json.load(open(args.kairos))
+
+    label_emb = predictor._model._span_typing.label_emb.weight.clone().detach()
+    idx2label = predictor._model.vocab.get_index_to_token_vocabulary('span_label')
+
+    emb_sim = shift_grid_cos_sim(label_emb)
+    fr2definition = {fr.name: (fr.URL, fr.definition) for fr in all_frames()}
+
+    last_mlp = predictor._model._span_typing.MLPs[-1].weight.detach().clone()
+    mlp_sim = shift_grid_cos_sim(last_mlp)
+
+    def rank_frame(sim):
+        rank = sim.argsort(1, True)
+        scores = sim.gather(1, rank)
+        mapping = {
+            fr.name: {
+                'similarity': list(),
+                'ontology': extract_relations(fr),
+                'URL': fr.URL,
+                'definition': fr.definition
+            } for fr in all_frames()
+        }
+        for left_idx, (right_indices, match_scores) in enumerate(zip(rank, scores)):
+            left_label = idx2label[left_idx]
+            if left_label not in mapping:
+                continue
+            for right_idx, s in zip(right_indices, match_scores):
+                right_label = idx2label[int(right_idx)]
+                if right_label not in mapping or right_idx == left_idx:
+                    continue
+                mapping[left_label]['similarity'].append((right_label, float(s)))
+        return mapping
+
+    emb_map = rank_frame(emb_sim)
+    mlp_map = rank_frame(mlp_sim)
+
+    def dump(mapping, folder_path):
+        os.makedirs(folder_path, exist_ok=True)
+        json.dump(mapping, open(os.path.join(folder_path, 'raw.json'), 'w'))
+        sim_lines, onto_lines = list(), list()
+
+        for fr, values in mapping.items():
+            sim_line = [
+                fr,
+                values['definition'],
+                values['URL'],
+            ]
+            onto_line = deepcopy(sim_line)
+            for rel_fr_name, rel_type in values['ontology']:
+                onto_line.append(f'{rel_fr_name} ({rel_type})')
+                onto_lines.append('\t'.join(onto_line))
+            if len(values['similarity']) > 0:
+                for sim_fr_name, score in values['similarity'][:args.topk]:
+                    sim_line.append(f'{sim_fr_name} ({score:.3f})')
+                sim_lines.append('\t'.join(sim_line))
+
+        with open(os.path.join(folder_path, 'similarity.tsv'), 'w') as fp:
+            fp.write('\n'.join(sim_lines))
+        with open(os.path.join(folder_path, 'ontology.tsv'), 'w') as fp:
+            fp.write('\n'.join(onto_lines))
+
+        kairos_dump = list()
+        for kairos_event, kairos_content in kairos_gold_mapping.items():
+            for gold_fr in kairos_content['framenet']:
+                gold_fr = gold_fr['label']
+                if gold_fr not in fr2definition:
+                    continue
+                kairos_dump.append([
+                    'GOLD',
+                    gold_fr,
+                    kairos_event,
+                    fr2definition[gold_fr][0],
+                    fr2definition[gold_fr][1],
+                    str(kairos_content['description']),
+                    '1.00'
+                ])
+                for ass_fr, sim_score in mapping[gold_fr]['similarity'][:args.topk]:
+                    kairos_dump.append([
+                        '',
+                        ass_fr,
+                        kairos_event,
+                        fr2definition[ass_fr][0],
+                        fr2definition[ass_fr][1],
+                        str(kairos_content['description']),
+                        f'{sim_score:.2f}'
+                    ])
+        kairos_dump = list(map(lambda line: '\t'.join(line), kairos_dump))
+        open(os.path.join(folder_path, 'kairos_sheet.tsv'), 'w').write('\n'.join(kairos_dump))
+
+    dump(mlp_map, os.path.join(args.dst, 'mlp'))
+    dump(emb_map, os.path.join(args.dst, 'emb'))
+
+
+if __name__ == '__main__':
+    run()

spanfinder/scripts/archive/kairos_mapping.py

@@ -0,0 +1,43 @@
+import argparse
+import os
+import json
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('map', metavar='MappingFile', type=str, help="Mapping JSON file.")
+    parser.add_argument('src', metavar='SourceFile', type=str, help="Results of span finder.")
+    parser.add_argument('dst', metavar='Destination', type=str, help="Output path.")
+    args = parser.parse_args()
+    assert os.path.exists(args.map), "Mapping file doesn't exist."
+    assert os.path.exists(args.src), "Rouce file not found."
+
+    k_raw = json.load(open(args.map))
+    k_map = dict()
+    for kairos_event, content in k_raw.items():
+        for fr in content['framenet']:
+            if fr['label'] in k_map:
+                print("Duplicate frame: " + fr['label'])
+            k_map[fr['label']] = kairos_event
+    inputs = list(map(json.loads, open(args.src).readlines()))
+
+    n_total = n_mapped = 0
+
+    for line in inputs:
+        new_frames = list()
+        n_total += len(line['prediction'])
+        for fr in line['prediction']:
+            if fr['label'] in k_map:
+                fr['label'] = k_map[fr['label']]
+                new_frames.append(fr)
+                n_mapped += 1
+        line['prediction'] = new_frames
+
+    with open(args.dst, 'w') as fp:
+        fp.write('\n'.join(map(json.dumps, inputs)))
+
+    print(f'Done. Among {n_total} frames, {n_mapped} are mapped to KAIROS ontology, others are omitted.')
+
+
+if __name__ == '__main__':
+    main()

spanfinder/scripts/archive/onto_test.py

@@ -0,0 +1,34 @@
+import json
+from tools.framenet.naive_identifier import FrameIdentifier
+
+test_file_path = '/home/gqin2/data/framenet/full/test.jsonl'
+test_sentences = [
+    json.loads(line) for line in open(test_file_path)
+]
+test_set = []
+for ann in test_sentences:
+    for fr in ann['frame']:
+        test_set.append((fr['name'], ann['text'][fr['target'][0]: fr['target'][-1]+1], fr['lu']))
+
+fi = FrameIdentifier()
+
+
+tp = fp = fn = 0
+fails = []
+for frame, target_words, lu in test_set:
+    pred = fi(target_words)
+    if frame in pred:
+        tp += 1
+        fp += len(pred) - 1
+    else:
+        fp += len(pred)
+        fn += 1
+        fails.append((frame, target_words, pred, lu))
+
+fails.sort(key=lambda x: x[0])
+for frame, target_words, pred, lu in fails:
+    print(frame, ' '.join(target_words), ' '.join(pred), lu, sep='\t')
+
+print(f'tp={tp}, fp={fp}, fn={fn}')
+print(f'precision={tp/(tp+fp)}')
+print(f'recall={tp/(tp+fn)}')