Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1 @@
+./saved_models

ReadMe.md

@@ -0,0 +1,47 @@
+Official code for the paper ["Systematic Investigation of Strategies Tailored for Low-Resource Settings for Low-Resource Dependency Parsing"](https://arxiv.org/abs/2201.11374).
+If you use this code please cite our paper.
+
+## Requirements
+
+* Python 3.7 
+* Pytorch 1.1.0 
+* Cuda 9.0 
+* Gensim 3.8.1
+
+We assume that you have installed conda beforehand. 
+
+```
+conda install pytorch==1.1.0 torchvision==0.3.0 cudatoolkit=9.0 -c pytorch
+pip install gensim==3.8.1
+```
+
+## Pretrained embeddings for Sanskrit
+* Pretrained FastText embeddings for STBC/VST can be obtained from [here](https://drive.google.com/drive/folders/1SwdEqikTq-N2vOL7QSUX2vqi3faZE7bq?usp=sharing). Make sure that `.txt` file is placed at `data/`
+* The main results are reported on the systems trained by combining train and dev splits. 
+
+
+## How to train model for Sanskrit
+To run proposed system: (1) Pretraining (2) Integration, then simply run bash script `run_STBC.sh` or `run_VST.sh` for the respective dataset. With these scripts you will be able to reproduce our results reported in Section-3 and Table 2.
+
+```bash
+bash run_STBC.sh
+
+```
+
+## Citations
+```
+@misc{sandhan_systematic,
+  doi = {10.48550/ARXIV.2201.11374},
+  url = {https://arxiv.org/abs/2201.11374},
+  author = {Sandhan, Jivnesh and Behera, Laxmidhar and Goyal, Pawan},
+  keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences},
+  title = {Systematic Investigation of Strategies Tailored for Low-Resource Settings for Low-Resource Dependency Parsing},
+  publisher = {arXiv},
+  year = {2022},
+  copyright = {Creative Commons Attribution 4.0 International}
+}
+
+```
+
+## Acknowledgements
+Our ensembled system is built on the top of ["DCST Implementation"](https://github.com/rotmanguy/DCST)

examples/BiAFF_macro_UAS_LAS.py

@@ -0,0 +1,108 @@
+import sys
+
+
+def load_results(filename):
+
+    results = []
+    sent = []
+    with open(filename, 'r') as fp:
+        for i, line in enumerate(fp):
+            if i == 0:
+                continue
+            splits = line.strip().split('\t')
+            if len(line.strip()) == 0:
+                if len(sent) != 0:
+                    results.append(sent)
+                    sent = []
+                continue
+            gold_head = splits[-4]
+            gold_label = splits[-3]
+            pred_head = splits[-2]
+            pred_label = splits[-1]
+            sent.append((gold_head, gold_label, pred_head, pred_label))
+    print('Total Number of sentences ' + str(len(results)))
+    return results
+
+def calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels):
+
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    for i in range(len(gold_heads)):
+        if gold_heads[i] == pred_heads[i]:
+            u_correct +=1
+        u_total +=1
+        l_total +=1
+        if gold_heads[i] == pred_heads[i] and gold_labels[i] == pred_labels[i]:
+            l_correct +=1
+    return u_correct, u_total, l_correct, l_total
+
+
+def calculate_stats(results,path):
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    sent_uas = []
+    sent_las = []
+
+    for i in range(len(results)):
+        gold_heads, gold_labels, pred_heads, pred_labels = zip(*results[i])
+        u_c, u_t, l_c, l_t = calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels)
+        if u_t >0:
+            uas = float(u_c)/u_t
+            las = float(l_c)/l_t
+            sent_uas.append(uas)
+            sent_las.append(las)
+        u_correct += u_c
+        l_correct += l_c
+        u_total += u_t
+        l_total += l_t
+
+    UAS = float(u_correct)/u_total
+    LAS = float(l_correct)/l_total
+    path = path.replace('combined_1300_test.txt','Macro-UAS-LAS-score.txt')
+    f = open(path,'w')
+    f.write('Word level UAS : ' + str(UAS) +'\n')
+    f.write('Word level LAS : ' + str(LAS)+'\n')
+    f.write('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas))+'\n')
+    f.write('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las))+'\n')
+    f.close()
+    print('Word level UAS : ' + str(UAS))
+    print('Word level LAS : ' + str(LAS))
+    print('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas)))
+    print('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las)))
+
+    return sent_uas, sent_las, UAS, LAS
+
+def write_results(sent_uas, sent_las, filename_uas, filename_las):
+
+    fp_uas = open(filename_uas, 'w')
+    fp_las = open(filename_las, 'w')
+
+    for i in range(len(sent_uas)):
+        fp_uas.write(str(sent_uas[i]) + '\n')
+        fp_las.write(str(sent_las[i]) + '\n')
+
+    fp_uas.close()
+    fp_las.close()
+
+
+if __name__=="__main__":
+    dirs = sys.argv[1]
+    # results_2 = load_results(sys.argv[2])
+    ##path = "Predictions/Yap/"+dirs
+    # path = "/home/jivnesh/Documents/San-SOTA/saved_models/"+dirs+"/final_ensembled/dev_combined_1000.txt"
+    path = "./saved_models/"+dirs+"/combined_1300_test.txt"
+    result = load_results(path)
+
+
+    sent_uas1, sent_las1, UAS1, LAS1 = calculate_stats(result,path)
+    # sent_uas2, sent_las2, UAS2, LAS2 = calculate_stats(results_2)
+
+
+    write_results(sent_uas1, sent_las1, 'results1_uas.txt', 'results1_las.txt')
+    # write_results(sent_uas2, sent_las2, 'results2_uas.txt', 'results2_las.txt')

examples/BiAFF_write_1300_combined.py

@@ -0,0 +1,48 @@
+import sys
+
+def write_combined(dirs):
+    path = "./saved_models/"+dirs+'/'
+    f = open(path+'domain_san_test_model_domain_san_data_domain_san_gold.txt','r')
+    gold = f.readlines()
+    f.close()
+    f = open(path+'domain_san_test_model_domain_san_data_domain_san_pred.txt','r')
+    pred = f.readlines()
+    f.close()
+
+    for i in range(len(gold)):
+        if gold[i] == '\n':
+            continue
+        if gold[i].split('\t')[0] == pred[i].split('\t')[0]:
+            gold[i] = gold[i].replace('\n','\t')
+            gold[i] = gold[i]+'\t'.join(pred[i].split('\t')[-2:])
+
+    f = open(path+'domain_san_prose_model_domain_san_data_domain_san_gold.txt','r')
+    prose_gold = f.readlines()
+    f.close()
+    f = open(path+'domain_san_prose_model_domain_san_data_domain_san_pred.txt','r')
+    prose_pred = f.readlines()
+    f.close()
+
+    for i in range(len(prose_gold)):
+        if prose_gold[i] == '\n':
+            gold.append('\n')
+            continue
+        if prose_gold[i].split('\t')[0] == prose_pred[i].split('\t')[0]:
+            line = prose_gold[i].replace('\n','\t')
+            line =line+'\t'.join(prose_pred[i].split('\t')[-2:])
+            gold.append(line)
+    gold.insert(0,'word_id\tword\tpostag\tlemma\tgold_head\tgold_label\tpred_head\tpred_label\n\n')
+
+
+    f = open(path+'combined_1300_test.txt','w')
+    for line in gold:
+        f.write(line)
+    f.close()
+
+
+if __name__=="__main__":
+
+    dir_path = sys.argv[1]
+
+    write_combined(dir_path)
+

examples/GraphParser.py

@@ -0,0 +1,599 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs
+
+sys.path.append(".")
+sys.path.append("..")
+
+import argparse
+from copy import deepcopy
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from utils.io_ import seeds, Writer, get_logger, prepare_data, rearrange_splits
+from utils.models.parsing_gating import BiAffine_Parser_Gated
+from utils import load_word_embeddings
+from utils.tasks import parse
+import time
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+import uuid
+
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('GraphParser')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling GraphParser')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain/language', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--gating',action='store_true', help='use gated mechanism')
+    args_.add_argument('--num_gates', type=int, default=0, help='number of gates for gating mechanism')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--arc_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--arc_tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--arc_decode', choices=['mst', 'greedy'], help='arc decoding algorithm', required=True)
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--freeze_sequence_taggers', action='store_true', help='frozen the BiLSTMs of the pre-trained taggers.')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--set_num_training_samples', type=int, help='downsampling training set to a fixed number of samples')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--load_sequence_taggers_paths', nargs='+', help='path for loading saved sequence_tagger saved_models files.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contin '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['gating'] = args.gating
+    args_dict['num_gates'] = args.num_gates
+    args_dict['arc_decode'] = args.arc_decode
+    # args_dict['splits'] = ['train', 'dev', 'test']
+    args_dict['splits'] = ['train', 'dev', 'test','poetry','prose']
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['data_paths'] = {}
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/onto_pos_ner_dp'
+    else:
+        data_path = 'data/ud_pos_ner_dp'
+    for split in args_dict['splits']:
+        args_dict['data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain']
+    ###################################    
+    args_dict['data_paths']['poetry'] = 'data/ud_pos_ner_dp' + '_' + 'poetry' + '_' + args_dict['domain']
+    args_dict['data_paths']['prose'] = 'data/ud_pos_ner_dp' + '_' + 'prose' + '_' + args_dict['domain']
+    ###################################
+    args_dict['alphabet_data_paths'] = {}
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                args_dict['alphabet_data_paths'][split] = args_dict['data_paths'][split]
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['load_path'] = args.load_path
+    args_dict['load_sequence_taggers_paths'] = args.load_sequence_taggers_paths
+    if args_dict['load_sequence_taggers_paths'] is not None:
+        args_dict['gating'] = True
+        args_dict['num_gates'] = len(args_dict['load_sequence_taggers_paths']) + 1
+    else:
+        if not args_dict['gating']:
+            args_dict['num_gates'] = 0
+    args_dict['strict'] = args.strict
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['arc_space'] = args.arc_space
+    args_dict['arc_tag_space'] = args.arc_tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['set_num_training_samples'] = args.set_num_training_samples
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['freeze_sequence_taggers'] = args.freeze_sequence_taggers
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_data_paths'], args_dict['word_dict'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_data_paths, word_dict):
+    train_paths = alphabet_data_paths['train']
+    extra_paths = [v for k,v in alphabet_data_paths.items() if k != 'train']
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets(alphabet_path,
+                                                               train_paths,
+                                                               extra_paths=extra_paths,
+                                                               max_vocabulary_size=100000,
+                                                               embedd_dict=word_dict)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in tokens_dict:
+            embedding = tokens_dict[token]
+        elif token.lower() in tokens_dict:
+            embedding = tokens_dict[token.lower()]
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt,
+             'dev_eval_dict': dev_eval_dict,
+             'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=True):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+
+    if strict:
+        generate_optimizer(args, args.learning_rate, model.parameters())
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        for state in optimizer.state.values():
+            for k, v in state.items():
+                if isinstance(v, torch.Tensor):
+                    state[k] = v.to(args.device)
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = BiAffine_Parser_Gated(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.num_arc,
+                            args.arc_space, args.arc_tag_space, args.num_gates,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out,  p_rnn=args.p_rnn,
+                            biaffine=True, arc_decode=args.arc_decode, initializer=args.initializer)
+    print(model)
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=args.strict)
+    if args.load_sequence_taggers_paths:
+        pretrained_dict = {}
+        model_dict = model.state_dict()
+        for idx, path in enumerate(args.load_sequence_taggers_paths):
+            print('Loading saved sequence_tagger from: %s' % path)
+            checkpoint = torch.load(path, map_location=args.device)
+            for k, v in checkpoint['model_state_dict'].items():
+                if 'rnn_encoder.' in k:
+                    pretrained_dict['extra_rnn_encoders.' + str(idx) + '.' + k.replace('rnn_encoder.', '')] = v
+        model_dict.update(pretrained_dict)
+        model.load_state_dict(model_dict)
+    if args.freeze_sequence_taggers:
+        print('Freezing Classifiers')
+        for name, parameter in model.named_parameters():
+            if 'extra_rnn_encoders' in name:
+                parameter.requires_grad = False
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['dp_uas'] = 0.0
+    eval_dict['dp_las'] = 0.0
+    eval_dict['epoch'] = 0
+    eval_dict['dp_ucorrect'] = 0.0
+    eval_dict['dp_lcorrect'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_ucomplete_match'] = 0.0
+    eval_dict['dp_lcomplete_match'] = 0.0
+    eval_dict['dp_ucorrect_nopunc'] = 0.0
+    eval_dict['dp_lcorrect_nopunc'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_ucomplete_match_nopunc'] = 0.0
+    eval_dict['dp_lcomplete_match_nopunc'] = 0.0
+    eval_dict['dp_root_correct'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] <= curr_dev_eval_dict['dp_lcorrect_nopunc'] or \
+              (dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] == curr_dev_eval_dict['dp_lcorrect_nopunc'] and
+               dev_eval_dict['in_domain']['dp_ucorrect_nopunc'] <= curr_dev_eval_dict['dp_ucorrect_nopunc'])
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        if args.set_num_training_samples is not None:
+            splits_to_write = datasets.keys()
+        else:
+            splits_to_write = ['dev', 'test']
+        for split in splits_to_write:
+            if split == 'dev':
+                eval_dict = dev_eval_dict['in_domain']
+            elif split == 'test':
+                eval_dict = test_eval_dict['in_domain']
+            else:
+                eval_dict = None
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        print("Saving best model")
+        save_checkpoint(best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        stats, stats_nopunc, stats_root, num_inst = parse.eval_(word, pos, heads_pred, arc_tags_pred, heads,
+                                                                arc_tags, args.alphabets['word_alphabet'], args.alphabets['pos_alphabet'],
+                                                                lengths, punct_set=args.punct_set, symbolic_root=True)
+        ucorr, lcorr, total, ucm, lcm = stats
+        ucorr_nopunc, lcorr_nopunc, total_nopunc, ucm_nopunc, lcm_nopunc = stats_nopunc
+        corr_root, total_root = stats_root
+        eval_dict['dp_ucorrect'] += ucorr
+        eval_dict['dp_lcorrect'] += lcorr
+        eval_dict['dp_total'] += total
+        eval_dict['dp_ucomplete_match'] += ucm
+        eval_dict['dp_lcomplete_match'] += lcm
+        eval_dict['dp_ucorrect_nopunc'] += ucorr_nopunc
+        eval_dict['dp_lcorrect_nopunc'] += lcorr_nopunc
+        eval_dict['dp_total_nopunc'] += total_nopunc
+        eval_dict['dp_ucomplete_match_nopunc'] += ucm_nopunc
+        eval_dict['dp_lcomplete_match_nopunc'] += lcm_nopunc
+        eval_dict['dp_root_correct'] += corr_root
+        eval_dict['dp_total_root'] += total_root
+        eval_dict['dp_total_inst'] += num_inst
+
+    eval_dict['dp_uas'] = eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    eval_dict['dp_las'] = eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- Dependency Parsing - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + '  W. Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect'], eval_dict['dp_lcorrect'], eval_dict['dp_total'],
+            eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_ucomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(
+        str_res + ' on ' + split + '  Wo Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect_nopunc'], eval_dict['dp_lcorrect_nopunc'], eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_lcorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(str_res + ' on ' + split + '  Root: corr: %d, total: %d, acc: %.2f%% (epoch: %d)' % (
+        eval_dict['dp_root_correct'], eval_dict['dp_total_root'],
+        eval_dict['dp_root_correct'] * 100 / eval_dict['dp_total_root'], eval_dict['epoch']))
+    print('\n')
+
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+    if eval_dict is not None:
+        # save results dictionary into a file
+        with open(res_filename, 'w') as f:
+            json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads_pred, arc_tags_pred, lengths, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        print("Splits are:",split)
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+    if args.set_num_training_samples is not None:
+        print('Setting train and dev to %d samples' % args.set_num_training_samples)
+        datasets = rearrange_splits.rearranging_splits(datasets, args.set_num_training_samples)
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_arc_loss = 0.0
+            total_arc_tag_loss = 0.0
+            total_train_inst = 0.0
+
+            train_iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(train_iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss_arc, loss_arc_tag = model.loss(out_arc, out_arc_tag, heads, arc_tags, mask=masks, length=lengths)
+                loss = loss_arc + loss_arc_tag
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_arc_loss += loss_arc.item() * num_insts
+                total_arc_tag_loss += loss_arc_tag.item() * num_insts
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                                total_arc_tag_loss / total_train_inst, time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                   total_arc_tag_loss / total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+        for split in datasets.keys():
+            eval_dict = evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train', 'dev', 'test','poetry','prose']:
+            eval_dict = evaluation(args, datasets[split], split, model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+
+if __name__ == '__main__':
+    main()

examples/GraphParser_MTL_POS.py

@@ -0,0 +1,633 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs
+
+sys.path.append(".")
+sys.path.append("..")
+
+import argparse
+from copy import deepcopy
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from utils.io_ import seeds, Writer, get_logger, prepare_data, rearrange_splits
+from utils.models.parsing_gating_mtl_pos import BiAffine_Parser_Gated
+# from utils.models.sequence_tagger import Sequence_Tagger
+from utils import load_word_embeddings
+from utils.tasks import parse
+import time
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+import uuid
+
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('GraphParser')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling GraphParser')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain/language', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--gating',action='store_true', help='use gated mechanism')
+    args_.add_argument('--num_gates', type=int, default=0, help='number of gates for gating mechanism')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--arc_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--arc_tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--arc_decode', choices=['mst', 'greedy'], help='arc decoding algorithm', required=True)
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--freeze_sequence_taggers', action='store_true', help='frozen the BiLSTMs of the pre-trained taggers.')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--set_num_training_samples', type=int, help='downsampling training set to a fixed number of samples')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--load_sequence_taggers_paths', nargs='+', help='path for loading saved sequence_tagger saved_models files.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contin '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['gating'] = args.gating
+    args_dict['num_gates'] = args.num_gates
+    args_dict['arc_decode'] = args.arc_decode
+    # args_dict['splits'] = ['train', 'dev', 'test']
+    args_dict['splits'] = ['train', 'dev', 'test','poetry','prose']
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['data_paths'] = {}
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/onto_pos_ner_dp'
+    else:
+        data_path = 'data/ud_pos_ner_dp'
+    for split in args_dict['splits']:
+        args_dict['data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain']
+    ###################################    
+    args_dict['data_paths']['poetry'] = data_path + '_' + 'poetry' + '_'  + args_dict['domain']
+    args_dict['data_paths']['prose'] = data_path + '_' + 'prose' + '_' + args_dict['domain']
+    # args_dict['data_paths']['poetry'] = 'data/Shishu_300' + '_' + 'poetry' + '_' + args_dict['domain']
+    # args_dict['data_paths']['prose'] = 'data/Shishu_300' + '_' + 'prose' + '_' + args_dict['domain']
+
+    ###################################
+    args_dict['alphabet_data_paths'] = {}
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                args_dict['alphabet_data_paths'][split] = args_dict['data_paths'][split]
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['load_path'] = args.load_path
+    args_dict['load_sequence_taggers_paths'] = args.load_sequence_taggers_paths
+    if args_dict['load_sequence_taggers_paths'] is not None:
+        args_dict['gating'] = True
+        args_dict['num_gates'] = len(args_dict['load_sequence_taggers_paths']) + 1
+    else:
+        if not args_dict['gating']:
+            args_dict['num_gates'] = 0
+    args_dict['strict'] = args.strict
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['arc_space'] = args.arc_space
+    args_dict['arc_tag_space'] = args.arc_tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['set_num_training_samples'] = args.set_num_training_samples
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['freeze_sequence_taggers'] = args.freeze_sequence_taggers
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_data_paths'], args_dict['word_dict'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_data_paths, word_dict):
+    train_paths = alphabet_data_paths['train']
+    extra_paths = [v for k,v in alphabet_data_paths.items() if k != 'train']
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets(alphabet_path,
+                                                               train_paths,
+                                                               extra_paths=extra_paths,
+                                                               max_vocabulary_size=100000,
+                                                               embedd_dict=word_dict)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in tokens_dict:
+            embedding = tokens_dict[token]
+        elif token.lower() in tokens_dict:
+            embedding = tokens_dict[token.lower()]
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt,
+             'dev_eval_dict': dev_eval_dict,
+             'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=False):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+
+    if strict:
+        generate_optimizer(args, args.learning_rate, model.parameters())
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        for state in optimizer.state.values():
+            for k, v in state.items():
+                if isinstance(v, torch.Tensor):
+                    state[k] = v.to(args.device)
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = BiAffine_Parser_Gated(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.num_arc,
+                            args.arc_space, args.arc_tag_space, args.num_gates,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out,  p_rnn=args.p_rnn,
+                            biaffine=True, arc_decode=args.arc_decode, initializer=args.initializer)
+    # Tagger = Sequence_Tagger(args.word_dim, args.num_word, args.char_dim, args.num_char,
+    #                         args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+    #                         args.num_filters, args.kernel_size, args.rnn_mode,
+    #                         args.hidden_size, args.num_layers, args.arc_tag_space, 519,
+    #                         embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+    #                         p_in=args.p_in, p_out=args.p_out, p_rnn=args.p_rnn,
+    #                         initializer=args.initializer)
+    print(model)
+    # print(Tagger)
+    # Tagger.rnn_encoder = model.rnn_encoder
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=False)
+    if args.load_sequence_taggers_paths:
+        pretrained_dict = {}
+        model_dict = model.state_dict()
+        for idx, path in enumerate(args.load_sequence_taggers_paths):
+            print('Loading saved sequence_tagger from: %s' % path)
+            checkpoint = torch.load(path, map_location=args.device)
+            for k, v in checkpoint['model_state_dict'].items():
+                if 'rnn_encoder.' in k:
+                    pretrained_dict['extra_rnn_encoders.' + str(idx) + '.' + k.replace('rnn_encoder.', '')] = v
+        model_dict.update(pretrained_dict)
+        model.load_state_dict(model_dict)
+    if args.freeze_sequence_taggers:
+        print('Freezing Classifiers')
+        for name, parameter in model.named_parameters():
+            if 'extra_rnn_encoders' in name:
+                parameter.requires_grad = False
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    # Tagger.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['dp_uas'] = 0.0
+    eval_dict['dp_las'] = 0.0
+    eval_dict['epoch'] = 0
+    eval_dict['dp_ucorrect'] = 0.0
+    eval_dict['dp_lcorrect'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_ucomplete_match'] = 0.0
+    eval_dict['dp_lcomplete_match'] = 0.0
+    eval_dict['dp_ucorrect_nopunc'] = 0.0
+    eval_dict['dp_lcorrect_nopunc'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_ucomplete_match_nopunc'] = 0.0
+    eval_dict['dp_lcomplete_match_nopunc'] = 0.0
+    eval_dict['dp_root_correct'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] <= curr_dev_eval_dict['dp_lcorrect_nopunc'] or \
+              (dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] == curr_dev_eval_dict['dp_lcorrect_nopunc'] and
+               dev_eval_dict['in_domain']['dp_ucorrect_nopunc'] <= curr_dev_eval_dict['dp_ucorrect_nopunc'])
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        if args.set_num_training_samples is not None:
+            splits_to_write = datasets.keys()
+        else:
+            splits_to_write = ['dev', 'test']
+        for split in splits_to_write:
+            if split == 'dev':
+                eval_dict = dev_eval_dict['in_domain']
+            elif split == 'test':
+                eval_dict = test_eval_dict['in_domain']
+            else:
+                eval_dict = None
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        print("Saving best model")
+        save_checkpoint(best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        stats, stats_nopunc, stats_root, num_inst = parse.eval_(word, pos, heads_pred, arc_tags_pred, heads,
+                                                                arc_tags, args.alphabets['word_alphabet'], args.alphabets['pos_alphabet'],
+                                                                lengths, punct_set=args.punct_set, symbolic_root=True)
+        ucorr, lcorr, total, ucm, lcm = stats
+        ucorr_nopunc, lcorr_nopunc, total_nopunc, ucm_nopunc, lcm_nopunc = stats_nopunc
+        corr_root, total_root = stats_root
+        eval_dict['dp_ucorrect'] += ucorr
+        eval_dict['dp_lcorrect'] += lcorr
+        eval_dict['dp_total'] += total
+        eval_dict['dp_ucomplete_match'] += ucm
+        eval_dict['dp_lcomplete_match'] += lcm
+        eval_dict['dp_ucorrect_nopunc'] += ucorr_nopunc
+        eval_dict['dp_lcorrect_nopunc'] += lcorr_nopunc
+        eval_dict['dp_total_nopunc'] += total_nopunc
+        eval_dict['dp_ucomplete_match_nopunc'] += ucm_nopunc
+        eval_dict['dp_lcomplete_match_nopunc'] += lcm_nopunc
+        eval_dict['dp_root_correct'] += corr_root
+        eval_dict['dp_total_root'] += total_root
+        eval_dict['dp_total_inst'] += num_inst
+
+    eval_dict['dp_uas'] = eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    eval_dict['dp_las'] = eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- Dependency Parsing - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + '  W. Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect'], eval_dict['dp_lcorrect'], eval_dict['dp_total'],
+            eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_ucomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(
+        str_res + ' on ' + split + '  Wo Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect_nopunc'], eval_dict['dp_lcorrect_nopunc'], eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_lcorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(str_res + ' on ' + split + '  Root: corr: %d, total: %d, acc: %.2f%% (epoch: %d)' % (
+        eval_dict['dp_root_correct'], eval_dict['dp_total_root'],
+        eval_dict['dp_root_correct'] * 100 / eval_dict['dp_total_root'], eval_dict['epoch']))
+    print('\n')
+
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+    if eval_dict is not None:
+        # save results dictionary into a file
+        with open(res_filename, 'w') as f:
+            json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads_pred, arc_tags_pred, lengths, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        print("Splits are:",split)
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+    if args.set_num_training_samples is not None:
+        print('Setting train and dev to %d samples' % args.set_num_training_samples)
+        datasets = rearrange_splits.rearranging_splits(datasets, args.set_num_training_samples)
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_mtl_pos_loss =0.0
+            total_mtl_case_loss =0.0
+            total_mtl_label_loss =0.0
+            total_arc_loss = 0.0
+            total_arc_tag_loss = 0.0
+            total_train_inst = 0.0
+
+            train_iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(train_iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss_arc, loss_arc_tag = model.loss(out_arc, out_arc_tag, heads, arc_tags, mask=masks, length=lengths)
+                ############################################################
+                loss_morph = model.loss_morph(word, char, pos, mask=masks, length=lengths)
+                # loss_case = model.loss_case(word, char, ner_tags, mask=masks, length=lengths)
+                # loss_label = model.loss_label(word, char, arc_tags, mask=masks, length=lengths)
+
+                ## Adding multi-tasking
+                # Tag_output, Tag_masks, Tag_lengths = Tagger.forward(word, char, pos, mask=masks, length=lengths)
+                # Tag_loss = Tagger.loss(Tag_output, pos, mask=Tag_masks, length=Tag_lengths)
+                # # update losses
+                # Tag_num_insts = Tag_masks.data.sum() - word.size(0)
+                # Tag_total_loss += Tag_loss.item() * Tag_num_insts
+                # Tag_total_train_inst += Tag_num_insts
+
+                #############################################################
+                loss = loss_arc + loss_arc_tag + loss_morph
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_arc_loss += loss_arc.item() * num_insts
+                total_arc_tag_loss += loss_arc_tag.item() * num_insts
+                total_mtl_pos_loss += loss_morph.item() * num_insts
+                # total_mtl_case_loss += loss_case.item() * num_insts
+                # total_mtl_label_loss += loss_label.item() * num_insts
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f,morph_loss: %.2f, case_loss: %.2f, label_loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                                total_arc_tag_loss / total_train_inst,total_mtl_pos_loss/total_train_inst,total_mtl_case_loss/total_train_inst,total_mtl_label_loss/total_train_inst,  time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f,morph_loss: %.2f, case_loss: %.2f, label_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                   total_arc_tag_loss / total_train_inst,total_mtl_pos_loss/total_train_inst,total_mtl_case_loss/total_train_inst,total_mtl_label_loss/total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+        for split in datasets.keys():
+            eval_dict = evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train', 'dev', 'test','poetry','prose']:
+            eval_dict = evaluation(args, datasets[split], split, model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+
+if __name__ == '__main__':
+    main()

examples/SequenceTagger.py

@@ -0,0 +1,589 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs, system, remove
+
+sys.path.append(".")
+sys.path.append("..")
+
+import time
+import argparse
+import uuid
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from copy import deepcopy
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+from utils.io_ import seeds, Writer, get_logger, Index2Instance, prepare_data, write_extra_labels
+from utils.models.sequence_tagger import Sequence_Tagger
+from utils import load_word_embeddings
+from utils.tasks.seqeval import accuracy_score, f1_score, precision_score, recall_score,classification_report
+
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('SequenceTagger')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling SequenceTagger')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--task', default='distance_from_the_root', choices=['distance_from_the_root', 'number_of_children',\
+     'relative_pos_based', 'language_model','add_label','add_head_coarse_pos','Multitask_POS_predict','Multitask_case_predict',\
+     'Multitask_label_predict','Multitask_coarse_predict','MRL_case','MRL_POS','MRL_no','MRL_label',\
+     'predict_coarse_of_modifier','predict_ma_tag_of_modifier','add_head_ma','predict_case_of_modifier'], help='sequence_tagger task')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--use_unlabeled_data', action='store_true', help='flag to use unlabeled data.')
+    args_.add_argument('--use_labeled_data', action='store_true', help='flag to use labeled data.')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--parser_path', help='path for loading parser files.', default=None)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contain '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['task'] = args.task
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['load_path'] = args.load_path
+    args_dict['strict'] = args.strict
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['parser_path'] = args.parser_path
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['use_unlabeled_data'] = args.use_unlabeled_data
+    args_dict['use_labeled_data'] = args.use_labeled_data
+    print(args_dict['parser_path'])
+    if args_dict['task'] == 'number_of_children':
+        args_dict['data_paths'] = write_extra_labels.add_number_of_children(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                            use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                            use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'distance_from_the_root':
+        args_dict['data_paths'] = write_extra_labels.add_distance_from_the_root(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_label_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_label_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_coarse_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_coarse_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_POS_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_POS_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'relative_pos_based':
+        args_dict['data_paths'] = write_extra_labels.add_relative_pos_based(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_label':
+        args_dict['data_paths'] = write_extra_labels.add_label(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_relative_TAG':
+        args_dict['data_paths'] = write_extra_labels.add_relative_TAG(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_head_coarse_pos':
+        args_dict['data_paths'] = write_extra_labels.add_head_coarse_pos(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_ma_tag_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_ma_tag_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_case_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_case_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_coarse_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_coarse_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_case_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_case_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_head_ma':
+        args_dict['data_paths'] = write_extra_labels.add_head_ma(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_case':
+        args_dict['data_paths'] = write_extra_labels.MRL_case(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_POS':
+        args_dict['data_paths'] = write_extra_labels.MRL_POS(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_no':
+        args_dict['data_paths'] = write_extra_labels.MRL_no(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_label':
+        args_dict['data_paths'] = write_extra_labels.MRL_label(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    else: #args_dict['task'] == 'language_model':
+        args_dict['data_paths'] = write_extra_labels.add_language_model(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                            use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                            use_labeled_data=args_dict['use_labeled_data'])
+    args_dict['splits'] = args_dict['data_paths'].keys()
+    alphabet_data_paths = deepcopy(args_dict['data_paths'])
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/onto_pos_ner_dp'
+    else:
+        data_path = 'data/ud_pos_ner_dp'
+    # Adding more resources to make sure equal alphabet size for all domains
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            alphabet_data_paths['additional_' + split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                alphabet_data_paths[split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                alphabet_data_paths[split] = args_dict['data_paths'][split]
+    args_dict['alphabet_data_paths'] = alphabet_data_paths
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['tag_space'] = args.tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['alphabet_parser_path'] = path.join(args_dict['parser_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_parser_path'], args_dict['alphabet_data_paths'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_parser_path, alphabet_data_paths):
+    data_paths_list = alphabet_data_paths.values()
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets_for_sequence_tagger(alphabet_path, alphabet_parser_path, data_paths_list)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_alphabet')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in tokens_dict:
+            embedding = tokens_dict[token]
+        elif token.lower() in tokens_dict:
+            embedding = tokens_dict[token.lower()]
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def get_free_gpu():
+    system('nvidia-smi -q -d Memory |grep -A4 GPU|grep Free > tmp.txt')
+    memory_available = [int(x.split()[2]) for x in open('tmp.txt', 'r').readlines()]
+    remove("tmp.txt")
+    free_device = 'cuda:' + str(np.argmax(memory_available))
+    return free_device
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt, 'dev_eval_dict': dev_eval_dict, 'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=True):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+    if strict:
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = Sequence_Tagger(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.tag_space, args.num_auto_label,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out, p_rnn=args.p_rnn,
+                            initializer=args.initializer)
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=args.strict)
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['auto_label_accuracy'] = 0.0
+    eval_dict['auto_label_precision'] = 0.0
+    eval_dict['auto_label_recall'] = 0.0
+    eval_dict['auto_label_f1'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['auto_label_f1'] <= curr_dev_eval_dict['auto_label_f1']
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        for split in ['dev', 'test']:
+            eval_dict = dev_eval_dict['in_domain'] if split == 'dev' else test_eval_dict['in_domain']
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        save_checkpoint(best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient, curr_dev_eval_dict
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+    auto_label_idx2inst = Index2Instance(args.alphabets['auto_label_alphabet'])
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    pred_labels = []
+    gold_labels = []
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        auto_label_preds = model.decode(output, mask=masks, length=lengths, leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        auto_label = auto_label.data.cpu().numpy()
+        auto_label_preds = auto_label_preds.data.cpu().numpy()
+        gold_labels += auto_label_idx2inst.index2instance(auto_label, lengths, symbolic_root=True)
+        pred_labels += auto_label_idx2inst.index2instance(auto_label_preds, lengths, symbolic_root=True)
+
+    eval_dict['auto_label_accuracy'] = accuracy_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_precision'] = precision_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_recall'] = recall_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_f1'] = f1_score(gold_labels, pred_labels) * 100
+    eval_dict['classification_report'] = classification_report(gold_labels, pred_labels)
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- sequence_tagger - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + ' accuracy: %.2f%%, precision: %.2f%%, recall: %.2f%%, F1: %.2f%% (epoch: %d)'
+        % (eval_dict['auto_label_accuracy'], eval_dict['auto_label_precision'], eval_dict['auto_label_recall'], eval_dict['auto_label_f1'],
+           eval_dict['epoch']))
+    print(eval_dict['classification_report'])
+
+
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+
+    # save results dictionary into a file
+    with open(res_filename, 'w') as f:
+        json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        auto_label_preds = model.decode(output, mask=masks, length=lengths,
+                                        leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        auto_label_preds = auto_label_preds.data.cpu().numpy()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads, arc_tags, lengths, auto_label=auto_label_preds, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, auto_label=auto_label, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    print(model)
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        terminal_patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_train_inst = 0.0
+
+            iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss = model.loss(output, auto_label, mask=masks, length=lengths)
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient,curr_dev_eval_dict = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            store ={'dev_eval_dict':curr_dev_eval_dict }
+            ############################################# 
+            str_file = args.full_model_name + '_' +'all_epochs'
+            with open(str_file,'a') as f:
+                f.write(str(store)+'\n')
+            if patient == 0:
+                terminal_patient = 0
+            else:
+                terminal_patient += 1
+            if terminal_patient >= 4 * args.schedule:
+                # Save best model and terminate learning
+                cur_epoch = epoch
+                epoch = args.num_epochs
+                in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model,
+                                     best_optimizer, patient)
+                log_info = 'Terminating training in epoch %d' % (cur_epoch)
+                sys.stdout.write(log_info)
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+                return
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train', 'dev', 'test']:
+            evaluation(args, datasets[split], split, model, args.domain, epoch, 'best_results')
+
+
+if __name__ == '__main__':
+    main()

examples/VST_Pred_Prepare.py

@@ -0,0 +1,34 @@
+import sys
+
+def write_combined(dirs):
+    path = "./saved_models/"+dirs+'/final_ensembled_TranSeq/'
+    f = open(path+'domain_VST_test_model_domain_VST_data_domain_VST_gold.txt','r')
+    gold = f.readlines()
+    f.close()
+    f = open(path+'domain_VST_test_model_domain_VST_data_domain_VST_pred.txt','r')
+    pred = f.readlines()
+    f.close()
+
+    for i in range(len(gold)):
+        if gold[i] == '\n':
+            continue
+        if gold[i].split('\t')[0] == pred[i].split('\t')[0]:
+            gold[i] = gold[i].replace('\n','\t')
+            gold[i] = gold[i]+'\t'.join(pred[i].split('\t')[-2:])
+
+
+    gold.insert(0,'word_id\tword\tpostag\tlemma\tgold_head\tgold_label\tpred_head\tpred_label\n\n')
+
+
+    f = open(path+'VST_test.txt','w')
+    for line in gold:
+        f.write(line)
+    f.close()
+
+
+if __name__=="__main__":
+
+    dir_path = sys.argv[1]
+
+    write_combined(dir_path)
+

examples/VST_macro_score.py

@@ -0,0 +1,107 @@
+import sys
+
+
+def load_results(filename):
+
+    results = []
+    sent = []
+    with open(filename, 'r') as fp:
+        for i, line in enumerate(fp):
+            if i == 0:
+                continue
+            splits = line.strip().split('\t')
+            if len(line.strip()) == 0:
+                if len(sent) != 0:
+                    results.append(sent)
+                    sent = []
+                continue
+            gold_head = splits[-4]
+            gold_label = splits[-3]
+            pred_head = splits[-2]
+            pred_label = splits[-1]
+            sent.append((gold_head, gold_label, pred_head, pred_label))
+    print('Total Number of sentences ' + str(len(results)))
+    return results
+
+def calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels):
+
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    for i in range(len(gold_heads)):
+        if gold_heads[i] == pred_heads[i]:
+            u_correct +=1
+        u_total +=1
+        l_total +=1
+        if gold_heads[i] == pred_heads[i] and gold_labels[i] == pred_labels[i]:
+            l_correct +=1
+    return u_correct, u_total, l_correct, l_total
+
+
+def calculate_stats(results,path):
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    sent_uas = []
+    sent_las = []
+
+    for i in range(len(results)):
+        gold_heads, gold_labels, pred_heads, pred_labels = zip(*results[i])
+        u_c, u_t, l_c, l_t = calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels)
+        if u_t >0:
+            uas = float(u_c)/u_t
+            las = float(l_c)/l_t
+            sent_uas.append(uas)
+            sent_las.append(las)
+        u_correct += u_c
+        l_correct += l_c
+        u_total += u_t
+        l_total += l_t
+
+    UAS = float(u_correct)/u_total
+    LAS = float(l_correct)/l_total
+    path = path.replace('VST_test.txt','Macro-UAS-LAS-score.txt')
+    f = open(path,'w')
+    f.write('Word level UAS : ' + str(UAS) +'\n')
+    f.write('Word level LAS : ' + str(LAS)+'\n')
+    f.write('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas))+'\n')
+    f.write('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las))+'\n')
+    f.close()
+    print('Word level UAS : ' + str(UAS))
+    print('Word level LAS : ' + str(LAS))
+    print('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas)))
+    print('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las)))
+
+    return sent_uas, sent_las, UAS, LAS
+
+def write_results(sent_uas, sent_las, filename_uas, filename_las):
+
+    fp_uas = open(filename_uas, 'w')
+    fp_las = open(filename_las, 'w')
+
+    for i in range(len(sent_uas)):
+        fp_uas.write(str(sent_uas[i]) + '\n')
+        fp_las.write(str(sent_las[i]) + '\n')
+
+    fp_uas.close()
+    fp_las.close()
+
+
+if __name__=="__main__":
+    dirs = sys.argv[1]
+    # results_2 = load_results(sys.argv[2])
+    ##path = "Predictions/Yap/"+dirs
+    path = "./saved_models/"+dirs+"/final_ensembled_TranSeq/VST_test.txt"
+    result = load_results(path)
+
+
+    sent_uas1, sent_las1, UAS1, LAS1 = calculate_stats(result,path)
+    # sent_uas2, sent_las2, UAS2, LAS2 = calculate_stats(results_2)
+
+
+    write_results(sent_uas1, sent_las1, 'results1_uas.txt', 'results1_las.txt')
+    # write_results(sent_uas2, sent_las2, 'results2_uas.txt', 'results2_las.txt')

examples/eval/conll03eval.v2

@@ -0,0 +1,336 @@
+#!/usr/bin/perl -w
+# conlleval: evaluate result of processing CoNLL-2000 shared task
+# usage:     conlleval [-l] [-r] [-d delimiterTag] [-o oTag] < file
+#            README: http://cnts.uia.ac.be/conll2000/chunking/output.html
+# options:   l: generate LaTeX output for tables like in
+#               http://cnts.uia.ac.be/conll2003/ner/example.tex
+#            r: accept raw result tags (without B- and I- prefix;
+#                                       assumes one word per chunk)
+#            d: alternative delimiter tag (default is single space)
+#            o: alternative outside tag (default is O)
+# note:      the file should contain lines with items separated
+#            by $delimiter characters (default space). The final
+#            two items should contain the correct tag and the 
+#            guessed tag in that order. Sentences should be
+#            separated from each other by empty lines or lines
+#            with $boundary fields (default -X-).
+# url:       http://lcg-www.uia.ac.be/conll2000/chunking/
+# started:   1998-09-25
+# version:   2004-01-26
+# author:    Erik Tjong Kim Sang <erikt@uia.ua.ac.be>
+
+use strict;
+
+my $false = 0;
+my $true = 42;
+
+my $boundary = "-X-";     # sentence boundary
+my $correct;              # current corpus chunk tag (I,O,B)
+my $correctChunk = 0;     # number of correctly identified chunks
+my $correctTags = 0;      # number of correct chunk tags
+my $correctType;          # type of current corpus chunk tag (NP,VP,etc.)
+my $delimiter = " ";      # field delimiter
+my $FB1 = 0.0;            # FB1 score (Van Rijsbergen 1979)
+my $firstItem;            # first feature (for sentence boundary checks)
+my $foundCorrect = 0;     # number of chunks in corpus
+my $foundGuessed = 0;     # number of identified chunks
+my $guessed;              # current guessed chunk tag
+my $guessedType;          # type of current guessed chunk tag
+my $i;                    # miscellaneous counter
+my $inCorrect = $false;   # currently processed chunk is correct until now
+my $lastCorrect = "O";    # previous chunk tag in corpus
+my $latex = 0;            # generate LaTeX formatted output
+my $lastCorrectType = ""; # type of previously identified chunk tag
+my $lastGuessed = "O";    # previously identified chunk tag
+my $lastGuessedType = ""; # type of previous chunk tag in corpus
+my $lastType;             # temporary storage for detecting duplicates
+my $line;                 # line
+my $nbrOfFeatures = -1;   # number of features per line
+my $precision = 0.0;      # precision score
+my $oTag = "O";           # outside tag, default O
+my $raw = 0;              # raw input: add B to every token
+my $recall = 0.0;         # recall score
+my $tokenCounter = 0;     # token counter (ignores sentence breaks)
+
+my %correctChunk = ();    # number of correctly identified chunks per type
+my %foundCorrect = ();    # number of chunks in corpus per type
+my %foundGuessed = ();    # number of identified chunks per type
+
+my @features;             # features on line
+my @sortedTypes;          # sorted list of chunk type names
+
+# sanity check
+while (@ARGV and $ARGV[0] =~ /^-/) {
+   if ($ARGV[0] eq "-l") { $latex = 1; shift(@ARGV); }
+   elsif ($ARGV[0] eq "-r") { $raw = 1; shift(@ARGV); }
+   elsif ($ARGV[0] eq "-d") { 
+      shift(@ARGV); 
+      if (not defined $ARGV[0]) { 
+         die "conlleval: -d requires delimiter character"; 
+      }
+      $delimiter = shift(@ARGV);
+   } elsif ($ARGV[0] eq "-o") {
+      shift(@ARGV);
+      if (not defined $ARGV[0]) {
+         die "conlleval: -o requires delimiter character";
+      }
+      $oTag = shift(@ARGV);
+   } else { die "conlleval: unknown argument $ARGV[0]\n"; }
+}
+if (@ARGV) { die "conlleval: unexpected command line argument\n"; }
+# process input
+while (<STDIN>) {
+   chomp($line = $_);
+   @features = split(/$delimiter/,$line);
+   if ($nbrOfFeatures < 0) { $nbrOfFeatures = $#features; }
+   elsif ($nbrOfFeatures != $#features and @features != 0) {
+      printf STDERR "unexpected number of features: %d (%d)\n",
+         $#features+1,$nbrOfFeatures+1;
+      exit(1);
+   }
+   if (@features == 0 or 
+       $features[0] eq $boundary) { @features = ($boundary,"O","O"); }
+   if (@features < 2) { 
+      die "conlleval: unexpected number of features in line $line\n"; 
+   }
+   if ($raw) {
+      if ($features[$#features] eq $oTag) { $features[$#features] = "O"; } 
+      if ($features[$#features-1] eq $oTag) { $features[$#features-1] = "O"; } 
+      if ($features[$#features] ne "O") { 
+         $features[$#features] = "B-$features[$#features]";
+      }
+      if ($features[$#features-1] ne "O") { 
+         $features[$#features-1] = "B-$features[$#features-1]";
+      }
+   }
+   # 20040126 ET code which allows hyphens in the types
+   if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
+      $guessed = $1;
+      $guessedType = $2;
+   } else { 
+      $guessed = $features[$#features]; 
+      $guessedType = ""; 
+   }
+   pop(@features);
+   if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
+      $correct = $1;
+      $correctType = $2;
+   } else { 
+      $correct = $features[$#features]; 
+      $correctType = ""; 
+   }
+   pop(@features);
+#  ($guessed,$guessedType) = split(/-/,pop(@features));
+#  ($correct,$correctType) = split(/-/,pop(@features));
+   $guessedType = $guessedType ? $guessedType : "";
+   $correctType = $correctType ? $correctType : "";
+   $firstItem = shift(@features);
+
+   # 1999-06-26 sentence breaks should always be counted as out of chunk
+   if ( $firstItem eq $boundary ) { $guessed = "O"; }
+
+   if ($inCorrect) {
+      if ( &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and
+           &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
+           $lastGuessedType eq $lastCorrectType) {
+         $inCorrect=$false;
+         $correctChunk++;
+         $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
+             $correctChunk{$lastCorrectType}+1 : 1;
+      } elsif ( 
+           &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) != 
+           &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) or
+           $guessedType ne $correctType ) {
+         $inCorrect=$false; 
+      }
+   }
+
+   if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and 
+        &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
+        $guessedType eq $correctType) { $inCorrect = $true; }
+
+   if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) ) {
+      $foundCorrect++; 
+      $foundCorrect{$correctType} = $foundCorrect{$correctType} ?
+          $foundCorrect{$correctType}+1 : 1;
+   }
+   if ( &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) ) {
+      $foundGuessed++; 
+      $foundGuessed{$guessedType} = $foundGuessed{$guessedType} ?
+          $foundGuessed{$guessedType}+1 : 1;
+   }
+   if ( $firstItem ne $boundary ) { 
+      if ( $correct eq $guessed and $guessedType eq $correctType ) { 
+         $correctTags++; 
+      }
+      $tokenCounter++; 
+   }
+
+   $lastGuessed = $guessed;
+   $lastCorrect = $correct;
+   $lastGuessedType = $guessedType;
+   $lastCorrectType = $correctType;
+}
+if ($inCorrect) { 
+   $correctChunk++;
+   $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
+       $correctChunk{$lastCorrectType}+1 : 1;
+}
+
+if (not $latex) {
+   # compute overall precision, recall and FB1 (default values are 0.0)
+   $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
+   $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
+   $FB1 = 2*$precision*$recall/($precision+$recall)
+      if ($precision+$recall > 0);
+   
+   # print overall performance
+   printf "processed $tokenCounter tokens with $foundCorrect phrases; ";
+   printf "found: $foundGuessed phrases; correct: $correctChunk.\n";
+   if ($tokenCounter>0) {
+      printf "accuracy: %6.2f%%; ",100*$correctTags/$tokenCounter;
+      printf "precision: %6.2f%%; ",$precision;
+      printf "recall: %6.2f%%; ",$recall;
+      printf "FB1: %6.2f\n",$FB1;
+   }
+}
+
+# sort chunk type names
+undef($lastType);
+@sortedTypes = ();
+foreach $i (sort (keys %foundCorrect,keys %foundGuessed)) {
+   if (not($lastType) or $lastType ne $i) { 
+      push(@sortedTypes,($i));
+   }
+   $lastType = $i;
+}
+# print performance per chunk type
+if (not $latex) {
+   for $i (@sortedTypes) {
+      $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
+      if (not($foundGuessed{$i})) { $foundGuessed{$i} = 0; $precision = 0.0; }
+      else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
+      if (not($foundCorrect{$i})) { $recall = 0.0; }
+      else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
+      if ($precision+$recall == 0.0) { $FB1 = 0.0; }
+      else { $FB1 = 2*$precision*$recall/($precision+$recall); }
+      printf "%17s: ",$i;
+      printf "precision: %6.2f%%; ",$precision;
+      printf "recall: %6.2f%%; ",$recall;
+      printf "FB1: %6.2f  %d\n",$FB1,$foundGuessed{$i};
+   }
+} else {
+   print "        & Precision &  Recall  & F\$_{\\beta=1} \\\\\\hline";
+   for $i (@sortedTypes) {
+      $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
+      if (not($foundGuessed{$i})) { $precision = 0.0; }
+      else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
+      if (not($foundCorrect{$i})) { $recall = 0.0; }
+      else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
+      if ($precision+$recall == 0.0) { $FB1 = 0.0; }
+      else { $FB1 = 2*$precision*$recall/($precision+$recall); }
+      printf "\n%-7s &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\",
+             $i,$precision,$recall,$FB1;
+   }
+   print "\\hline\n";
+   $precision = 0.0;
+   $recall = 0;
+   $FB1 = 0.0;
+   $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
+   $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
+   $FB1 = 2*$precision*$recall/($precision+$recall)
+      if ($precision+$recall > 0);
+   printf "Overall &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\\\hline\n",
+          $precision,$recall,$FB1;
+}
+
+exit 0;
+
+# endOfChunk: checks if a chunk ended between the previous and current word
+# arguments:  previous and current chunk tags, previous and current types
+# note:       this code is capable of handling other chunk representations
+#             than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
+#             Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
+
+sub endOfChunk {
+   my $prevTag = shift(@_);
+   my $tag = shift(@_);
+   my $prevType = shift(@_);
+   my $type = shift(@_);
+   my $chunkEnd = $false;
+
+   if ( $prevTag eq "B" and $tag eq "B" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "B" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "B" and $tag eq "S" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "I" and $tag eq "B" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "I" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "I" and $tag eq "O" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "E" and $tag eq "E" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "I" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "B" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "S" and $tag eq "E" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "I" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "B" ) { $chunkEnd = $true; }
+   
+
+   if ($prevTag ne "O" and $prevTag ne "." and $prevType ne $type) { 
+      $chunkEnd = $true; 
+   }
+
+   # corrected 1998-12-22: these chunks are assumed to have length 1
+   if ( $prevTag eq "]" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "[" ) { $chunkEnd = $true; }
+
+   return($chunkEnd);   
+}
+
+# startOfChunk: checks if a chunk started between the previous and current word
+# arguments:    previous and current chunk tags, previous and current types
+# note:         this code is capable of handling other chunk representations
+#               than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
+#               Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
+
+sub startOfChunk {
+   my $prevTag = shift(@_);
+   my $tag = shift(@_);
+   my $prevType = shift(@_);
+   my $type = shift(@_);
+   my $chunkStart = $false;
+
+   if ( $prevTag eq "B" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "I" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "B" ) { $chunkStart = $true; }
+   
+   if ( $prevTag eq "B" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "I" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "S" ) { $chunkStart = $true; }
+
+   if ( $prevTag eq "O" and $tag eq "I" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "I" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "I" ) { $chunkStart = $true; }
+
+   if ( $prevTag eq "S" and $tag eq "E" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "E" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "E" ) { $chunkStart = $true; }
+
+   if ($tag ne "O" and $tag ne "." and $prevType ne $type) { 
+      $chunkStart = $true; 
+   }
+
+   # corrected 1998-12-22: these chunks are assumed to have length 1
+   if ( $tag eq "[" ) { $chunkStart = $true; }
+   if ( $tag eq "]" ) { $chunkStart = $true; }
+
+   return($chunkStart);   
+}

examples/eval/conll06eval.pl

@@ -0,0 +1,1826 @@
+#!/usr/bin/env perl
+
+# Author: Yuval Krymolowski
+# Addition of precision and recall 
+#   and of frame confusion list: Sabine Buchholz
+# Addition of DEPREL + ATTACHMENT:
+#   Prokopis Prokopidis (prokopis at ilsp dot gr)
+# Acknowledgements: 
+#   to Markus Kuhn for suggesting the use of 
+#   the Unicode category property
+
+if ($] < 5.008001)
+{
+  printf STDERR <<EOM
+
+ This script requires PERL 5.8.1 for running.
+ The new version is needed for proper handling
+ of Unicode characters.
+
+ Please obtain a new version or contact the shared task team
+ if you are unable to upgrade PERL.
+
+EOM
+;
+  exit(1) ;
+}
+
+require Encode;
+
+use strict ;
+use warnings;
+use Getopt::Std ;
+
+my ($usage) = <<EOT
+
+  CoNLL-X evaluation script:
+
+   [perl] eval.pl [OPTIONS] -g <gold standard> -s <system output>
+
+  This script evaluates a system output with respect to a gold standard.
+  Both files should be in UTF-8 encoded CoNLL-X tabular format.
+
+  Punctuation tokens (those where all characters have the Unicode
+  category property "Punctuation") are ignored for scoring (unless the
+  -p flag is used).
+
+  The output breaks down the errors according to their type and context.
+
+  Optional parameters:
+     -o FILE : output: print output to FILE (default is standard output)
+     -q : quiet:       only print overall performance, without the details
+     -b : evalb:       produce output in a format similar to evalb 
+                       (http://nlp.cs.nyu.edu/evalb/); use together with -q
+     -p : punctuation: also score on punctuation (default is not to score on it)
+     -v : version:     show the version number
+     -h : help:        print this help text and exit
+
+EOT
+;
+
+my ($line_num) ;
+my ($sep) = '0x01' ;
+
+my ($START) = '.S' ;
+my ($END) = '.E' ;
+
+my ($con_err_num) = 3 ;
+my ($freq_err_num) = 10 ;
+my ($spec_err_loc_con) = 8 ;
+
+################################################################################
+###                              subfunctions                                ###
+################################################################################
+
+# Whether a string consists entirely of characters with the Unicode
+# category property "Punctuation" (see "man perlunicode")
+sub is_uni_punct
+{
+  my ($word) = @_ ;
+
+  return scalar(Encode::decode_utf8($word)=~ /^\p{Punctuation}+$/) ;
+}
+
+# The length of a unicode string, excluding non-spacing marks
+# (for example vowel marks in Arabic)
+
+sub uni_len
+{
+  my ($word) = @_ ;
+  my ($ch, $l) ;
+
+  $l = 0 ;
+  foreach $ch (split(//,  Encode::decode_utf8($word)))
+  {
+    if ($ch !~ /^\p{NonspacingMark}/)
+    {
+      $l++ ;
+    }
+  }
+
+  return $l ;
+}
+
+sub filter_context_counts
+{ # filter_context_counts
+
+  my ($vec, $num, $max_len) = @_ ;
+  my ($con, $l, $thresh) ;
+
+  $thresh = (sort {$b <=> $a} values %{$vec})[$num-1] ;
+
+  foreach $con (keys %{$vec})
+  {
+    if (${$vec}{$con} < $thresh)
+    {
+      delete ${$vec}{$con} ;
+      next ;
+    }
+
+    $l = uni_len($con) ;
+
+    if ($l > ${$max_len})
+    {
+      ${$max_len} = $l ;
+    }
+  }
+
+} # filter_context_counts
+
+sub print_context
+{ # print_context
+
+  my ($counts, $counts_pos, $max_con_len, $max_con_pos_len) = @_ ;
+  my (@v_con, @v_con_pos, $con, $con_pos, $i, $n) ;
+
+  printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s", $max_con_pos_len, 'CPOS', 'any', 'head', 'dep', 'both' ;
+  printf OUT "  ||" ;
+  printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s", $max_con_len, 'word', 'any', 'head', 'dep', 'both' ;
+  printf OUT "\n" ;
+  printf OUT "  %s-+------+------+------+-----", '-' x $max_con_pos_len;
+  printf OUT "--++" ;
+  printf OUT "--%s-+------+------+------+-----", '-' x $max_con_len;
+  printf OUT "\n" ;
+
+  @v_con = sort {${$counts}{tot}{$b} <=> ${$counts}{tot}{$a}} keys %{${$counts}{tot}} ;
+  @v_con_pos = sort {${$counts_pos}{tot}{$b} <=> ${$counts_pos}{tot}{$a}} keys %{${$counts_pos}{tot}} ;
+
+  $n = scalar @v_con ;
+  if (scalar @v_con_pos > $n)
+  {
+    $n = scalar @v_con_pos ;
+  }
+
+  foreach $i (0 .. $n-1)
+  {
+    if (defined $v_con_pos[$i])
+    {
+      $con_pos = $v_con_pos[$i] ;
+      printf OUT "  %-*s | %4d | %4d | %4d | %4d",
+	$max_con_pos_len, $con_pos, ${$counts_pos}{tot}{$con_pos},
+	  ${$counts_pos}{err_head}{$con_pos}, ${$counts_pos}{err_dep}{$con_pos},
+	    ${$counts_pos}{err_dep}{$con_pos}+${$counts_pos}{err_head}{$con_pos}-${$counts_pos}{tot}{$con_pos} ;
+    }
+    else
+    {
+      printf OUT "  %-*s | %4s | %4s | %4s | %4s",
+	$max_con_pos_len, ' ', ' ', ' ', ' ', ' ' ;
+    }
+
+    printf OUT "  ||" ;
+
+    if (defined $v_con[$i])
+    {
+      $con = $v_con[$i] ;
+      printf OUT "  %-*s | %4d | %4d | %4d | %4d",
+	$max_con_len+length($con)-uni_len($con), $con, ${$counts}{tot}{$con},
+	  ${$counts}{err_head}{$con}, ${$counts}{err_dep}{$con},
+	    ${$counts}{err_dep}{$con}+${$counts}{err_head}{$con}-${$counts}{tot}{$con} ;
+    }
+    else
+    {
+      printf OUT "  %-*s | %4s | %4s | %4s | %4s",
+	$max_con_len, ' ', ' ', ' ', ' ', ' ' ;
+    }
+
+    printf OUT "\n" ;
+  }
+
+  printf OUT "  %s-+------+------+------+-----", '-' x $max_con_pos_len;
+  printf OUT "--++" ;
+  printf OUT "--%s-+------+------+------+-----", '-' x $max_con_len;
+  printf OUT "\n" ;
+
+  printf OUT "\n\n" ;
+
+} # print_context
+
+sub num_as_word
+{
+  my ($num) = @_ ;
+
+  $num = abs($num) ;
+
+  if ($num == 1)
+  {
+    return ('one word') ;
+  }
+  elsif ($num == 2)
+  {
+    return ('two words') ;
+  }
+  elsif ($num == 3)
+  {
+    return ('three words') ;
+  }
+  elsif ($num == 4)
+  {
+    return ('four words') ;
+  }
+  else
+  {
+    return ($num.' words') ;
+  }
+}
+
+sub describe_err
+{ # describe_err
+
+  my ($head_err, $head_aft_bef, $dep_err) = @_ ;
+  my ($dep_g, $dep_s, $desc) ;
+  my ($head_aft_bef_g, $head_aft_bef_s) = split(//, $head_aft_bef) ;
+
+  if ($head_err eq '-')
+  {
+    $desc = 'correct head' ;
+
+    if ($head_aft_bef_s eq '0')
+    {
+      $desc .= ' (0)' ;
+    }
+    elsif ($head_aft_bef_s eq 'e')
+    {
+      $desc .= ' (the focus word)' ;
+    }
+    elsif ($head_aft_bef_s eq 'a')
+    {
+      $desc .= ' (after the focus word)' ;
+    }
+    elsif ($head_aft_bef_s eq 'b')
+    {
+      $desc .= ' (before the focus word)' ;
+    }
+  }
+  elsif ($head_aft_bef_s eq '0')
+  {
+    $desc = 'head = 0 instead of ' ;
+    if ($head_aft_bef_g eq 'a')
+    {
+      $desc.= 'after ' ;
+    }
+    if ($head_aft_bef_g eq 'b')
+    {
+      $desc.= 'before ' ;
+    }
+    $desc .= 'the focus word' ;
+  }
+  elsif ($head_aft_bef_g eq '0')
+  {
+    $desc = 'head is ' ;
+    if ($head_aft_bef_g eq 'a')
+    {
+      $desc.= 'after ' ;
+    }
+    if ($head_aft_bef_g eq 'b')
+    {
+      $desc.= 'before ' ;
+    }
+    $desc .= 'the focus word instead of 0' ;
+  }
+  else
+  {
+    $desc = num_as_word($head_err) ;
+    if ($head_err < 0)
+    {
+      $desc .= ' before' ;
+    }
+    else
+    {
+      $desc .= ' after' ;
+    }
+
+    $desc = 'head '.$desc.' the correct head ' ;
+
+    if ($head_aft_bef_s eq '0')
+    {
+      $desc .= '(0' ;
+    }
+    elsif ($head_aft_bef_s eq 'e')
+    {
+      $desc .= '(the focus word' ;
+    }
+    elsif ($head_aft_bef_s eq 'a')
+    {
+      $desc .= '(after the focus word' ;
+    }
+    elsif ($head_aft_bef_s eq 'b')
+    {
+      $desc .= '(before the focus word' ;
+    }
+
+    if ($head_aft_bef_g ne $head_aft_bef_s)
+    {
+      $desc .= ' instead of' ;
+      if ($head_aft_bef_s eq '0')
+      {
+	$desc .= '0' ;
+      }
+      elsif ($head_aft_bef_s eq 'e')
+      {
+	$desc .= 'the focus word' ;
+      }
+      elsif ($head_aft_bef_s eq 'a')
+      {
+	$desc .= 'after the focus word' ;
+      }
+      elsif ($head_aft_bef_s eq 'b')
+      {
+	$desc .= 'before the focus word' ;
+      }
+    }
+
+    $desc .= ')' ;
+  }
+
+  $desc .= ', ' ;
+
+  if ($dep_err eq '-')
+  {
+    $desc .= 'correct dependency' ;
+  }
+  else
+  {
+    ($dep_g, $dep_s) = ($dep_err =~ /^(.*)->(.*)$/) ;
+    $desc .= sprintf('dependency "%s" instead of "%s"', $dep_s, $dep_g) ;
+  }
+
+  return($desc) ;
+
+} # describe_err
+
+sub get_context
+{ # get_context
+
+  my ($sent, $i_w) = @_ ;
+  my ($w_2, $w_1, $w1, $w2) ;
+  my ($p_2, $p_1, $p1, $p2) ;
+
+  if ($i_w >= 2)
+  {
+    $w_2 = ${${$sent}[$i_w-2]}{word} ;
+    $p_2 = ${${$sent}[$i_w-2]}{pos} ;
+  }
+  else
+  {
+    $w_2 = $START ;
+    $p_2 = $START ;
+  }
+
+  if ($i_w >= 1)
+  {
+    $w_1 = ${${$sent}[$i_w-1]}{word} ;
+    $p_1 = ${${$sent}[$i_w-1]}{pos} ;
+  }
+  else
+  {
+    $w_1 = $START ;
+    $p_1 = $START ;
+  }
+
+  if ($i_w <= scalar @{$sent}-2)
+  {
+    $w1 = ${${$sent}[$i_w+1]}{word} ;
+    $p1 = ${${$sent}[$i_w+1]}{pos} ;
+  }
+  else
+  {
+    $w1 = $END ;
+    $p1 = $END ;
+  }
+
+  if ($i_w <= scalar @{$sent}-3)
+  {
+    $w2 = ${${$sent}[$i_w+2]}{word} ;
+    $p2 = ${${$sent}[$i_w+2]}{pos} ;
+  }
+  else
+  {
+    $w2 = $END ;
+    $p2 = $END ;
+  }
+
+  return ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) ;
+
+} # get_context
+
+sub read_sent
+{ # read_sent
+
+  my ($sent_gold, $sent_sys) = @_ ;
+  my ($line_g, $line_s, $new_sent) ;
+  my (%fields_g, %fields_s) ;
+
+  $new_sent = 1 ;
+
+  @{$sent_gold} = () ;
+  @{$sent_sys} = () ;
+
+  while (1)
+  { # main reading loop
+
+    $line_g = <GOLD> ;
+    $line_s = <SYS> ;
+
+    $line_num++ ;
+
+    # system output has fewer lines than gold standard
+    if ((defined $line_g) && (! defined $line_s))
+    {
+	printf STDERR "line mismatch, line %d:\n", $line_num ;
+	printf STDERR " gold: %s", $line_g ;
+	printf STDERR " sys : past end of file\n" ;
+	exit(1) ;
+    }
+
+    # system output has more lines than gold standard
+    if ((! defined $line_g) && (defined $line_s))
+    {
+	printf STDERR "line mismatch, line %d:\n", $line_num ;
+	printf STDERR " gold: past end of file\n" ;
+	printf STDERR " sys : %s", $line_s ;
+	exit(1) ;
+    }
+    
+    # end of file reached for both
+    if ((! defined $line_g) && (! defined $line_s))
+    {
+	return (1) ;
+    }
+
+    # one contains end of sentence but other one does not
+    if (($line_g =~ /^\s+$/) != ($line_s =~ /^\s+$/))
+    {
+      printf STDERR "line mismatch, line %d:\n", $line_num ;
+      printf STDERR " gold: %s", $line_g ;
+      printf STDERR " sys : %s", $line_s ;
+      exit(1) ;
+    }
+
+    # end of sentence reached
+    if ($line_g =~ /^\s+$/)
+    {
+	return(0) ;
+    }
+
+    # now both lines contain information
+
+    if ($new_sent)
+    {
+      $new_sent = 0 ;
+    }
+
+    # 'official' column names
+    # options.output = ['id','form','lemma','cpostag','postag',
+    #                   'feats','head','deprel','phead','pdeprel']
+
+    @fields_g{'word', 'pos', 'head', 'dep'} = (split (/\s+/, $line_g))[1, 3, 6, 7] ;
+
+    push @{$sent_gold}, { %fields_g } ;
+
+    @fields_s{'word', 'pos', 'head', 'dep'} = (split (/\s+/, $line_s))[1, 3, 6, 7] ;
+
+    if (($fields_g{word} ne $fields_s{word})
+	||
+	($fields_g{pos} ne $fields_s{pos}))
+    {
+      printf STDERR "Word/pos mismatch, line %d:\n", $line_num ;
+      printf STDERR " gold: %s", $line_g ;
+      printf STDERR " sys : %s", $line_s ;
+      exit(1) ;
+    }
+
+    push @{$sent_sys}, { %fields_s } ;
+
+  } # main reading loop
+  
+} # read_sent
+
+################################################################################
+###                                  main                                    ###
+################################################################################
+
+our ($opt_g, $opt_s, $opt_o, $opt_h, $opt_v, $opt_q, $opt_p, $opt_b) ;
+
+my ($sent_num, $eof, $word_num, @err_sent) ;
+my (@sent_gold, @sent_sys, @starts) ;
+my ($word, $pos, $wp, $head_g, $dep_g, $head_s, $dep_s) ;
+my (%counts, $err_head, $err_dep, $con, $con1, $con_pos, $con_pos1, $thresh) ;
+my ($head_err, $dep_err, @cur_err, %err_counts, $err_counter, $err_desc) ;
+my ($loc_con, %loc_con_err_counts, %err_desc) ;
+my ($head_aft_bef_g, $head_aft_bef_s, $head_aft_bef) ;
+my ($con_bef, $con_aft, $con_bef_2, $con_aft_2, @bits, @e_bits, @v_con, @v_con_pos) ;
+my ($con_pos_bef, $con_pos_aft, $con_pos_bef_2, $con_pos_aft_2) ;
+my ($max_word_len, $max_pos_len, $max_con_len, $max_con_pos_len) ;
+my ($max_word_spec_len, $max_con_bef_len, $max_con_aft_len) ;
+my (%freq_err, $err) ;
+
+my ($i, $j, $i_w, $l, $n_args) ;
+my ($w_2, $w_1, $w1, $w2) ;
+my ($wp_2, $wp_1, $wp1, $wp2) ;
+my ($p_2, $p_1, $p1, $p2) ;
+
+my ($short_output) ;
+my ($score_on_punct) ;
+$counts{punct} = 0; # initialize
+
+getopts("g:o:s:qvhpb") ;
+
+if (defined $opt_v)
+{
+    my $id = '$Id: eval.pl,v 1.9 2006/05/09 20:30:01 yuval Exp $';
+    my @parts = split ' ',$id;
+    print "Version $parts[2]\n";
+    exit(0);
+}
+
+if ((defined $opt_h) || ((! defined $opt_g) && (! defined $opt_s)))
+{
+  die $usage ;
+}
+
+if (! defined $opt_g)
+{
+  die "Gold standard file (-g) missing\n" ;
+}
+
+if (! defined $opt_s)
+{
+  die "System output file (-s) missing\n" ;
+}
+
+if (! defined $opt_o)
+{
+  $opt_o = '-' ;
+}
+
+if (defined $opt_q)
+{
+    $short_output = 1 ;
+} else {
+    $short_output = 0 ;
+}
+
+if (defined $opt_p)
+{
+    $score_on_punct = 1 ;
+} else {
+    $score_on_punct = 0 ;
+}
+
+$line_num = 0 ;
+$sent_num = 0 ;
+$eof = 0 ;
+
+@err_sent = () ;
+@starts = () ;
+
+%{$err_sent[0]} = () ;
+
+$max_pos_len = length('CPOS') ;
+
+################################################################################
+###                              reading input                               ###
+################################################################################
+
+open (GOLD, "<$opt_g") || die "Could not open gold standard file $opt_g\n" ;
+open (SYS,  "<$opt_s") || die "Could not open system output file $opt_s\n" ;
+open (OUT,  ">$opt_o") || die "Could not open output file $opt_o\n" ;
+
+
+if (defined $opt_b) {  # produce output similar to evalb
+    print OUT "     Sent.          Attachment      Correct        Scoring          \n";
+    print OUT "    ID Tokens  -   Unlab. Lab.   HEAD HEAD+DEPREL   tokens   - - - -\n";
+    print OUT "  ============================================================================\n";
+}
+
+
+while (! $eof)
+{ # main reading loop
+
+  $starts[$sent_num] = $line_num+1 ;
+  $eof = read_sent(\@sent_gold, \@sent_sys) ;
+
+  $sent_num++ ;
+
+  %{$err_sent[$sent_num]} = () ;
+  $word_num = scalar @sent_gold ;
+
+  # for accuracy per sentence
+  my %sent_counts = ( tot      => 0,
+		      err_any  => 0,
+		      err_head => 0
+		      ); 
+
+  # printf "$sent_num $word_num\n" ;
+
+  my @frames_g = ('** '); # the initial frame for the virtual root
+  my @frames_s = ('** '); # the initial frame for the virtual root
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+      push @frames_g, ''; # initialize
+      push @frames_s, ''; # initialize
+  }
+
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+
+    ($word, $pos, $head_g, $dep_g)
+      = @{$sent_gold[$i_w]}{'word', 'pos', 'head', 'dep'} ;
+    $wp = $word.' / '.$pos ;
+
+    # printf "%d: %s %s %s %s\n", $i_w,  $word, $pos, $head_g, $dep_g ;
+
+    if ((! $score_on_punct) && is_uni_punct($word))
+    {
+      $counts{punct}++ ;
+      # ignore punctuations
+      next ;
+    }
+
+    if (length($pos) > $max_pos_len)
+    {
+      $max_pos_len = length($pos) ;
+    }
+
+    ($head_s, $dep_s) = @{$sent_sys[$i_w]}{'head', 'dep'} ;
+
+    $counts{tot}++ ;
+    $counts{word}{$wp}{tot}++ ;
+    $counts{pos}{$pos}{tot}++ ;
+    $counts{head}{$head_g-$i_w-1}{tot}++ ;
+
+    # for frame confusions
+    # add child to frame of parent
+    $frames_g[$head_g] .= "$dep_g ";
+    $frames_s[$head_s] .= "$dep_s ";
+    # add to frame of token itself
+    $frames_g[$i_w+1] .= "*$dep_g* "; # $i_w+1 because $i_w starts counting at zero
+    $frames_s[$i_w+1] .= "*$dep_g* ";
+
+    # for precision and recall of DEPREL
+    $counts{dep}{$dep_g}{tot}++ ;     # counts for gold standard deprels
+    $counts{dep2}{$dep_g}{$dep_s}++ ; # counts for confusions
+    $counts{dep_s}{$dep_s}{tot}++ ;   # counts for system deprels
+    $counts{all_dep}{$dep_g} = 1 ;    # list of all deprels that occur ...
+    $counts{all_dep}{$dep_s} = 1 ;    # ... in either gold or system output
+
+    # for precision and recall of HEAD direction
+    my $dir_g;
+    if ($head_g == 0) {
+	$dir_g = 'to_root';
+    } elsif ($head_g < $i_w+1) { # $i_w+1 because $i_w starts counting at zero
+                                 # also below
+	$dir_g = 'left';
+    } elsif ($head_g > $i_w+1) {
+	$dir_g = 'right';
+    } else {
+        # token links to itself; should never happen in correct gold standard
+	$dir_g = 'self'; 
+    }
+    my $dir_s;
+    if ($head_s == 0) {
+	$dir_s = 'to_root';
+    } elsif ($head_s < $i_w+1) {
+	$dir_s = 'left';
+    } elsif ($head_s > $i_w+1) {
+	$dir_s = 'right';
+    } else {
+        # token links to itself; should not happen in good system 
+        # (but not forbidden in shared task)
+	$dir_s = 'self'; 
+    }
+    $counts{dir_g}{$dir_g}{tot}++ ;   # counts for gold standard head direction
+    $counts{dir2}{$dir_g}{$dir_s}++ ; # counts for confusions
+    $counts{dir_s}{$dir_s}{tot}++ ;   # counts for system head direction
+
+    # for precision and recall of HEAD distance
+    my $dist_g;
+    if ($head_g == 0) {
+	$dist_g = 'to_root';
+    } elsif ( abs($head_g - ($i_w+1)) <= 1 ) {
+	$dist_g = '1'; # includes the 'self' cases
+    } elsif ( abs($head_g - ($i_w+1)) <= 2 ) {
+	$dist_g = '2';
+    } elsif ( abs($head_g - ($i_w+1)) <= 6 ) {
+	$dist_g = '3-6';
+    } else {
+	$dist_g = '7-...';
+    }
+    my $dist_s;
+    if ($head_s == 0) {
+	$dist_s = 'to_root';
+    } elsif ( abs($head_s - ($i_w+1)) <= 1 ) {
+	$dist_s = '1'; # includes the 'self' cases
+    } elsif ( abs($head_s - ($i_w+1)) <= 2 ) {
+	$dist_s = '2';
+    } elsif ( abs($head_s - ($i_w+1)) <= 6 ) {
+	$dist_s = '3-6';
+    } else {
+	$dist_s = '7-...';
+    }
+    $counts{dist_g}{$dist_g}{tot}++ ;    # counts for gold standard head distance
+    $counts{dist2}{$dist_g}{$dist_s}++ ; # counts for confusions
+    $counts{dist_s}{$dist_s}{tot}++ ;    # counts for system head distance
+
+
+    $err_head = ($head_g ne $head_s) ; # error in head
+    $err_dep = ($dep_g ne $dep_s) ;    # error in deprel
+
+    $head_err = '-' ;
+    $dep_err = '-' ;
+
+    # for accuracy per sentence
+    $sent_counts{tot}++ ;
+    if ($err_dep || $err_head) {
+	$sent_counts{err_any}++ ;
+    }
+    if ($err_head) {
+	$sent_counts{err_head}++ ;
+    }
+
+    # total counts and counts for CPOS involved in errors
+
+    if ($head_g eq '0')
+    {
+      $head_aft_bef_g = '0' ;
+    }
+    elsif ($head_g eq $i_w+1)
+    {
+      $head_aft_bef_g = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_g = ($head_g <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    if ($head_s eq '0')
+    {
+      $head_aft_bef_s = '0' ;
+    }
+    elsif ($head_s eq $i_w+1)
+    {
+      $head_aft_bef_s = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_s = ($head_s <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    $head_aft_bef = $head_aft_bef_g.$head_aft_bef_s ;
+
+    if ($err_head)
+    {
+      if ($head_aft_bef_s eq '0')
+      {
+	$head_err = 0 ;
+      }
+      else
+      {
+	$head_err = $head_s-$head_g ;
+      }
+
+      $err_sent[$sent_num]{head}++ ;
+      $counts{err_head}{tot}++ ;
+      $counts{err_head}{$head_err}++ ;
+
+      $counts{word}{err_head}{$wp}++ ;
+      $counts{pos}{$pos}{err_head}{tot}++ ;
+      $counts{pos}{$pos}{err_head}{$head_err}++ ;
+    }
+
+    if ($err_dep)
+    {
+      $dep_err = $dep_g.'->'.$dep_s ;
+      $err_sent[$sent_num]{dep}++ ;
+      $counts{err_dep}{tot}++ ;
+      $counts{err_dep}{$dep_err}++ ;
+
+      $counts{word}{err_dep}{$wp}++ ;
+      $counts{pos}{$pos}{err_dep}{tot}++ ;
+      $counts{pos}{$pos}{err_dep}{$dep_err}++ ;
+
+      if ($err_head)
+      {
+	$counts{err_both}++ ;
+	$counts{pos}{$pos}{err_both}++ ;
+      }
+    }
+
+    ### DEPREL + ATTACHMENT
+    if ((!$err_dep) && ($err_head)) {
+	$counts{err_head_corr_dep}{tot}++ ;
+	$counts{err_head_corr_dep}{$dep_s}++ ;
+    }
+    ### DEPREL + ATTACHMENT
+
+    # counts for words involved in errors
+
+    if (! ($err_head || $err_dep))
+    {
+      next ;
+    }
+
+    $err_sent[$sent_num]{word}++ ;
+    $counts{err_any}++ ;
+    $counts{word}{err_any}{$wp}++ ;
+    $counts{pos}{$pos}{err_any}++ ;
+
+    ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) = get_context(\@sent_gold, $i_w) ;
+
+    if ($w_2 ne $START)
+    {
+      $wp_2 = $w_2.' / '.$p_2 ;
+    }
+    else
+    {
+      $wp_2 = $w_2 ;
+    }
+
+    if ($w_1 ne $START)
+    {
+      $wp_1 = $w_1.' / '.$p_1 ;
+    }
+    else
+    {
+      $wp_1 = $w_1 ;
+    }
+
+    if ($w1 ne $END)
+    {
+      $wp1 = $w1.' / '.$p1 ;
+    }
+    else
+    {
+      $wp1 = $w1 ;
+    }
+
+    if ($w2 ne $END)
+    {
+      $wp2 = $w2.' / '.$p2 ;
+    }
+    else
+    {
+      $wp2 = $w2 ;
+    }
+
+    $con_bef = $wp_1 ;
+    $con_bef_2 = $wp_2.' + '.$wp_1 ;
+    $con_aft = $wp1 ;
+    $con_aft_2 = $wp1.' + '.$wp2 ;
+
+    $con_pos_bef = $p_1 ;
+    $con_pos_bef_2 = $p_2.'+'.$p_1 ;
+    $con_pos_aft = $p1 ;
+    $con_pos_aft_2 = $p1.'+'.$p2 ;
+
+    if ($w_1 ne $START)
+    {
+      # do not count '.S' as a word context
+      $counts{con_bef_2}{tot}{$con_bef_2}++ ;
+      $counts{con_bef_2}{err_head}{$con_bef_2} += $err_head ;
+      $counts{con_bef_2}{err_dep}{$con_bef_2} += $err_dep ;
+      $counts{con_bef}{tot}{$con_bef}++ ;
+      $counts{con_bef}{err_head}{$con_bef} += $err_head ;
+      $counts{con_bef}{err_dep}{$con_bef} += $err_dep ;
+    }
+
+    if ($w1 ne $END)
+    {
+      # do not count '.E' as a word context
+      $counts{con_aft_2}{tot}{$con_aft_2}++ ;
+      $counts{con_aft_2}{err_head}{$con_aft_2} += $err_head ;
+      $counts{con_aft_2}{err_dep}{$con_aft_2} += $err_dep ;
+      $counts{con_aft}{tot}{$con_aft}++ ;
+      $counts{con_aft}{err_head}{$con_aft} += $err_head ;
+      $counts{con_aft}{err_dep}{$con_aft} += $err_dep ;
+    }
+
+    $counts{con_pos_bef_2}{tot}{$con_pos_bef_2}++ ;
+    $counts{con_pos_bef_2}{err_head}{$con_pos_bef_2} += $err_head ;
+    $counts{con_pos_bef_2}{err_dep}{$con_pos_bef_2} += $err_dep ;
+    $counts{con_pos_bef}{tot}{$con_pos_bef}++ ;
+    $counts{con_pos_bef}{err_head}{$con_pos_bef} += $err_head ;
+    $counts{con_pos_bef}{err_dep}{$con_pos_bef} += $err_dep ;
+
+    $counts{con_pos_aft_2}{tot}{$con_pos_aft_2}++ ;
+    $counts{con_pos_aft_2}{err_head}{$con_pos_aft_2} += $err_head ;
+    $counts{con_pos_aft_2}{err_dep}{$con_pos_aft_2} += $err_dep ;
+    $counts{con_pos_aft}{tot}{$con_pos_aft}++ ;
+    $counts{con_pos_aft}{err_head}{$con_pos_aft} += $err_head ;
+    $counts{con_pos_aft}{err_dep}{$con_pos_aft} += $err_dep ;
+
+    $err = $head_err.$sep.$head_aft_bef.$sep.$dep_err ;
+    $freq_err{$err}++ ;
+
+  } # loop on words
+
+  foreach $i_w (0 .. $word_num) # including one for the virtual root
+  { # loop on words
+      if ($frames_g[$i_w] ne $frames_s[$i_w]) {
+	  $counts{frame2}{"$frames_g[$i_w]/ $frames_s[$i_w]"}++ ;
+      }
+  }
+
+  if (defined $opt_b) { # produce output similar to evalb
+      if ($word_num > 0) {
+	  my ($unlabeled,$labeled) = ('NaN', 'NaN');
+	  if ($sent_counts{tot} > 0) { # there are scoring tokens
+	      $unlabeled = 100-$sent_counts{err_head}*100.0/$sent_counts{tot};
+	      $labeled   = 100-$sent_counts{err_any} *100.0/$sent_counts{tot};
+	  }
+	  printf OUT "  %4d %4d    0  %6.2f %6.2f  %4d    %4d        %4d    0 0 0 0\n", 
+	  $sent_num, $word_num, 
+	  $unlabeled, $labeled, 
+	  $sent_counts{tot}-$sent_counts{err_head}, 
+	  $sent_counts{tot}-$sent_counts{err_any}, 
+	  $sent_counts{tot},;
+      }
+  }
+
+} # main reading loop
+
+################################################################################
+###                             printing output                              ###
+################################################################################
+
+if (defined $opt_b) {  # produce output similar to evalb
+    print OUT "\n\n";
+}
+printf OUT "  Labeled   attachment score: %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_any},      $counts{tot}, 100-$counts{err_any}*100.0/$counts{tot} ;
+printf OUT "  Unlabeled attachment score: %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_head}{tot}, $counts{tot}, 100-$counts{err_head}{tot}*100.0/$counts{tot} ;
+printf OUT "  Label accuracy score:       %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_dep}{tot}, $counts{tot}, 100-$counts{err_dep}{tot}*100.0/$counts{tot} ;
+
+if ($short_output)
+{
+    exit(0) ;
+}
+printf OUT "\n  %s\n\n", '=' x 80 ;
+printf OUT "  Evaluation of the results in %s\n  vs. gold standard %s:\n\n", $opt_s, $opt_g ;
+
+printf OUT "  Legend: '%s' - the beginning of a sentence, '%s' - the end of a sentence\n\n", $START, $END ;
+
+printf OUT "  Number of non-scoring tokens: $counts{punct}\n\n";
+
+printf OUT "  The overall accuracy and its distribution over CPOSTAGs\n\n" ;
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %-5s | %-5s |   %%  | %-5s |   %%  | %-5s |   %%\n",
+  'Accuracy', 'words', 'right', 'right', 'both' ;
+printf OUT "  %-10s | %-5s | %-5s |      | %-5s |      | %-5s |\n",
+  ' ', ' ', 'head', ' dep', 'right' ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+  'total', $counts{tot},
+  $counts{tot}-$counts{err_head}{tot}, 100-$counts{err_head}{tot}*100.0/$counts{tot},
+  $counts{tot}-$counts{err_dep}{tot}, 100-$counts{err_dep}{tot}*100.0/$counts{tot},
+  $counts{tot}-$counts{err_any}, 100-$counts{err_any}*100.0/$counts{tot} ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+foreach $pos (sort {$counts{pos}{$b}{tot} <=> $counts{pos}{$a}{tot}} keys %{$counts{pos}})
+{
+    if (! defined($counts{pos}{$pos}{err_head}{tot}))
+    {
+	$counts{pos}{$pos}{err_head}{tot} = 0 ;
+    }
+    if (! defined($counts{pos}{$pos}{err_dep}{tot}))
+    {
+	$counts{pos}{$pos}{err_dep}{tot} = 0 ;
+    }
+    if (! defined($counts{pos}{$pos}{err_any}))
+    {
+	$counts{pos}{$pos}{err_any} = 0 ;
+    }
+
+    printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+    $pos, $counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_head}{tot}, 100-$counts{pos}{$pos}{err_head}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_dep}{tot}, 100-$counts{pos}{$pos}{err_dep}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_any}, 100-$counts{pos}{$pos}{err_any}*100.0/$counts{pos}{$pos}{tot} ;
+}
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  The overall error rate and its distribution over CPOSTAGs\n\n" ;
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %-5s | %-5s |   %%  | %-5s |   %%  | %-5s |   %%\n",
+  'Error', 'words', 'head', ' dep', 'both' ;
+printf OUT "  %-10s | %-5s | %-5s |      | %-5s |      | %-5s |\n",
+
+  'Rate', ' ', 'err', ' err', 'wrong' ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+  'total', $counts{tot},
+  $counts{err_head}{tot}, $counts{err_head}{tot}*100.0/$counts{tot},
+  $counts{err_dep}{tot}, $counts{err_dep}{tot}*100.0/$counts{tot},
+  $counts{err_both}, $counts{err_both}*100.0/$counts{tot} ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+foreach $pos (sort {$counts{pos}{$b}{tot} <=> $counts{pos}{$a}{tot}} keys %{$counts{pos}})
+{
+    if (! defined($counts{pos}{$pos}{err_both}))
+    {
+	$counts{pos}{$pos}{err_both} = 0 ;
+    }
+
+    printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+    $pos, $counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_head}{tot}, $counts{pos}{$pos}{err_head}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_dep}{tot}, $counts{pos}{$pos}{err_dep}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_both}, $counts{pos}{$pos}{err_both}*100.0/$counts{pos}{$pos}{tot} ;
+    
+}
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+### added by Sabine Buchholz
+printf OUT "\n\n";
+printf OUT "  Precision and recall of DEPREL\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  deprel          | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dep (sort keys %{$counts{all_dep}}) {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dep2}{$dep}{$dep})) {
+	$tot_corr = $counts{dep2}{$dep}{$dep};
+    } 
+    if (defined($counts{dep}{$dep}{tot})) {
+    	$tot_g = $counts{dep}{$dep}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dep_s}{$dep}{tot})) {
+	$tot_s = $counts{dep_s}{$dep}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dep, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+### DEPREL + ATTACHMENT:
+### Same as Sabine's DEPREL apart from $tot_corr calculation
+printf OUT "\n\n";
+printf OUT "  Precision and recall of DEPREL + ATTACHMENT\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  deprel          | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dep (sort keys %{$counts{all_dep}}) {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dep2}{$dep}{$dep})) {
+	if (defined($counts{err_head_corr_dep}{$dep})) {
+	    $tot_corr = $counts{dep2}{$dep}{$dep} - $counts{err_head_corr_dep}{$dep};
+	} else {
+	    $tot_corr = $counts{dep2}{$dep}{$dep};
+	}
+    } 
+    if (defined($counts{dep}{$dep}{tot})) {
+    	$tot_g = $counts{dep}{$dep}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dep_s}{$dep}{tot})) {
+	$tot_s = $counts{dep_s}{$dep}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dep, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+### DEPREL + ATTACHMENT
+
+printf OUT "\n\n";
+printf OUT "  Precision and recall of binned HEAD direction\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  direction       | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dir ('to_root', 'left', 'right', 'self') {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dir2}{$dir}{$dir})) {
+	$tot_corr = $counts{dir2}{$dir}{$dir};
+    } 
+    if (defined($counts{dir_g}{$dir}{tot})) {
+    	$tot_g = $counts{dir_g}{$dir}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dir_s}{$dir}{tot})) {
+	$tot_s = $counts{dir_s}{$dir}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dir, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+printf OUT "\n\n";
+printf OUT "  Precision and recall of binned HEAD distance\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  distance        | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dist ('to_root', '1', '2', '3-6', '7-...') {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dist2}{$dist}{$dist})) {
+	$tot_corr = $counts{dist2}{$dist}{$dist};
+    } 
+    if (defined($counts{dist_g}{$dist}{tot})) {
+    	$tot_g = $counts{dist_g}{$dist}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dist_s}{$dist}{tot})) {
+	$tot_s = $counts{dist_s}{$dist}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dist, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+printf OUT "\n\n";
+printf OUT "  Frame confusions (gold versus system; *...* marks the head token)\n\n";
+foreach my $frame (sort {$counts{frame2}{$b} <=> $counts{frame2}{$a}} keys %{$counts{frame2}})
+{
+    if ($counts{frame2}{$frame} >= 5) # (make 5 a changeable threshold later)
+    {
+	printf OUT "  %3d  %s\n", $counts{frame2}{$frame}, $frame;
+    }
+}
+### end of: added by Sabine Buchholz
+
+
+#
+# Leave only the 5 words mostly involved in errors
+#
+
+
+$thresh = (sort {$b <=> $a} values %{$counts{word}{err_any}})[4] ;
+
+# ensure enough space for title
+$max_word_len = length('word') ;
+
+foreach $word (keys %{$counts{word}{err_any}})
+{
+  if ($counts{word}{err_any}{$word} < $thresh)
+  {
+    delete $counts{word}{err_any}{$word} ;
+    next ;
+  }
+
+  $l = uni_len($word) ;
+  if ($l > $max_word_len)
+  {
+    $max_word_len = $l ;
+  }
+}
+
+# filter a case when the difference between the error counts
+# for 2-word and 1-word contexts is small
+# (leave the 2-word context)
+
+foreach $con (keys %{$counts{con_aft_2}{tot}})
+{
+  ($w1) = split(/\+/, $con) ;
+  
+  if (defined $counts{con_aft}{tot}{$w1} &&
+      $counts{con_aft}{tot}{$w1}-$counts{con_aft_2}{tot}{$con} <= 1)
+  {
+    delete $counts{con_aft}{tot}{$w1} ;
+  }
+}
+
+foreach $con (keys %{$counts{con_bef_2}{tot}})
+{
+  ($w_2, $w_1) = split(/\+/, $con) ;
+
+  if (defined $counts{con_bef}{tot}{$w_1} &&
+      $counts{con_bef}{tot}{$w_1}-$counts{con_bef_2}{tot}{$con} <= 1)
+  {
+    delete $counts{con_bef}{tot}{$w_1} ;
+  }
+}
+
+foreach $con_pos (keys %{$counts{con_pos_aft_2}{tot}})
+{
+  ($p1) = split(/\+/, $con_pos) ;
+
+  if (defined($counts{con_pos_aft}{tot}{$p1}) &&
+      $counts{con_pos_aft}{tot}{$p1}-$counts{con_pos_aft_2}{tot}{$con_pos} <= 1)
+  {
+    delete $counts{con_pos_aft}{tot}{$p1} ;
+  }
+}
+
+foreach $con_pos (keys %{$counts{con_pos_bef_2}{tot}})
+{
+  ($p_2, $p_1) = split(/\+/, $con_pos) ;
+
+  if (defined($counts{con_pos_bef}{tot}{$p_1}) &&
+      $counts{con_pos_bef}{tot}{$p_1}-$counts{con_pos_bef_2}{tot}{$con_pos} <= 1)
+  {
+    delete $counts{con_pos_bef}{tot}{$p_1} ;
+  }
+}
+
+# for each context type, take the three contexts most involved in errors
+
+$max_con_len = 0 ;
+
+filter_context_counts($counts{con_bef_2}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_bef}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_aft}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_aft_2}{tot}, $con_err_num, \$max_con_len) ;
+
+# for each CPOS context type, take the three CPOS contexts most involved in errors
+
+$max_con_pos_len = 0 ;
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_bef_2}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_bef_2}{tot}})
+{
+  if ($counts{con_pos_bef_2}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_bef_2}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_bef}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_bef}{tot}})
+{
+  if ($counts{con_pos_bef}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_bef}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_aft}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_aft}{tot}})
+{
+  if ($counts{con_pos_aft}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_aft}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_aft_2}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_aft_2}{tot}})
+{
+  if ($counts{con_pos_aft_2}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_aft_2}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+# printing
+
+# ------------- focus words
+
+printf OUT "\n\n" ;
+printf OUT "  %d focus words where most of the errors occur:\n\n", scalar keys %{$counts{word}{err_any}} ;
+
+printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s\n", $max_word_len, ' ', 'any', 'head', 'dep', 'both' ;
+printf OUT "  %s-+------+------+------+------\n", '-' x $max_word_len;
+
+foreach $word (sort {$counts{word}{err_any}{$b} <=> $counts{word}{err_any}{$a}} keys %{$counts{word}{err_any}})
+{
+    if (!defined($counts{word}{err_head}{$word}))
+    {
+	$counts{word}{err_head}{$word} = 0 ;
+    }
+    if (! defined($counts{word}{err_dep}{$word}))
+    {
+	$counts{word}{err_dep}{$word} = 0 ;
+    }
+    if (! defined($counts{word}{err_any}{$word}))
+    {
+	$counts{word}{err_any}{$word} = 0;
+    }
+    printf OUT "  %-*s | %4d | %4d | %4d | %4d\n",
+    $max_word_len+length($word)-uni_len($word), $word, $counts{word}{err_any}{$word},
+    $counts{word}{err_head}{$word},
+    $counts{word}{err_dep}{$word},
+    $counts{word}{err_dep}{$word}+$counts{word}{err_head}{$word}-$counts{word}{err_any}{$word} ;
+}
+
+printf OUT "  %s-+------+------+------+------\n", '-' x $max_word_len;
+
+# ------------- contexts
+
+printf OUT "\n\n" ;
+
+printf OUT "  one-token preceeding contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_bef}, $counts{con_pos_bef}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  two-token preceeding contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_bef_2}, $counts{con_pos_bef_2}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  one-token following contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_aft}, $counts{con_pos_aft}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  two-token following contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_aft_2}, $counts{con_pos_aft_2}, $max_con_len, $max_con_pos_len) ;
+
+# ------------- Sentences
+
+printf OUT "  Sentence with the highest number of word errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{word}) && $err_sent[$b]{word})
+		 <=> (defined($err_sent[$a]{word}) && $err_sent[$a]{word}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  Sentence with the highest number of head errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{head}) && $err_sent[$b]{head}) 
+		 <=> (defined($err_sent[$a]{head}) && $err_sent[$a]{head}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  Sentence with the highest number of dependency errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{dep}) && $err_sent[$b]{dep}) 
+		 <=> (defined($err_sent[$a]{dep}) && $err_sent[$a]{dep}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+#
+# Second pass, collect statistics of the frequent errors
+#
+
+# filter the errors, leave the most frequent $freq_err_num errors
+
+$i = 0 ;
+
+$thresh = (sort {$b <=> $a} values %freq_err)[$freq_err_num-1] ;
+
+foreach $err (keys %freq_err)
+{
+  if ($freq_err{$err} < $thresh)
+  {
+    delete $freq_err{$err} ;
+  }
+}
+
+# in case there are several errors with the threshold count
+
+$freq_err_num = scalar keys %freq_err ;
+
+%err_counts = () ;
+
+$eof = 0 ;
+
+seek (GOLD, 0, 0) ;
+seek (SYS, 0, 0) ;
+
+while (! $eof)
+{ # second reading loop
+
+  $eof = read_sent(\@sent_gold, \@sent_sys) ;
+  $sent_num++ ;
+
+  $word_num = scalar @sent_gold ;
+
+  # printf "$sent_num $word_num\n" ;
+  
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+    ($word, $pos, $head_g, $dep_g)
+      = @{$sent_gold[$i_w]}{'word', 'pos', 'head', 'dep'} ;
+
+    # printf "%d: %s %s %s %s\n", $i_w,  $word, $pos, $head_g, $dep_g ;
+
+    if ((! $score_on_punct) && is_uni_punct($word))
+    {
+      # ignore punctuations
+      next ;
+    }
+
+    ($head_s, $dep_s) = @{$sent_sys[$i_w]}{'head', 'dep'} ;
+
+    $err_head = ($head_g ne $head_s) ;
+    $err_dep = ($dep_g ne $dep_s) ;
+
+    $head_err = '-' ;
+    $dep_err = '-' ;
+
+    if ($head_g eq '0')
+    {
+      $head_aft_bef_g = '0' ;
+    }
+    elsif ($head_g eq $i_w+1)
+    {
+      $head_aft_bef_g = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_g = ($head_g <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    if ($head_s eq '0')
+    {
+      $head_aft_bef_s = '0' ;
+    }
+    elsif ($head_s eq $i_w+1)
+    {
+      $head_aft_bef_s = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_s = ($head_s <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    $head_aft_bef = $head_aft_bef_g.$head_aft_bef_s ;
+
+    if ($err_head)
+    {
+      if ($head_aft_bef_s eq '0')
+      {
+	$head_err = 0 ;
+      }
+      else
+      {
+	$head_err = $head_s-$head_g ;
+      }
+    }
+
+    if ($err_dep)
+    {
+      $dep_err = $dep_g.'->'.$dep_s ;
+    }
+
+    if (! ($err_head || $err_dep))
+    {
+      next ;
+    }
+
+    # handle only the most frequent errors
+
+    $err = $head_err.$sep.$head_aft_bef.$sep.$dep_err ;
+
+    if (! exists $freq_err{$err})
+    {
+      next ;
+    }
+
+    ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) = get_context(\@sent_gold, $i_w) ;
+
+    $con_bef = $w_1 ;
+    $con_bef_2 = $w_2.' + '.$w_1 ;
+    $con_aft = $w1 ;
+    $con_aft_2 = $w1.' + '.$w2 ;
+
+    $con_pos_bef = $p_1 ;
+    $con_pos_bef_2 = $p_2.'+'.$p_1 ;
+    $con_pos_aft = $p1 ;
+    $con_pos_aft_2 = $p1.'+'.$p2 ;
+
+    @cur_err = ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) ;
+
+    # printf "# %-25s %-15s %-10s %-25s %-3s %-30s\n",
+    #  $con_bef, $word, $pos, $con_aft, $head_err, $dep_err ;
+    
+    @bits = (0, 0, 0, 0, 0, 0) ;
+    $j = 0 ;
+
+    while ($j == 0)
+    {
+      for ($i = 0; $i <= $#bits; $i++)
+      {
+	if ($bits[$i] == 0)
+	{
+	  $bits[$i] = 1 ;
+	  $j = 0 ;
+	  last ;
+	}
+	else
+	{
+	  $bits[$i] = 0 ;
+	  $j = 1 ;
+	}
+      }
+
+      @e_bits = @cur_err ;
+
+      for ($i = 0; $i <= $#bits; $i++)
+      {
+	if (! $bits[$i])
+	{
+	  $e_bits[$i] = '*' ;
+	}
+      }
+
+      # include also the last case which is the most general
+      # (wildcards for everything)
+      $err_counts{$err}{join($sep, @e_bits)}++ ;
+
+    }
+
+  } # loop on words
+} # second reading loop
+
+printf OUT "\n\n" ;
+printf OUT "  Specific errors, %d most frequent errors:", $freq_err_num ;
+printf OUT "\n  %s\n", '=' x 41 ;
+
+
+# deleting local contexts which are too general
+
+foreach $err (keys %err_counts)
+{
+  foreach $loc_con (sort {$err_counts{$err}{$b} <=> $err_counts{$err}{$a}}
+		    keys %{$err_counts{$err}})
+  {
+    @cur_err = split(/\Q$sep\E/, $loc_con) ;
+
+    # In this loop, one or two elements of the local context are
+    # replaced with '*' to make it more general. If the entry for
+    # the general context has the same count it is removed.
+
+    foreach $i (0 .. $#cur_err)
+    {
+      $w1 = $cur_err[$i] ;
+      if ($cur_err[$i] eq '*')
+      {
+	next ;
+      }
+      $cur_err[$i] = '*' ;
+      $con1 = join($sep, @cur_err) ;
+      if ( defined($err_counts{$err}{$con1}) && defined($err_counts{$err}{$loc_con})
+	   && ($err_counts{$err}{$con1} == $err_counts{$err}{$loc_con}))
+      {
+	delete $err_counts{$err}{$con1} ;
+      }
+      for ($j = $i+1; $j <=$#cur_err; $j++)
+      {
+	if ($cur_err[$j] eq '*')
+	{
+	  next ;
+	}
+	$w2 = $cur_err[$j] ;
+	$cur_err[$j] = '*' ;
+	$con1 = join($sep, @cur_err) ;
+	if ( defined($err_counts{$err}{$con1}) && defined($err_counts{$err}{$loc_con})
+	     && ($err_counts{$err}{$con1} == $err_counts{$err}{$loc_con}))
+	{
+	  delete $err_counts{$err}{$con1} ;
+	}
+	$cur_err[$j] = $w2 ;
+      }
+      $cur_err[$i] = $w1 ;
+    }
+  }
+}
+
+# Leaving only the topmost local contexts for each error
+
+foreach $err (keys %err_counts)
+{
+  $thresh = (sort {$b <=> $a} values %{$err_counts{$err}})[$spec_err_loc_con-1] || 0 ;
+
+  # of the threshold is too low, take the 2nd highest count
+  # (the highest may be the total which is the generic case
+  #   and not relevant for printing)
+
+  if ($thresh < 5)
+  {
+    $thresh = (sort {$b <=> $a} values %{$err_counts{$err}})[1] ;
+  }
+
+  foreach $loc_con (keys %{$err_counts{$err}})
+  {
+    if ($err_counts{$err}{$loc_con} < $thresh)
+    {
+      delete $err_counts{$err}{$loc_con} ;
+    }
+    else
+    {
+      if ($loc_con ne join($sep, ('*', '*', '*', '*', '*', '*')))
+      {
+	$loc_con_err_counts{$loc_con}{$err} = $err_counts{$err}{$loc_con} ;
+      }
+    }
+  }
+}
+
+# printing an error summary
+
+# calculating the context field length
+
+$max_word_spec_len= length('word') ;
+$max_con_aft_len = length('word') ;
+$max_con_bef_len = length('word') ;
+$max_con_pos_len = length('CPOS') ;
+
+foreach $err (keys %err_counts)
+{
+  foreach $loc_con (sort keys %{$err_counts{$err}})
+  {
+    ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $loc_con) ;
+
+    $l = uni_len($word) ;
+    if ($l > $max_word_spec_len)
+    {
+      $max_word_spec_len = $l ;
+    }
+
+    $l = uni_len($con_bef) ;
+    if ($l > $max_con_bef_len)
+    {
+      $max_con_bef_len = $l ;
+    }
+
+    $l = uni_len($con_aft) ;
+    if ($l > $max_con_aft_len)
+    {
+      $max_con_aft_len = $l ;
+    }
+
+    if (length($con_pos_aft) > $max_con_pos_len)
+    {
+      $max_con_pos_len = length($con_pos_aft) ;
+    }
+
+    if (length($con_pos_bef) > $max_con_pos_len)
+    {
+      $max_con_pos_len = length($con_pos_bef) ;
+    }
+  }
+}
+
+$err_counter = 0 ;
+
+foreach $err (sort {$freq_err{$b} <=> $freq_err{$a}} keys %freq_err)
+{
+
+  ($head_err, $head_aft_bef, $dep_err) = split(/\Q$sep\E/, $err) ;
+
+  $err_counter++ ;
+  $err_desc{$err} = sprintf("%2d. ", $err_counter).
+    describe_err($head_err, $head_aft_bef, $dep_err) ;
+  
+  # printf OUT "  %-3s %-30s %d\n", $head_err, $dep_err, $freq_err{$err} ;
+  printf OUT "\n" ;
+  printf OUT "  %s : %d times\n", $err_desc{$err}, $freq_err{$err} ;
+
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  printf OUT "  %-*s | %-*s | %-*s | %s\n",
+      $max_con_pos_len+$max_con_bef_len+3, '  Before',
+	$max_word_spec_len+$max_pos_len+3, '   Focus',
+	  $max_con_pos_len+$max_con_aft_len+3, '  After',
+	    'Count' ;
+
+  printf OUT "  %-*s   %-*s | %-*s   %-*s | %-*s   %-*s |\n",
+    $max_con_pos_len, 'CPOS', $max_con_bef_len, 'word',
+       $max_pos_len, 'CPOS', $max_word_spec_len, 'word',
+	$max_con_pos_len, 'CPOS', $max_con_aft_len, 'word' ;
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  foreach $loc_con (sort {$err_counts{$err}{$b} <=> $err_counts{$err}{$a}}
+		    keys %{$err_counts{$err}})
+  {
+    if ($loc_con eq join($sep, ('*', '*', '*', '*', '*', '*')))
+    {
+      next ;
+    }
+
+    $con1 = $loc_con ;
+    $con1 =~ s/\*/ /g ;
+
+    ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $con1) ;
+
+    printf OUT "  %-*s | %-*s | %-*s | %-*s | %-*s | %-*s | %3d\n",
+      $max_con_pos_len, $con_pos_bef, $max_con_bef_len+length($con_bef)-uni_len($con_bef), $con_bef,
+	  $max_pos_len, $pos, $max_word_spec_len+length($word)-uni_len($word), $word,
+	    $max_con_pos_len, $con_pos_aft, $max_con_aft_len+length($con_aft)-uni_len($con_aft), $con_aft,
+	      $err_counts{$err}{$loc_con} ;
+  }
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+}
+
+printf OUT "\n\n" ;
+printf OUT "  Local contexts involved in several frequent errors:" ;
+printf OUT "\n  %s\n", '=' x 51 ;
+printf OUT "\n\n" ;
+
+foreach $loc_con (sort {scalar keys %{$loc_con_err_counts{$b}} <=>
+			  scalar keys %{$loc_con_err_counts{$a}}}
+		  keys %loc_con_err_counts)
+{
+
+  if (scalar keys %{$loc_con_err_counts{$loc_con}} == 1)
+  {
+    next ;
+  }
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  printf OUT "  %-*s | %-*s | %-*s \n",
+      $max_con_pos_len+$max_con_bef_len+3, '  Before',
+	$max_word_spec_len+$max_pos_len+3, '   Focus',
+	  $max_con_pos_len+$max_con_aft_len+3, '  After' ;
+
+  printf OUT "  %-*s   %-*s | %-*s   %-*s | %-*s   %-*s \n",
+    $max_con_pos_len, 'CPOS', $max_con_bef_len, 'word',
+       $max_pos_len, 'CPOS', $max_word_spec_len, 'word',
+	$max_con_pos_len, 'CPOS', $max_con_aft_len, 'word' ;
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  $con1 = $loc_con ;
+  $con1 =~ s/\*/ /g ;
+
+  ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $con1) ;
+
+  printf OUT "  %-*s | %-*s | %-*s | %-*s | %-*s | %-*s \n",
+    $max_con_pos_len, $con_pos_bef, $max_con_bef_len+length($con_bef)-uni_len($con_bef), $con_bef,
+      $max_pos_len, $pos, $max_word_spec_len+length($word)-uni_len($word), $word,
+	$max_con_pos_len, $con_pos_aft, $max_con_aft_len+length($con_aft)-uni_len($con_aft), $con_aft ;
+	  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  foreach $err (sort {$loc_con_err_counts{$loc_con}{$b} <=>
+			$loc_con_err_counts{$loc_con}{$a}}
+		keys %{$loc_con_err_counts{$loc_con}})
+  {
+    printf OUT "  %s : %d times\n", $err_desc{$err},
+      $loc_con_err_counts{$loc_con}{$err} ;
+  }
+
+  printf OUT "\n" ;
+}
+
+close GOLD ;
+close SYS ;
+
+close OUT ;

examples/macro_UAS_LAS.py

@@ -0,0 +1,107 @@
+import sys
+
+
+def load_results(filename):
+
+    results = []
+    sent = []
+    with open(filename, 'r') as fp:
+        for i, line in enumerate(fp):
+            if i == 0:
+                continue
+            splits = line.strip().split('\t')
+            if len(line.strip()) == 0:
+                if len(sent) != 0:
+                    results.append(sent)
+                    sent = []
+                continue
+            gold_head = splits[-4]
+            gold_label = splits[-3]
+            pred_head = splits[-2]
+            pred_label = splits[-1]
+            sent.append((gold_head, gold_label, pred_head, pred_label))
+    print('Total Number of sentences ' + str(len(results)))
+    return results
+
+def calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels):
+
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    for i in range(len(gold_heads)):
+        if gold_heads[i] == pred_heads[i]:
+            u_correct +=1
+        u_total +=1
+        l_total +=1
+        if gold_heads[i] == pred_heads[i] and gold_labels[i] == pred_labels[i]:
+            l_correct +=1
+    return u_correct, u_total, l_correct, l_total
+
+
+def calculate_stats(results,path):
+    u_correct = 0
+    l_correct = 0
+    u_total = 0
+    l_total = 0
+
+    sent_uas = []
+    sent_las = []
+
+    for i in range(len(results)):
+        gold_heads, gold_labels, pred_heads, pred_labels = zip(*results[i])
+        u_c, u_t, l_c, l_t = calculate_las_uas(gold_heads, gold_labels, pred_heads, pred_labels)
+        if u_t >0:
+            uas = float(u_c)/u_t
+            las = float(l_c)/l_t
+            sent_uas.append(uas)
+            sent_las.append(las)
+        u_correct += u_c
+        l_correct += l_c
+        u_total += u_t
+        l_total += l_t
+
+    UAS = float(u_correct)/u_total
+    LAS = float(l_correct)/l_total
+    path = path.replace('combined_1300_test.txt','Macro-UAS-LAS-score.txt')
+    f = open(path,'w')
+    f.write('Word level UAS : ' + str(UAS) +'\n')
+    f.write('Word level LAS : ' + str(LAS)+'\n')
+    f.write('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas))+'\n')
+    f.write('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las))+'\n')
+    f.close()
+    print('Word level UAS : ' + str(UAS))
+    print('Word level LAS : ' + str(LAS))
+    print('Sentence level UAS : ' + str(float(sum(sent_uas))/len(sent_uas)))
+    print('Sentence level LAS : ' + str(float(sum(sent_las))/len(sent_las)))
+
+    return sent_uas, sent_las, UAS, LAS
+
+def write_results(sent_uas, sent_las, filename_uas, filename_las):
+
+    fp_uas = open(filename_uas, 'w')
+    fp_las = open(filename_las, 'w')
+
+    for i in range(len(sent_uas)):
+        fp_uas.write(str(sent_uas[i]) + '\n')
+        fp_las.write(str(sent_las[i]) + '\n')
+
+    fp_uas.close()
+    fp_las.close()
+
+
+if __name__=="__main__":
+    dirs = sys.argv[1]
+    # results_2 = load_results(sys.argv[2])
+    ##path = "Predictions/Yap/"+dirs
+    path = "./saved_models/"+dirs+"/final_ensembled/combined_1300_test.txt"
+    result = load_results(path)
+
+
+    sent_uas1, sent_las1, UAS1, LAS1 = calculate_stats(result,path)
+    # sent_uas2, sent_las2, UAS2, LAS2 = calculate_stats(results_2)
+
+
+    write_results(sent_uas1, sent_las1, 'results1_uas.txt', 'results1_las.txt')
+    # write_results(sent_uas2, sent_las2, 'results2_uas.txt', 'results2_las.txt')

examples/write_1300_combined.py

@@ -0,0 +1,48 @@
+import sys
+
+def write_combined(dirs):
+    path = "./saved_models/"+dirs+"/final_ensembled/"
+    f = open(path+'domain_san_test_model_domain_san_data_domain_san_gold.txt','r')
+    gold = f.readlines()
+    f.close()
+    f = open(path+'domain_san_test_model_domain_san_data_domain_san_pred.txt','r')
+    pred = f.readlines()
+    f.close()
+
+    for i in range(len(gold)):
+        if gold[i] == '\n':
+            continue
+        if gold[i].split('\t')[0] == pred[i].split('\t')[0]:
+            gold[i] = gold[i].replace('\n','\t')
+            gold[i] = gold[i]+'\t'.join(pred[i].split('\t')[-2:])
+
+    f = open(path+'domain_san_prose_model_domain_san_data_domain_san_gold.txt','r')
+    prose_gold = f.readlines()
+    f.close()
+    f = open(path+'domain_san_prose_model_domain_san_data_domain_san_pred.txt','r')
+    prose_pred = f.readlines()
+    f.close()
+
+    for i in range(len(prose_gold)):
+        if prose_gold[i] == '\n':
+            gold.append('\n')
+            continue
+        if prose_gold[i].split('\t')[0] == prose_pred[i].split('\t')[0]:
+            line = prose_gold[i].replace('\n','\t')
+            line =line+'\t'.join(prose_pred[i].split('\t')[-2:])
+            gold.append(line)
+    gold.insert(0,'word_id\tword\tpostag\tlemma\tgold_head\tgold_label\tpred_head\tpred_label\n\n')
+
+
+    f = open(path+'combined_1300_test.txt','w')
+    for line in gold:
+        f.write(line)
+    f.close()
+
+
+if __name__=="__main__":
+
+    dir_path = sys.argv[1]
+
+    write_combined(dir_path)
+

run_STBC.sh

@@ -0,0 +1,75 @@
+
+domain="san"
+# Path of pretrained embedding file
+word_path=data/cc.STBC.300.txt
+# Path to store model and predictions  
+saved_models=saved_models
+declare -i num_epochs=100
+declare -i word_dim=300
+start_time=`date +%s`
+current_time=$(date "+%Y.%m.%d-%H.%M.%S")
+model_path="STBC_"$current_time
+touch $saved_models/base_log.txt
+
+###############################################################
+# Running the base Biaffine Parser
+echo "#################################################################"
+echo "Currently base model in progress..."
+echo "#################################################################"
+python examples/GraphParser_MTL_POS.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs  --batch_size 16 --hidden_size 512 --arc_space 512 \
+--arc_tag_space 128 --num_layers 2 --num_filters 100 --use_char --use_pos \
+--word_dim $word_dim --char_dim 100 --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 \
+--epsilon 1e-6 --p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext --char_embedding random  --pos_embedding random --word_path $word_path \
+--model_path $saved_models/$model_path 2>&1 | tee $saved_models/base_log.txt
+
+mv $saved_models/base_log.txt $saved_models/$model_path/base_log.txt
+python examples/BiAFF_write_1300_combined.py $model_path
+python examples/BiAFF_macro_UAS_LAS.py $model_path
+
+# ###################################################################
+# Pretraining step: Running the Sequence Tagger
+# Auxiliary tasks : 'Multitask_case_predict' 'Multitask_POS_predict' 'Multitask_label_predict'
+for task in  'Multitask_POS_predict' 'Multitask_label_predict' 'Multitask_case_predict'; do
+	touch $saved_models/$model_path/log.txt
+	echo "#################################################################"
+	echo "Currently $task in progress..."
+	echo "#################################################################"  
+    python examples/SequenceTagger.py --dataset ud --domain $domain --task $task \
+    --rnn_mode LSTM --num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+	--tag_space 128 --num_layers 2 --num_filters 100 --use_char --use_pos  --char_dim 100 \
+	--pos_dim 100 --initializer xavier --opt adam --learning_rate 0.002 --decay_rate 0.5 \
+	--schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 --p_rnn 0.33 0.33 \
+	--p_in 0.33 --p_out 0.33 --punct_set '.' '``'  ':' ','  \
+	--word_dim $word_dim --word_embedding fasttext --word_path $word_path --pos_embedding random \
+	--parser_path $saved_models/$model_path/ \
+	--use_unlabeled_data --char_embedding random \
+	--model_path $saved_models/$model_path/$task/ 2>&1 | tee $saved_models/$model_path/log.txt
+	mv $saved_models/$model_path/log.txt $saved_models/$model_path/$task/log.txt
+done
+
+######################################################################
+## Integration step: The final ensembled proposed system
+echo "#################################################################"
+echo "Currently final model in progress..."
+echo "#################################################################"
+touch $saved_models/$model_path/log.txt
+python examples/GraphParser_MTL_POS.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+--arc_space 512 --arc_tag_space 128 --num_layers 2 --num_filters 100 --use_char --use_pos \
+--word_dim $word_dim --char_dim 100 --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 \
+--p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst --pos_embedding random \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext --char_embedding random --word_path $word_path  \
+--gating --num_gates 4 \
+--load_sequence_taggers_paths $saved_models/$model_path/Multitask_case_predict/domain_$domain.pt \
+$saved_models/$model_path/Multitask_POS_predict/domain_$domain.pt \
+$saved_models/$model_path/Multitask_label_predict/domain_$domain.pt \
+--model_path $saved_models/$model_path/final_ensembled 2>&1 | tee $saved_models/$model_path/log.txt
+mv $saved_models/$model_path/log.txt $saved_models/$model_path/final_ensembled/log.txt
+python examples/write_1300_combined.py $model_path
+python examples/macro_UAS_LAS.py $model_path
+end_time=`date +%s`
+echo execution time was `expr $end_time - $start_time` s.