File size: 3,060 Bytes
e8f4897 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import argparse
import os
# /home/jivnesh/Documents/DCST/d
# 'ar','eu','cs','de','hu','lv','pl','sv','fi','ru'
# python utils/io_/convert_ud_to_onto_format.py --ud_data_path d
def write_ud_files(args):
languages_for_low_resource = ['el']
languages = sorted(list(set(languages_for_low_resource)))
splits = ['train', 'dev', 'test']
lng_to_files = dict((language, {}) for language in languages)
for language, d in lng_to_files.items():
for split in splits:
d[split] = []
lng_to_files[language] = d
sub_folders = os.listdir(args.ud_data_path)
for sub_folder in sub_folders:
folder = os.path.join(args.ud_data_path, sub_folder)
files = os.listdir(folder)
for file in files:
for language in languages:
if file.startswith(language) and file.endswith('conllu'):
for split in splits:
if split in file:
full_path = os.path.join(folder, file)
lng_to_files[language][split].append(full_path)
break
for language, split_dict in lng_to_files.items():
for split, files in split_dict.items():
if split == 'dev' and len(files) == 0:
files = split_dict['train']
print('No dev files were found, copying train files instead')
sentences = []
num_sentences = 0
for file in files:
with open(file, 'r') as file:
for line in file:
new_line = []
line = line.strip()
if len(line) == 0:
sentences.append(new_line)
num_sentences += 1
continue
tokens = line.split('\t')
if not tokens[0].isdigit():
continue
id = tokens[0]
word = tokens[1]
pos = tokens[3]
ner = tokens[5]
head = tokens[6]
arc_tag = tokens[7]
new_line = [id, word, pos, ner, head, arc_tag]
sentences.append(new_line)
print('Language: %s Split: %s Num. Sentences: %s ' % (language, split, num_sentences))
if not os.path.exists('data'):
os.makedirs('data')
write_data_path = 'data/MRL/ud_pos_ner_dp_' + split + '_' + language
print('creating %s' % write_data_path)
with open(write_data_path, 'w') as f:
for line in sentences:
f.write('\t'.join(line) + '\n')
def main():
# Parse arguments
args_ = argparse.ArgumentParser()
args_.add_argument('--ud_data_path', help='Directory path of the UD treebanks.', required=True)
args = args_.parse_args()
write_ud_files(args)
if __name__ == "__main__":
main()
|