|
|
|
|
|
|
|
|
|
|
|
from abc import ABC, abstractmethod |
|
|
|
import jsonlines |
|
import json |
|
import copy |
|
import random |
|
import fire |
|
|
|
|
|
class Converter(ABC): |
|
|
|
def __init__(self, filepath) -> None: |
|
super().__init__() |
|
|
|
self.filepath = filepath |
|
|
|
def convert(self): |
|
""" |
|
Implement your convert logics in this function |
|
""" |
|
self.start() |
|
self.process() |
|
self.end() |
|
pass |
|
|
|
def start(self): |
|
print(f'Start processing {self.__class__.__name__} at {self.filepath}') |
|
|
|
def end(self): |
|
print( |
|
f'Finish processing {self.__class__.__name__} at {self.filepath}') |
|
|
|
@abstractmethod |
|
def process(self): |
|
""" |
|
Implement your convert logics in this function |
|
""" |
|
|
|
|
|
class WoWConverter(Converter): |
|
|
|
def process(self): |
|
|
|
train_data = json.load(open(f'{self.filepath}/train.json')) |
|
topic_data = {} |
|
for i in train_data: |
|
chosen_topic = i['chosen_topic'] |
|
if not chosen_topic in topic_data.keys(): |
|
topic_data[chosen_topic] = [] |
|
else: |
|
topic_data[chosen_topic].append((i['persona'], i['dialog'])) |
|
|
|
topic_data_sorted = sorted( |
|
topic_data.items(), key=lambda k: -len(k[1])) |
|
|
|
examples = [] |
|
for topic, dialogs in topic_data_sorted[1:100:2]: |
|
for persona, dialog in dialogs[:1]: |
|
history = [persona] |
|
history = [] |
|
example = {} |
|
checked_sentence = '' |
|
for i in dialog: |
|
speaker = i['speaker'] |
|
text = i['text'] |
|
if 'Wizard' in speaker: |
|
|
|
try: |
|
checked_sentence = next( |
|
iter(i['checked_sentence'].values())) |
|
except Exception: |
|
checked_sentence = '' |
|
response = text |
|
example['Context'] = ' EOS '.join(history) |
|
example['Knowledge'] = checked_sentence |
|
example['Response'] = response.strip() |
|
examples.append(copy.deepcopy(example)) |
|
example = {} |
|
else: |
|
text = text |
|
history.append(text.strip()) |
|
|
|
with jsonlines.open('../data/wow/wow_train.jsonl', mode='w') as writer: |
|
for i in examples: |
|
writer.write(i) |
|
|
|
for split in ['valid', 'test']: |
|
data = json.load( |
|
open(f'{self.filepath}/{split}_random_split.json')) |
|
examples = [] |
|
for dialog in data: |
|
history = [] |
|
example = {} |
|
checked_sentence = '' |
|
persona = dialog['persona'] |
|
history = [persona] |
|
for i in dialog['dialog']: |
|
speaker = i['speaker'] |
|
text = i['text'] |
|
if 'Wizard' in speaker: |
|
try: |
|
checked_sentence = next( |
|
iter(i['checked_sentence'].values())) |
|
except Exception: |
|
checked_sentence = '' |
|
|
|
text = text |
|
response = text |
|
example['Context'] = ' EOS '.join(history) |
|
example['Knowledge'] = checked_sentence |
|
example['Response'] = response.strip() |
|
examples.append(copy.deepcopy(example)) |
|
example = {} |
|
else: |
|
text = text |
|
history.append(text) |
|
|
|
with jsonlines.open(f'../data/wow/wow_{split}.jsonl', mode='w') as writer: |
|
for i in examples: |
|
writer.write(i) |
|
|
|
return super().process() |
|
|
|
|
|
class WoIConverter(Converter): |
|
|
|
def process(self): |
|
for split in ['train', 'valid', 'test']: |
|
reader = jsonlines.open(f'{self.filepath}/{split}.jsonl') |
|
examples = [] |
|
num_of_dialogs = 0 |
|
for dialog in reader: |
|
num_of_dialogs += 1 |
|
example = {} |
|
history = [] |
|
turn = '' |
|
data = list(dialog.values())[0] |
|
persona = data['apprentice_persona'] |
|
history = [persona.replace('\n', ' ')] |
|
|
|
for i in data['dialog_history']: |
|
if 'SearchAgent' in i['action']: |
|
continue |
|
|
|
else: |
|
if i['action'] == 'Wizard => Apprentice': |
|
|
|
contents = [] |
|
selected = [] |
|
|
|
for content_ in i['context']['contents']: |
|
contents.extend(content_['content']) |
|
|
|
for selected_ in i['context']['selected_contents']: |
|
selected.extend(selected_) |
|
|
|
knowledge = [] |
|
for c, s in zip(contents, selected[1:]): |
|
if s: |
|
knowledge.append(c) |
|
|
|
turn = i['text'].strip() |
|
example['Context'] = ' EOS '.join(history) |
|
example['Knowledge'] = ' '.join(knowledge) |
|
example['Response'] = turn.strip() |
|
examples.append(copy.deepcopy(example)) |
|
else: |
|
turn = i['text'].strip() |
|
history.append(turn) |
|
|
|
with jsonlines.open(f'../data/woi/woi_{split}.jsonl', mode='w') as writer: |
|
for i in examples: |
|
if split == 'train': |
|
if random.random() < 0.006: |
|
writer.write(i) |
|
else: |
|
writer.write(i) |
|
|
|
return super().process() |
|
|
|
|
|
class CoQAConverter(Converter): |
|
|
|
def process(self): |
|
|
|
for split in ['train', 'dev']: |
|
source = open(f'{self.filepath}/seq2seq-{split}-h2-src.txt') |
|
target = open(f'{self.filepath}/seq2seq-{split}-h2-tgt.txt') |
|
|
|
source_ = [] |
|
for line in source: |
|
if line.strip() != '': |
|
sotry, question = line.strip().split('||') |
|
source_.append((sotry, question)) |
|
|
|
target_ = [] |
|
for line in target: |
|
if line.strip() != '': |
|
target_.append(line.strip()) |
|
examples = [] |
|
for context, response in zip(source_, target_): |
|
story, question = context |
|
examples.append( |
|
{'Context': question, 'Response': response, 'Knowledge': story}) |
|
|
|
if split == 'dev': |
|
split = 'valid' |
|
with jsonlines.open(f'../data/coqa/coqa_{split}.jsonl', mode='w') as writer: |
|
for i in examples: |
|
if split == 'train': |
|
if random.random() < 0.006: |
|
writer.write(i) |
|
else: |
|
writer.write(i) |
|
|
|
return super().process() |
|
|
|
|
|
class MultiWOZConverter(Converter): |
|
|
|
def process(self): |
|
|
|
for split in ['train', 'val', 'test']: |
|
data = json.load(open(f'{self.filepath}/{split}.json')) |
|
examples = [] |
|
for i in data: |
|
name = i['file'].lower() |
|
history = [] |
|
for turn in i['info']: |
|
history.append(turn['user_orig']) |
|
bs = turn['BS'] |
|
bs_str = [] |
|
for domain, states in bs.items(): |
|
domain_str = [] |
|
for state in states: |
|
domain_str.append(state[0] + ' = ' + state[1]) |
|
domain_str = ' ; '.join(domain_str) |
|
bs_str.append(domain + ' ' + domain_str) |
|
bs_str = ' | '.join(bs_str) |
|
|
|
db_str = 'kb ' |
|
db = turn['KB'] |
|
if db == 0: |
|
db_str += 'zero' |
|
elif db_str == 1: |
|
db_str += 'one' |
|
elif db_str == 2: |
|
db_str += 'two' |
|
else: |
|
db_str += 'more than two' |
|
|
|
act_seq = ' '.join(turn['act'].keys()) |
|
example = {} |
|
example['Context'] = ' EOS '.join(history[:]) |
|
example['Knowledge'] = bs_str + ' | ' + db_str |
|
example['Response'] = act_seq + ' | ' + turn['sys'].strip() |
|
|
|
history.append(turn['sys'].strip()) |
|
examples.append(copy.copy(example)) |
|
|
|
if split == 'val': |
|
split = 'valid' |
|
with jsonlines.open(f'../data/multiwoz/multiwoz_{split}.jsonl', mode='w') as writer: |
|
for i in examples: |
|
if split == 'train': |
|
if random.random() < 0.006: |
|
writer.write(i) |
|
else: |
|
writer.write(i) |
|
|
|
return super().process() |
|
|
|
|
|
def convert(class_name, file_path): |
|
eval(class_name)(file_path).convert() |
|
|
|
|
|
def main(): |
|
fire.Fire(convert) |
|
|
|
|
|
if __name__ == '__main__': |
|
main() |
|
|
