dataset.py

import re
import tensorflow as tf
import tensorflow_datasets as tfds
import nltk
from nltk.stem import WordNetLemmatizer

nltk.download('wordnet')
nltk.download('punkt')

lemmatizer = WordNetLemmatizer()


def preprocess_sentence(sentence):
    sentence = sentence.lower().strip()
    
    sentence = re.sub(r"([?.!¿])", r" \1 ", sentence)
    sentence = re.sub(r'[" "]+', " ", sentence)
    sentence = re.sub(r"[-()\"#/@;:<>{}+=~|.?,]", "", sentence)

    sentence = re.sub(r"[^a-zA-ZğüşöçıİĞÜŞÖÇ?.!,¿]+", " ", sentence)
    sentence = sentence.strip()  
    
    sentence = ' '.join([lemmatizer.lemmatize(w) for w in nltk.word_tokenize(sentence)])
    
    return sentence

def load_conversations(hparams, lines_file, conversations_file):
    id2line = {}
    
    with open(lines_file, encoding = "utf-8", errors="ignore") as file:
        lines = file.readlines()
    
    for line in lines:
        parts = line.replace("\n", "").split(" +++$+++ ")
        id2line[parts[0]] = parts[4]
        
    questions = []
    answers = []
    
    with open(conversations_file, "r") as file:
        lines = file.readlines()
    for line in lines:
        parts = line.replace("\n", "").split(" +++$+++ ")
        conversation = [line[1:-1] for line in parts[3][1:-1].split(", ")]
        for i in range(len(conversation) - 1):
            questions.append(preprocess_sentence(id2line[conversation[i]]))
            answers.append(preprocess_sentence(id2line[conversation[i + 1]]))
            if len(questions) >= hparams.max_samples:
                return questions, answers
            
    return questions, answers


def tokenize(hparams, tokenizer, questions, answers):
    tokenized_inputs, tokenized_outputs = [], []
    
    for (question, answer) in zip(questions, answers):
        sentence1 = hparams.start_token + tokenizer.encode(question) + hparams.end_token
        sentence2 = hparams.start_token + tokenizer.encode(answer) + hparams.end_token
        
        if (len(sentence1) <= hparams.max_length and len(sentence2) <= hparams.max_length):
            tokenized_inputs.append(sentence1)
            tokenized_outputs.append(sentence2)
    
    tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(
        tokenized_inputs, maxlen=hparams.max_length, padding="post")
    tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(
        tokenized_outputs, maxlen=hparams.max_length, padding="post")

    return tokenized_inputs, tokenized_outputs


def get_dataset(hparams):
    lines_file ="data/lines.txt"
    conversations_file = "data/conversations.txt"
    
    questions, answers = load_conversations(hparams, lines_file, conversations_file)
    
    tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(questions + answers, target_vocab_size=2**13)
    
    tokenizer.save_to_file('tokenizer')
    
    hparams.start_token = [tokenizer.vocab_size]
    hparams.end_token = [tokenizer.vocab_size + 1]
    hparams.vocab_size = tokenizer.vocab_size + 2
    
    questions, answers = tokenize(hparams, tokenizer, questions, answers)
    
    dataset = tf.data.Dataset.from_tensor_slices(
        ({"inputs": questions, "dec_inputs": answers[:, :-1]}, answers[:, 1:])
    )
    
    dataset = dataset.cache()
    dataset = dataset.shuffle(len(questions))
    dataset = dataset.batch(hparams.batch_size)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
    
    return dataset, tokenizer