sampling_methods.py

import re
from nltk.corpus import stopwords
import random
from termcolor import colored

# Function to Watermark a Word Take Randomly Between Each lcs Point (Random Sampling)
def random_sampling(original_sentence, paraphrased_sentences):
    stop_words = set(stopwords.words('english'))
    original_sentence_lower = original_sentence.lower()
    paraphrased_sentences_lower = [s.lower() for s in paraphrased_sentences]
    paraphrased_sentences_no_stopwords = []

    for sentence in paraphrased_sentences_lower:
        words = re.findall(r'\b\w+\b', sentence)
        filtered_sentence = ' '.join([word for word in words if word not in stop_words])
        paraphrased_sentences_no_stopwords.append(filtered_sentence)

    results = []
    for idx, sentence in enumerate(paraphrased_sentences_no_stopwords):
        common_words = set(original_sentence_lower.split()) & set(sentence.split())
        common_substrings = ', '.join(sorted(common_words))

        words_to_replace = [word for word in sentence.split() if word not in common_words]
        if words_to_replace:
            word_to_mark = random.choice(words_to_replace)
            sentence = sentence.replace(word_to_mark, colored(word_to_mark, 'red'))

        for word in common_words:
            sentence = sentence.replace(word, colored(word, 'green'))

        results.append({
            f"Paraphrased Sentence {idx+1}": sentence,
            "Common Substrings": common_substrings
        })
    return results

# Function for Inverse Transform Sampling
def inverse_transform_sampling(original_sentence, paraphrased_sentences):
    stop_words = set(stopwords.words('english'))
    original_sentence_lower = original_sentence.lower()
    paraphrased_sentences_lower = [s.lower() for s in paraphrased_sentences]
    paraphrased_sentences_no_stopwords = []

    for sentence in paraphrased_sentences_lower:
        words = re.findall(r'\b\w+\b', sentence)
        filtered_sentence = ' '.join([word for word in words if word not in stop_words])
        paraphrased_sentences_no_stopwords.append(filtered_sentence)

    results = []
    for idx, sentence in enumerate(paraphrased_sentences_no_stopwords):
        common_words = set(original_sentence_lower.split()) & set(sentence.split())
        common_substrings = ', '.join(sorted(common_words))

        words_to_replace = [word for word in sentence.split() if word not in common_words]
        if words_to_replace:
            probabilities = [1 / len(words_to_replace)] * len(words_to_replace)
            chosen_word = random.choices(words_to_replace, weights=probabilities)[0]
            sentence = sentence.replace(chosen_word, colored(chosen_word, 'magenta'))

        for word in common_words:
            sentence = sentence.replace(word, colored(word, 'green'))

        results.append({
            f"Paraphrased Sentence {idx+1}": sentence,
            "Common Substrings": common_substrings
        })
    return results

# Function for Contextual Sampling
def contextual_sampling(original_sentence, paraphrased_sentences):
    stop_words = set(stopwords.words('english'))
    original_sentence_lower = original_sentence.lower()
    paraphrased_sentences_lower = [s.lower() for s in paraphrased_sentences]
    paraphrased_sentences_no_stopwords = []

    for sentence in paraphrased_sentences_lower:
        words = re.findall(r'\b\w+\b', sentence)
        filtered_sentence = ' '.join([word for word in words if word not in stop_words])
        paraphrased_sentences_no_stopwords.append(filtered_sentence)

    results = []
    for idx, sentence in enumerate(paraphrased_sentences_no_stopwords):
        common_words = set(original_sentence_lower.split()) & set(sentence.split())
        common_substrings = ', '.join(sorted(common_words))

        words_to_replace = [word for word in sentence.split() if word not in common_words]
        if words_to_replace:
            context = " ".join([word for word in sentence.split() if word not in common_words])
            chosen_word = random.choice(words_to_replace)
            sentence = sentence.replace(chosen_word, colored(chosen_word, 'red'))

        for word in common_words:
            sentence = sentence.replace(word, colored(word, 'green'))

        results.append({
            f"Paraphrased Sentence {idx+1}": sentence,
            "Common Substrings": common_substrings
        })
    return results

# Function for Exponential Minimum Sampling
def exponential_minimum_sampling(original_sentence, paraphrased_sentences):
    stop_words = set(stopwords.words('english'))
    original_sentence_lower = original_sentence.lower()
    paraphrased_sentences_lower = [s.lower() for s in paraphrased_sentences]
    paraphrased_sentences_no_stopwords = []

    for sentence in paraphrased_sentences_lower:
        words = re.findall(r'\b\w+\b', sentence)
        filtered_sentence = ' '.join([word for word in words if word not in stop_words])
        paraphrased_sentences_no_stopwords.append(filtered_sentence)

    results = []
    for idx, sentence in enumerate(paraphrased_sentences_no_stopwords):
        common_words = set(original_sentence_lower.split()) & set(sentence.split())
        common_substrings = ', '.join(sorted(common_words))

        words_to_replace = [word for word in sentence.split() if word not in common_words]
        if words_to_replace:
            num_words = len(words_to_replace)
            probabilities = [2 ** (-i) for i in range(num_words)]
            chosen_word = random.choices(words_to_replace, weights=probabilities)[0]
            sentence = sentence.replace(chosen_word, colored(chosen_word, 'red'))

        for word in common_words:
            sentence = sentence.replace(word, colored(word, 'green'))

        results.append({
            f"Paraphrased Sentence {idx+1}": sentence,
            "Common Substrings": common_substrings
        })
    return results



    #---------------------------------------------------------------------------
    # aryans implementation please refactor it as you see fit 

import torch
import random

def sample_word(words, logits, sampling_technique='inverse_transform', temperature=1.0):
    if sampling_technique == 'inverse_transform':
        probs = torch.softmax(torch.tensor(logits), dim=-1)
        cumulative_probs = torch.cumsum(probs, dim=-1)
        random_prob = random.random()
        sampled_index = torch.where(cumulative_probs >= random_prob)[0][0]
    elif sampling_technique == 'exponential_minimum':
        probs = torch.softmax(torch.tensor(logits), dim=-1)
        exp_probs = torch.exp(-torch.log(probs))
        random_probs = torch.rand_like(exp_probs)
        sampled_index = torch.argmax(random_probs * exp_probs)
    elif sampling_technique == 'temperature':
        scaled_logits = torch.tensor(logits) / temperature
        probs = torch.softmax(scaled_logits, dim=-1)
        sampled_index = torch.multinomial(probs, 1).item()
    elif sampling_technique == 'greedy':
        sampled_index = torch.argmax(torch.tensor(logits)).item()
    else:
        raise ValueError("Invalid sampling technique. Choose 'inverse_transform', 'exponential_minimum', 'temperature', or 'greedy'.")
    
    sampled_word = words[sampled_index]
    return sampled_word