File size: 3,779 Bytes
0db33af |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
# The purpose of this file is to take given texts
# Put AI ones into negative and human ones into positive
# While making sure to split all the texts into word by word
# To ensure searching before the text has finished streaming
# Example this: "The dog walked over the pavement." will be turned into:
# The
# The dog
# The dog walked
# The dog walked over
# The dog walked over the
# The dog walked over the pavement
# The dog walked over the pavement.
# Example data row:
# {"query": "Write a story about dogs", "pos": ["lorem ipsum..."], "neg": ["lorem ipsum..."]}
import re
import ujson as json
import random
from tqdm import tqdm
def split_string(text):
"""Split a given text by spaces and punctuation"""
# Split the text by spaces
words = text.split()
# For now we disabled further splitting because of issues
# # Further split each word by punctuation using regex
# split_words = []
# for word in words:
# # Find all substrings that match the pattern: either a word or a punctuation mark
# split_words.extend(re.findall(r'\w+|[^\w\s]', word))
return words
reddit_vs_synth_writing_prompts = []
with open("writing_prompts/reddit_vs_synth_writing_prompts.jsonl", "r") as f:
temp = f.read()
for line in temp.splitlines():
loaded_object = json.loads(line)
if not "story_human" in loaded_object: # Remove ones where we don't have human data
continue
reddit_vs_synth_writing_prompts.append(loaded_object)
dataset_entries = []
SAVE_FILE_NAME = "bert_reddit_vs_synth_writing_prompts.jsonl"
def add_streamed_data(data):
entries = []
data_parts = split_string(data)
for i in range(len(data_parts)):
streamed_so_far = " ".join(data_parts[:i + 1]) # Since python slicing is exclusive toward the end
entries.append({"text": streamed_so_far, "label": HUMAN_LABEL})
return entries
with open(SAVE_FILE_NAME, "w") as f:
f.write("")
NUM_OF_TURNS_TO_DUMP = 200
i = 0
for data in tqdm(reddit_vs_synth_writing_prompts):
# {"text": "AI-generated text example 1", "label": 1},
# Assuming 1 means AI generated, 0 means human
HUMAN_LABEL = 0
AI_LABEL = 1
i += 1
# Below is to enable writing dataset part by part
if i == NUM_OF_TURNS_TO_DUMP:
i = 0
dumped_string = ""
dumped_entries = []
for entry in dataset_entries:
dumped_entries.append(json.dumps(entry))
dumped_string = "\n".join(dumped_entries) + "\n"
with open(SAVE_FILE_NAME, "a") as f:
f.write(dumped_string)
dataset_entries = []
if False: # Disable Streaming
# Add streamed data
human_entries = add_streamed_data(data["story_human"])
dataset_entries.extend(human_entries)
ai_data = []
if data.get("story_opus"):
ai_data.extend(add_streamed_data(data["story_opus"]))
if data.get("story_gpt_3_5"):
ai_data.extend(add_streamed_data(data["story_gpt_3_5"]))
dataset_entries.extend(ai_data)
else:
# Add without streaming
dataset_entries.append({"text": data["story_human"], "label": HUMAN_LABEL})
ai_data = []
if data.get("story_opus"):
dataset_entries.append({"text": data["story_opus"], "label": AI_LABEL})
if data.get("story_gpt_3_5"):
dataset_entries.append({"text": data["story_gpt_3_5"], "label": AI_LABEL})
# Dump as JSONL
dumped_string = ""
dumped_entries = []
for entry in dataset_entries:
dumped_entries.append(json.dumps(entry))
dumped_string = "\n".join(dumped_entries) + "\n"
with open(SAVE_FILE_NAME, "a") as f:
f.write(dumped_string)
|