Spaces:
Runtime error
Runtime error
File size: 6,150 Bytes
970607e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
import os
import cv2
import json
import time
import pickle
import openai
import re
from word2number import w2n
def create_dir(output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
def read_csv(file):
data = []
with open(file, 'r') as f:
for line in f:
data.append(line.strip())
return data
def read_pandas_csv(csv_path):
# read a pandas csv sheet
import pandas as pd
df = pd.read_csv(csv_path)
return df
def read_json(path):
with open(path, 'r', encoding='utf-8') as f:
return json.load(f)
def read_jsonl(file):
with open(file, 'r') as f:
data = [json.loads(line) for line in f]
return data
def read_pickle(path):
with open(path, 'rb') as f:
return pickle.load(f)
def save_json(data, path):
with open(path, 'w') as f:
json.dump(data, f, indent=4)
def save_array_img(path, image):
cv2.imwrite(path, image)
def contains_digit(text):
# check if text contains a digit
if any(char.isdigit() for char in text):
return True
return False
def contains_number_word(text):
# check if text contains a number word
ignore_words = ["a", "an", "point"]
words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text
for word in words:
if word in ignore_words:
continue
try:
w2n.word_to_num(word)
return True # If the word can be converted to a number, return True
except ValueError:
continue # If the word can't be converted to a number, continue with the next word
# check if text contains a digit
if any(char.isdigit() for char in text):
return True
return False # If none of the words could be converted to a number, return False
def contains_quantity_word(text, special_keep_words=[]):
# check if text contains a quantity word
quantity_words = ["most", "least", "fewest"
"more", "less", "fewer",
"largest", "smallest", "greatest",
"larger", "smaller", "greater",
"highest", "lowest", "higher", "lower",
"increase", "decrease",
"minimum", "maximum", "max", "min",
"mean", "average", "median",
"total", "sum", "add", "subtract",
"difference", "quotient", "gap",
"half", "double", "twice", "triple",
"square", "cube", "root",
"approximate", "approximation",
"triangle", "rectangle", "circle", "square", "cube", "sphere", "cylinder", "cone", "pyramid",
"multiply", "divide",
"percentage", "percent", "ratio", "proportion", "fraction", "rate",
]
quantity_words += special_keep_words # dataset specific words
words = re.findall(r'\b\w+\b', text) # This regex pattern matches any word in the text
if any(word in quantity_words for word in words):
return True
return False # If none of the words could be converted to a number, return False
def is_bool_word(text):
if text in ["Yes", "No", "True", "False",
"yes", "no", "true", "false",
"YES", "NO", "TRUE", "FALSE"]:
return True
return False
def is_digit_string(text):
# remove ".0000"
text = text.strip()
text = re.sub(r'\.0+$', '', text)
try:
int(text)
return True
except ValueError:
return False
def is_float_string(text):
# text is a float string if it contains a "." and can be converted to a float
if "." in text:
try:
float(text)
return True
except ValueError:
return False
return False
def copy_image(image_path, output_image_path):
from shutil import copyfile
copyfile(image_path, output_image_path)
def copy_dir(src_dir, dst_dir):
from shutil import copytree
# copy the source directory to the target directory
copytree(src_dir, dst_dir)
import PIL.Image as Image
def get_image_size(img_path):
img = Image.open(img_path)
width, height = img.size
return width, height
def get_chat_response(promot, api_key, api_base, model="gpt-3.5-turbo", temperature=0, max_tokens=256, n=1, patience=10000000,
sleep_time=0):
messages = [
{"role": "user", "content": promot},
]
# print("I am here")
while patience > 0:
patience -= 1
try:
response = openai.ChatCompletion.create(model=model,
messages=messages,
api_key=api_key,
api_base=api_base,
temperature=temperature,
max_tokens=max_tokens,
n=n)
if n == 1:
prediction = response['choices'][0]['message']['content'].strip()
if prediction != "" and prediction != None:
return prediction
else:
prediction = [choice['message']['content'].strip() for choice in response['choices']]
if prediction[0] != "" and prediction[0] != None:
return prediction
except Exception as e:
if "Rate limit" not in str(e):
print(e)
if "Please reduce the length of the messages" in str(e):
print("!!Reduce promot size")
# reduce input prompt and keep the tail
new_size = int(len(promot) * 0.9)
new_start = len(promot) - new_size
promot = promot[new_start:]
messages = [
{"role": "user", "content": promot},
]
if sleep_time > 0:
time.sleep(sleep_time)
return "" |