Spaces:
Paused
Paused
File size: 3,346 Bytes
9189e38 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
import pandas as pd
from sentence_transformers import SentenceTransformer, util
import re
# Load pre-trained SBERT model
model = SentenceTransformer('all-mpnet-base-v2') # Larger and more accurate model
# Load dictionary from CSV file
csv_file_path = './dictionary/dictionary.csv'
df = pd.read_csv(csv_file_path)
dictionary = df['description'].tolist()
# Method to preprocess the input word
def preprocess_input(input_word):
# Remove text within parentheses
input_word = re.sub(r'\(.*?\)', '', input_word).strip()
# Handle broad category and specific item separately when there's a hyphen
if ' - ' in input_word:
broad_category, specific_item = input_word.split(' - ', 1)
return specific_item.strip(), broad_category.strip()
return input_word
# Method to create regex pattern for filtering
def create_regex_pattern(input_word):
words = re.findall(r'\w+', input_word.lower())
pattern = '|'.join([re.escape(word) for word in words])
return pattern
# Method to find the best match for the input word in the dictionary
def match_word(input_word, dictionary):
processed_input = preprocess_input(input_word)
if isinstance(processed_input, tuple):
specific_item, broad_category = processed_input
specific_pattern = create_regex_pattern(specific_item)
broad_pattern = create_regex_pattern(broad_category)
filtered_dictionary = [desc for desc in dictionary if re.search(specific_pattern, desc.lower()) or re.search(broad_pattern, desc.lower())]
else:
specific_item = processed_input
specific_pattern = create_regex_pattern(specific_item)
filtered_dictionary = [desc for desc in dictionary if re.search(specific_pattern, desc.lower())]
print(f"Filtered dictionary size: {len(filtered_dictionary)}")
input_embedding = model.encode(input_word, convert_to_tensor=True)
similarities = []
for entry in filtered_dictionary:
entry_embedding = model.encode(entry, convert_to_tensor=True)
similarity_score = util.pytorch_cos_sim(input_embedding, entry_embedding).item()
similarities.append((entry, similarity_score))
if similarities:
best_match = max(similarities, key=lambda x: x[1])
return best_match if best_match[1] > 0.7 else None
else:
return None
# Example usage
input_words = [
"Carrot (10 lbs )",
"Pepper - Habanero Pepper", "Bananas (12 lbs)", "Squash - Yellow Squash", "Cauliflower",
"Squash mix italian/yellow (30 lbs)", "Tomato - Roma Tomato", "Tomato - Grape Tomato",
"Squash - Mexican Squash", "Pepper - Bell Pepper", "Squash - Italian Squash",
"Pepper - Red Fresno Pepper", "Tomato - Cherry Tomato", "Pepper - Serrano Pepper",
"Kale ( 5 lbs)", "Tomato - Beefsteak Tomato", "Pepper - Anaheim Pepper",
"Banana - Burro Banana", "Squash - Butternut Squash", "Apricot ( 10 lbs)",
"Squash - Acorn Squash", "Tomato - Heirloom Tomato", "Pepper - Pasilla Pepper",
"Pepper - Jalapeno Pepper"
]
for input_word in input_words:
print("Input word:", input_word)
matched_entry = match_word(input_word, dictionary)
if matched_entry:
print("Matched entry:", matched_entry[0])
print("Similarity score:", matched_entry[1])
else:
print("Matched entry: None")
print()
|