given the following code, i need you to:
1. making it professional, efficient and production ready.
2. require only the path for input file 
3 the clustered csv file and onehot file  can be saved in a subfolder of the original input path e.g. /processed/clustered_{originalname}.csv and /processed/onehot_{originalname}.tsv
4. the lemmatized csv and labels counting csv can be saved just if required by used. always in the /processed/lemmatized.csv path
5. the cli print should also be saved in a /processed/log_{originalname}.log

this is the code:

import os
import re
import csv
import nltk
import pandas as pd
from collections import defaultdict, Counter
from nltk.stem import WordNetLemmatizer
from gensim.models import KeyedVectors, FastText
from sklearn.cluster import AgglomerativeClustering
import numpy as np
from sklearn.preprocessing import OneHotEncoder
from nltk import pos_tag
from nltk.corpus import wordnet as wn
from nltk.corpus import stopwords
# Ensure NLTK data is downloaded
nltk.download('wordnet')
nltk.download('stopwords')
# Initialize the lemmatizer
lemmatizer = WordNetLemmatizer()
# Set of stopwords to filter out
stop_words = set(stopwords.words('english'))
# ------------------------------#
# Cleaning Functions #
# ------------------------------#
def clean_csv(input_file, labels):
total_row_count = 0
empty_row_count = 0
na_count = 0
no_comma_count = 0
no_comma_rows = []
cleaned_data = []
# Check if the input file exists
if not os.path.exists(input_file):
raise FileNotFoundError(f"The input file '{input_file}' does not exist.")
# Read the entire content of the file
with open(input_file, 'r', newline='', encoding='utf-8') as infile:
content = infile.read()
# Remove newline characters inside quoted strings
content = re.sub(r'("\[.*?")\n', r'\1 ', content)
# Use StringIO to read the cleaned content as a CSV
import io
temp_file = io.StringIO(content)
reader = csv.reader(temp_file)
# Skip the first line (header)
header = next(reader)
# Process each row
for row in reader:
total_row_count += 1
jpg_file = row[0]
keywords = extract_keywords(row[1], labels)
# Count occurrences of "N/A" and empty spaces
na_count += keywords.count("N/A")
na_count += sum(1 for k in keywords if k.strip() == "")
# Check if the row is empty or contains only "N/A" or empty spaces
if not keywords or all(k.strip().upper() == 'N/A' or k.strip() == '' for k in keywords):
empty_row_count += 1
continue # Skip this row
# Concatenate the keywords into a single string
keywords_str = ', '.join(keywords)
# Replace multiple spaces with a comma and a single space
keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
# Check if the row has no commas
if ',' not in keywords_str:
no_comma_count += 1
no_comma_rows.append(f'{jpg_file},{keywords_str}')
# Add to cleaned data
cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
# Now, create a DataFrame from the cleaned_data
df = pd.DataFrame(cleaned_data)
# Clean the 'Keywords' column further
df = clean_keywords_in_df(df, labels)
# Optional: Print statistics (comment out if not needed)
# print(f"Total rows: {total_row_count}")
# print(f"Number of empty or N/A rows: {empty_row_count}")
# print(f"Occurrences of N/A (including empty spaces): {na_count}")
# print(f"Number of rows without commas: {no_comma_count}")
# print("\nRows without commas:")
# for row in no_comma_rows:
# print(row)
return df
def clean_keywords_in_df(df, labels):
"""
 Function to clean the 'Keywords' column in the DataFrame.
 - Removes occurrences of specified labels.
 - Removes '- ' (dash and space).
 - Removes single quotes around keywords.
 - Replaces multiple spaces with a comma and a single space.
 """
# Construct a regex pattern from the labels with word boundaries
# Example: r'\b(Type|Color|Wear Location|Material|Style)\b\s*'
labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*'
# Replace labels with an empty string using regex
df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True)
# Remove any '- ' (dash and space) using exact string replacement
df['Keywords'] = df['Keywords'].str.replace('- ', '', regex=False)
# Remove single quotes around keywords using regex
df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True)
# Replace multiple spaces with a comma and a single space using regex
df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True)
# Optionally, remove leading/trailing commas and spaces
df['Keywords'] = df['Keywords'].str.strip(' ,')
return df
def extract_keywords(line, labels):
"""
 Extracts keywords from a line.
 - First, tries to extract keywords between curly braces.
 - If not found, removes labels and other unwanted characters.
 - Splits keywords by commas and strips whitespace.
 """
# First, try to extract keywords between curly braces
keywords = re.findall(r'\{([^}]+)\}', line)
if keywords:
# Further split if multiple keywords are within braces
return [kw.strip() for group in keywords for kw in group.split(',')]
# Remove newline characters from the keywords section
line = line.replace('\n', ' ')
# Remove the labels using word boundaries and considering possible formats
for label in labels:
# Patterns like "Label:", "Label -", "[Label]:", etc.
pattern = rf'\b{re.escape(label)}\b[: -]?'
line = re.sub(pattern, '', line, flags=re.IGNORECASE)
# Remove any '- ' (dash and space) using exact string replacement
line = line.replace('- ', '')
# Remove single quotes around keywords using regex
line = re.sub(r"'([^']+)'", r'\1', line)
# Remove any remaining brackets
line = line.replace('[', '').replace(']', '').replace('{', '').replace('}', '')
# Split keywords by commas and strip whitespace
keywords = [keyword.strip() for keyword in line.split(',')]
return keywords
# ------------------------------#
# Lemmatization Functions #
# ------------------------------#
def lemmatize_keyword(keyword):
return lemmatizer.lemmatize(keyword.lower())
def get_wordnet_pos(treebank_tag):
if treebank_tag.startswith('J'):
return wn.ADJ # Adjective
elif treebank_tag.startswith('V'):
return wn.VERB # Verb
elif treebank_tag.startswith('N'):
return wn.NOUN # Noun
elif treebank_tag.startswith('R'):
return wn.ADV # Adverb
else:
return None
# ------------------------------#
# Model Loading Functions #
# ------------------------------#
def load_model(model_type, model_path):
if model_type == 'word2vec':
print("Loading Word2Vec model. This may take a few minutes...")
model = KeyedVectors.load_word2vec_format(model_path, binary=True)
print("Word2Vec model loaded successfully.")
elif model_type == 'glove':
print("Loading GloVe model. This may take a few minutes...")
model = {}
with open(model_path, 'r', encoding='utf-8') as f:
for line in f:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], dtype='float32')
model[word] = vector
print("GloVe model loaded successfully.")
elif model_type == 'fasttext':
print("Loading FastText model. This may take a few minutes...")
model = FastText.load_fasttext_format(model_path)
print("FastText model loaded successfully.")
else:
raise ValueError(f"Unsupported model type: {model_type}")
return model
def get_word_vector(word, model, model_type):
if model_type in ['word2vec', 'fasttext']:
if word in model:
return model[word]
elif model_type == 'glove':
if word in model:
return model[word]
# Handle multi-word phrases by averaging their word vectors
words = word.split()
vectors = []
for w in words:
if model_type in ['word2vec', 'fasttext'] and w in model:
vectors.append(model[w])
elif model_type == 'glove' and w in model:
vectors.append(model[w])
if vectors:
return np.mean(vectors, axis=0)
else:
return None
# ------------------------------#
# Lemmatize and Count Keywords#
# ------------------------------#
def lemmatize_keywords(df, lemmatized_counts_output_file):
per_row_data = []
keyword_counts = Counter()
# Iterate through each row to extract and lemmatize keywords
for index, row in df.iterrows():
image_name = row.get('Image Name', '')
keywords_str = row.get('Keywords', '')
if pd.isna(keywords_str):
keywords_str = ''
# Split keywords by comma and strip whitespace
extracted_keywords = [kw.strip() for kw in keywords_str.split(',')]
# Lemmatize keywords
lemmatized_keywords = [lemmatize_keyword(kw) for kw in extracted_keywords if kw]
# Update global counts
keyword_counts.update(lemmatized_keywords)
# Store per-row data
per_row_data.append({
'Image Name': image_name,
'Keywords': ', '.join(extracted_keywords),
'Lemmatized Keywords': ', '.join(lemmatized_keywords)
})
# Create a DataFrame from per_row_data
per_row_df = pd.DataFrame(per_row_data)
# Write the lemmatized keyword counts to CSV
lemmatized_counts = [{'Lemmatized Keyword': kw, 'Count': count}
for kw, count in keyword_counts.items()]
lemmatized_counts_df = pd.DataFrame(lemmatized_counts)
# Sort by Count in descending order
lemmatized_counts_df = lemmatized_counts_df.sort_values(by='Count', ascending=False)
# Write to CSV
lemmatized_counts_df.to_csv(lemmatized_counts_output_file, index=False)
print(f"Lemmatized keyword counts saved to '{lemmatized_counts_output_file}'.")
return per_row_df, keyword_counts
# ------------------------------#
# Clustering Functions #
# ------------------------------#
def cluster_infrequent_keywords(keywords, keyword_counts, model, model_type, num_clusters=5, distance_threshold=None):
vectors = []
valid_keywords = []
for keyword in keywords:
vector = get_word_vector(keyword, model, model_type)
if vector is not None:
vectors.append(vector)
valid_keywords.append(keyword)
else:
print(f"Vector not found for keyword: '{keyword}'")
if not vectors:
print("No valid vectors found for clustering.")
return {}
vectors = np.array(vectors)
# Perform Hierarchical Clustering with distance threshold
if distance_threshold is not None:
print(f"Performing Agglomerative Clustering with distance_threshold={distance_threshold}")
clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold)
else:
print(f"Performing Agglomerative Clustering with n_clusters={num_clusters}")
clustering_model = AgglomerativeClustering(n_clusters=num_clusters)
labels = clustering_model.fit_predict(vectors)
# Organize keywords by clusters
clusters = defaultdict(list)
for label, keyword in zip(labels, valid_keywords):
clusters[label].append(keyword)
# Select the most frequent keyword in each cluster as the representative
representative_keywords = {}
for label, words in clusters.items():
most_frequent = max(words, key=lambda w: keyword_counts[w])
representative_keywords[label] = most_frequent
# Create a mapping from keyword to representative
keyword_to_representative = {}
for label, words in clusters.items():
rep = representative_keywords[label]
for word in words:
keyword_to_representative[word] = rep
return keyword_to_representative
def perform_clustering_with_threshold(keyword_counts, model, model_type, threshold=5, num_clusters=5, distance_threshold=None):
# Separate frequent and infrequent keywords
frequent_keywords = [kw for kw, count in keyword_counts.items() if count > threshold]
infrequent_keywords = [kw for kw, count in keyword_counts.items() if count <= threshold]
print(f"Number of frequent keywords (> {threshold}): {len(frequent_keywords)}")
print(f"Number of infrequent keywords (<= {threshold}): {len(infrequent_keywords)}")
# Keep frequent keywords as-is
clustered = {kw: kw for kw in frequent_keywords}
# Cluster infrequent keywords and map them to their representatives
if infrequent_keywords:
mapping = cluster_infrequent_keywords(
infrequent_keywords, keyword_counts, model, model_type,
num_clusters=num_clusters, distance_threshold=distance_threshold
)
clustered.update(mapping)
print(f"Number of cluster representatives: {len(set(mapping.values()))}")
else:
print("No infrequent keywords to cluster.")
return clustered
# ------------------------------#
# Processing Clustered Data #
# ------------------------------#
def categorize_and_count_clustered_keywords(per_row_df, labels, Category_Limit=50):
category_keyword_counts = {label: Counter() for label in labels}
for idx, row in per_row_df.iterrows():
clustered_keywords = row['Clustered Keywords'].split(', ')
for i, label in enumerate(labels):
if i < len(clustered_keywords):
keyword = clustered_keywords[i]
category_keyword_counts[label][keyword] += 1
# Limit to top N keywords per category
for label in labels:
category_keyword_counts[label] = Counter(dict(category_keyword_counts[label].most_common(Category_Limit)))
return category_keyword_counts
def generate_clustered_keywords_and_categories(per_row_df, clustered_keyword_mapping, category_keyword_counts, labels, remove_duplicates=False):
top_keywords_per_category = {label: set(counts.keys()) for label, counts in category_keyword_counts.items()}
# Initialize lists for categories and counters for "Other"
category_keywords_list = {label: [] for label in labels}
clustered_keywords_list = []
# Initialize a counter for the number of rows with specific "Other" counts
max_other_count = len(labels)
other_count_rows = {i: 0 for i in range(max_other_count + 1)} # from 0 to len(labels)
for idx, row in per_row_df.iterrows():
lemmatized_keywords = row['Lemmatized Keywords'].split(', ')
clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords]
if remove_duplicates:
# Remove duplicates while preserving order
clustered_keywords = list(dict.fromkeys(clustered_keywords))
clustered_keywords_list.append(', '.join(clustered_keywords))
# Initialize "Other" counter for each row
row_other_count = 0
# Assign keywords to categories based on positions
for i, label in enumerate(labels):
if i < len(clustered_keywords):
keyword = clustered_keywords[i]
# Map to 'Other' if not in top keywords
if keyword not in top_keywords_per_category[label]:
keyword = 'Other'
row_other_count += 1 # Increase the row counter for "Other"
category_keywords_list[label].append(keyword)
else:
category_keywords_list[label].append('') # or 'Other'
# Count the number of rows with the specific number of "Other" keywords
if row_other_count <= max_other_count:
other_count_rows[row_other_count] += 1
per_row_df['Clustered Keywords'] = clustered_keywords_list
# Add the category columns to per_row_df
for label in labels:
per_row_df[label] = category_keywords_list[label]
# Optional: Print summary of "Other" counts per category
other_count = {label: 0 for label in labels} # Counter for "Other" per category
print("\nSummary of 'Other' counts per category:")
for label in labels:
other_count[label] = sum(1 for keyword in category_keywords_list[label] if keyword == 'Other')
print(f"{label}: {other_count[label]}")
# Print the summary of rows having specific "Other" counts
print("\nSummary of 'Other' count rows:")
for count in range(max_other_count + 1):
print(f"Rows with {count} 'Other' keywords: {other_count_rows[count]}")
return per_row_df
# ------------------------------#
# One-Hot Encoding #
# ------------------------------#
def encode_categorical_features(input_df, output_tsv_path, id_column='Image Name', remove_extension='.jpg', labels=[], sep='\t', header=False, index=False):
"""
 Encodes categorical features using One-Hot Encoding and saves them to a TSV file.
 """
if not labels:
raise ValueError("Labels for encoding must be provided.")
# Read only the necessary columns to improve efficiency
columns_to_encode = labels
columns_to_read = [id_column] + columns_to_encode
df = input_df[columns_to_read]
# Initialize OneHotEncoder with dtype set to integer to output int values
enc = OneHotEncoder(handle_unknown='ignore', dtype=np.int8)
# Fit and transform the categorical columns
emb = enc.fit_transform(df[columns_to_encode])
# Retrieve the names of the one-hot encoded features
feature_names = enc.get_feature_names_out(columns_to_encode)
# Create a DataFrame with the encoded features and ensure they are integers
emb_df = pd.DataFrame(emb.toarray(), columns=feature_names, dtype=np.int8)
# Concatenate the id_column with the one-hot encoded features
result_df = pd.concat([df[[id_column]].reset_index(drop=True), emb_df.reset_index(drop=True)], axis=1)
# Remove specified extension from id_column
if remove_extension:
result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False)
# Save the final DataFrame to a TSV file without headers and index
result_df.to_csv(output_tsv_path, index=index, header=header, sep=sep)
# ------------------------------#
# Main #
# ------------------------------#
def main():
# ------------------------------#
# File Paths & Params #
# ------------------------------#
# Input and output file paths
input_file = '/home/danilo/Projects/IR_MD/utils/_testing_publication_cleaned_code/clothing_qwen_original.csv' # Replace with your actual input file path
# Since we are not writing cleaned output to a file, we no longer have 'cleaned_output_file'
# The lemmatized_counts_output_file can be saved in the same location as the input file
input_dir = os.path.dirname(input_file)
input_filename = os.path.basename(input_file)
input_name_no_ext = os.path.splitext(input_filename)[0]
output_file = os.path.join(input_dir, f'clustered_{input_filename}')
one_hot_encoded_file = os.path.join(input_dir, f'onehot_{input_name_no_ext}.tsv')
lemmatized_counts_output_file = os.path.join(input_dir, f'lemmatized_keyword_counts.csv')
# Clustering parameters
model_type = 'glove' # Choose from 'word2vec', 'glove', 'fasttext'
model_path = '/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt' # Path to your model file
frequency_threshold = 10 # Keywords with count > threshold are kept as-is
num_clusters = 50 # Number of clusters for infrequent keywords
Category_Limit = 100 # Limit to top N keywords per category
distance_threshold = None # Set distance threshold for clustering
remove_duplicates = False # Set to True to remove duplicates in clustered keywords per row
# Labels (now customizable)
labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style'] # Modify as needed
# ------------------------------#
# Step 1: Cleaning #
# ------------------------------#
print("Starting CSV cleaning...")
df_clean = clean_csv(input_file, labels)
print("CSV cleaning completed.")
# ------------------------------#
# Step 2: Lemmatization #
# ------------------------------#
print("Starting lemmatization...")
per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatized_counts_output_file)
if per_row_df is None:
return # Exit if there was an error
# ------------------------------#
# Step 3: Load Model #
# ------------------------------#
model = load_model(model_type, model_path)
# ------------------------------#
# Step 4: Clustering #
# ------------------------------#
clustered_keyword_mapping = perform_clustering_with_threshold(
keyword_counts, model, model_type,
threshold=frequency_threshold, num_clusters=num_clusters,
distance_threshold=distance_threshold
)
# ------------------------------#
# Step 5: Clustered Keywords #
# ------------------------------#
per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
)
# ------------------------------#
# Step 6: Categorization #
# ------------------------------#
category_keyword_counts = categorize_and_count_clustered_keywords(per_row_df, labels=labels, Category_Limit=Category_Limit)
# ------------------------------#
# Step 7: Final Processing #
# ------------------------------#
per_row_df = generate_clustered_keywords_and_categories(
per_row_df, clustered_keyword_mapping, category_keyword_counts, labels=labels, remove_duplicates=remove_duplicates
)
# Optionally, write category counts to CSV files
for category, counts in category_keyword_counts.items():
category_counts_df = pd.DataFrame({'Keyword': list(counts.keys()), 'Count': list(counts.values())})
category_counts_df = category_counts_df.sort_values(by='Count', ascending=False)
category_counts_output_file = os.path.join(input_dir, f'{category}_keywords_counts.csv')
category_counts_df.to_csv(category_counts_output_file, index=False)
print(f"{category} keyword counts saved to '{category_counts_output_file}'.")
# ------------------------------#
# Step 8: Save Results #
# ------------------------------#
per_row_df.to_csv(output_file, index=False)
print(f"\nProcess completed. Output saved to '{output_file}'.")
# ------------------------------#
# Step 9: One-Hot Encoding #
# ------------------------------#
encode_categorical_features(
per_row_df, one_hot_encoded_file,
id_column='Image Name', remove_extension='.jpg',
labels=labels,
sep='\t', header=False, index=False
)
print(f"One-hot encoded features saved to '{one_hot_encoded_file}'.")
if __name__ == "__main__":
main()