This is my current code. i need you to make it more professional and remove unnecessary steps. i need  can you help me making it better? what could i do to improve it?you to add more logging info


import os
import re
import csv
import argparse
import logging
import nltk
import pandas as pd
from collections import defaultdict, Counter
from nltk.stem import WordNetLemmatizer
from gensim.models import KeyedVectors, FastText
from sklearn.cluster import AgglomerativeClustering
import numpy as np
from sklearn.preprocessing import OneHotEncoder
from nltk.corpus import wordnet as wn
from nltk.corpus import stopwords

def download_nltk_data():
    """
    Ensures that required NLTK data packages are downloaded.
    """
    nltk_data_packages = ['wordnet', 'stopwords']
    for package in nltk_data_packages:
        try:
            nltk.data.find(f'corpora/{package}')
        except LookupError:
            nltk.download(package)

def parse_arguments():
    """
    Parses command-line arguments.
    """
    parser = argparse.ArgumentParser(description='Process and cluster keywords from a CSV file.')
    parser.add_argument('--input_file', type=str, help='Path to the input CSV file.')
    parser.add_argument('--save_lemmatized', action='store_true', help='Save lemmatized keywords and counts.')
    parser.add_argument('--model_type', type=str, choices=['word2vec', 'glove', 'fasttext'], default='glove',
                        help='Type of model to use for clustering. Default is "glove".')
    parser.add_argument('--model_path', type=str, default='/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt', required=False, help='Path to the word embedding model file.')
    parser.add_argument('--frequency_threshold', type=int, default=10,
                        help='Frequency threshold for keywords. Default is 10.')
    parser.add_argument('--num_clusters', type=int, default=50,
                        help='Number of clusters for infrequent keywords. Default is 50.')
    parser.add_argument('--distance_threshold', type=float, default=None,
                        help='Distance threshold for clustering. Overrides num_clusters if set.')
    parser.add_argument('--category_limit', type=int, default=100,
                        help='Limit of top N keywords per category. Default is 100.')
    parser.add_argument('--remove_duplicates', action='store_true',
                        help='Remove duplicates in clustered keywords per row.')
    return parser.parse_args()

def main():
    # Parse command-line arguments
    args = parse_arguments()
    input_file = args.input_file
    save_lemmatized = args.save_lemmatized
    model_type = args.model_type
    model_path = args.model_path
    frequency_threshold = args.frequency_threshold
    num_clusters = args.num_clusters
    distance_threshold = args.distance_threshold
    category_limit = args.category_limit
    remove_duplicates = args.remove_duplicates

    # Check if input file exists
    if not os.path.exists(input_file):
        raise FileNotFoundError(f"Input file '{input_file}' does not exist.")

    # Set up logging
    input_dir = os.path.dirname(os.path.abspath(input_file))
    input_filename = os.path.basename(input_file)
    input_name_no_ext = os.path.splitext(input_filename)[0]
    processed_dir = os.path.join(input_dir, 'processed')
    os.makedirs(processed_dir, exist_ok=True)
    log_file = os.path.join(processed_dir, f'log_{input_name_no_ext}.log')
    logging.basicConfig(filename=log_file, level=logging.INFO,
                        format='%(asctime)s %(levelname)s:%(message)s')
    # Also log to stdout
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.INFO)
    console_formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s')
    console_handler.setFormatter(console_formatter)
    logging.getLogger().addHandler(console_handler)

    logging.info(f"Processing input file: {input_file}")

    # Download NLTK data if necessary
    download_nltk_data()

    # Initialize the lemmatizer
    lemmatizer = WordNetLemmatizer()
    # Set of stopwords to filter out
    stop_words = set(stopwords.words('english'))

    # Labels (customizable)
    labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style']  # Modify as needed

    # Define output file paths
    output_file = os.path.join(processed_dir, f'clustered_{input_filename}')
    one_hot_encoded_file = os.path.join(processed_dir, f'onehot_{input_name_no_ext}.tsv')
    lemmatized_counts_output_file = os.path.join(processed_dir, f'lemmatized_keyword_counts.csv')

    # ------------------------------#
    # Step 1: Cleaning #
    # ------------------------------#
    logging.info("Starting CSV cleaning...")
    df_clean = clean_csv(input_file, labels)
    logging.info("CSV cleaning completed.")

    # ------------------------------#
    # Step 2: Lemmatization #
    # ------------------------------#
    logging.info("Starting lemmatization...")
    if save_lemmatized:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatized_counts_output_file, lemmatizer)
    else:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, None, lemmatizer)
    # Check if per_row_df is None
    if per_row_df is None:
        logging.error("Error in lemmatization.")
        return

    # ------------------------------#
    # Step 3: Load Model #
    # ------------------------------#
    model = load_model(model_type, model_path)

    # ------------------------------#
    # Step 4: Clustering #
    # ------------------------------#
    clustered_keyword_mapping = perform_clustering_with_threshold(
        keyword_counts, model, model_type,
        threshold=frequency_threshold, num_clusters=num_clusters,
        distance_threshold=distance_threshold
    )

    # ------------------------------#
    # Step 5: Clustered Keywords #
    # ------------------------------#
    per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
        lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
    )

    # ------------------------------#
    # Step 6: Categorization #
    # ------------------------------#
    category_keyword_counts = categorize_and_count_clustered_keywords(
        per_row_df, labels=labels, category_limit=category_limit
    )

    # ------------------------------#
    # Step 7: Final Processing #
    # ------------------------------#
    per_row_df = generate_clustered_keywords_and_categories(
        per_row_df, clustered_keyword_mapping, category_keyword_counts, labels=labels,
        remove_duplicates=remove_duplicates
    )

    # Optionally, write category counts to CSV files
    if save_lemmatized:
        for category, counts in category_keyword_counts.items():
            category_counts_df = pd.DataFrame({'Keyword': list(counts.keys()), 'Count': list(counts.values())})
            category_counts_df = category_counts_df.sort_values(by='Count', ascending=False)
            category_counts_output_file = os.path.join(processed_dir, f'{category}_keywords_counts.csv')
            category_counts_df.to_csv(category_counts_output_file, index=False)
            logging.info(f"{category} keyword counts saved to '{category_counts_output_file}'.")

    # ------------------------------#
    # Step 8: Save Results #
    # ------------------------------#
    per_row_df.to_csv(output_file, index=False)
    logging.info(f"Process completed. Output saved to '{output_file}'.")

    # ------------------------------#
    # Step 9: One-Hot Encoding #
    # ------------------------------#
    encode_categorical_features(
        per_row_df, one_hot_encoded_file,
        id_column='Image Name', remove_extension='.jpg',
        labels=labels,
        sep='\t', header=False, index=False
    )
    logging.info(f"One-hot encoded features saved to '{one_hot_encoded_file}'.")

def clean_csv(input_file, labels):
    """
    Cleans the input CSV file and returns a DataFrame.
    """
    total_row_count = 0
    empty_row_count = 0
    na_count = 0
    no_comma_count = 0
    no_comma_rows = []
    cleaned_data = []
    # Read the entire content of the file
    try:
        with open(input_file, 'r', newline='', encoding='utf-8') as infile:
            content = infile.read()
    except Exception as e:
        logging.error(f"Error reading input file: {e}")
        return None
    # Remove newline characters inside quoted strings
    content = re.sub(r'("\[.*?")\n', r'\1 ', content)
    # Use StringIO to read the cleaned content as a CSV
    import io
    temp_file = io.StringIO(content)
    reader = csv.reader(temp_file)

    # Skip the first line (header)
    try:
        header = next(reader)
    except StopIteration:
        logging.error("Input file is empty.")
        return None

    # Process each row
    for row in reader:
        total_row_count += 1
        if not row:
            continue
        jpg_file = row[0]
        if len(row) < 2:
            logging.warning(f"Row {total_row_count} doesn't have enough columns: {row}")
            keywords = []
        else:
            keywords = extract_keywords(row[1], labels)
        # Count occurrences of "N/A" and empty spaces
        na_count += keywords.count("N/A")
        na_count += sum(1 for k in keywords if k.strip() == "")
        # Check if the row is empty or contains only "N/A" or empty spaces
        if not keywords or all(k.strip().upper() == 'N/A' or k.strip() == '' for k in keywords):
            empty_row_count += 1
            continue  # Skip this row
        # Concatenate the keywords into a single string
        keywords_str = ', '.join(keywords)
        # Replace multiple spaces with a comma and a single space
        keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
        # Check if the row has no commas
        if ',' not in keywords_str:
            no_comma_count += 1
            no_comma_rows.append(f'{jpg_file},{keywords_str}')
        # Add to cleaned data
        cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
    # Now, create a DataFrame from the cleaned_data
    df = pd.DataFrame(cleaned_data)
    # Clean the 'Keywords' column further
    df = clean_keywords_in_df(df, labels)
    # Log statistics
    logging.info(f"Total rows processed: {total_row_count}")
    logging.info(f"Number of empty or N/A rows: {empty_row_count}")
    logging.info(f"Occurrences of N/A (including empty spaces): {na_count}")
    logging.info(f"Number of rows without commas: {no_comma_count}")
    if no_comma_rows:
        logging.info("Rows without commas:")
        for row in no_comma_rows:
            logging.info(row)
    return df

def extract_keywords(line, labels):
    """
    Extracts keywords from a line.
    """
    # First, try to extract keywords between curly braces
    keywords = re.findall(r'\{([^}]+)\}', line)
    if keywords:
        # Further split if multiple keywords are within braces
        return [kw.strip() for group in keywords for kw in group.split(',')]
    # Remove newline characters from the keywords section
    line = line.replace('\n', ' ')
    # Remove the labels using word boundaries and considering possible formats
    for label in labels:
        # Patterns like "Label:", "Label -", "[Label]:", etc.
        pattern = rf'\b{re.escape(label)}\b[: -]?'
        line = re.sub(pattern, '', line, flags=re.IGNORECASE)
    # Remove any '- ' (dash and space) using exact string replacement
    line = line.replace('- ', '')
    # Remove single quotes around keywords using regex
    line = re.sub(r"'([^']+)'", r'\1', line)
    # Remove any remaining brackets
    line = line.replace('[', '').replace(']', '').replace('{', '').replace('}', '')
    # Split keywords by commas and strip whitespace
    keywords = [keyword.strip() for keyword in line.split(',')]
    return keywords

def clean_keywords_in_df(df, labels):
    """
    Cleans the 'Keywords' column in the DataFrame.
    """
    # Construct a regex pattern from the labels with word boundaries
    labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*'
    # Replace labels with an empty string using regex
    df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True)
    # Remove any '- ' (dash and space)
    df['Keywords'] = df['Keywords'].str.replace('- ', '', regex=False)
    # Remove single quotes around keywords
    df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True)
    # Replace multiple spaces with a comma and a single space
    df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True)
    # Remove leading/trailing commas and spaces
    df['Keywords'] = df['Keywords'].str.strip(' ,')
    return df

def lemmatize_keywords(df, lemmatized_counts_output_file, lemmatizer):
    """
    Lemmatizes the keywords in the DataFrame and returns a DataFrame with lemmatized keywords.
    Optionally saves the lemmatized keyword counts to a CSV file.
    """
    per_row_data = []
    keyword_counts = Counter()
    # Iterate through each row to extract and lemmatize keywords
    for index, row in df.iterrows():
        image_name = row.get('Image Name', '')
        keywords_str = row.get('Keywords', '')
        if pd.isna(keywords_str):
            keywords_str = ''
        # Split keywords by comma and strip whitespace
        extracted_keywords = [kw.strip() for kw in keywords_str.split(',')]
        # Lemmatize keywords
        lemmatized_keywords = [lemmatizer.lemmatize(kw.lower()) for kw in extracted_keywords if kw]
        # Update global counts
        keyword_counts.update(lemmatized_keywords)
        # Store per-row data
        per_row_data.append({
            'Image Name': image_name,
            'Keywords': ', '.join(extracted_keywords),
            'Lemmatized Keywords': ', '.join(lemmatized_keywords)
        })
    # Create a DataFrame from per_row_data
    per_row_df = pd.DataFrame(per_row_data)
    if lemmatized_counts_output_file:
        # Write the lemmatized keyword counts to CSV
        lemmatized_counts = [{'Lemmatized Keyword': kw, 'Count': count}
                             for kw, count in keyword_counts.items()]
        lemmatized_counts_df = pd.DataFrame(lemmatized_counts)
        # Sort by Count in descending order
        lemmatized_counts_df = lemmatized_counts_df.sort_values(by='Count', ascending=False)
        # Write to CSV
        lemmatized_counts_df.to_csv(lemmatized_counts_output_file, index=False)
        logging.info(f"Lemmatized keyword counts saved to '{lemmatized_counts_output_file}'.")
    return per_row_df, keyword_counts

def load_model(model_type, model_path):
    """
    Loads a word embedding model based on the specified type and path.
    """
    if model_type == 'word2vec':
        logging.info("Loading Word2Vec model. This may take a few minutes...")
        model = KeyedVectors.load_word2vec_format(model_path, binary=True)
        logging.info("Word2Vec model loaded successfully.")
    elif model_type == 'glove':
        logging.info("Loading GloVe model. This may take a few minutes...")
        model = {}
        with open(model_path, 'r', encoding='utf-8') as f:
            for line in f:
                values = line.split()
                word = values[0]
                vector = np.asarray(values[1:], dtype='float32')
                model[word] = vector
        logging.info("GloVe model loaded successfully.")
    elif model_type == 'fasttext':
        logging.info("Loading FastText model. This may take a few minutes...")
        model = FastText.load_fasttext_format(model_path)
        logging.info("FastText model loaded successfully.")
    else:
        raise ValueError(f"Unsupported model type: {model_type}")
    return model

def get_word_vector(word, model, model_type):
    """
    Retrieves the vector representation of a word from the model.
    """
    if word in model:
        return model[word]
    # Handle multi-word phrases by averaging their word vectors
    words = word.split()
    vectors = []
    for w in words:
        if w in model:
            vectors.append(model[w])
    if vectors:
        return np.mean(vectors, axis=0)
    else:
        return None

def perform_clustering_with_threshold(keyword_counts, model, model_type, threshold=5, num_clusters=5, distance_threshold=None):
    """
    Clusters infrequent keywords using hierarchical clustering.
    """
    # Separate frequent and infrequent keywords
    frequent_keywords = [kw for kw, count in keyword_counts.items() if count > threshold]
    infrequent_keywords = [kw for kw, count in keyword_counts.items() if count <= threshold]
    logging.info(f"Number of frequent keywords (> {threshold}): {len(frequent_keywords)}")
    logging.info(f"Number of infrequent keywords (<= {threshold}): {len(infrequent_keywords)}")
    # Keep frequent keywords as-is
    clustered = {kw: kw for kw in frequent_keywords}
    # Cluster infrequent keywords and map them to their representatives
    if infrequent_keywords:
        mapping = cluster_infrequent_keywords(
            infrequent_keywords, keyword_counts, model, model_type,
            num_clusters=num_clusters, distance_threshold=distance_threshold
        )
        clustered.update(mapping)
        logging.info(f"Number of cluster representatives: {len(set(mapping.values()))}")
    else:
        logging.info("No infrequent keywords to cluster.")
    return clustered

def cluster_infrequent_keywords(keywords, keyword_counts, model, model_type, num_clusters=5, distance_threshold=None):
    """
    Clusters infrequent keywords and maps them to cluster representatives.
    """
    vectors = []
    valid_keywords = []
    for keyword in keywords:
        vector = get_word_vector(keyword, model, model_type)
        if vector is not None:
            vectors.append(vector)
            valid_keywords.append(keyword)
        else:
            logging.warning(f"Vector not found for keyword: '{keyword}'")
    if not vectors:
        logging.warning("No valid vectors found for clustering.")
        return {}
    vectors = np.array(vectors)
    # Perform Hierarchical Clustering with distance threshold
    if distance_threshold is not None:
        logging.info(f"Performing Agglomerative Clustering with distance_threshold={distance_threshold}")
        clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold)
    else:
        logging.info(f"Performing Agglomerative Clustering with n_clusters={num_clusters}")
        clustering_model = AgglomerativeClustering(n_clusters=num_clusters)
    labels = clustering_model.fit_predict(vectors)
    # Organize keywords by clusters
    clusters = defaultdict(list)
    for label, keyword in zip(labels, valid_keywords):
        clusters[label].append(keyword)
    # Select the most frequent keyword in each cluster as the representative
    representative_keywords = {}
    for label, words in clusters.items():
        most_frequent = max(words, key=lambda w: keyword_counts[w])
        representative_keywords[label] = most_frequent
    # Create a mapping from keyword to representative
    keyword_to_representative = {}
    for label, words in clusters.items():
        rep = representative_keywords[label]
        for word in words:
            keyword_to_representative[word] = rep
    return keyword_to_representative

def categorize_and_count_clustered_keywords(per_row_df, labels, category_limit=50):
    """
    Categorizes clustered keywords and counts them per category.
    """
    category_keyword_counts = {label: Counter() for label in labels}
    for idx, row in per_row_df.iterrows():
        clustered_keywords = row['Clustered Keywords'].split(', ')
        for i, label in enumerate(labels):
            if i < len(clustered_keywords):
                keyword = clustered_keywords[i]
                category_keyword_counts[label][keyword] += 1
    # Limit to top N keywords per category
    for label in labels:
        original_count = len(category_keyword_counts[label])
        category_keyword_counts[label] = Counter(dict(category_keyword_counts[label].most_common(category_limit)))
        logging.info(f"Category '{label}': Reduced from {original_count} to {len(category_keyword_counts[label])} keywords (top {category_limit})")

    return category_keyword_counts


def generate_clustered_keywords_and_categories(per_row_df, clustered_keyword_mapping, category_keyword_counts, labels, remove_duplicates=False):
    """
    Generates the final DataFrame with clustered keywords and category assignments.
    """
    top_keywords_per_category = {label: set(counts.keys()) for label, counts in category_keyword_counts.items()}
    # Initialize lists for categories and counters for "Other"
    category_keywords_list = {label: [] for label in labels}
    clustered_keywords_list = []
    # Initialize a counter for the number of rows with specific "Other" counts
    max_other_count = len(labels)
    other_count_rows = {i: 0 for i in range(max_other_count + 1)}  # from 0 to len(labels)
    for idx, row in per_row_df.iterrows():
        lemmatized_keywords = row['Lemmatized Keywords'].split(', ')
        clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords]
        if remove_duplicates:
            # Remove duplicates while preserving order
            clustered_keywords = list(dict.fromkeys(clustered_keywords))
        clustered_keywords_list.append(', '.join(clustered_keywords))
        # Initialize "Other" counter for each row
        row_other_count = 0
        # Assign keywords to categories based on positions
        for i, label in enumerate(labels):
            if i < len(clustered_keywords):
                keyword = clustered_keywords[i]
                # Map to 'Other' if not in top keywords
                if keyword not in top_keywords_per_category[label]:
                    keyword = 'Other'
                    row_other_count += 1  # Increase the row counter for "Other"
                category_keywords_list[label].append(keyword)
            else:
                category_keywords_list[label].append('')  # or 'Other'
        # Count the number of rows with the specific number of "Other" keywords
        if row_other_count <= max_other_count:
            other_count_rows[row_other_count] += 1
    per_row_df['Clustered Keywords'] = clustered_keywords_list
    # Add the category columns to per_row_df
    for label in labels:
        per_row_df[label] = category_keywords_list[label]
    # Log summary of "Other" counts per category
    other_count = {label: 0 for label in labels}  # Counter for "Other" per category
    logging.info("\nSummary of 'Other' counts per category:")
    for label in labels:
        other_count[label] = sum(1 for keyword in category_keywords_list[label] if keyword == 'Other')
        logging.info(f"{label}: {other_count[label]}")
    # Log the summary of rows having specific "Other" counts
    logging.info("\nSummary of 'Other' count rows:")
    for count in range(max_other_count + 1):
        logging.info(f"Rows with {count} 'Other' keywords: {other_count_rows[count]}")
    return per_row_df

def encode_categorical_features(input_df, output_tsv_path, id_column='Image Name', remove_extension='.jpg', labels=None, sep='\t', header=False, index=False):
    """
    Encodes categorical features using One-Hot Encoding and saves them to a TSV file.
    """
    if labels is None:
        raise ValueError("Labels for encoding must be provided.")
    # Read only the necessary columns
    columns_to_encode = labels
    columns_to_read = [id_column] + columns_to_encode
    df = input_df[columns_to_read]
    # Initialize OneHotEncoder
    enc = OneHotEncoder(handle_unknown='ignore', dtype=np.int8)
    # Fit and transform the categorical columns
    emb = enc.fit_transform(df[columns_to_encode])
    # Retrieve the names of the one-hot encoded features
    feature_names = enc.get_feature_names_out(columns_to_encode)
    # Create a DataFrame with the encoded features
    emb_df = pd.DataFrame(emb.toarray(), columns=feature_names, dtype=np.int8)
    # Concatenate the id_column with the one-hot encoded features
    result_df = pd.concat([df[[id_column]].reset_index(drop=True), emb_df.reset_index(drop=True)], axis=1)
    # Remove specified extension from id_column
    if remove_extension:
        result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False)
    # Save to TSV file
    result_df.to_csv(output_tsv_path, index=index, header=header, sep=sep)

if __name__ == "__main__":
    main()