app.py

import gradio as gr
import pandas as pd

def data_pre_processing(file_responses):
    # Financial Weights can be anything (ultimately the row-wise weights are aggregated and the corresponding fractions are obtained from that rows' total tax payed)
    
    try: # Define the columns to be processed
        
        # Developing Numeric Columns
        # Convert columns to numeric and fill NaN values with 0
        file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Latest estimated Tax payment?'] = pd.to_numeric(file_responses['Latest estimated Tax payment?'], errors='coerce').fillna(0)
        
        # Adding a new column 'TotalWeightageAllocated' by summing specific columns by their names
        file_responses['TotalWeightageAllocated'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_3_TaxWeightageAllocated']

        

        # Creating Datasets (we assume everything has been provided to us in English, or the translations have been done already)
        # Renaming the datasets into similar column headings
        initial_dataset_1 = file_responses.rename(columns={
            'Personal_TaxDirection_1_Wish': 'Problem_Description',
            'Personal_TaxDirection_1_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_1_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]
        
        initial_dataset_2 = file_responses.rename(columns={
            'Personal_TaxDirection_2_Wish': 'Problem_Description',
            'Personal_TaxDirection_2_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_2_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]
        
        initial_dataset_3 = file_responses.rename(columns={
            'Personal_TaxDirection_3_Wish': 'Problem_Description',
            'Personal_TaxDirection_3_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_3_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]


        
        # Calculating the actual TaxAmount to be allocated against each WISH (by overwriting the newly created columns)
        initial_dataset_1['Financial_Weight'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        initial_dataset_2['Financial_Weight'] = file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        initial_dataset_3['Financial_Weight'] = file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        
        # Removing useless rows
        # Drop rows where Problem_Description is NaN or an empty string
        initial_dataset_1 = initial_dataset_1.dropna(subset=['Problem_Description'], axis=0)
        initial_dataset_2 = initial_dataset_2.dropna(subset=['Problem_Description'], axis=0)
        initial_dataset_3 = initial_dataset_3.dropna(subset=['Problem_Description'], axis=0)
        
        # Convert 'Problem_Description' column to string type
        initial_dataset_1['Problem_Description'] = initial_dataset_1['Problem_Description'].astype(str)
        initial_dataset_2['Problem_Description'] = initial_dataset_2['Problem_Description'].astype(str)
        initial_dataset_3['Problem_Description'] = initial_dataset_3['Problem_Description'].astype(str)
        
        # Merging the Datasets
        # Vertically concatenating (merging) the 3 DataFrames
        merged_dataset = pd.concat([initial_dataset_1, initial_dataset_2, initial_dataset_3], ignore_index=True)

        
        # Different return can be used to check the processing
        # return file_responses
        return merged_dataset
        
    except Exception as e:
        return str(e)






import spacy
from transformers import AutoTokenizer, AutoModel
import torch

# Load SpaCy model
nlp = spacy.load('en_core_web_sm')

# Load Hugging Face Transformers model
tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")

# def combined_text_processing(text):
#     # Basic NLP processing using SpaCy
#     doc = nlp(text)
#     lemmatized_text = ' '.join([token.lemma_ for token in doc])
    
#     # Advanced text representation using Hugging Face Transformers
#     inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True)
#     with torch.no_grad():
#         outputs = model(**inputs)
    
#     return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()


import re
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

def combined_text_processing(text):
    # Remove punctuation, numbers, URLs, and special characters
    text = re.sub(r'[^\w\s]', '', text)  # Remove punctuation and special characters
    text = re.sub(r'\d+', '', text)  # Remove numbers
    text = re.sub(r'http\S+', '', text)  # Remove URLs
    
    # Tokenize and remove stopwords
    tokens = word_tokenize(text.lower())  # Convert to lowercase
    stop_words = set(stopwords.words('english'))
    tokens = [word for word in tokens if word not in stop_words]
    
    # Lemmatize tokens using SpaCy
    doc = nlp(' '.join(tokens))
    lemmatized_text = ' '.join([token.lemma_ for token in doc])
    
    # Apply Hugging Face Transformers
    inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True)
    with torch.no_grad():
        outputs = model(**inputs)
    
    return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()

















def nlp_pipeline(original_df):
    # Data Preprocessing
    processed_df = data_pre_processing(original_df)
    
    
    # Apply the combined function to your DataFrame
    processed_df['Processed_ProblemDescription'] = processed_df['Problem_Description'].apply(combined_text_processing)
    
    
    
    return processed_df

def process_excel(file):
    try:
        # Ensure the file path is correct
        file_path = file.name if hasattr(file, 'name') else file
        # Read the Excel file
        df = pd.read_excel(file_path)
        
        # Process the DataFrame
        result_df = nlp_pipeline(df)

        output_file = "Output_ProjectProposals.xlsx"
        result_df.to_excel(output_file, index=False)
        
        return output_file  # Return the processed DataFrame as Excel file
        
    except Exception as e:
        return str(e)  # Return the error message






example_files = ['#TaxDirection (Responses)_BasicExample.xlsx',
                 '#TaxDirection (Responses)_IntermediateExample.xlsx',
                 '#TaxDirection (Responses)_UltimateExample.xlsx'
                ]


import random
a_random_object = random.choice(["⇒", "↣", "↠", "→"])



# Define the Gradio interface
interface = gr.Interface(
    fn=process_excel,  # The function to process the uploaded file
    inputs=gr.File(type="filepath", label="Upload Excel File here. \t Be sure to check that the column headings in your upload are the same as in the Example files below. \t (Otherwise there will be Error during the processing)"),  # File upload input
    
    examples=example_files,  # Add the example files
    
    # outputs=gr.File(label="Download Processed Excel File"),  # File download output
    outputs=gr.File(label="Download the processed Excel File containing the ** Project Proposals ** for each Location~Problem paired combination"),  # File download output
    
    
    # title="Excel File Uploader",
    # title="Upload Excel file containing #TaxDirections → Download HyperLocal Project Proposals\n",
    
    title = (
        "<p style='font-weight: bold; font-size: 25px; text-align: center;'>"
        "<span style='color: blue;'>Upload Excel file containing #TaxDirections</span> "
        
        # "<span style='color: brown; font-size: 35px;'>&rarr; </span>"
        # "<span style='color: brown; font-size: 35px;'>&rArr;  &rarrtl; &Rarr; </span>"
        "<span style='color: brown; font-size: 35px;'> " +a_random_object +" </span>"
        
        "<span style='color: green;'>Download HyperLocal Project Proposals</span>"
        "</p>\n"
    ),



    description=(
        "<p style='font-size: 12px; color: gray; text-align: center'>This tool allows for the systematic evaluation and proposal of solutions tailored to specific location-problem pairs, ensuring efficient resource allocation and project planning. For more information, visit <a href='https://santanban.github.io/TaxDirection/' target='_blank'>#TaxDirection weblink</a>.</p>"
        
        "<p style='font-weight: bold; font-size: 16px; color: blue;'>Upload an Excel file to process and download the result or use the Example files:</p>"
        "<p style='font-weight: bold; font-size: 15px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>"
        
        "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Processed output contains a Project Proposal for each Location~Problem paired combination (i.e. each cell).</p>"
        "<p style='font-weight: bold; font-size: 13px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>"

        
        "<p style='font-size: 12px; color: gray; text-align: center'>Note: The example files provided above are for demonstration purposes. Feel free to upload your own Excel files to see the results. If you have any questions, refer to the documentation-links or contact <a href='https://www.change.org/p/democracy-evolution-ensuring-humanity-s-eternal-existence-through-taxdirection' target='_blank'>support</a>.</p>"
        
    )  # Solid description with right-aligned second sentence

)



# Launch the interface
if __name__ == "__main__":
    interface.launch()