Merge pull request #4 from Shrutisd1003/data-transformation

app.py

@@ -5,27 +5,62 @@ from data_filter import DataFilter
 from data_transformer import DataTransformer
 from data_visualizer import DataVisualizer
 from data_QA import DataQA
+import os
+from streamlit_option_menu import option_menu
 
 def main():
+    if os.path.exists("data.csv"):
+        os.remove("data.csv")
+    with open("data.csv", 'w'):
+        pass
     st.title('Insights 📶')
 
     data_loader = DataLoader()
     data = data_loader.load_data()
 
-    data_transformer = DataTransformer(data)
-    modified_data = data_transformer.perform_column_operation()
+    if os.path.getsize("data.csv") != 0:
+        with st.sidebar:
+            selected = option_menu(
+                menu_title="Main Menu",
+                options=["Data Loader", "Exploratory Data Analysis", "Data Cleaning", "Q/A", "Data Party"])
 
-    data_analyzer = DataAnalyzer(modified_data)
-    data_analyzer.show_summary_statistics()
-    data_analyzer.show_data_types()
+        # --- DATA LOADER ---
+        if selected == "Data Loader":
+            st.toast("Data Loaded")
+            st.write(data.head())
 
-    data_filter = DataFilter(modified_data)
-    data = data_filter.filter_rows()
+        # --- EDA ---
+        if selected == "Exploratory Data Analysis":
+            data_analyzer = DataAnalyzer(data)
+            data_analyzer.show_eda()
+            data_analyzer.show_null_value_statistics()
+            data_analyzer.show_count_plots()
+            data_analyzer.show_summary_statistics()
 
-    data_visualizer = DataVisualizer(modified_data)
-    data_visualizer.visualize_data()
+            data_visualizer = DataVisualizer(data)
+            data_visualizer.visualize_data()
+
+        # --- DATA CLEANING ---
+        if selected == "Data Cleaning":
+            data_transformer = DataTransformer(data)
+            data_analyzer = DataAnalyzer(data)
+            modified_data = data_transformer.perform_column_operation()
+            data_analyzer.show_null_value_statistics()
+            modified_data = data_transformer.remove_null()
+            modified_data = data_transformer.impute_null()
+            modified_data = data_transformer.remove_columns()
+
+            # data_filter = DataFilter(modified_data)
+            # data = data_filter.filter_rows()
+
+        # --- QUESTION AND ANSWER ---
+        if selected == "Q/A":
+            data_QA = DataQA(data)
+            data_QA.ask_csv()
+
+        # --- DATA PARTY ---
+        if selected == "Data Party":
+            st.write("To be continued... :)")
 
-    data_QA = DataQA(data)
-    data_QA.ask_csv()
 if __name__ == "__main__":
     main()

data_analyzer.py

@@ -1,13 +1,66 @@
 import streamlit as st
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
 
 class DataAnalyzer:
     def __init__(self, data):
         self.data = data
+        st.header("Exploratory Data Analysis")
+
+    def show_eda(self):
+        st.write("Number of rows:", self.data.shape[0])
+        st.write("Number of columns:", self.data.shape[1])
+        columns_by_dtype = {}
+        for column_name, dtype in self.data.dtypes.items():
+            dtype_str = str(dtype)
+            if dtype_str not in columns_by_dtype:
+                columns_by_dtype[dtype_str] = [column_name]
+            else:
+                columns_by_dtype[dtype_str].append(column_name)
+        col_type_df = []
+        for dtype, columns in columns_by_dtype.items():
+            col_type_df.append([dtype, ', '.join(columns)])
+        df = pd.DataFrame(col_type_df, columns=["Data Type", "Column Names"])
+        st.subheader("Columns by Data Type")
+        st.dataframe(df, hide_index=True, use_container_width=True)
 
     def show_summary_statistics(self):
         if st.button('Show Summary Statistics'):
             st.write(self.data.describe())
+            st.write(self.data.describe(include=object))
+
+    def show_null_value_statistics(self):
+        st.subheader("Null Value Statistics")
+        null_counts = self.data.isnull().sum()
+        total_null = null_counts.sum()
+        total_rows = self.data.shape[0]
+        null_percentages = (null_counts / total_rows) * 100
+        null_stats_df = pd.DataFrame({
+            'Column Name': null_counts.index,
+            'Null Values': null_counts.values,
+            'Percentage Null': null_percentages.values
+        })
+        null_stats_df.loc[len(null_stats_df)] = ['Total', total_null, (total_null / (total_rows * self.data.shape[1])) * 100]
+        st.dataframe(null_stats_df, hide_index=True, use_container_width=True)
+
+    def show_count_plots(self):
+        st.subheader("Count Plots")
+        sns.set(style="whitegrid")
+
+        for column_name in self.data.columns:
+            unique_values = self.data[column_name].nunique()
+
+            if unique_values <= 12:
+                fig, ax = plt.subplots(figsize=(10, 6))
+                sns.countplot(data=self.data, x=column_name, ax=ax)
+                ax.set_title(f'Count Plot of {column_name}')
+                ax.set_xticklabels(ax.get_xticklabels())
+                st.pyplot(fig)
 
-    def show_data_types(self):
-        if st.button('Show Data Types'):
-            st.write(self.data.dtypes)
+            else:
+                fig, ax = plt.subplots(figsize=(10, 6))
+                sns.histplot(data=self.data, x=column_name, bins=20, ax=ax)
+                ax.set_title(f'Histogram of {column_name}')
+                ax.set_xlabel(column_name)
+                st.pyplot(fig)

data_transformer.py

@@ -4,6 +4,7 @@ import pandas as pd
 class DataTransformer:
     def __init__(self, data):
         self.data = data
+        st.header("Data Cleaning")
 
     def perform_column_operation(self):
         column_operation = st.sidebar.text_input('Column operation (e.g., age * 2)')
@@ -13,8 +14,39 @@ class DataTransformer:
             st.write(self.data)
         return self.data
 
+    def remove_null(self):
+        if st.button('Remove Null'):
+            col = st.multiselect('Choose columns to remove nulls', self.data.columns)
+            self.data.dropna(subset=col, inplace=True)
+            st.toast("Null values removed")
+        return self.data
+
+    def impute_null(self):
+        if st.button('Impute Null'):
+            col = st.multiselect('Choose columns to impute nulls', self.data.select_dtypes(include=[np.number]).columns)
+            option = st.selectbox('Impute nulls with', ('mean', 'mode', '0'))
+            if option == "mean":
+                self.data.fillna(df.mean())
+            elif option == "mode":
+                self.data.fillna(df.mode())
+            elif option == "0":
+                self.data.fillna("0")
+            st.toast("Null values filled")
+        return self.data
+
+    def remove_columns(self):
+        if st.button('Remove Columns'):
+            col = st.multiselect('Choose columns to remove', self.data.columns)
+            self.data.drop(columns=col, inplace=True)
+            st.toast("Columns removed")
+        return self.data
+
+        # PROBLEMS RESOLVED
         #transformed data is not retained
         #null values handling
         #2 options - to remove or to impute that is the question
+
+        # PROBLEMS TO BE ADDRESSED
+        #categorical to numerical
         #give option to analyse the transformed dataset or save it.
 

data_visualizer.py

@@ -7,9 +7,11 @@ import seaborn as sns
 class DataVisualizer:
     def __init__(self, data):
         self.data = data
+        st.subheader("Data Visualizer")
 
     def visualize_data(self):
         plot_type = st.selectbox('Choose a type of plot', ['Histogram', 'Box Plot', 'Pie Chart', 'Scatter Plot', 'Heatmap'])
+        
         if plot_type == 'Histogram':
             numeric_columns = self.data.select_dtypes(include=[np.number]).columns
             if numeric_columns.empty:
@@ -18,7 +20,11 @@ class DataVisualizer:
                 column_to_visualize = st.selectbox('Choose a column to visualize', numeric_columns)
                 fig, ax = plt.subplots()
                 ax.hist(self.data[column_to_visualize])
+                ax.set_title(f'Histogram of {column_to_visualize}')
+                ax.set_xlabel(column_to_visualize)
+                ax.set_ylabel('Frequency')
                 st.pyplot(fig)
+        
         elif plot_type == 'Box Plot':
             numeric_columns = self.data.select_dtypes(include=[np.number]).columns
             if numeric_columns.empty:
@@ -27,23 +33,42 @@ class DataVisualizer:
                 column_to_visualize = st.selectbox('Choose a column to visualize', numeric_columns)
                 fig, ax = plt.subplots()
                 ax.boxplot(self.data[column_to_visualize].dropna())
+                ax.set_title(f'Box Plot of {column_to_visualize}')
+                ax.set_ylabel(column_to_visualize)
                 st.pyplot(fig)
+        
         elif plot_type == 'Pie Chart':
-            column_to_visualize = st.selectbox('Choose a column to visualize', self.data.select_dtypes(include=['object']).columns)
-            fig, ax = plt.subplots()
-            self.data[column_to_visualize].value_counts().plot(kind='pie', ax=ax, autopct='%1.1f%%', textprops={'fontsize': 'small'})
-            st.pyplot(fig)
+            nonnumeric_columns = self.data.select_dtypes(include=['object']).columns
+            if nonnumeric_columns.empty:
+                st.warning('No non numeric columns in the data to visualize.')
+            else:
+                column_to_visualize = st.selectbox('Choose a column to visualize', nonnumeric_columns)
+                fig, ax = plt.subplots()
+                self.data[column_to_visualize].value_counts().plot(kind='pie', ax=ax, autopct='%1.1f%%', textprops={'fontsize': 'small'})
+                ax.set_title(f'Pie Chart of {column_to_visualize}')
+                ax.set_ylabel('')
+                st.pyplot(fig)
+        
         elif plot_type == 'Scatter Plot':
-            columns_to_visualize = st.multiselect('Choose two columns to visualize', self.data.select_dtypes(include=[np.number]).columns)
-            if len(columns_to_visualize) != 2:
-                st.warning('Please select exactly two columns for scatter plot.')
+            left, right = st.columns(2)
+            with left:
+                x_col = st.selectbox('Choose values on X axis', self.data.select_dtypes(include=[np.number]).columns)
+            with right:
+                y_col = st.selectbox('Choose values on Y axis', self.data.select_dtypes(include=[np.number]).columns)
+            if x_col == y_col:
+                st.warning('Please select two different columns for scatter plot.')
             else:
                 fig, ax = plt.subplots()
-                ax.scatter(self.data[columns_to_visualize[0]], self.data[columns_to_visualize[1]])
+                ax.scatter(self.data[x_col], self.data[y_col])
+                ax.set_title(f'Scatter Plot of {x_col} vs {y_col}')
+                ax.set_xlabel(x_col)
+                ax.set_ylabel(y_col)
                 st.pyplot(fig)
+        
         elif plot_type == 'Heatmap':
             numeric_data = self.data.select_dtypes(include=[np.number])
             corr = numeric_data.corr()
             fig, ax = plt.subplots()
             sns.heatmap(corr, annot=True, ax=ax)
+            ax.set_title('Correlation Heatmap')
             st.pyplot(fig)

requirements.txt

@@ -6,4 +6,5 @@ seaborn
 langchain-google-genai
 langchain-experimental
 python-dotenv
-tabulate
+tabulate
+streamlit_option_menu

test.py

@@ -1,16 +1,42 @@
-from langchain_google_genai import GoogleGenerativeAI
-from langchain_experimental.agents import create_csv_agent
-import pandas as pd
-from dotenv import load_dotenv
+import streamlit as st
+from data_loader import DataLoader
+from data_analyzer import DataAnalyzer
+from data_filter import DataFilter
+from data_transformer import DataTransformer
+from data_visualizer import DataVisualizer
+from data_QA import DataQA
 import os
-load_dotenv()  # take environment variables from .env.
 
-GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+def main():
+    if os.path.exists("data.csv"):
+        os.remove("data.csv")
+    with open("data.csv", 'w'):
+        pass
+    st.title('Insights 📶')
 
-data = pd.read_csv("data.csv")
-llm = GoogleGenerativeAI(model="gemini-pro", google_api_key=GOOGLE_API_KEY)
-csv_agent = create_csv_agent(llm,"data.csv", verbose=True)
-question = "describe the dataset"
-response = csv_agent.run(question)
+    data_loader = DataLoader()
+    data = data_loader.load_data()
 
-print(response)
+    if os.path.getsize("data.csv") != 0:
+        data_analyzer = DataAnalyzer(data)
+        data_analyzer.show_summary_statistics()
+        data_analyzer.show_data_types()
+        data_analyzer.show_null_value_statistics()
+
+        data_filter = DataFilter(data)
+        data = data_filter.filter_rows()
+
+        data_transformer = DataTransformer(data)
+        data = data_transformer.perform_column_operation()
+        data = data_transformer.remove_null()
+        data = data_transformer.impute_null()
+        data = data_transformer.remove_columns()
+
+        data_visualizer = DataVisualizer(data)
+        data_visualizer.visualize_data()
+
+        data_QA = DataQA(data)
+        data_QA.ask_csv()
+
+if __name__ == "__main__":
+    main()