Data retaintion problem solved

.gitignore

@@ -9,6 +9,8 @@ __pycache__/
 
 # data set
 data.csv
+original_data.csv
+
 #Env variables
 .env
 # Distribution / packaging

Experiment.py

@@ -0,0 +1,57 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+
+# Function to upload dataset
+def upload_dataset():
+    uploaded_file = st.file_uploader("Upload CSV file", type=["csv"])
+    if uploaded_file is not None:
+        df = pd.read_csv(uploaded_file)
+        return df
+
+# Function to impute null values
+def impute_null(df):
+    # Implement your logic for null value imputation
+    col = st.multiselect('Choose columns to impute nulls', df.select_dtypes(include=[np.number]).columns)
+    option = st.selectbox('Impute nulls with', ('mean', 'mode', '0'))
+    if st.button('Impute Null'):
+        if option == "mean":
+                df[col] = df[col].fillna(df[col].mean())
+        elif option == "mode":
+                df[col] = df[col].fillna(df[col].mode().iloc[0])  # mode() returns a DataFrame, so we select the first row
+        elif option == "0":
+                df[col] = df[col].fillna(0)
+        st.success("Null values filled")
+    return df
+
+# Function to display transformed data
+def display_data(df):
+    st.write(df)
+
+def main():
+    st.title("Data Transformation App")
+    
+    # Step 1: Upload Dataset
+    st.sidebar.title("Upload Dataset")
+    df = upload_dataset()
+    
+    if df is not None:
+        # Step 2: Perform Data Transformation
+        st.sidebar.title("Data Transformation")
+        if st.sidebar.button("Impute Null Values"):
+            df = impute_null(df)
+            st.success("Null values imputed successfully!")
+        
+        # Step 3: Display Transformed Data
+        st.sidebar.title("Transformed Data")
+        if st.sidebar.checkbox("Show Transformed Data"):
+            display_data(df)
+        
+        # Step 4: Store Transformed Data
+        # You can store the transformed data in a variable or a data structure here
+    
+    # Step 5: Use Transformed Data
+    # You can utilize the transformed data for further analysis, visualization, etc.
+
+if __name__ == "__main__":
+    main()

app.py

@@ -8,15 +8,18 @@ from data_QA import DataQA
 import os
 from streamlit_option_menu import option_menu
 
+
+import pandas as pd
+
 def main():
-    if os.path.exists("data.csv"):
-        os.remove("data.csv")
-    with open("data.csv", 'w'):
-        pass
     st.title('Insights 📶')
-
+    
     data_loader = DataLoader()
-    data = data_loader.load_data()
+    load = data_loader.load_data()
+    if load:
+        data = pd.read_csv('data.csv')
+
+    
 
     if os.path.getsize("data.csv") != 0:
         with st.sidebar:
@@ -31,6 +34,7 @@ def main():
 
         # --- EDA ---
         if selected == "Exploratory Data Analysis":
+            data = pd.read_csv("data.csv")
             data_analyzer = DataAnalyzer(data)
             data_analyzer.show_eda()
             data_analyzer.show_null_value_statistics()
@@ -43,13 +47,18 @@ def main():
         # --- DATA CLEANING ---
         if selected == "Data Cleaning":
             data_transformer = DataTransformer(data)
-            data_analyzer = DataAnalyzer(data)
+            
             modified_data = data_transformer.perform_column_operation()
-            data_analyzer.show_null_value_statistics()
             modified_data = data_transformer.remove_null()
             modified_data = data_transformer.impute_null()
-            modified_data = data_transformer.remove_columns()
+            data = modified_data
+            data_analyzer = DataAnalyzer(data)
+            data_analyzer.show_null_value_statistics()
+            new_data_analyzer = DataAnalyzer(modified_data)
+            data_analyzer.show_null_value_statistics()
 
+            # modified_data = data_transformer.remove_columns()
+            
             # data_filter = DataFilter(modified_data)
             # data = data_filter.filter_rows()
 

data_loader.py

@@ -3,21 +3,28 @@ import pandas as pd
 
 class DataLoader:
     def __init__(self):
-        self.data = pd.DataFrame()  # Initialize data as an empty DataFrame
+        pass
 
-    def load_data(self):
-        data_source = st.selectbox('Select data source', ['Upload a CSV file', 'Input a URL'])
-        if data_source == 'Upload a CSV file':
-            uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
-            if uploaded_file is not None:
-                self.data = pd.read_csv(uploaded_file)
-                file_path = './data.csv'
-                self.data.to_csv(file_path, index=False)
-        elif data_source == 'Input a URL':
-            url = st.text_input('Enter the URL of a CSV file')
-            if url:
-                try:
-                    self.data = pd.read_csv(url)
-                except:
-                    st.error('Could not load data from the provided URL. Please make sure the URL is correct and points to a CSV file.')
-        return self.data
+    @st.cache_data(experimental_allow_widgets=True)
+    def load_data(_):
+        if True:
+            data = pd.DataFrame()
+            data_source = st.selectbox('Select data source', ['Upload a CSV file', 'Input a URL'])
+            if data_source == 'Upload a CSV file':
+                uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
+                if uploaded_file is not None:
+                    data = pd.read_csv(uploaded_file)
+                    data.to_csv('./original_data.csv', index=False)
+                    data.to_csv('./data.csv',index=False)
+                    return True
+            elif data_source == 'Input a URL':
+                url = st.text_input('Enter the URL of a CSV file')
+                if url:
+                    try:
+                        data = pd.read_csv(url)
+                        data_loaded = True
+                    except:
+                        st.error('Could not load data from the provided URL. Please make sure the URL is correct and points to a CSV file.')
+                return True
+            print("data loader ran once")
+        return data

data_transformer.py

@@ -1,5 +1,6 @@
 import streamlit as st
 import pandas as pd
+import numpy as np
 
 class DataTransformer:
     def __init__(self, data):
@@ -22,16 +23,17 @@ class DataTransformer:
         return self.data
 
     def impute_null(self):
+        col = st.multiselect('Choose columns to impute nulls', self.data.select_dtypes(include=[np.number]).columns)
+        option = st.selectbox('Impute nulls with', ('mean', 'mode', '0'))
         if st.button('Impute Null'):
-            col = st.multiselect('Choose columns to impute nulls', self.data.select_dtypes(include=[np.number]).columns)
-            option = st.selectbox('Impute nulls with', ('mean', 'mode', '0'))
             if option == "mean":
-                self.data.fillna(df.mean())
+                self.data[col] = self.data[col].fillna(self.data[col].mean())
             elif option == "mode":
-                self.data.fillna(df.mode())
+                self.data[col] = self.data[col].fillna(self.data[col].mode().iloc[0])  # mode() returns a DataFrame, so we select the first row
             elif option == "0":
-                self.data.fillna("0")
-            st.toast("Null values filled")
+                self.data[col] = self.data[col].fillna(0)
+            st.success("Null values filled")
+            self.data.to_csv("data.csv", index=False)
         return self.data
 
     def remove_columns(self):

test.py

@@ -1,42 +0,0 @@
-import streamlit as st
-from data_loader import DataLoader
-from data_analyzer import DataAnalyzer
-from data_filter import DataFilter
-from data_transformer import DataTransformer
-from data_visualizer import DataVisualizer
-from data_QA import DataQA
-import os
-
-def main():
-    if os.path.exists("data.csv"):
-        os.remove("data.csv")
-    with open("data.csv", 'w'):
-        pass
-    st.title('Insights 📶')
-
-    data_loader = DataLoader()
-    data = data_loader.load_data()
-
-    if os.path.getsize("data.csv") != 0:
-        data_analyzer = DataAnalyzer(data)
-        data_analyzer.show_summary_statistics()
-        data_analyzer.show_data_types()
-        data_analyzer.show_null_value_statistics()
-
-        data_filter = DataFilter(data)
-        data = data_filter.filter_rows()
-
-        data_transformer = DataTransformer(data)
-        data = data_transformer.perform_column_operation()
-        data = data_transformer.remove_null()
-        data = data_transformer.impute_null()
-        data = data_transformer.remove_columns()
-
-        data_visualizer = DataVisualizer(data)
-        data_visualizer.visualize_data()
-
-        data_QA = DataQA(data)
-        data_QA.ask_csv()
-
-if __name__ == "__main__":
-    main()