Added tests for data-transformer module

Modules/data_transformer.py

@@ -5,7 +5,7 @@ import numpy as np
 class DataTransformer:
     def __init__(self, data):
         self.data = data
-        st.header("Data Cleaning")
+        
 
     def perform_column_operation(self):
         column_operation = st.sidebar.text_input('Column operation (e.g., age * 2)')
@@ -21,26 +21,17 @@ class DataTransformer:
             st.subheader("Remove Null Values")
             col = st.multiselect('Choose columns to remove nulls', self.data.columns)
             if st.button('Remove Null'):
-                self.data.dropna(subset=col, inplace=True)
+                self.handle_null_remove(col)
                 st.success("Null values removed")
-                self.data.to_csv("data.csv", index=False)
             st.subheader("Impute Null Values")
             col = st.multiselect('Choose columns to impute nulls', self.data.select_dtypes(include=[np.number]).columns)
             option = st.selectbox('Impute nulls with', ('-Select-','mean', 'mode', '0'))
             if st.button('Impute Null'):
                 try:
-                    if option == "mean":
-                        self.data[col] = self.data[col].fillna(self.data[col].mean())
-                    elif option == "mode":
-                        self.data[col] = self.data[col].fillna(self.data[col].mode().iloc[0])
-                    elif option == "0":
-                        self.data[col] = self.data[col].fillna(0)
-                    elif option == "-Select-":
-                        raise ValueError("Select a value")
+                    self.handle_null_impute(col,option)
                     st.success("Null values filled")
                 except ValueError as e:
                     st.error(str(e))
-                self.data.to_csv("data.csv", index=False)
         with right:
             st.write("Null Stats")
             null_counts = self.data.isnull().sum()
@@ -65,12 +56,8 @@ class DataTransformer:
         st.subheader("Convert Categorical to Numerical")
         columns_to_encode = st.multiselect('Choose columns to convert', self.data.select_dtypes(include=object).columns)
         if st.button('Convert'):
-            for col in columns_to_encode:
-                one_hot_encoded = pd.get_dummies(self.data[col], prefix=col).astype(int)
-                self.data = pd.concat([self.data, one_hot_encoded], axis=1)
-                self.data.drop(col, axis=1, inplace=True)
+            self.categorical_to_numerical_func(columns_to_encode)
             st.success("Converted categoricals variables")
-            self.data.to_csv("data.csv", index=False)
             st.write(self.data.head())
         return self.data
 
@@ -78,17 +65,46 @@ class DataTransformer:
         st.subheader("Remove Columns")
         col = st.multiselect('Choose columns to remove', self.data.columns)
         if st.button('Remove Columns'):
-            self.data.drop(columns=col, inplace=True)
+            self.remove_columns_func(col)
             st.success("Columns removed")
+        return self.data
+
+
+    #---CORE FUNCTIONALITY---
+    def remove_columns_func(self,col):
+        self.data.drop(columns=col, inplace=True)
         self.data.to_csv("data.csv", index=False)
         return self.data
+    
+    def handle_null_remove(self,col):
+        self.data.dropna(subset=col, inplace=True)
+        print(self.data)
+        self.data.to_csv("data.csv", index=False)
+
+    def handle_null_impute(self,col,option):
+        if option == "mean":
+            self.data[col] = self.data[col].fillna(self.data[col].mean())
+        elif option == "mode":
+            self.data[col] = self.data[col].fillna(self.data[col].mode().iloc[0])
+        elif option == "0":
+            self.data[col] = self.data[col].fillna(0)
+        elif option == "-Select-":
+            raise ValueError("Select an option")
+        self.data.to_csv("data.csv", index=False)
+
 
-        # PROBLEMS RESOLVED
+    def categorical_to_numerical_func(self,columns_to_encode):
+        for col in columns_to_encode:
+            one_hot_encoded = pd.get_dummies(self.data[col], prefix=col).astype(int)
+            self.data = pd.concat([self.data, one_hot_encoded], axis=1)
+            self.data.drop(col, axis=1, inplace=True)
+        self.data.to_csv("data.csv", index=False)
+
+    # PROBLEMS RESOLVED
         #transformed data is not retained
         #null values handling
         #2 options - to remove or to impute that is the question
         #categorical to numerical
 
         # PROBLEMS TO BE ADDRESSED
-        #give option to analyse the transformed dataset or save it.
-
+        #give option to analyse the transformed dataset or save it.

test_app.py → Testing/test_app.py

@@ -1,4 +1,6 @@
 from streamlit.testing.v1 import AppTest
+import sys
+sys.path.append("..")
 
 def test_smoke():
     """Basic smoke test"""

Testing/test_data_transformer.py

@@ -0,0 +1,69 @@
+import sys
+import os
+import numpy as np
+
+
+sys.path.append("..")
+
+import unittest
+import pandas as pd
+from Modules.data_transformer import DataTransformer
+
+
+class TestDataTransformer(unittest.TestCase):
+
+    def setUp(self):
+        # Initialize DataTransformer object with sample data
+        data = {
+            'A': [1, 2, 3, None, 5],
+            'B': [4, 5, None, 7, 8],
+            'C': ['X', 'Y', 'Z', 'X', 'Y'],
+            'D': ['M', 'N', 'O', 'N', 'P'],
+            'E': [10.1, 20.2, None, 40.4, 50.5],
+            'F': [10.1, 20.2, None, 40.4, None],
+            'G': [None, 20.2, None, 40.4, 50.5]
+        }
+        self.sample_data = pd.DataFrame(data)
+        self.sample_data.to_csv("data.csv", index=False)
+        self.transformer = DataTransformer(self.sample_data.copy())
+
+    def test_handle_null_remove(self):
+        # Test removing rows with null values
+        self.transformer.handle_null_remove(['G'])
+        self.assertNotIn(None, self.transformer.data['G'])
+        # self.assertTrue(pd.read_csv("data.csv").equals(self.transformer.data))
+
+    def test_remove_columns_func(self):
+        # Test removing columns
+        self.transformer.remove_columns_func(['D'])
+        self.assertNotIn('D', self.transformer.data.columns)
+
+    def test_handle_null_impute(self):
+        # Test imputing null values with mean
+        self.transformer.handle_null_impute('A', 'mean')
+        self.assertFalse(self.transformer.data['A'].isnull().any())
+        self.assertTrue(pd.read_csv("data.csv").equals(self.transformer.data))
+
+        # Test imputing null values with mode
+        self.transformer.handle_null_impute('F', 'mode')
+        self.assertFalse(self.transformer.data['F'].isnull().any())
+        self.assertTrue(pd.read_csv("data.csv").equals(self.transformer.data))
+
+        # Test imputing null values with 0
+        self.transformer.handle_null_impute('G', '0')
+        self.assertFalse(self.transformer.data['G'].isnull().any())
+        self.assertTrue(pd.read_csv("data.csv").equals(self.transformer.data))
+
+    def test_categorical_to_numerical_func(self):
+        # Test converting categorical columns to numerical
+        self.transformer.categorical_to_numerical_func(['C'])
+        self.assertTrue(any(col.startswith('C_') for col in self.transformer.data.columns))
+        self.assertTrue(pd.read_csv("data.csv").equals(self.transformer.data))
+
+    def tearDown(self):
+        # Clean up temporary files generated during tests
+        import os
+        os.remove("data.csv")
+
+if __name__ == '__main__':
+    unittest.main()

app.py

@@ -52,6 +52,7 @@ def main():
 
         # --- DATA CLEANING ---
         if selected == "Data Cleaning":
+            st.header("Data Cleaning")
             data_transformer = DataTransformer(data)
             
             # modified_data = data_transformer.perform_column_operation()