Update report.py

report.py

@@ -1,208 +1,206 @@
-import os
-import numpy as np
-import pandas as pd
-import seaborn as sns
-import matplotlib.pyplot as plt
-from datetime import datetime
-
-REPORT_DIR = f"cleaning_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-os.makedirs(REPORT_DIR, exist_ok=True)
-
-def save_plot(fig, filename):
-    fig.savefig(os.path.join(REPORT_DIR, filename), dpi=400, bbox_inches='tight')
-    plt.close(fig)
-
-def plot_heatmap(df, title):
-    plt.figure(figsize=(12, 8))
-    sns.heatmap(df.isnull(), cbar=False, cmap='viridis')
-    plt.title(title)
-    plt.tight_layout()
-    save_plot(plt.gcf(), f'{title.lower().replace(" ", "_")}.png')
-
-
-def plot_valid_data_percentage(original_df, cleaned_df):
-    original_valid = (original_df.notna().sum() / len(original_df)) * 100
-    cleaned_valid = (cleaned_df.notna().sum() / len(cleaned_df)) * 100
-
-    # Create a DataFrame with both original and cleaned percentages
-    combined_df = pd.DataFrame({
-        'Original': original_valid,
-        'Cleaned': cleaned_valid
-    }).fillna(0)  # Fill NaN with 0 for columns that were removed
-
-    plt.figure(figsize=(15, 8))
-    combined_df.plot(kind='bar', width=0.8, alpha=0.8)
-
-    plt.xlabel('Columns')
-    plt.ylabel('Percentage of Valid Data')
-    plt.title('Percentage of Valid Data Before and After Cleaning')
-    plt.xticks(rotation=90)
-    plt.legend(['Before Cleaning', 'After Cleaning'])
-
-    # Add percentage labels on the bars
-    for i, (index, row) in enumerate(combined_df.iterrows()):
-        plt.text(i, row['Original'], f'{row["Original"]:.1f}%', ha='center', va='bottom')
-        if row['Cleaned'] > 0:  # Only add label if column exists in cleaned data
-            plt.text(i, row['Cleaned'], f'{row["Cleaned"]:.1f}%', ha='center', va='bottom')
-
-    plt.tight_layout()
-    plt.savefig(os.path.join(REPORT_DIR, 'valid_data_percentage.png'))
-    plt.close()
-
-def plot_column_schemas(df):
-    schemas = df.dtypes.astype(str).value_counts()
-    fig, ax = plt.subplots(figsize=(10, 6))
-    sns.barplot(x=schemas.index, y=schemas.values, ax=ax)
-    ax.set_title('Column Data Types')
-    ax.set_xlabel('Data Type')
-    ax.set_ylabel('Count')
-    save_plot(fig, 'column_schemas.png')
-
-def plot_nonconforming_cells(nonconforming_cells):
-    # Ensure that nonconforming_cells is a dictionary
-    if isinstance(nonconforming_cells, dict):
-        # Proceed with plotting if it's a dictionary
-        fig, ax = plt.subplots(figsize=(12, 6))
-        sns.barplot(x=list(nonconforming_cells.keys()), y=list(nonconforming_cells.values()), ax=ax)
-        ax.set_title('Nonconforming Cells by Column')
-        ax.set_xlabel('Columns')
-        ax.set_ylabel('Number of Nonconforming Cells')
-        plt.xticks(rotation=90)
-        save_plot(fig, 'nonconforming_cells.png')
-    else:
-        print(f"Expected nonconforming_cells to be a dictionary, but got {type(nonconforming_cells)}.")
-
-
-def plot_column_distributions(original_df, cleaned_df):
-    numeric_columns = original_df.select_dtypes(include=[np.number]).columns
-    num_columns = len(numeric_columns)
-
-    if num_columns == 0:
-        print("No numeric columns found for distribution plots.")
-        return
-
-    # Create subplots for distributions
-    fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(18, 5 * ((num_columns + 2) // 3)))
-    axes = axes.flatten() if num_columns > 1 else [axes]
-
-    for i, column in enumerate(numeric_columns):
-        if column in cleaned_df.columns:
-            sns.histplot(original_df[column].dropna(), ax=axes[i], kde=True, color='blue', label='Before Cleaning', alpha=0.5)
-            sns.histplot(cleaned_df[column].dropna(), ax=axes[i], kde=True, color='orange', label='After Cleaning', alpha=0.5)
-            axes[i].set_title(f'{column} - Distribution Before & After Cleaning')
-            axes[i].legend()
-
-    # Remove any unused subplots
-    for j in range(i + 1, len(axes)):
-        fig.delaxes(axes[j])
-
-    plt.tight_layout()
-    save_plot(fig, 'distributions_before_after_cleaning.png')
-
-
-def plot_boxplot_with_outliers(df):
-    print("Plotting boxplots with outliers...")
-    numeric_columns = df.select_dtypes(include=[np.number]).columns
-    num_columns = len(numeric_columns)
-
-    if num_columns == 0:
-        print("No numeric columns found for boxplot.")
-        return
-
-    # Create subplots based on the number of numeric columns
-    fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(15, 5 * ((num_columns + 2) // 3)))
-    axes = axes.flatten() if num_columns > 1 else [axes]
-
-    for i, column in enumerate(numeric_columns):
-        sns.boxplot(x=df[column], ax=axes[i])
-        axes[i].set_title(f'Boxplot of {column} with Outliers')
-
-    # Remove any unused subplots
-    for j in range(i + 1, len(axes)):
-        fig.delaxes(axes[j])
-
-    plt.tight_layout()
-    save_plot(fig, 'boxplots_with_outliers.png')
-
-
-def plot_correlation_heatmap(df):
-    # Select only numeric, float, and integer columns
-    numeric_df = df.select_dtypes(include=[np.number])
-
-    # Compute the correlation matrix
-    correlation_matrix = numeric_df.corr()
-
-    # Plot the heatmap
-    fig, ax = plt.subplots(figsize=(15, 10))
-    sns.heatmap(correlation_matrix, annot=True, fmt=".2f", cmap='coolwarm', ax=ax, cbar_kws={'label': 'Correlation'})
-    ax.set_title('Correlation Heatmap')
-    save_plot(fig, 'correlation_heatmap.png')
-
-
-
-def plot_process_times(process_times):
-    # Convert seconds to minutes
-    process_times_minutes = {k: v / 60 for k, v in process_times.items()}
-
-    # Separate main processes and column cleaning processes
-    main_processes = {k: v for k, v in process_times_minutes.items() if not k.startswith("Clean column:")}
-    column_processes = {k: v for k, v in process_times_minutes.items() if k.startswith("Clean column:")}
-
-    # Create the plot
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 10))
-
-    # Plot main processes
-    bars1 = ax1.bar(main_processes.keys(), main_processes.values())
-    ax1.set_title('Main Process Times')
-    ax1.set_ylabel('Time (minutes)')
-    ax1.tick_params(axis='x', rotation=45)
-
-    # Plot column cleaning processes
-    bars2 = ax2.bar(column_processes.keys(), column_processes.values())
-    ax2.set_title('Column Cleaning Times')
-    ax2.set_ylabel('Time (minutes)')
-    ax2.tick_params(axis='x', rotation=90)
-
-    # Add value labels on top of each bar
-    for ax, bars in zip([ax1, ax2], [bars1, bars2]):
-        for bar in bars:
-            height = bar.get_height()
-            ax.text(bar.get_x() + bar.get_width() / 2., height,
-                    f'{height:.2f}', ha='center', va='bottom')
-
-    # Add total time to the plot
-    total_time = sum(process_times_minutes.values())
-    fig.suptitle(f'Process Times (Total: {total_time:.2f} minutes)', fontsize=16)
-
-    plt.tight_layout()
-    save_plot(fig, 'process_times.png')
-
-
-def create_full_report(original_df, cleaned_df, nonconforming_cells_before, process_times, removed_columns, removed_rows):
-    os.makedirs(REPORT_DIR, exist_ok=True)
-
-    sns.set_style("whitegrid")
-    plt.rcParams['figure.dpi'] = 400
-
-    print("Plotting valid data percentages...")
-    plot_valid_data_percentage(original_df, cleaned_df)
-
-    print("Plotting column schemas...")
-    plot_column_schemas(cleaned_df)
-
-    print("Plotting nonconforming cells before cleaning...")
-    plot_nonconforming_cells(nonconforming_cells_before)
-
-    print("Plotting column distributions...")
-    plot_column_distributions(original_df, cleaned_df)
-
-    print("Plotting process times...")
-    plot_process_times(process_times)
-
-    print("Plotting heatmaps...")
-    plot_heatmap(original_df, "Missing Values Before Cleaning")
-
-    print("Plotting correlation heatmap...")
-    plot_correlation_heatmap(cleaned_df)
-
+import os
+import numpy as np
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+REPORT_DIR = f"cleaning_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+os.makedirs(REPORT_DIR, exist_ok=True)
+
+def save_plot(fig, filename):
+    fig.savefig(os.path.join(REPORT_DIR, filename), dpi=400, bbox_inches='tight')
+    plt.close(fig)
+
+def plot_heatmap(df, title):
+    plt.figure(figsize=(12, 8))
+    sns.heatmap(df.isnull(), cbar=False, cmap='viridis')
+    plt.title(title)
+    plt.tight_layout()
+    save_plot(plt.gcf(), f'{title.lower().replace(" ", "_")}.png')
+
+
+def plot_valid_data_percentage(original_df, cleaned_df):
+    original_valid = (original_df.notna().sum() / len(original_df)) * 100
+    cleaned_valid = (cleaned_df.notna().sum() / len(cleaned_df)) * 100
+    
+    plt.figure(figsize=(15, 8))
+    x = range(len(original_valid))
+    width = 0.35
+    
+    plt.bar(x, original_valid, width, label='Before Cleaning', alpha=0.8)
+    plt.bar([i + width for i in x], cleaned_valid, width, label='After Cleaning', alpha=0.8)
+    
+    plt.xlabel('Columns')
+    plt.ylabel('Percentage of Valid Data')
+    plt.title('Percentage of Valid Data Before and After Cleaning')
+    plt.xticks([i + width/2 for i in x], original_valid.index, rotation=90)
+    plt.legend()
+    
+    # Add percentage labels on the bars with smaller font size
+    for i, v in enumerate(original_valid):
+        plt.text(i, v, f'{v:.1f}%', ha='center', va='bottom', fontsize=6)
+    for i, v in enumerate(cleaned_valid):
+        plt.text(i + width, v, f'{v:.1f}%', ha='center', va='bottom', fontsize=6)
+    
+    plt.tight_layout()
+    plt.savefig(os.path.join(REPORT_DIR, 'valid_data_percentage.png'))
+    plt.close()
+
+def plot_column_schemas(df):
+    schemas = df.dtypes.astype(str).value_counts()
+    fig, ax = plt.subplots(figsize=(10, 6))
+    sns.barplot(x=schemas.index, y=schemas.values, ax=ax)
+    ax.set_title('Column Data Types')
+    ax.set_xlabel('Data Type')
+    ax.set_ylabel('Count')
+    save_plot(fig, 'column_schemas.png')
+
+def plot_nonconforming_cells(nonconforming_cells):
+    # Ensure that nonconforming_cells is a dictionary
+    if isinstance(nonconforming_cells, dict):
+        # Proceed with plotting if it's a dictionary
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.barplot(x=list(nonconforming_cells.keys()), y=list(nonconforming_cells.values()), ax=ax)
+        ax.set_title('Nonconforming Cells by Column')
+        ax.set_xlabel('Columns')
+        ax.set_ylabel('Number of Nonconforming Cells')
+        plt.xticks(rotation=90)
+        save_plot(fig, 'nonconforming_cells.png')
+    else:
+        print(f"Expected nonconforming_cells to be a dictionary, but got {type(nonconforming_cells)}.")
+
+
+def plot_column_distributions(original_df, cleaned_df):
+    numeric_columns = original_df.select_dtypes(include=[np.number]).columns
+    num_columns = len(numeric_columns)
+
+    if num_columns == 0:
+        print("No numeric columns found for distribution plots.")
+        return
+
+    # Create subplots for distributions
+    fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(18, 5 * ((num_columns + 2) // 3)))
+    axes = axes.flatten() if num_columns > 1 else [axes]
+
+    for i, column in enumerate(numeric_columns):
+        if column in cleaned_df.columns:
+            sns.histplot(original_df[column].dropna(), ax=axes[i], kde=True, color='blue', label='Before Cleaning', alpha=0.5)
+            sns.histplot(cleaned_df[column].dropna(), ax=axes[i], kde=True, color='orange', label='After Cleaning', alpha=0.5)
+            axes[i].set_title(f'{column} - Distribution Before & After Cleaning')
+            axes[i].legend()
+
+    # Remove any unused subplots
+    for j in range(i + 1, len(axes)):
+        fig.delaxes(axes[j])
+
+    plt.tight_layout()
+    save_plot(fig, 'distributions_before_after_cleaning.png')
+
+
+def plot_boxplot_with_outliers(df):
+    print("Plotting boxplots with outliers...")
+    numeric_columns = df.select_dtypes(include=[np.number]).columns
+    num_columns = len(numeric_columns)
+
+    if num_columns == 0:
+        print("No numeric columns found for boxplot.")
+        return
+
+    # Create subplots based on the number of numeric columns
+    fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(15, 5 * ((num_columns + 2) // 3)))
+    axes = axes.flatten() if num_columns > 1 else [axes]
+
+    for i, column in enumerate(numeric_columns):
+        sns.boxplot(x=df[column], ax=axes[i])
+        axes[i].set_title(f'Boxplot of {column} with Outliers')
+
+    # Remove any unused subplots
+    for j in range(i + 1, len(axes)):
+        fig.delaxes(axes[j])
+
+    plt.tight_layout()
+    save_plot(fig, 'boxplots_with_outliers.png')
+
+
+def plot_correlation_heatmap(df):
+    # Select only numeric, float, and integer columns
+    numeric_df = df.select_dtypes(include=[np.number])
+
+    # Compute the correlation matrix
+    correlation_matrix = numeric_df.corr()
+
+    # Plot the heatmap
+    fig, ax = plt.subplots(figsize=(15, 10))
+    sns.heatmap(correlation_matrix, annot=True, fmt=".2f", cmap='coolwarm', ax=ax, cbar_kws={'label': 'Correlation'})
+    ax.set_title('Correlation Heatmap')
+    save_plot(fig, 'correlation_heatmap.png')
+
+
+
+def plot_process_times(process_times):
+    # Convert seconds to minutes
+    process_times_minutes = {k: v / 60 for k, v in process_times.items()}
+
+    # Separate main processes and column cleaning processes
+    main_processes = {k: v for k, v in process_times_minutes.items() if not k.startswith("Clean column:")}
+    column_processes = {k: v for k, v in process_times_minutes.items() if k.startswith("Clean column:")}
+
+    # Create the plot
+    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 10))
+
+    # Plot main processes
+    bars1 = ax1.bar(main_processes.keys(), main_processes.values())
+    ax1.set_title('Main Process Times')
+    ax1.set_ylabel('Time (minutes)')
+    ax1.tick_params(axis='x', rotation=45)
+
+    # Plot column cleaning processes
+    bars2 = ax2.bar(column_processes.keys(), column_processes.values())
+    ax2.set_title('Column Cleaning Times')
+    ax2.set_ylabel('Time (minutes)')
+    ax2.tick_params(axis='x', rotation=90)
+
+    # Add value labels on top of each bar
+    for ax, bars in zip([ax1, ax2], [bars1, bars2]):
+        for bar in bars:
+            height = bar.get_height()
+            ax.text(bar.get_x() + bar.get_width() / 2., height,
+                    f'{height:.2f}', ha='center', va='bottom')
+
+    # Add total time to the plot
+    total_time = sum(process_times_minutes.values())
+    fig.suptitle(f'Process Times (Total: {total_time:.2f} minutes)', fontsize=16)
+
+    plt.tight_layout()
+    save_plot(fig, 'process_times.png')
+
+
+def create_full_report(original_df, cleaned_df, nonconforming_cells_before, process_times, removed_columns, removed_rows):
+    os.makedirs(REPORT_DIR, exist_ok=True)
+
+    sns.set_style("whitegrid")
+    plt.rcParams['figure.dpi'] = 400
+
+    print("Plotting valid data percentages...")
+    plot_valid_data_percentage(original_df, cleaned_df)
+
+    print("Plotting column schemas...")
+    plot_column_schemas(cleaned_df)
+
+    print("Plotting nonconforming cells before cleaning...")
+    plot_nonconforming_cells(nonconforming_cells_before)
+
+    print("Plotting column distributions...")
+    plot_column_distributions(original_df, cleaned_df)
+
+    print("Plotting process times...")
+    plot_process_times(process_times)
+
+    print("Plotting heatmaps...")
+    plot_heatmap(original_df, "Missing Values Before Cleaning")
+
+    print("Plotting correlation heatmap...")
+    plot_correlation_heatmap(cleaned_df)
+
     print(f"All visualization reports saved in directory: {REPORT_DIR}")