create pdf for this code to teach  me the code and summarize and library summary and tutorial for advanced this code and learn more about ml in  cybeersecuirty and how to improve the code to make him abetter for anomly detection import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import LabelEncoder
from collections import Counter
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
import joblib
import os


def data_cleaning(df):
    # Custom function to process column names
    def process_column_name(col_name):
        # Convert to lowercase and remove leading/trailing spaces
        col_name = col_name.lower().strip()
        if col_name.startswith(' '):
            col_name = col_name.lstrip()  # Remove leading spaces
        else:
            # Replace remaining spaces with underscores
            col_name = col_name.replace(' ', '_')
        return col_name

    # Apply the custom function to all column names
    df.columns = [process_column_name(col) for col in df.columns]
    
    # Define important columns with lowercase names and underscores instead of spaces
    important_columns = ['flow_id', 'destination_ip', 'source_ip', 'timestamp',
                         'label', 'source_port', 'destination_port', 'protocol', 'flow_duration', 'flow_bytes/s', 'flow_packets/s']

    # Ensure all important columns exist in the DataFrame
    missing_columns = [col for col in important_columns if col not in df.columns]
    if missing_columns:
        print(f"Missing columns in the DataFrame: {missing_columns}")
        # Handle missing columns as needed, e.g., by removing them from the list
        important_columns = [col for col in important_columns if col in df.columns]

    # Replace infinities with NaN
    df.replace([np.inf, -np.inf], np.nan, inplace=True)

    # Separate numeric and non-numeric columns
    numeric_cols = df.select_dtypes(include='number').columns
    non_numeric_cols = df.select_dtypes(exclude='number').columns

    # Impute missing values in numeric columns using SimpleImputer
    imputer = SimpleImputer(strategy='mean')  # or strategy='median', 'most_frequent', etc.
    df[numeric_cols] = imputer.fit_transform(df[numeric_cols])

    # For non-numeric columns, you can choose to impute differently or handle them as needed
    # Use ffill() directly on the DataFrame
    df[non_numeric_cols] = df[non_numeric_cols].ffill()

    # Feature Engineering: Example features (adjust based on your dataset)
    if 'packet_size' in df.columns and 'duration' in df.columns:
        df['packet_rate'] = df['packet_size'] / df['duration']

    # Feature Selection: Using Variance Threshold to remove low-variance features
    numeric_data = df[numeric_cols]
    selector = VarianceThreshold(threshold=0.1)  # Adjust threshold as needed
    selected_features = selector.fit_transform(numeric_data)

    # Create a DataFrame with selected features
    selected_df = pd.DataFrame(selected_features, columns=numeric_data.columns[selector.get_support()])

    # Combine selected features with important columns
    final_df = pd.concat([df[important_columns].reset_index(drop=True), selected_df.reset_index(drop=True)], axis=1)

    return final_df

def data_train(df):
    # Call the data_cleaning function
    df = data_cleaning(df)
    
    # Perform model training
    def label_anomalous(text):
        """Binarize target labels into normal or anomalous."""
        if text == "BENIGN":
            return 0
        else:
            return 1

    df["label"] = df["label"].apply(label_anomalous)
    y = df["label"].values
    counts = Counter(y).most_common()

    # Ensure there are at least two different labels before calculating contamination parameter
    if len(counts) > 1:
        contamination_parameter = counts[1][1] / (counts[0][1] + counts[1][1])
    else:
        contamination_parameter = 0  # or some default value
    
    encodings_dictionary = dict()
    for c in df.columns:
        # Use is_object_dtype to check if the column type is object
        if pd.api.types.is_object_dtype(df[c]):
            encodings_dictionary[c] = LabelEncoder()
            df[c] = encodings_dictionary[c].fit_transform(df[c].astype(str))

    # Split the data into normal and abnormal subsets
    df_abnormal = df[df['label'] == 1]
    df_normal = df[df['label'] == 0]

    X_normal = df_normal.drop(columns=['label']).values
    y_normal = df_normal['label'].values
    X_anomaly = df_abnormal.drop(columns=['label']).values
    y_anomaly = df_abnormal['label'].values


    X_normal_train, X_normal_test, y_normal_train, y_normal_test = train_test_split(
        df_normal.drop('label', axis=1), 
        df_normal['label'], test_size=0.3, random_state=11
    )
    
    X_anomaly_train, X_anomaly_test, y_anomaly_train, y_anomaly_test = train_test_split(
        df_abnormal.drop('label', axis=1), df_abnormal['label'],
         test_size=0.3, random_state=11
    )

    X_train = np.concatenate((X_normal_train, X_anomaly_train))
    y_train = np.concatenate((y_normal_train, y_anomaly_train))
    X_test = np.concatenate((X_normal_test, X_anomaly_test))
    y_test = np.concatenate((y_normal_test, y_anomaly_test))

    # Handle NaN values using SimpleImputer
    imputer = SimpleImputer(strategy='mean')
    X_train = imputer.fit_transform(X_train)
    X_test = imputer.transform(X_test)

    # Check for NaN values in training data
    if np.any(np.isnan(X_train)):
        print("Training data contains NaN values. Please check data processing steps.")
        return

    # Standardize the data
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    # Train Isolation Forest
    IF = IsolationForest(contamination=contamination_parameter)
    IF.fit(X_train)
    
     # Save the model for deployment
    joblib.dump(IF, 'isolation_forest_model.pkl')
    joblib.dump(imputer, 'imputer.pkl')
    joblib.dump(scaler, 'scaler.pkl')
    # Save the feature names used in training
    feature_names = df.columns.tolist()
    joblib.dump(feature_names, 'isolation_forest_features.pkl')

    decisionScores_train_normal = IF.decision_function(X_normal_train)
    decisionScores_train_anomaly = IF.decision_function(X_anomaly_train)

     # Visualize the decision scores
    plt.figure(figsize=(20, 10))
    plt.hist(decisionScores_train_anomaly, bins=50)
    plt.title('Decision Scores for Anomaly Training Data')
    plt.xlabel('Decision Score')
    plt.ylabel('Frequency')
    plt.show()


       
   # Save normal and anomaly data
    y_pred_train = IF.predict(X_train)
    y_pred_train = np.where(y_pred_train == 1, 0, 1)
    normal_data = X_train[y_pred_train == 0]
    anomaly_data = X_train[y_pred_train == 1]
    pd.DataFrame(normal_data).to_csv('normal_data.csv', index=False)
    pd.DataFrame(anomaly_data).to_csv('anomaly_data.csv', index=False)
    
    # Evaluate on test set
    y_pred_test = IF.predict(X_test)
    y_pred_test = np.where(y_pred_test == 1, 0, 1)


   # Calculate evaluation metrics
    from sklearn.metrics import classification_report, confusion_matrix
    print("Classification Report:")
    print(classification_report(y_test, y_pred_test))
    print("Confusion Matrix:")
    print(confusion_matrix(y_test, y_pred_test))

    return IF, imputer, scaler

def handle_new_scenario(new_data):
    """
    Handles new data by preprocessing, transforming, and predicting anomalies.

    Args:
        new_data (pd.DataFrame): The new data to be processed.

    Returns:
        pd.DataFrame: The processed data with anomaly labels.
    """
    # Load trained model and feature names
    isolation_forest = joblib.load("isolation_forest_model.pkl")
    trained_features = joblib.load("isolation_forest_features.pkl")

    # Ensure the new data has all required columns
    missing_features = [f for f in trained_features if f not in new_data.columns]
    extra_features = [f for f in new_data.columns if f not in trained_features]

    if missing_features:
        print(f"Warning: Missing features in new_data: {missing_features}")
        for feature in missing_features:
            new_data[feature] = np.nan  # Add missing columns with NaN values

    if extra_features:
        print(f"Warning: Extra features in new_data: {extra_features}")
        new_data.drop(columns=extra_features, inplace=True)

    # Ensure no duplicate columns
    new_data = new_data.loc[:, ~new_data.columns.duplicated()].copy()

    # Ensure the new data only has the features that were used for training
    new_data = new_data[trained_features].copy()

    # Replace infinite values with NaN
    new_data.replace([np.inf, -np.inf], np.nan, inplace=True)

    # Separate numeric and non-numeric columns
    numeric_cols = new_data.select_dtypes(include=['number']).columns
    non_numeric_cols = new_data.select_dtypes(exclude=['number']).columns

    # Clip large values to a reasonable range for numeric columns
    if not numeric_cols.empty:
        new_data[numeric_cols] = new_data[numeric_cols].clip(lower=-1e10, upper=1e10)

    # Define the transformers for numeric and non-numeric columns
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='mean')),
        ('scaler', StandardScaler())
    ])

    non_numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent'))
    ])

    # Combine the transformers into a ColumnTransformer
    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_cols),
            ('cat', non_numeric_transformer, non_numeric_cols)
        ]
    )

    # Transform the new data
    new_data_transformed = preprocessor.fit_transform(new_data)

    # Convert the transformed data back to a DataFrame
    new_data_transformed = pd.DataFrame(new_data_transformed, columns=numeric_cols.tolist() + non_numeric_cols.tolist())

    # Ensure the new data contains exactly the features the model was trained on
    new_data_transformed = new_data_transformed[trained_features]

    # Predict anomalies using the isolation forest model
    predictions = isolation_forest.predict(new_data_transformed)

    # Add predictions to the new data
    new_data['anomaly'] = predictions

    return new_data



#make the new data is used to real time detection and create csv file every 24 hours with new data to train the model with it.abs



# Run the training
# Load the data
#df = pd.read_csv("Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv", index_col=None)
#data_train(df)
new_data = pd.read_csv("Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv")


# Run the new scenario
result = handle_new_scenario(new_data)

print(result.head())