import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.tree import DecisionTreeRegressor, plot_tree, export_graphviz
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
import graphviz
import os

# Check if graphviz is installed
if not os.system("dot -V"):
    print("Graphviz is installed correctly.")
else:
    print("Please install Graphviz from https://graphviz.org/download/")

# a. Load the Dataset
data_path = '/Users/rahulkulkarni/Downloads/Data/merged.csv'
df = pd.read_csv(data_path)
print("Initial Data:")
print(df.head())

# b. Data Preprocessing
df = df.drop(['LGA Name', 'LGA'], axis=1, errors='ignore')
df = pd.get_dummies(df, columns=['Region'], drop_first=True)
print("\nData after preprocessing:")
print(df.head())

# c. Select Features and Target
target = 'EGM expenditure per capita'
# Creating a dummy target variable for demonstration
df[target] = np.random.rand(len(df)) * 1000  # You should replace this with your actual target data

feature_columns = [
    '2012',
    'IRSD (avg)',
    'population',
    'Personal income <$400/week, persons',
    'Number of families',
    'Female-headed lone parent families',
    'Male-headed lone parent families',
    'Holds degree or higher, persons',
    'Did not complete year 12, persons'
]
categorical_features = [col for col in df.columns if 'Region_' in col]
feature_columns.extend(categorical_features)

# Check if all selected features are available
available_features = [col for col in feature_columns if col in df.columns]
if len(available_features) != len(feature_columns):
    missing_features = list(set(feature_columns) - set(available_features))
    print(f"Warning: The following features are missing and will be ignored: {missing_features}")

# Adjust the feature set accordingly
X = df[available_features]
y = df[target]

# Handle Missing Values
X = X.fillna(X.median())
y = y.fillna(y.median())

# d. Split the Dataset
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# e. Train the Decision Tree Model
dt_regressor = DecisionTreeRegressor(random_state=42, max_depth=5, min_samples_split=10)
dt_regressor.fit(X_train, y_train)

# f. Make Predictions and Evaluate
y_pred = dt_regressor.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
print(f"\nDecision Tree MSE: {mse:.2f}")
print(f"Decision Tree R² Score: {r2:.2f}")

# g. Feature Importance
importances = dt_regressor.feature_importances_
feature_importances = pd.DataFrame({'Feature': available_features, 'Importance': importances}).sort_values(by='Importance', ascending=False)
print("\nFeature Importances:")
print(feature_importances)

# h. Visualize the Decision Tree
plt.figure(figsize=(20, 10))
plot_tree(dt_regressor, feature_names=available_features, filled=True, rounded=True, fontsize=12)
plt.title("Decision Tree for EGM Expenditure per Capita")
plt.show()

# Optional: Enhanced Visualization with Graphviz
dot_data = export_graphviz(dt_regressor, out_file=None, feature_names=available_features, filled=True, rounded=True, special_characters=True)
graph = graphviz.Source(dot_data)
output_file = "decision_tree_egm_expenditure"
try:
    graph.render(output_file, format="png", cleanup=True)
    print(f"\nDecision Tree visualization saved as '{output_file}.png'.")
except Exception as e:
    print(f"Error in rendering the graphviz tree: {e}")

# i. Hyperparameter Tuning (Optional)
param_grid = {
    'max_depth': [3, 5, 7, 10],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}
grid_search = GridSearchCV(estimator=DecisionTreeRegressor(random_state=42), param_grid=param_grid, cv=5, scoring='r2', n_jobs=-1)
grid_search.fit(X_train, y_train)
print("\nBest Parameters from Grid Search:")
print(grid_search.best_params_)

# j. Cross-Validation
cv_scores = cross_val_score(dt_regressor, X_train, y_train, cv=5, scoring='r2')
print(f"\nCross-Validation R² Scores: {cv_scores}")
print(f"Average CV R² Score: {cv_scores.mean():.2f}")

# k. Ensemble Method: Random Forest (Optional)
rf_regressor = RandomForestRegressor(n_estimators=100, random_state=42, max_depth=5, min_samples_split=10)
rf_regressor.fit(X_train, y_train)
y_pred_rf = rf_regressor.predict(X_test)
mse_rf = mean_squared_error(y_test, y_pred_rf)
r2_rf = r2_score(y_test, y_pred_rf)
print(f"\nRandom Forest MSE: {mse_rf:.2f}")
print(f"Random Forest R² Score: {r2_rf:.2f}")

Removing all variables...
 



runfile('/Users/rahulkulkarni/.spyder-py3/untitled9.py', wdir='/Users/rahulkulkarni/.spyder-py3')
Please install Graphviz from https://graphviz.org/download/
sh: dot: command not found
Initial Data:
             LGA Name Region  ...  2012 ERP age 85+, persons   IRSD (avg)
0          whittlesea      M  ...                       1522   991.986560
1  northern grampians      C  ...                        379   939.210476
2     greater geelong      C  ...                       6080   992.193318
3         colac-otway      C  ...                        588   964.493561
4           moorabool      C  ...                        678  1015.812630

[5 rows x 27 columns]

Data after preprocessing:
           2012  Personal income <$400/week, persons  ...   IRSD (avg)  Region_M
0  1.115836e+08                                46539  ...   991.986560      True
1  1.121941e+07                                 4554  ...   939.210476     False
2  1.183161e+08                                75898  ...   992.193318     False
3  1.091608e+07                                 6598  ...   964.493561     False
4  1.116534e+07                                13686  ...  1015.812630     False

[5 rows x 25 columns]

Decision Tree MSE: 95708.43
Decision Tree R² Score: -0.68

Feature Importances:
                               Feature  Importance
0                                 2012    0.347439
7      Holds degree or higher, persons    0.232138
1                           IRSD (avg)    0.203325
6     Male-headed lone parent families    0.113497
4                   Number of families    0.103601
2                           population    0.000000
3  Personal income <$400/week, persons    0.000000
5   Female-headed lone parent families    0.000000
8    Did not complete year 12, persons    0.000000
9                             Region_M    0.000000
Error in rendering the graphviz tree: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH

Best Parameters from Grid Search:
{'max_depth': 3, 'min_samples_leaf': 1, 'min_samples_split': 2}

Cross-Validation R² Scores: [-0.84350201 -0.27124243 -1.23986129 -0.88635615 -0.11849693]
Average CV R² Score: -0.67

Random Forest MSE: 71256.23
Random Forest R² Score: -0.25
Note that each of the above features should have an importance, so improve the decision tree so that it captures all the features to the appropriate extent