Upload 21 files

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/trained_model/data_processor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9510e6be685fb7b5fdfd38517a116a0403faa75c922b2e81a9f215921ac2e0be
+size 217195

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/trained_model/model_metrics.json

@@ -0,0 +1,8 @@
+{
+    "mse": 0.05486344948816889,
+    "rmse": 0.23422948039939143,
+    "mae": 0.1660625786187038,
+    "r2": 0.29007536468986816,
+    "max_error": 0.7487417459487915,
+    "mape": 27499842257.400738
+}

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/trained_model/performance_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c0de80b55e64dc99f1e1ffcc69f7ff3799341b393000728bcf641c64ea02b27
+size 50035

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/trained_model/performance_model_20250709_221148_params.json

@@ -0,0 +1,10 @@
+{
+    "subsample": 0.8,
+    "reg_lambda": 10,
+    "reg_alpha": 1,
+    "min_child_weight": 1,
+    "max_depth": 9,
+    "learning_rate": 0.1,
+    "gamma": 0.2,
+    "colsample_bytree": 1.0
+}

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/trained_model/performance_model_shap_values.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de8239f80d8fb314e44e471031c7d93c12ed854bcb692f041b89d65ad19c136c
+size 9728

Prediksi Performa Akademik/edtech/backend/models/performance_predictor/training_logs/feature_importance.csv

@@ -0,0 +1,5 @@
+feature,importance
+f18,22.0
+f4,19.0
+f12,3.0
+f3,1.0

Prediksi Performa Akademik/edtech/backend/src/app.py

@@ -0,0 +1,431 @@
+from fastapi import FastAPI, HTTPException, Depends, status
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field, field_validator
+from typing import List, Optional
+import pandas as pd
+import joblib
+from pathlib import Path
+import numpy as np
+import sys
+import logging
+import time
+from prometheus_fastapi_instrumentator import Instrumentator
+import uvicorn
+import xgboost as xgb
+import shap
+import json
+from contextlib import asynccontextmanager
+from datetime import datetime
+import os
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('api.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Setup paths - Disesuaikan dengan struktur folder Anda
+BASE_DIR = Path(__file__).parent.parent  # Menyesuaikan dengan lokasi app.py
+MODEL_DIR = BASE_DIR / "models" / "performance_predictor" / "trained_model"
+MODEL_PATH = MODEL_DIR / "performance_model.pkl"
+PREPROCESSOR_PATH = MODEL_DIR / "data_processor.pkl"
+METRICS_PATH = MODEL_DIR / "model_metrics.json"
+
+# Pastikan direktori model ada
+os.makedirs(MODEL_DIR, exist_ok=True)
+
+# Lifespan handler untuk manajemen siklus hidup aplikasi
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Mengelola startup dan shutdown aplikasi"""
+    try:
+        # Muat komponen saat startup
+        app.state.model_components = await load_components()
+        
+        # Muat metrik model
+        if METRICS_PATH.exists():
+            with open(METRICS_PATH) as f:
+                app.state.model_metrics = json.load(f)
+        else:
+            app.state.model_metrics = {
+                "mse": 0.05486344948816889,
+                "rmse": 0.23422948039939143,
+                "mae": 0.1660625786187038,
+                "r2": 0.29007536468986816,
+                "max_error": 0.7487417459487915
+            }
+            logger.warning("File metrik model tidak ditemukan, menggunakan nilai default")
+        
+        logger.info("Aplikasi siap menerima request")
+        yield
+        
+    except Exception as e:
+        logger.error(f"Startup error: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Gagal memulai aplikasi"
+        )
+
+# Inisialisasi FastAPI
+app = FastAPI(
+    title="EdTech Performance Prediction API",
+    description="API untuk memprediksi performa akademik siswa menggunakan model XGBoost",
+    version="2.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    lifespan=lifespan
+)
+
+# Enable CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3024", "http://192.168.56.1:3024"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Setup Prometheus metrics
+Instrumentator().instrument(app).expose(app)
+
+# Definisi Model Pydantic
+class FeatureInput(BaseModel):
+    grade: float = Field(..., gt=0, le=12, description="Kelas siswa (1-12)")
+    tech_savvy: int = Field(..., ge=1, le=5, description="Kemampuan teknologi (skala 1-5)")
+    duration_minutes: float = Field(..., gt=0, description="Durasi belajar dalam menit")
+    engagement_score: float = Field(..., ge=0, le=1, description="Skor engagement (0-1)")
+    completion_rate: float = Field(..., ge=0, le=1, description="Tingkat penyelesaian materi (0-1)")
+    material_rating: float = Field(..., ge=1, le=5, description="Rating materi (skala 1-5)")
+    interaction_duration: float = Field(..., gt=0, description="Durasi interaksi dengan materi")
+    material_engagement_score: float = Field(..., ge=0, le=1, description="Skor engagement dengan materi")
+    feature_engagement: float = Field(..., ge=0, le=1, description="Engagement dengan fitur platform")
+    jam_belajar: float = Field(..., ge=0, le=24, description="Jam belajar (0-24)")
+    hari_dalam_minggu: float = Field(..., ge=0, le=6, description="Hari dalam minggu (0-6)")
+    akhir_pekan: float = Field(..., ge=0, le=1, description="Indikator akhir pekan (0/1)")
+    efisiensi_belajar: float = Field(..., ge=0, description="Indeks efisiensi belajar")
+    rasio_penyelesaian: float = Field(..., ge=0, le=1, description="Rasio penyelesaian tugas")
+    interaksi_total: float = Field(..., ge=0, description="Total interaksi dengan platform")
+    preferensi_materi: float = Field(..., ge=0, le=1, description="Preferensi jenis materi")
+    jumlah_pengakses: float = Field(..., ge=0, description="Jumlah pengakses materi")
+    engagement_rata2: float = Field(..., ge=0, le=1, description="Rata-rata engagement")
+    performance_label_encoded: int = Field(..., ge=0, description="Label performa (encoded)")
+    learning_speed_encoded: int = Field(..., ge=0, description="Kecepatan belajar (encoded)")
+    student_feedback_encoded: int = Field(..., ge=0, description="Feedback siswa (encoded)")
+    achievement_status_encoded: int = Field(..., ge=0, description="Status pencapaian (encoded)")
+
+    @field_validator('engagement_score', 'completion_rate', 'material_engagement_score', 
+                   'feature_engagement', 'efisiensi_belajar', 'rasio_penyelesaian',
+                   'preferensi_materi', 'engagement_rata2')
+    @classmethod
+    def check_proportion(cls, v):
+        if not 0 <= v <= 1:
+            raise ValueError("Nilai harus antara 0 dan 1")
+        return v
+
+class PredictionInput(BaseModel):
+    features: FeatureInput
+
+class BatchPredictionInput(BaseModel):
+    samples: List[FeatureInput]
+
+class FeatureContribution(BaseModel):
+    feature: str
+    value: float
+    contribution: float
+
+class PredictionResponse(BaseModel):
+    prediction: float = Field(..., description="Nilai prediksi skor kuis")
+    confidence_interval: List[float] = Field(..., description="Interval kepercayaan prediksi")
+    feature_contributions: Optional[List[FeatureContribution]] = Field(
+        None, 
+        description="Kontribusi masing-masing fitur terhadap prediksi"
+    )
+    execution_time_ms: float = Field(..., description="Waktu eksekusi dalam milidetik")
+    model_version: str = Field(..., description="Versi model yang digunakan")
+
+class BatchPredictionResponse(BaseModel):
+    predictions: List[float]
+    confidence_intervals: List[List[float]]
+    feature_contributions: Optional[List[List[FeatureContribution]]]
+    execution_time_ms: float
+    model_version: str
+    total_samples: int
+    avg_time_per_sample_ms: float
+
+class HealthCheckResponse(BaseModel):
+    status: str
+    model_version: str
+    model_metrics: dict
+    uptime_seconds: float
+
+class ModelInfoResponse(BaseModel):
+    features: List[str]
+    model_type: str
+    training_date: Optional[str]
+    performance_metrics: dict
+
+# Dependency untuk memuat komponen model
+async def load_components():
+    """Memuat model dan preprocessor dari file"""
+    try:
+        start_time = time.time()
+        
+        # Verifikasi file ada
+        if not MODEL_PATH.exists():
+            raise FileNotFoundError(f"File model tidak ditemukan di {MODEL_PATH}")
+        if not PREPROCESSOR_PATH.exists():
+            raise FileNotFoundError(f"File preprocessor tidak ditemukan di {PREPROCESSOR_PATH}")
+        
+        # Load model
+        model = joblib.load(MODEL_PATH)
+        logger.info(f"Model berhasil dimuat dari {MODEL_PATH}")
+        
+        # Load preprocessor
+        processor_data = joblib.load(PREPROCESSOR_PATH)
+        preprocessor = processor_data['preprocessor']
+        feature_names = processor_data['feature_names']
+        logger.info(f"Preprocessor berhasil dimuat dari {PREPROCESSOR_PATH}")
+        
+        load_time = time.time() - start_time
+        logger.info(f"Komponen model berhasil dimuat dalam {load_time:.2f} detik")
+        
+        return {
+            "model": model,
+            "preprocessor": preprocessor,
+            "feature_names": feature_names,
+            "load_time": load_time
+        }
+    except FileNotFoundError as e:
+        logger.error(f"File tidak ditemukan: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"File model/preprocessor tidak ditemukan: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Gagal memuat model/preprocessor: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Gagal memuat komponen model: {str(e)}"
+        )
+
+# Endpoint Utama
+@app.get("/", include_in_schema=False)
+async def root():
+    """Endpoint root untuk informasi dasar API"""
+    return {
+        "message": "Selamat datang di EdTech Performance Prediction API",
+        "version": app.version,
+        "endpoints": {
+            "docs": "/docs",
+            "health": "/health",
+            "model_info": "/model/info",
+            "predict": "/predict",
+            "batch_predict": "/predict/batch"
+        }
+    }
+
+@app.get("/health", response_model=HealthCheckResponse)
+async def health_check():
+    """Endpoint untuk health check dan monitoring"""
+    return {
+        "status": "healthy",
+        "model_version": app.version,
+        "model_metrics": app.state.model_metrics,
+        "uptime_seconds": time.time() - app.state.model_components.get("load_time", time.time())
+    }
+
+@app.get("/model/info", response_model=ModelInfoResponse)
+async def model_info():
+    """Endpoint untuk mendapatkan informasi tentang model"""
+    return {
+        "features": app.state.model_components["feature_names"],
+        "model_type": "XGBoost Regressor",
+        "training_date": datetime.fromtimestamp(MODEL_PATH.stat().st_mtime).isoformat(),
+        "performance_metrics": app.state.model_metrics
+    }
+
+@app.post("/predict", response_model=PredictionResponse)
+async def predict_performance(
+    input_data: PredictionInput
+):
+    """Endpoint untuk prediksi tunggal performa siswa"""
+    start_time = time.time()
+    
+    try:
+        components = app.state.model_components
+        model = components["model"]
+        preprocessor = components["preprocessor"]
+        feature_names = components["feature_names"]
+        
+        # Konversi input ke DataFrame
+        input_dict = input_data.features.dict()
+        input_df = pd.DataFrame([input_dict])
+        
+        # Validasi fitur
+        missing_cols = set(feature_names) - set(input_df.columns)
+        if missing_cols:
+            raise ValueError(f"Kolom berikut tidak ditemukan: {missing_cols}")
+        
+        # Urutkan kolom sesuai dengan yang diharapkan model
+        input_df = input_df[feature_names]
+        
+        # Preprocess input
+        processed_input = preprocessor.transform(input_df)
+        
+        # Buat prediksi
+        if isinstance(model, xgb.Booster):
+            dmatrix = xgb.DMatrix(processed_input)
+            prediction = model.predict(dmatrix)[0]
+        else:
+            prediction = model.predict(processed_input)[0]
+        
+        # Hitung confidence interval berdasarkan metrik model
+        std_dev = np.sqrt(app.state.model_metrics.get('mse', 0.05486344948816889))
+        confidence = [max(0, prediction - 1.96*std_dev), min(1, prediction + 1.96*std_dev)]
+        
+        # Hitung feature contributions menggunakan SHAP
+        feature_contributions = None
+        if hasattr(model, 'feature_names_in_'):
+            try:
+                explainer = shap.Explainer(model)
+                shap_values = explainer(processed_input)
+                
+                feature_contributions = []
+                for i, feature in enumerate(feature_names):
+                    feature_contributions.append({
+                        "feature": feature,
+                        "value": input_df.iloc[0][feature],
+                        "contribution": float(shap_values[0].values[i])
+                    })
+                # Urutkan berdasarkan kontribusi absolut terbesar
+                feature_contributions.sort(key=lambda x: abs(x["contribution"]), reverse=True)
+            except Exception as e:
+                logger.warning(f"Tidak dapat menghitung SHAP values: {str(e)}")
+        
+        # Hitung waktu response
+        exec_time = (time.time() - start_time) * 1000  # dalam milidetik
+        
+        return {
+            "prediction": float(prediction),
+            "confidence_interval": confidence,
+            "feature_contributions": feature_contributions,
+            "execution_time_ms": exec_time,
+            "model_version": app.version
+        }
+        
+    except ValueError as e:
+        logger.error(f"Input validation error: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Input tidak valid: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Error dalam prediksi: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Error dalam prediksi: {str(e)}"
+        )
+
+@app.post("/predict/batch", response_model=BatchPredictionResponse)
+async def batch_predict_performance(
+    input_data: BatchPredictionInput
+):
+    """Endpoint untuk prediksi batch performa siswa"""
+    start_time = time.time()
+    
+    try:
+        components = app.state.model_components
+        model = components["model"]
+        preprocessor = components["preprocessor"]
+        feature_names = components["feature_names"]
+        
+        # Konversi input ke DataFrame
+        samples = [sample.dict() for sample in input_data.samples]
+        input_df = pd.DataFrame(samples)
+        
+        # Validasi fitur
+        missing_cols = set(feature_names) - set(input_df.columns)
+        if missing_cols:
+            raise ValueError(f"Kolom berikut tidak ditemukan: {missing_cols}")
+        
+        # Urutkan kolom
+        input_df = input_df[feature_names]
+        
+        # Preprocess input
+        processed_input = preprocessor.transform(input_df)
+        
+        # Buat prediksi
+        if isinstance(model, xgb.Booster):
+            dmatrix = xgb.DMatrix(processed_input)
+            predictions = model.predict(dmatrix)
+        else:
+            predictions = model.predict(processed_input)
+        
+        # Hitung confidence intervals
+        std_dev = np.sqrt(app.state.model_metrics.get('mse', 0.05486344948816889))
+        conf_intervals = [
+            [max(0, p - 1.96*std_dev), min(1, p + 1.96*std_dev)] 
+            for p in predictions
+        ]
+        
+        # Hitung feature contributions
+        feature_contributions_list = None
+        if hasattr(model, 'feature_names_in_'):
+            try:
+                explainer = shap.Explainer(model)
+                shap_values = explainer(processed_input)
+                
+                feature_contributions_list = []
+                for i in range(len(predictions)):
+                    contributions = []
+                    for j, feature in enumerate(feature_names):
+                        contributions.append({
+                            "feature": feature,
+                            "value": input_df.iloc[i][feature],
+                            "contribution": float(shap_values[i].values[j])
+                        })
+                    # Urutkan berdasarkan kontribusi absolut terbesar
+                    contributions.sort(key=lambda x: abs(x["contribution"]), reverse=True)
+                    feature_contributions_list.append(contributions)
+            except Exception as e:
+                logger.warning(f"Tidak dapat menghitung SHAP values untuk batch: {str(e)}")
+        
+        # Hitung waktu response
+        exec_time = (time.time() - start_time) * 1000  # dalam milidetik
+        avg_time_per_sample = exec_time / len(predictions)
+        
+        return {
+            "predictions": [float(p) for p in predictions],
+            "confidence_intervals": conf_intervals,
+            "feature_contributions": feature_contributions_list,
+            "execution_time_ms": exec_time,
+            "model_version": app.version,
+            "total_samples": len(predictions),
+            "avg_time_per_sample_ms": avg_time_per_sample
+        }
+        
+    except ValueError as e:
+        logger.error(f"Input validation error: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Input tidak valid: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Error dalam batch prediction: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Error dalam batch prediction: {str(e)}"
+        )
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "app:app",
+        host="192.168.56.1",
+        port=8024,
+        reload=True
+    )

Prediksi Performa Akademik/edtech/backend/src/models/recommenders/collaborative/collab_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4aef73c6272415cb11002c1ff5c96f65587498acaa7c86ad4f7167d1d73fe48
+size 6080

Prediksi Performa Akademik/edtech/backend/src/models/recommenders/content_based/content_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63d1a2f5acb72fa4e6c3825586d578da46d850c31d82883ef50f618789722977
+size 5211833

Prediksi Performa Akademik/edtech/backend/src/models/recommenders/hybrid/hybrid_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d173427052471e467df306ab61013e0599cfb0a80ff3805e464f9b7a25166933
+size 32

Prediksi Performa Akademik/edtech/backend/src/performance_prediction/__init__.py

@@ -0,0 +1,11 @@
+from .data_processor import PerformanceDataProcessor
+from .model_trainer import PerformanceModelTrainer
+from .evaluator import PerformanceEvaluator
+from .predictor import PerformancePredictor
+
+__all__ = [
+    'PerformanceDataProcessor',
+    'PerformanceModelTrainer', 
+    'PerformanceEvaluator',
+    'PerformancePredictor'
+]

Prediksi Performa Akademik/edtech/backend/src/performance_prediction/data_processor.py

@@ -0,0 +1,180 @@
+# backend/src/performance_prediction/data_processor.py
+
+import pandas as pd
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, RobustScaler, OneHotEncoder
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.compose import ColumnTransformer
+import joblib
+import os
+from datetime import datetime
+from pathlib import Path
+import logging
+import json
+
+class PerformanceDataProcessor:
+    def __init__(self, data_path, config_path=None):
+        self.data_path = data_path
+        self.config_path = config_path
+        self.features = None
+        self.target = None
+        self.preprocessor = None
+        self.logger = self._setup_logger()
+        
+    def _setup_logger(self):
+        logger = logging.getLogger(__name__)
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        return logger
+    
+    def load_data(self):
+        """Memuat data dengan penanganan error yang lebih baik"""
+        try:
+            data = pd.read_csv(self.data_path)
+            
+            # Log informasi dasar data
+            self.logger.info(f"Data berhasil dimuat. Shape: {data.shape}")
+            self.logger.info(f"Kolom yang tersedia: {list(data.columns)}")
+            self.logger.info(f"Contoh data:\n{data.head(2)}")
+            
+            return data
+        except Exception as e:
+            self.logger.error(f"Gagal memuat data: {str(e)}")
+            raise
+            
+    def prepare_features_target(self, data, target_col='quiz_score'):
+        """
+        Menyiapkan fitur dan target dengan penanganan data yang lebih komprehensif
+        """
+        try:
+            # Load feature configuration if available
+            if self.config_path:
+                with open(self.config_path) as f:
+                    config = json.load(f)
+                relevant_features = config.get('features', [])
+            else:
+                # Default features
+                relevant_features = [
+                    'grade', 'tech_savvy', 'duration_minutes', 'engagement_score',
+                    'completion_rate', 'material_rating', 'interaction_duration',
+                    'material_engagement_score', 'feature_engagement', 'jam_belajar',
+                    'hari_dalam_minggu', 'akhir_pekan', 'efisiensi_belajar',
+                    'rasio_penyelesaian', 'interaksi_total', 'preferensi_materi',
+                    'jumlah_pengakses', 'engagement_rata2', 'performance_label_encoded',
+                    'learning_speed_encoded', 'student_feedback_encoded',
+                    'achievement_status_encoded'
+                ]
+            
+            # Tambahkan fitur interaksi baru
+            data['efisiensi_engagement'] = data['engagement_score'] / (data['duration_minutes'] + 1e-6)
+            data['learning_consistency'] = data['completion_rate'] * data['material_rating']
+            relevant_features.extend(['efisiensi_engagement', 'learning_consistency'])
+            
+            # Pastikan kolom target ada
+            if target_col not in data.columns:
+                raise ValueError(f"Kolom target '{target_col}' tidak ditemukan")
+                
+            # Handle missing values
+            data[relevant_features] = data[relevant_features].fillna(data[relevant_features].median())
+            
+            self.features = data[relevant_features]
+            self.target = data[target_col]
+            
+            # Setup preprocessing pipeline
+            numeric_features = self.features.select_dtypes(include=['int64', 'float64']).columns
+            categorical_features = self.features.select_dtypes(include=['object', 'category']).columns
+            
+            numeric_transformer = Pipeline(steps=[
+                ('imputer', SimpleImputer(strategy='median')),
+                ('scaler', RobustScaler())  # Lebih robust terhadap outlier
+            ])
+            
+            categorical_transformer = Pipeline(steps=[
+                ('imputer', SimpleImputer(strategy='most_frequent')),
+                ('onehot', OneHotEncoder(handle_unknown='ignore'))
+            ])
+            
+            self.preprocessor = ColumnTransformer(
+                transformers=[
+                    ('num', numeric_transformer, numeric_features),
+                    ('cat', categorical_transformer, categorical_features)
+                ])
+            
+            return self.features, self.target
+            
+        except Exception as e:
+            self.logger.error(f"Error dalam menyiapkan fitur: {str(e)}")
+            raise
+    
+    def split_data(self, test_size=0.2, val_size=0.2, random_state=42):
+        """Membagi data menjadi train, validation, dan test set"""
+        try:
+            if self.features is None or self.target is None:
+                raise ValueError("Fitur atau target belum disiapkan")
+                
+            # Bagi data menjadi train+val dan test
+            X_train_val, X_test, y_train_val, y_test = train_test_split(
+                self.features, self.target, 
+                test_size=test_size, 
+                random_state=random_state
+            )
+            
+            # Bagi train_val menjadi train dan validation
+            val_size_adjusted = val_size / (1 - test_size)  # Adjust untuk ukuran asli dataset
+            X_train, X_val, y_train, y_val = train_test_split(
+                X_train_val, y_train_val,
+                test_size=val_size_adjusted,
+                random_state=random_state
+            )
+            
+            # Preprocess data
+            X_train = self.preprocessor.fit_transform(X_train)
+            X_val = self.preprocessor.transform(X_val)
+            X_test = self.preprocessor.transform(X_test)
+            
+            # Validasi data
+            self._validate_data(X_train, y_train)
+            self._validate_data(X_val, y_val)
+            self._validate_data(X_test, y_test)
+            
+            return X_train, X_val, X_test, y_train, y_val, y_test
+            
+        except Exception as e:
+            self.logger.error(f"Error dalam membagi data: {str(e)}")
+            raise
+
+    def _validate_data(self, X, y):
+        """Validasi kualitas data"""
+        if isinstance(X, np.ndarray):
+            if np.any(np.isnan(X)) or np.any(np.isinf(X)):
+                raise ValueError("Data mengandung NaN atau infinity")
+        if len(X) != len(y):
+            raise ValueError("Jumlah sampel X dan y tidak sama")
+        if len(y) == 0:
+            raise ValueError("Data target kosong")
+            
+    def save_processor(self, save_dir):
+        """Menyimpan processor dan preprocessing pipeline"""
+        try:
+            os.makedirs(save_dir, exist_ok=True)
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            save_path = Path(save_dir) / f"data_processor_{timestamp}.pkl"
+            
+            # Simpan seluruh objek processor
+            joblib.dump({
+                'processor': self,
+                'preprocessor': self.preprocessor,
+                'feature_names': list(self.features.columns) if self.features is not None else None
+            }, save_path)
+            
+            self.logger.info(f"Processor disimpan di: {save_path}")
+            return str(save_path)
+            
+        except Exception as e:
+            self.logger.error(f"Gagal menyimpan processor: {str(e)}")
+            raise

Prediksi Performa Akademik/edtech/backend/src/performance_prediction/evaluator.py

@@ -0,0 +1,255 @@
+# backend/src/performance_prediction/evaluator.py
+
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics import (
+    mean_squared_error,
+    mean_absolute_error,
+    r2_score,
+    explained_variance_score,
+    max_error,
+    mean_absolute_percentage_error
+)
+import pandas as pd
+import logging
+from typing import Dict, Tuple, Optional
+from pathlib import Path
+import json
+import shap
+
+class PerformanceEvaluator:
+    def __init__(self, y_true: np.ndarray, y_pred: np.ndarray, model=None, X_test=None):
+        """
+        Inisialisasi evaluator dengan tambahan SHAP values dan model interpretability
+        
+        Parameters:
+            y_true (np.ndarray): Nilai sebenarnya
+            y_pred (np.ndarray): Nilai prediksi
+            model (optional): Model yang sudah dilatih untuk interpretasi
+            X_test (optional): Data fitur untuk interpretasi model
+        """
+        self.y_true = y_true
+        self.y_pred = y_pred
+        self.model = model
+        self.X_test = X_test
+        self.shap_values = None
+        self.logger = self._setup_logger()
+        self.metrics = self.calculate_metrics()
+        
+    def _setup_logger(self):
+        """Setup logger untuk evaluator"""
+        logger = logging.getLogger(__name__)
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        return logger
+    
+    def calculate_metrics(self) -> Dict[str, float]:
+        """
+        Menghitung berbagai metrik evaluasi dengan penanganan kasus khusus
+        
+        Returns:
+            Dict berisi berbagai metrik evaluasi
+        """
+        metrics = {
+            'mse': mean_squared_error(self.y_true, self.y_pred),
+            'rmse': np.sqrt(mean_squared_error(self.y_true, self.y_pred)),
+            'mae': mean_absolute_error(self.y_true, self.y_pred),
+            'r2': r2_score(self.y_true, self.y_pred),
+            'explained_variance': explained_variance_score(self.y_true, self.y_pred),
+            'max_error': max_error(self.y_true, self.y_pred),
+            'mean_error': np.mean(self.y_true - self.y_pred),
+            'std_error': np.std(self.y_true - self.y_pred)
+        }
+        
+        # Hitung MAPE hanya jika tidak ada nilai 0 di y_true
+        try:
+            metrics['mape'] = mean_absolute_percentage_error(self.y_true, self.y_pred) * 100
+        except ValueError:
+            metrics['mape'] = np.inf
+            self.logger.warning("Terdapat nilai 0 pada y_true, MAPE tidak dapat dihitung")
+        
+        # Hitung metrik tambahan jika model tersedia
+        if self.model is not None and self.X_test is not None:
+            try:
+                self._calculate_shap_values()
+                metrics['mean_abs_shap'] = np.mean(np.abs(self.shap_values))
+            except Exception as e:
+                self.logger.warning(f"Tidak dapat menghitung SHAP values: {str(e)}")
+        
+        return metrics
+    
+    def _calculate_shap_values(self, sample_size: int = 100):
+        """Menghitung SHAP values untuk interpretasi model"""
+        if self.model is None or self.X_test is None:
+            raise ValueError("Model dan X_test diperlukan untuk menghitung SHAP values")
+        
+        # Sample data untuk efisiensi
+        if len(self.X_test) > sample_size:
+            sample_idx = np.random.choice(len(self.X_test), sample_size, replace=False)
+            X_sample = self.X_test[sample_idx]
+        else:
+            X_sample = self.X_test
+        
+        # Hitung SHAP values
+        if hasattr(self.model, 'predict_proba'):
+            explainer = shap.Explainer(self.model)
+            self.shap_values = explainer(X_sample).values
+        else:
+            explainer = shap.Explainer(self.model)
+            self.shap_values = explainer(X_sample).values
+    
+    def get_performance_report(self) -> str:
+        """Membuat laporan performa model dalam format string"""
+        report = "\n=== MODEL PERFORMANCE REPORT ===\n"
+        for name, value in self.metrics.items():
+            report += f"{name.upper():<20}: {value:.4f}\n"
+        return report
+    
+    def plot_residuals(self, save_path: Optional[str] = None) -> Optional[plt.Figure]:
+        """
+        Visualisasi residual plot dengan informasi tambahan
+        
+        Parameters:
+            save_path (optional): Path untuk menyimpan plot
+            
+        Returns:
+            plt.Figure jika save_path tidak ditentukan
+        """
+        residuals = self.y_true - self.y_pred
+        
+        plt.figure(figsize=(12, 8))
+        sns.scatterplot(x=self.y_pred, y=residuals, alpha=0.6)
+        
+        # Tambahkan garis referensi
+        plt.axhline(y=0, color='r', linestyle='--')
+        
+        # Tambahkan garis rata-rata residual
+        mean_residual = np.mean(residuals)
+        plt.axhline(y=mean_residual, color='b', linestyle='-', 
+                   label=f'Mean Residual: {mean_residual:.2f}')
+        
+        # Hitung dan plot interval kepercayaan
+        std_residual = np.std(residuals)
+        plt.axhline(y=mean_residual + 1.96*std_residual, color='g', linestyle=':',
+                   label='95% Confidence Interval')
+        plt.axhline(y=mean_residual - 1.96*std_residual, color='g', linestyle=':')
+        
+        plt.xlabel('Predicted Values')
+        plt.ylabel('Residuals')
+        plt.title('Residual Analysis')
+        plt.legend()
+        
+        if save_path:
+            plt.savefig(save_path, bbox_inches='tight')
+            plt.close()
+            self.logger.info(f"Residual plot disimpan di: {save_path}")
+        else:
+            return plt
+    
+    def plot_actual_vs_predicted(self, save_path: Optional[str] = None) -> Optional[plt.Figure]:
+        """Visualisasi aktual vs prediksi dengan informasi tambahan"""
+        plt.figure(figsize=(12, 8))
+        
+        # Scatter plot
+        ax = sns.scatterplot(x=self.y_true, y=self.y_pred, alpha=0.6)
+        
+        # Garis diagonal
+        min_val = min(self.y_true.min(), self.y_pred.min())
+        max_val = max(self.y_true.max(), self.y_pred.max())
+        plt.plot([min_val, max_val], [min_val, max_val], 'r--', label='Ideal Prediction')
+        
+        # Garis regresi
+        coef = np.polyfit(self.y_true, self.y_pred, 1)
+        poly1d_fn = np.poly1d(coef)
+        plt.plot(self.y_true, poly1d_fn(self.y_true), 'b-', 
+                label=f'Regression Line (slope={coef[0]:.2f})')
+        
+        plt.xlabel('Actual Values')
+        plt.ylabel('Predicted Values')
+        plt.title('Actual vs Predicted Values')
+        plt.legend()
+        
+        if save_path:
+            plt.savefig(save_path, bbox_inches='tight')
+            plt.close()
+            self.logger.info(f"Actual vs Predicted plot disimpan di: {save_path}")
+        else:
+            return plt
+    
+    def plot_error_distribution(self, save_path: Optional[str] = None) -> Optional[plt.Figure]:
+        """Visualisasi distribusi error dengan informasi statistik"""
+        errors = self.y_true - self.y_pred
+        
+        plt.figure(figsize=(12, 8))
+        
+        # Histogram dengan KDE
+        ax = sns.histplot(errors, kde=True, bins=30)
+        
+        # Tambahkan garis statistik
+        mean_error = np.mean(errors)
+        std_error = np.std(errors)
+        
+        plt.axvline(mean_error, color='r', linestyle='-', 
+                   label=f'Mean Error: {mean_error:.2f}')
+        plt.axvline(mean_error + std_error, color='g', linestyle='--',
+                   label=f'±1 Std Dev: {std_error:.2f}')
+        plt.axvline(mean_error - std_error, color='g', linestyle='--')
+        
+        plt.xlabel('Prediction Error')
+        plt.ylabel('Frequency')
+        plt.title('Prediction Error Distribution')
+        plt.legend()
+        
+        if save_path:
+            plt.savefig(save_path, bbox_inches='tight')
+            plt.close()
+            self.logger.info(f"Error distribution plot disimpan di: {save_path}")
+        else:
+            return plt
+    
+    def plot_shap_summary(self, feature_names: list = None, save_path: Optional[str] = None) -> Optional[plt.Figure]:
+        """Visualisasi SHAP summary plot"""
+        if self.shap_values is None:
+            self.logger.warning("SHAP values belum dihitung")
+            return None
+            
+        plt.figure(figsize=(14, 8))
+        shap.summary_plot(self.shap_values, self.X_test, feature_names=feature_names, show=False)
+        plt.title('SHAP Feature Importance')
+        plt.tight_layout()
+        
+        if save_path:
+            plt.savefig(save_path, bbox_inches='tight')
+            plt.close()
+            self.logger.info(f"SHAP summary plot disimpan di: {save_path}")
+        else:
+            return plt
+    
+    def save_evaluation_results(self, save_dir: str):
+        """
+        Menyimpan semua hasil evaluasi termasuk plot dan metrik
+        
+        Parameters:
+            save_dir: Direktori untuk menyimpan hasil
+        """
+        save_path = Path(save_dir)
+        save_path.mkdir(parents=True, exist_ok=True)
+        
+        # Simpan metrik
+        with open(save_path / 'evaluation_metrics.json', 'w') as f:
+            json.dump(self.metrics, f, indent=4)
+        
+        # Simpan plot
+        self.plot_residuals(save_path / 'residual_plot.png')
+        self.plot_actual_vs_predicted(save_path / 'actual_vs_predicted.png')
+        self.plot_error_distribution(save_path / 'error_distribution.png')
+        
+        # Simpan SHAP plot jika tersedia
+        if self.shap_values is not None:
+            self.plot_shap_summary(save_path=save_path / 'shap_summary.png')
+        
+        self.logger.info(f"Hasil evaluasi disimpan di: {save_path}")

Prediksi Performa Akademik/edtech/backend/src/performance_prediction/model_trainer.py

@@ -0,0 +1,412 @@
+# backend/src/performance_prediction/model_trainer.py
+
+import xgboost as xgb
+import optuna
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+from sklearn.model_selection import cross_val_score, KFold
+import numpy as np
+import joblib
+import os
+from datetime import datetime
+from pathlib import Path
+import pandas as pd
+import matplotlib.pyplot as plt
+import json
+import logging
+from functools import partial
+import shap
+import random
+
+class PerformanceModelTrainer:
+    def __init__(self):
+        self.model = None
+        self.feature_importance = None
+        self.shap_values = None
+        self.best_params = None
+        self.cv_results = None
+        self.logger = self._setup_logger()
+        self.study = None
+        
+    def _setup_logger(self):
+        logger = logging.getLogger(__name__)
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        return logger
+    
+    def objective(self, trial, X, y):
+        """Fungsi objective untuk Optuna dengan error handling yang lebih baik"""
+        try:
+            params = {
+                'objective': 'reg:squarederror',
+                'n_estimators': trial.suggest_int('n_estimators', 100, 1000),
+                'max_depth': trial.suggest_int('max_depth', 3, 12),
+                'learning_rate': trial.suggest_float('learning_rate', 1e-3, 0.3, log=True),  # Diperbarui range
+                'subsample': trial.suggest_float('subsample', 0.5, 1.0),
+                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
+                'gamma': trial.suggest_float('gamma', 0, 1.0),
+                'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
+                'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),
+                'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),
+                'random_state': 42,
+                'n_jobs': 1  
+            }
+            
+            model = xgb.XGBRegressor(**params)
+            
+            # Gunakan KFold cross-validation dengan error handling
+            kf = KFold(n_splits=3, shuffle=True, random_state=42)  # Kurangi splits untuk efisiensi
+            
+            try:
+                scores = cross_val_score(
+                    model, X, y, 
+                    cv=kf, 
+                    scoring='neg_mean_squared_error',
+                    n_jobs=1,  
+                    error_score='raise'
+                )
+                return np.mean(scores)
+            except Exception as e:
+                self.logger.warning(f"Trial gagal: {str(e)}")
+                return float('-inf')  # Return nilai terburuk jika gagal
+                
+        except Exception as e:
+            self.logger.error(f"Error dalam objective function: {str(e)}")
+            return float('-inf')
+
+        
+    def hyperparameter_tuning(self, X_train, y_train, n_trials=30):
+        """Alternatif sederhana jika Optuna bermasalah"""
+        param_grid = {
+            'n_estimators': [100, 200, 500],
+            'max_depth': [3, 6, 9],
+            'learning_rate': [0.01, 0.1, 0.2],
+            'subsample': [0.6, 0.8, 1.0],
+            'colsample_bytree': [0.6, 0.8, 1.0]
+        }
+        
+        best_score = float('-inf')
+        best_params = {}
+        
+        for _ in range(n_trials):
+            params = {k: random.choice(v) for k, v in param_grid.items()}
+            # Hapus n_estimators untuk xgb.train
+            train_params = params.copy()
+            train_params.pop('n_estimators', None)
+            
+            model = xgb.XGBRegressor(**params, random_state=42)
+            score = cross_val_score(model, X_train, y_train, 
+                                cv=3, scoring='neg_mean_squared_error').mean()
+            
+            if score > best_score:
+                best_score = score
+                best_params = params
+        
+        self.best_params = best_params
+        return best_params
+
+    def train_model(self, X_train, y_train, X_val=None, y_val=None, params=None):
+        """Melatih model final dengan early stopping"""
+        try:
+            self.logger.info("\n=== TRAINING FINAL MODEL ===")
+            
+            if params is None and self.best_params is not None:
+                params = self.best_params
+            
+            # Parameter default
+            default_params = {
+                'objective': 'reg:squarederror',
+                'random_state': 42,
+                'verbosity': 1
+            }
+            
+            # Hapus n_estimators jika menggunakan xgb.train
+            if 'n_estimators' in params:
+                params.pop('n_estimators')
+                
+            final_params = {**default_params, **(params or {})}
+            
+            if X_val is not None and y_val is not None:
+                self.logger.info("Menggunakan early stopping dengan validation set")
+                
+                dtrain = xgb.DMatrix(X_train, label=y_train)
+                dval = xgb.DMatrix(X_val, label=y_val)
+                
+                evals = [(dtrain, 'train'), (dval, 'val')]
+                evals_result = {}
+                model = xgb.train(
+                    final_params,
+                    dtrain,
+                    num_boost_round=1000,
+                    evals=evals,
+                    early_stopping_rounds=50,
+                    verbose_eval=50,
+                    evals_result=evals_result
+                )
+                
+                # Simpan evals_result
+                self.evals_result = evals_result
+            else:
+                self.logger.info("Training tanpa early stopping")
+                model = xgb.XGBRegressor(**final_params)
+                model.fit(X_train, y_train)
+            
+            self.model = model
+            
+            # Hitung feature importance dan SHAP values
+            self._calculate_feature_importance(X_train)
+            self._calculate_shap_values(X_train)
+            
+            return model
+        except Exception as e:
+            self.logger.error(f"Error dalam training model: {str(e)}")
+            raise
+
+    def _calculate_feature_importance(self, X_train):
+        """Menghitung feature importance"""
+        try:
+            if isinstance(self.model, xgb.Booster):
+                # Untuk model Booster (xgb.train)
+                importance = self.model.get_score(importance_type='weight')
+                # Konversi ke format yang konsisten
+                self.feature_importance = {k: float(v) for k, v in importance.items()}
+            elif hasattr(self.model, 'feature_importances_'):
+                # Untuk model scikit-learn API (XGBRegressor)
+                self.feature_importance = dict(zip(
+                    self.model.get_booster().feature_names,
+                    self.model.feature_importances_
+                ))
+            else:
+                self.logger.warning("Tipe model tidak dikenali untuk menghitung feature importance")
+                self.feature_importance = None
+        except Exception as e:
+            self.logger.error(f"Gagal menghitung feature importance: {str(e)}")
+            self.feature_importance = None
+            
+    def _calculate_shap_values(self, X_train, sample_size=100):
+        """Menghitung SHAP values untuk interpretasi model"""
+        try:
+            if self.model is None:
+                raise ValueError("Model belum dilatih")
+                
+            if isinstance(self.model, xgb.Booster):
+                explainer = shap.TreeExplainer(self.model)
+                X_sample = shap.utils.sample(X_train, sample_size)
+                self.shap_values = explainer.shap_values(X_sample)
+            else:
+                explainer = shap.Explainer(self.model)
+                self.shap_values = explainer(X_train)
+        except Exception as e:
+            self.logger.warning(f"Tidak dapat menghitung SHAP values: {str(e)}")
+            self.shap_values = None
+            
+    def evaluate_model(self, X_test, y_test):
+        """Evaluasi model dengan metrik lengkap"""
+        try:
+            if self.model is None:
+                raise ValueError("Model belum dilatih")
+            
+            dtest = xgb.DMatrix(X_test)
+            predictions = self.model.predict(dtest)
+            
+            # Hitung berbagai metrik evaluasi
+            metrics = self._calculate_all_metrics(y_test, predictions)
+            
+            self.logger.info("\n=== HASIL EVALUASI MODEL ===")
+            for name, value in metrics.items():
+                self.logger.info(f"{name}: {value:.4f}")
+            
+            return {
+                'metrics': metrics,
+                'predictions': predictions,
+                'shap_values': self.shap_values
+            }
+            
+        except Exception as e:
+            self.logger.error(f"Error dalam evaluasi model: {str(e)}")
+            raise
+    
+    def _calculate_all_metrics(self, y_true, y_pred):
+        """Menghitung semua metrik evaluasi"""
+        metrics = {
+            'mse': mean_squared_error(y_true, y_pred),
+            'rmse': np.sqrt(mean_squared_error(y_true, y_pred)),
+            'mae': mean_absolute_error(y_true, y_pred),
+            'r2': r2_score(y_true, y_pred),
+            'max_error': np.max(np.abs(y_true - y_pred))
+        }
+        
+        # Hitung MAPE dengan penanganan nilai 0
+        try:
+            # Tambahkan epsilon kecil untuk menghindari division by zero
+            y_true_adjusted = np.where(y_true == 0, 1e-10, y_true)
+            metrics['mape'] = np.mean(np.abs((y_true - y_pred) / y_true_adjusted)) * 100
+        except Exception as e:
+            metrics['mape'] = np.inf
+            self.logger.warning(f"Tidak dapat menghitung MAPE: {str(e)}")
+        
+        return metrics
+    
+    def plot_learning_curve(self, X_train, y_train, X_val, y_val, save_path=None):
+        """Visualisasi learning curve"""
+        try:
+            # Gunakan evals_result yang sudah disimpan
+            if not hasattr(self, 'evals_result') or not self.evals_result:
+                self.logger.warning("Tidak ada evals_result tersedia untuk learning curve")
+                return None
+            
+            results = self.evals_result
+            epochs = len(results['train']['rmse']) if 'train' in results else 0
+            
+            if epochs == 0:
+                self.logger.warning("Data learning curve kosong")
+                return None
+                
+            x_axis = range(0, epochs)
+            
+            fig, ax = plt.subplots(figsize=(12, 8))
+            ax.plot(x_axis, results['train']['rmse'], label='Train')
+            
+            if 'val' in results:
+                ax.plot(x_axis, results['val']['rmse'], label='Validation')
+                
+            ax.legend()
+            plt.ylabel('RMSE')
+            plt.xlabel('Epochs')
+            plt.title('XGBoost Learning Curve')
+            
+            if save_path:
+                plt.savefig(save_path, bbox_inches='tight')
+                plt.close()
+                self.logger.info(f"Learning curve disimpan di: {save_path}")
+            else:
+                return plt
+                
+        except Exception as e:
+            self.logger.error(f"Error membuat learning curve: {str(e)}")
+            raise
+    
+    def plot_feature_importance(self, feature_names=None, top_n=20, save_path=None):
+        """Visualisasi feature importance"""
+        try:
+            if self.feature_importance is None:
+                self._calculate_feature_importance(feature_names)  # Coba hitung lagi
+                
+            if self.feature_importance is None:
+                raise ValueError("Feature importance belum dihitung. Model mungkin belum dilatih atau terjadi error dalam perhitungan.")
+            
+            # Buat DataFrame dari feature importance
+            importance_df = pd.DataFrame({
+                'feature': list(self.feature_importance.keys()),
+                'importance': list(self.feature_importance.values())
+            }).sort_values('importance', ascending=False)
+            
+            # Jika ada feature_names, pastikan urutannya benar
+            if feature_names is not None:
+                importance_df = importance_df[importance_df['feature'].isin(feature_names)]
+            
+            # Ambil top N features
+            top_features = importance_df.head(top_n)
+            
+            # Plot
+            plt.figure(figsize=(14, 10))
+            bars = plt.barh(top_features['feature'], top_features['importance'])
+            plt.xlabel('Importance Score')
+            plt.title('Top Feature Importance')
+            
+            # Tambahkan nilai importance
+            for bar in bars:
+                width = bar.get_width()
+                plt.text(width + 0.001, bar.get_y() + bar.get_height()/2,
+                        f'{width:.4f}',
+                        va='center', ha='left')
+            
+            plt.gca().invert_yaxis()
+            plt.tight_layout()
+            
+            if save_path:
+                plt.savefig(save_path, bbox_inches='tight')
+                plt.close()
+                self.logger.info(f"Feature importance plot disimpan di: {save_path}")
+                return None, importance_df
+            else:
+                return plt, importance_df
+                
+        except Exception as e:
+            self.logger.error(f"Error membuat feature importance plot: {str(e)}")
+            raise
+        
+    
+    def plot_shap_summary(self, feature_names=None, save_path=None):
+        """Visualisasi SHAP summary plot"""
+        try:
+            if self.shap_values is None:
+                raise ValueError("SHAP values belum dihitung")
+            
+            plt.figure(figsize=(14, 10))
+            shap.summary_plot(self.shap_values, feature_names=feature_names, show=False)
+            plt.tight_layout()
+            
+            if save_path:
+                plt.savefig(save_path, bbox_inches='tight')
+                plt.close()
+                self.logger.info(f"SHAP summary plot disimpan di: {save_path}")
+            else:
+                return plt
+                
+        except Exception as e:
+            self.logger.error(f"Error membuat SHAP summary plot: {str(e)}")
+            raise
+    
+    def save_model(self, save_dir, model_name=None):
+        """Menyimpan model dan semua hasil terkait"""
+        try:
+            os.makedirs(save_dir, exist_ok=True)
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            
+            if not model_name:
+                model_name = f"performance_model_{timestamp}"
+            
+            # Path untuk berbagai file
+            model_path = Path(save_dir) / f"{model_name}.pkl"
+            params_path = Path(save_dir) / f"{model_name}_params.json"
+            cv_path = Path(save_dir) / f"{model_name}_cv_results.csv"
+            shap_path = Path(save_dir) / f"{model_name}_shap_values.npy"
+            study_path = Path(save_dir) / f"{model_name}_optuna_study.pkl"
+            
+            # Simpan model
+            joblib.dump(self.model, model_path)
+            
+            # Simpan parameter terbaik
+            with open(params_path, 'w') as f:
+                json.dump(self.best_params, f, indent=4)
+            
+            # Simpan hasil CV jika ada
+            if self.cv_results is not None:
+                pd.DataFrame(self.cv_results).to_csv(cv_path, index=False)
+            
+            # Simpan SHAP values jika ada
+            if self.shap_values is not None:
+                np.save(shap_path, self.shap_values, allow_pickle=True)
+            
+            # Simpan optuna study jika ada
+            if self.study is not None:
+                joblib.dump(self.study, study_path)
+            
+            self.logger.info("\n=== MODEL DISIMPAN ===")
+            self.logger.info(f"Model: {model_path}")
+            self.logger.info(f"Parameter: {params_path}")
+            if self.cv_results is not None:
+                self.logger.info(f"Hasil CV: {cv_path}")
+            if self.shap_values is not None:
+                self.logger.info(f"SHAP values: {shap_path}")
+            if self.study is not None:
+                self.logger.info(f"Optuna study: {study_path}")
+            
+            return str(model_path)
+            
+        except Exception as e:
+            self.logger.error(f"Error menyimpan model: {str(e)}")
+            raise

Prediksi Performa Akademik/edtech/backend/src/performance_prediction/predictor.py

@@ -0,0 +1,289 @@
+# backend/src/performance_prediction/predictor.py
+
+import joblib
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import logging
+from typing import Union, Dict, List, Optional
+import xgboost as xgb
+import shap
+from datetime import datetime
+
+class PerformancePredictor:
+    def __init__(self, model_path: str, preprocessor_path: Optional[str] = None):
+        """
+        Inisialisasi predictor dengan model dan preprocessor
+        
+        Parameters:
+            model_path: Path ke model yang sudah dilatih
+            preprocessor_path: Path ke preprocessor (opsional)
+        """
+        self.model_path = model_path
+        self.preprocessor_path = preprocessor_path
+        self.model = None
+        self.preprocessor = None
+        self.feature_names = None
+        self.shap_explainer = None
+        self.logger = self._setup_logger()
+        self._load_components()
+        
+    def _setup_logger(self):
+        """Setup logger untuk predictor"""
+        logger = logging.getLogger(__name__)
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        return logger
+    
+    def _load_components(self):
+        """Memuat model dan preprocessor"""
+        try:
+            # Load model
+            self.model = joblib.load(self.model_path)
+            self.logger.info(f"Model berhasil dimuat dari {self.model_path}")
+            
+            # Load preprocessor jika ada
+            if self.preprocessor_path:
+                processor_data = joblib.load(self.preprocessor_path)
+                self.preprocessor = processor_data['preprocessor']
+                self.feature_names = processor_data['feature_names']
+                self.logger.info(f"Preprocessor berhasil dimuat dari {self.preprocessor_path}")
+            
+            # Setup SHAP explainer
+            self._setup_shap_explainer()
+            
+        except Exception as e:
+            self.logger.error(f"Gagal memuat komponen: {str(e)}")
+            raise
+        
+    def _setup_shap_explainer(self):
+        """Mempersiapkan SHAP explainer untuk interpretasi"""
+        try:
+            if hasattr(self.model, 'predict_proba'):
+                self.shap_explainer = shap.Explainer(self.model)
+            else:
+                self.shap_explainer = shap.Explainer(self.model)
+            self.logger.info("SHAP explainer berhasil diinisialisasi")
+        except Exception as e:
+            self.logger.warning(f"Tidak dapat menginisialisasi SHAP explainer: {str(e)}")
+            self.shap_explainer = None
+    
+    def _prepare_input(self, input_data: Union[Dict, List[Dict]], return_dataframe: bool = False) -> Union[np.ndarray, pd.DataFrame]:
+        """
+        Mempersiapkan input data untuk prediksi
+
+        Parameters:
+            input_data: Input data dalam bentuk dict atau list of dicts
+            return_dataframe: Jika True kembalikan DataFrame, jika False kembalikan array
+
+        Returns:
+            Data yang sudah diproses dalam bentuk array atau DataFrame
+        """
+        # Konversi input ke DataFrame
+        if isinstance(input_data, dict):
+            input_df = pd.DataFrame([input_data])
+        elif isinstance(input_data, list):
+            input_df = pd.DataFrame(input_data)
+        elif isinstance(input_data, pd.DataFrame):
+            input_df = input_data.copy()
+        else:
+            raise ValueError("Input harus berupa dict, list of dicts, atau DataFrame")
+
+        # Validasi kolom
+        if self.feature_names is not None:
+            missing_cols = set(self.feature_names) - set(input_df.columns)
+            if missing_cols:
+                raise ValueError(f"Kolom berikut tidak ditemukan dalam input: {missing_cols}")
+
+            # Urutkan kolom sesuai dengan yang diharapkan model
+            input_df = input_df[self.feature_names]
+
+        # Preprocess data jika ada preprocessor
+        if self.preprocessor is not None:
+            processed_data = self.preprocessor.transform(input_df)
+        else:
+            processed_data = input_df.values if not return_dataframe else input_df
+
+        return processed_data if not return_dataframe else input_df
+
+    
+    def predict(self, input_data: Union[Dict, List[Dict]], 
+                return_contributions: bool = False) -> Dict:
+        """
+        Membuat prediksi dari input data dengan opsi interpretasi
+        
+        Parameters:
+            input_data: Input data dalam bentuk dict atau list of dicts
+            return_contributions: Jika True, kembalikan kontribusi fitur
+            
+        Returns:
+            Dict berisi prediksi dan informasi tambahan
+        """
+        start_time = datetime.now()
+        
+        try:
+            # Persiapkan input
+            processed_input = self._prepare_input(input_data)
+            
+            # Buat prediksi
+            if isinstance(self.model, xgb.Booster):
+                dmatrix = xgb.DMatrix(processed_input)
+                predictions = self.model.predict(dmatrix)
+            else:
+                predictions = self.model.predict(processed_input)
+            
+            # Hitung confidence interval (simplified)
+            if hasattr(self.model, 'predict_quantiles'):
+                quantiles = self.model.predict_quantiles(processed_input, quantiles=(0.025, 0.975))
+                confidence_intervals = list(zip(quantiles[0], quantiles[1]))
+            else:
+                # Fallback untuk model tanpa quantile prediction
+                std_dev = np.std(predictions)
+                confidence_intervals = [(p - 1.96*std_dev, p + 1.96*std_dev) for p in predictions]
+            
+            # Hitung feature contributions jika diminta
+            feature_contributions = None
+            if return_contributions and self.shap_explainer is not None:
+                feature_contributions = self._calculate_feature_contributions(processed_input)
+            
+            # Hitung waktu eksekusi
+            exec_time = (datetime.now() - start_time).total_seconds()
+            
+            # Format hasil
+            if isinstance(predictions, np.ndarray) and predictions.ndim == 1:
+                predictions = predictions.tolist()
+            
+            result = {
+                'predictions': predictions,
+                'confidence_intervals': confidence_intervals,
+                'execution_time_seconds': exec_time,
+                'timestamp': start_time.isoformat()
+            }
+            
+            if feature_contributions is not None:
+                result['feature_contributions'] = feature_contributions
+            
+            return result
+            
+        except Exception as e:
+            self.logger.error(f"Error dalam prediksi: {str(e)}")
+            raise
+    
+    def _calculate_feature_contributions(self, processed_input: np.ndarray) -> List[Dict]:
+        """
+        Menghitung kontribusi fitur menggunakan SHAP values
+        
+        Parameters:
+            processed_input: Input data yang sudah diproses
+            
+        Returns:
+            List berisi kontribusi setiap fitur untuk setiap sampel
+        """
+        if self.shap_explainer is None:
+            return None
+            
+        # Hitung SHAP values
+        shap_values = self.shap_explainer(processed_input)
+        
+        # Format hasil
+        contributions = []
+        for i in range(len(processed_input)):
+            sample_contributions = []
+            
+            for j, feature_name in enumerate(self.feature_names):
+                sample_contributions.append({
+                    'feature': feature_name,
+                    'value': processed_input[i][j] if isinstance(processed_input, np.ndarray) else processed_input.iloc[i][j],
+                    'contribution': float(shap_values.values[i][j]),
+                    'abs_contribution': float(np.abs(shap_values.values[i][j]))
+                })
+            
+            # Urutkan berdasarkan kontribusi absolut terbesar
+            sample_contributions.sort(key=lambda x: x['abs_contribution'], reverse=True)
+            contributions.append(sample_contributions)
+        
+        return contributions
+    
+    def batch_predict(self, input_data: List[Dict], batch_size: int = 100,
+                     return_contributions: bool = False) -> Dict:
+        """
+        Membuat prediksi dalam batch untuk efisiensi
+        
+        Parameters:
+            input_data: List of dicts berisi input data
+            batch_size: Ukuran batch untuk prediksi
+            return_contributions: Jika True, kembalikan kontribusi fitur
+            
+        Returns:
+            Dict berisi hasil prediksi untuk semua sampel
+        """
+        start_time = datetime.now()
+        total_samples = len(input_data)
+        results = []
+        
+        self.logger.info(f"Memulai batch prediction untuk {total_samples} sampel (batch_size={batch_size})")
+        
+        for i in range(0, total_samples, batch_size):
+            batch = input_data[i:i+batch_size]
+            try:
+                batch_result = self.predict(batch, return_contributions)
+                results.extend(batch_result['predictions'])
+            except Exception as e:
+                self.logger.error(f"Error pada batch {i//batch_size}: {str(e)}")
+                raise
+        
+        exec_time = (datetime.now() - start_time).total_seconds()
+        avg_time_per_sample = exec_time / total_samples
+        
+        self.logger.info(
+            f"Batch prediction selesai. Total waktu: {exec_time:.2f} detik "
+            f"({avg_time_per_sample:.4f} detik/sampel)"
+        )
+        
+        return {
+            'predictions': results,
+            'total_samples': total_samples,
+            'total_time_seconds': exec_time,
+            'avg_time_per_sample': avg_time_per_sample,
+            'timestamp': start_time.isoformat()
+        }
+    
+    def evaluate_model(self, X_test: np.ndarray, y_test: np.ndarray) -> Dict:
+        """
+        Evaluasi model pada dataset test
+        
+        Parameters:
+            X_test: Data fitur test
+            y_test: Target test
+            
+        Returns:
+            Dict berisi metrik evaluasi
+        """
+        from .evaluator import PerformanceEvaluator
+        
+        evaluator = PerformanceEvaluator(y_test, self.predict(X_test)['predictions'], 
+                                       self.model, X_test)
+        return evaluator.metrics
+    
+    def save_predictor(self, save_dir: str):
+        """
+        Menyimpan objek predictor untuk penggunaan nanti
+        
+        Parameters:
+            save_dir: Direktori untuk menyimpan predictor
+        """
+        save_path = Path(save_dir)
+        save_path.mkdir(parents=True, exist_ok=True)
+        
+        # Nama file berdasarkan timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        save_file = save_path / f"predictor_{timestamp}.pkl"
+        
+        # Simpan objek predictor
+        joblib.dump(self, save_file)
+        self.logger.info(f"Predictor disimpan di: {save_file}")
+        
+        return str(save_file)

Prediksi Performa Akademik/edtech/backend/src/train_performance_predictor.py

@@ -0,0 +1,164 @@
+# backend/src/train_performance_predictor.py
+import numpy as np
+import sys
+import os
+import json
+import logging
+from pathlib import Path
+import pandas as pd
+import joblib
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('training.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Setup paths
+current_dir = Path(__file__).parent
+sys.path.append(str(current_dir))
+
+from performance_prediction.data_processor import PerformanceDataProcessor
+from performance_prediction.model_trainer import PerformanceModelTrainer
+from performance_prediction.evaluator import PerformanceEvaluator
+
+def main():
+    try:
+        logger.info("=== MEMULAI PELATIHAN MODEL PREDIKSI PERFORMA ===")
+        
+        # Setup paths
+        BASE_DIR = current_dir.parent.parent
+        DATA_PATH = BASE_DIR / "backend/data/processed/cleaned_education_data.csv"  # Ensure this is the correct path
+        MODEL_SAVE_DIR = BASE_DIR / "models/performance_predictor/trained_model"
+        LOG_DIR = BASE_DIR / "models/performance_predictor/training_logs"
+        CONFIG_PATH = BASE_DIR / "config/model_config.json"
+
+        # Buat direktori jika belum ada
+        os.makedirs(MODEL_SAVE_DIR, exist_ok=True)
+        os.makedirs(LOG_DIR, exist_ok=True)
+        
+        # 1. Persiapan Data
+        logger.info("\n=== MEMUAT DAN MEMPROSES DATA ===")
+        processor = PerformanceDataProcessor(DATA_PATH, CONFIG_PATH)
+        data = processor.load_data()
+        
+        # Cek data
+        if data is None or data.empty:
+            logger.error("Data kosong atau gagal dimuat")
+            return
+            
+        # Siapkan fitur dan target
+        features, target = processor.prepare_features_target(data)
+        
+        # Bagi data menjadi train, validation, dan test set
+        X_train, X_val, X_test, y_train, y_val, y_test = processor.split_data(
+            test_size=0.2,
+            val_size=0.2
+        )
+        
+        # Gunakan subset data untuk testing jika perlu
+        # X_train, y_train = X_train[:1000], y_train[:1000]
+        # X_val, y_val = X_val[:1000], y_val[:1000]
+        
+        # 2. Pelatihan Model
+        logger.info("\n=== MELATIH MODEL ===")
+        trainer = PerformanceModelTrainer()
+        
+        # Gunakan parameter yang lebih konservatif untuk testing
+        best_params = {
+            'max_depth': 6,
+            'learning_rate': 0.1,
+            'subsample': 0.8,
+            'colsample_bytree': 0.8,
+            'reg_alpha': 0.1,
+            'reg_lambda': 1.0,
+            'min_child_weight': 1,
+            'gamma': 0
+        }
+        
+        # Latih model final dengan parameter
+        model = trainer.train_model(
+            X_train=X_train,
+            y_train=y_train,
+            X_val=X_val,
+            y_val=y_val,
+            params=best_params
+        )
+        
+        # 3. Evaluasi Model
+        logger.info("\n=== EVALUASI MODEL ===")
+        evaluation = trainer.evaluate_model(X_test, y_test)
+        
+        logger.info("\n=== DETAIL EVALUASI ===")
+        logger.info(f"Contoh 5 prediksi pertama: {evaluation['predictions'][:5]}")
+        logger.info(f"Contoh 5 nilai sebenarnya: {y_test[:5]}")
+        logger.info(f"Perbedaan prediksi dan aktual: {np.abs(y_test[:5] - evaluation['predictions'][:5])}")
+
+        # Simpan metrik evaluasi
+        metrics = evaluation['metrics']
+        with open(MODEL_SAVE_DIR / "model_metrics.json", 'w') as f:json.dump(metrics, f, indent=4)
+        
+        # Visualisasi evaluasi
+        evaluator = PerformanceEvaluator(y_test, evaluation['predictions'])
+        
+        # Plot dan simpan visualisasi
+        plots = {
+            "residual_plot": evaluator.plot_residuals(),
+            "actual_vs_predicted": evaluator.plot_actual_vs_predicted(),
+            "error_distribution": evaluator.plot_error_distribution()
+        }
+        
+        for name, plot in plots.items():
+            plot_path = LOG_DIR / f"{name}.png"
+            plot.savefig(plot_path, bbox_inches='tight')
+            plt.close()
+            logger.info(f"Plot {name} disimpan di: {plot_path}")
+        
+        # Plot dari model trainer
+        trainer.plot_learning_curve(
+            X_train=X_train,
+            y_train=y_train,
+            X_val=X_val,
+            y_val=y_val,
+            save_path=LOG_DIR / "learning_curve.png"
+        )
+
+        feature_plot, importance_df = trainer.plot_feature_importance(
+            feature_names=processor.features.columns,
+            save_path=LOG_DIR / "feature_importance.png"
+        )
+        
+        # Simpan feature importance
+        importance_df.to_csv(LOG_DIR / "feature_importance.csv", index=False)
+        
+        # SHAP summary plot
+        try:
+            trainer.plot_shap_summary(
+                feature_names=processor.features.columns,
+                save_path=LOG_DIR / "shap_summary.png"
+            )
+        except Exception as e:
+            logger.warning(f"Tidak dapat membuat SHAP plot: {str(e)}")
+        
+        # 4. Simpan Model dan Processor
+        logger.info("\n=== MENYIMPAN MODEL ===")
+        model_path = trainer.save_model(MODEL_SAVE_DIR)
+        processor_path = processor.save_processor(MODEL_SAVE_DIR)
+        
+        logger.info("\n=== PELATIHAN SELESAI ===")
+        logger.info(f"Model disimpan di: {model_path}")
+        logger.info(f"Processor disimpan di: {processor_path}")
+        print(f"Log dan visualisasi disimpan di: {LOG_DIR}")
+
+    except Exception as e:
+        logger.error(f"Terjadi kesalahan saat melatih model: {str(e)}")
+
+if __name__ == "__main__":
+    main()