Upload 2 files

Constrained-Track-Document-Bassline-Readability-Arabertv2-d3tok-reg.py

@@ -0,0 +1,563 @@
+# -*- coding: utf-8 -*-
+"""arabertv2-d3tok_Sentence_Constrained.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1vHngBRGkcXHJIwi317DS0AsZfsXguIc0
+"""
+
+# Authorize Colab to access your Google Drive
+from google.colab import drive
+drive.mount('/content/drive')
+
+import pandas as pd
+import numpy as np
+import os
+import torch
+import torch.nn as nn
+import zipfile
+from sklearn.metrics import cohen_kappa_score
+from torch.utils.data import Dataset as TorchDataset
+from transformers import (
+    AutoTokenizer,
+    AutoModel,
+    TrainingArguments,
+    Trainer,
+    EarlyStoppingCallback
+)
+import gc
+import ast # To safely evaluate string-formatted lists
+import json # Added import for json
+
+# --- Hugging Face Authentication ---
+from huggingface_hub import login
+from google.colab import userdata
+
+# Log in to Hugging Face using the token stored in Colab secrets
+try:
+    login(token=userdata.get('HF_TOKEN'))
+    print("✔️ Successfully logged into Hugging Face.")
+except (NameError, KeyError):
+    print("⚠️ Hugging Face token not found in Colab secrets. Please add it as 'HF_TOKEN'.")
+    # Fallback for local execution or if login() is preferred manually
+    login()
+
+# --- Model & Training ---
+MODEL_NAME = "CAMeL-Lab/readability-arabertv2-d3tok-reg"
+
+NUM_LABELS = 1
+TARGET_CLASSES = 19
+NUM_FEATURES = 7
+
+# --- IMPORTANT: Set the path to your project folder on Google Drive ---
+PROJECT_DRIVE_PATH = '/content/drive/MyDrive/BAREC_Competition'
+
+# --- File & Directory Paths (Now relative to your Google Drive) ---
+BASE_DIR = PROJECT_DRIVE_PATH
+PROCESSED_DATA_DIR = os.path.join(BASE_DIR, "lex")
+CHECKPOINT_DIR = os.path.join(BASE_DIR, "results", f"hybrid_constrained_samer_regression_v2_{MODEL_NAME.split('/')[-1]}")
+SUBMISSION_DIR = os.path.join(BASE_DIR, "submission")
+
+# Ensure the output directories exist on your Google Drive
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+os.makedirs(SUBMISSION_DIR, exist_ok=True)
+
+# Paths to the preprocessed input files on Google Drive
+TRAIN_PROCESSED_PATH = os.path.join(PROCESSED_DATA_DIR, 'train_processed_full.csv')
+DEV_PROCESSED_PATH = os.path.join(PROCESSED_DATA_DIR, 'dev_processed_full.csv')
+TEST_PROCESSED_PATH = os.path.join(PROCESSED_DATA_DIR, 'test_processed_full.csv')
+
+# --- Submission Paths on Google Drive ---
+SUBMISSION_PATH = os.path.join(SUBMISSION_DIR, "submission_hybrid_constrained_samer_regression.csv")
+ZIPPED_SUBMISSION_PATH = os.path.join(SUBMISSION_DIR, "submission_hybrid_constrained_samer_regression.zip")
+
+print(f"✔️ All paths configured to use Google Drive folder: {BASE_DIR}")
+
+"""# 2. DATA LOADING FUNCTION"""
+
+# =====================================================================================
+# 2. DATA LOADING FUNCTION
+# =====================================================================================
+
+def load_preprocessed_data():
+    """Loads the pre-processed CSV files directly from Google Drive."""
+    print("\n--- Loading Preprocessed Data from Google Drive ---")
+    try:
+        train_df = pd.read_csv(TRAIN_PROCESSED_PATH)
+        val_df = pd.read_csv(DEV_PROCESSED_PATH)
+
+        print("Converting 'features' column from string to list...")
+        train_df['features'] = train_df['features'].apply(ast.literal_eval)
+        val_df['features'] = val_df['features'].apply(ast.literal_eval)
+
+        # Convert labels for regression
+        train_df['label'] = (train_df['label'].astype(int) - 1).astype(float)
+        val_df['label'] = (val_df['label'].astype(int) - 1).astype(float)
+
+        print(f"✔ Successfully loaded {len(train_df)} training and {len(val_df)} validation records.")
+        return train_df, val_df
+    except FileNotFoundError as e:
+        print(f"❌ ERROR: Preprocessed file not found: {e}.")
+        print("Please make sure your data is uploaded to the correct Google Drive folder.")
+        return None, None
+    except Exception as e:
+        print(f"❌ ERROR during data loading: {e}")
+        return None, None
+
+"""# 3. MODEL, DATASET, AND METRICS DEFINITIONS"""
+
+# =====================================================================================
+# 3. MODEL, DATASET, AND METRICS DEFINITIONS
+# =====================================================================================
+
+class ReadabilityDataset(TorchDataset):
+    """
+    Custom PyTorch Dataset for readability prediction.
+    It takes text, extra numerical features, and optional labels.
+    """
+    def __init__(self, texts, features, labels=None, tokenizer_obj=None, max_len=256):
+        self.texts = texts
+        self.features = features
+        self.labels = labels
+        self.tokenizer = tokenizer_obj
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        text = str(self.texts[idx])
+        feature_vec = torch.tensor(self.features[idx], dtype=torch.float)
+
+        encoding = self.tokenizer.encode_plus(
+            text,
+            add_special_tokens=True,
+            max_length=self.max_len,
+            return_token_type_ids=False,
+            padding='max_length',
+            truncation=True,
+            return_attention_mask=True,
+            return_tensors='pt',
+        )
+
+        item = {
+            'input_ids': encoding['input_ids'].flatten(),
+            'attention_mask': encoding['attention_mask'].flatten(),
+            'extra_features': feature_vec
+        }
+
+        if self.labels is not None:
+            item['labels'] = torch.tensor(self.labels[idx], dtype=torch.float)
+
+        return item
+
+class HybridRegressionModel(nn.Module):
+    """
+    A hybrid model that combines a transformer base with additional numerical features.
+    The output is a single regression value.
+    """
+    def __init__(self, model_name, num_extra_features):
+        super(HybridRegressionModel, self).__init__()
+        self.transformer = AutoModel.from_pretrained(model_name)
+        # The regression head takes the transformer's output + extra features
+        self.regressor = nn.Linear(self.transformer.config.hidden_size + num_extra_features, 1)
+
+        # Add the attribute expected by Hugging Face Trainer for checkpoint loading
+        # Initialize it to None or an empty list if no keys should be ignored during saving
+        self._keys_to_ignore_on_save = None # Or []
+
+    def forward(self, input_ids, attention_mask, extra_features, labels=None):
+        outputs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)
+        pooler_output = outputs.pooler_output
+
+        # Concatenate transformer output with extra features
+        combined_features = torch.cat((pooler_output, extra_features), dim=1)
+
+        logits = self.regressor(combined_features)
+
+        loss = None
+        if labels is not None:
+            loss_fct = nn.MSELoss()
+            loss = loss_fct(logits.squeeze(), labels.squeeze())
+
+        return (loss, logits) if loss is not None else logits
+
+def compute_metrics(p):
+    """
+    Computes the Quadratic Weighted Kappa score for regression predictions.
+    Predictions are rounded and clipped to be valid class labels.
+    """
+    predictions, labels = p
+    # Round predictions to nearest integer and clip to valid range [0, 18]
+    clipped_preds = np.clip(np.round(predictions), 0, TARGET_CLASSES - 1)
+    qwk = cohen_kappa_score(labels, clipped_preds, weights='quadratic')
+    return {'qwk': qwk}
+
+"""# 4. & 5. MAIN EXECUTION FUNCTIONS"""
+
+# =====================================================================================
+# 4. & 5. MAIN EXECUTION FUNCTIONS
+# =====================================================================================
+
+def main_train():
+    print("===== 🚀 STARTING HYBRID REGRESSION MODEL PIPELINE =====\n")
+
+    print("Initializing Tokenizer...")
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+    train_df, val_df = load_preprocessed_data()
+    if train_df is None:
+        print("\n! Aborting script due to data loading failure.")
+        return
+
+    print("\nCreating Torch Datasets...")
+    train_dataset = ReadabilityDataset(train_df['d3tok_text'].tolist(), train_df['features'].tolist(), train_df['label'].tolist(), tokenizer)
+    val_dataset = ReadabilityDataset(val_df['d3tok_text'].tolist(), val_df['features'].tolist(), val_df['label'].tolist(), tokenizer)
+    print("✔ Datasets created.")
+
+    print("\nInitializing Hybrid Regression Model...")
+    model = HybridRegressionModel(MODEL_NAME, num_extra_features=NUM_FEATURES)
+
+    training_args = TrainingArguments(
+        output_dir=CHECKPOINT_DIR,
+        num_train_epochs=15,
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=64,
+        learning_rate=3e-5,
+        warmup_ratio=0.1,
+        weight_decay=0.01,
+        logging_steps=100,
+        eval_strategy="epoch",
+        save_strategy="epoch",
+        load_best_model_at_end=True,
+        metric_for_best_model="qwk",
+        greater_is_better=True,
+        save_total_limit=2,
+        fp16=torch.cuda.is_available(),
+        report_to="none"
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset,
+        compute_metrics=compute_metrics,
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=4)]
+    )
+
+    print("\nStarting model training... Checkpoints will be saved to Google Drive.")
+
+    latest_checkpoint = None
+    if os.path.exists(CHECKPOINT_DIR):
+        checkpoints = [d for d in os.listdir(CHECKPOINT_DIR) if d.startswith("checkpoint-")]
+        if checkpoints:
+            checkpoints.sort(key=lambda x: int(x.split('-')[-1]))
+            latest_checkpoint = os.path.join(CHECKPOINT_DIR, checkpoints[-1])
+            print(f"Resuming training from latest checkpoint: {latest_checkpoint}")
+        else:
+             print("No checkpoints found to resume training from. Starting from scratch.")
+    else:
+        print("Checkpoint directory not found. Starting training from scratch.")
+
+
+    trainer.train(resume_from_checkpoint=latest_checkpoint)
+    print("✔ Training finished.")
+
+    del model, trainer, train_dataset, val_dataset, train_df, val_df
+    gc.collect()
+    torch.cuda.empty_cache()
+
+
+def main_predict():
+    print("\n===== 🏆 GENERATING FINAL PREDICTIONS & SUBMISSION =====\n")
+    try:
+        print("Initializing Tokenizer for prediction...")
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+        print("Loading preprocessed test data from Google Drive...")
+        test_df = pd.read_csv(TEST_PROCESSED_PATH)
+        test_df['features'] = test_df['features'].apply(ast.literal_eval)
+
+        print("\nLoading the best trained model from Google Drive checkpoints...")
+        best_checkpoint_path = None
+        best_qwk = -1.0
+
+        if not os.path.exists(CHECKPOINT_DIR):
+             raise FileNotFoundError(f"Checkpoint directory not found on Google Drive: {CHECKPOINT_DIR}.")
+
+        checkpoints = [d for d in os.listdir(CHECKPOINT_DIR) if d.startswith("checkpoint-")]
+        if not checkpoints:
+             raise FileNotFoundError(f"No checkpoint found in the results directory on Google Drive: {CHECKPOINT_DIR}.")
+
+        checkpoints.sort(key=lambda x: int(x.split('-')[-1]), reverse=True)
+
+        for checkpoint in checkpoints:
+            checkpoint_path = os.path.join(CHECKPOINT_DIR, checkpoint)
+            trainer_state_path = os.path.join(checkpoint_path, "trainer_state.json")
+            model_file_path = os.path.join(checkpoint_path, "pytorch_model.bin")
+
+            if os.path.exists(model_file_path) and os.path.exists(trainer_state_path):
+                 try:
+                     with open(trainer_state_path, 'r') as f:
+                         trainer_state = json.load(f)
+                         if 'best_metric' in trainer_state and trainer_state['best_metric'] > best_qwk:
+                             best_qwk = trainer_state['best_metric']
+                             best_checkpoint_path = checkpoint_path
+                             print(f"Found improved eval_qwk {best_qwk} in {checkpoint_path}")
+                 except Exception as e:
+                    print(f"Warning: Could not parse trainer_state.json in {checkpoint_path}: {e}")
+
+        if not best_checkpoint_path:
+            print("Could not find best checkpoint via trainer_state.json. Falling back to the latest checkpoint with a model file.")
+            for checkpoint in checkpoints:
+                checkpoint_path = os.path.join(CHECKPOINT_DIR, checkpoint)
+                if os.path.exists(os.path.join(checkpoint_path, "pytorch_model.bin")):
+                    best_checkpoint_path = checkpoint_path
+                    print(f"Using latest valid checkpoint: {best_checkpoint_path}")
+                    break
+
+        if not best_checkpoint_path:
+            raise FileNotFoundError(f"No valid checkpoint with 'pytorch_model.bin' found in: {CHECKPOINT_DIR}.")
+
+        print(f"Loading model from: {best_checkpoint_path}")
+        model = HybridRegressionModel(MODEL_NAME, num_extra_features=NUM_FEATURES)
+        model.load_state_dict(torch.load(os.path.join(best_checkpoint_path, "pytorch_model.bin")))
+
+        trainer = Trainer(model=model, args=TrainingArguments(output_dir=CHECKPOINT_DIR))
+
+        print("Generating predictions on the test set...")
+        # Note: No labels for the test dataset
+        test_dataset = ReadabilityDataset(test_df['d3tok_text'].tolist(), test_df['features'].tolist(), tokenizer_obj=tokenizer)
+        predictions = trainer.predict(test_dataset)
+
+        clipped_preds = np.clip(np.round(predictions.predictions.flatten()), 0, TARGET_CLASSES - 1)
+        test_df['Prediction'] = (clipped_preds + 1).astype(int)
+
+        submission_df = test_df.rename(columns={'ID': 'id'})[['id', 'Prediction']]
+
+        print(f"\nSaving prediction file to: {SUBMISSION_PATH}")
+        submission_df.to_csv(SUBMISSION_PATH, index=False)
+
+        print(f"Compressing into {os.path.basename(ZIPPED_SUBMISSION_PATH)}...")
+        with zipfile.ZipFile(ZIPPED_SUBMISSION_PATH, 'w', zipfile.ZIP_DEFLATED) as zipf:
+            zipf.write(SUBMISSION_PATH, arcname=os.path.basename(SUBMISSION_PATH))
+
+        print(f"✔ Submission file '{os.path.basename(ZIPPED_SUBMISSION_PATH)}' created successfully in your Drive!")
+
+    except FileNotFoundError as e:
+        print(f"❌ ERROR: File not found: {e}. Ensure training was completed and checkpoints exist.")
+    except Exception as e:
+        print(f"❌ An error occurred during final prediction: {e}")
+
+"""# Running script"""
+
+# =====================================================================================
+# 6. SCRIPT RUNNER
+# =====================================================================================
+
+# Start the training process
+main_train()
+
+# Once training is done, generate predictions
+main_predict()
+
+print("\n--- ✅ All Done! Check your Google Drive for results and submission files. ---")
+
+BEST_CHECKPOINT_PATH = '/content/drive/MyDrive/BAREC_Competition/results/hybrid_constrained_samer_regression_v2_readability-arabertv2-d3tok-reg/checkpoint-42826'
+
+
+# --- Input & Output Directories (derived from base paths) ---
+PROCESSED_DATA_DIR = os.path.join(PROJECT_DRIVE_PATH, "lex")
+SUBMISSION_DIR = os.path.join(PROJECT_DRIVE_PATH, "submission")
+
+# Ensure the submission output directory exists
+os.makedirs(SUBMISSION_DIR, exist_ok=True)
+
+# --- File Paths ---
+TEST_PROCESSED_PATH = os.path.join(PROCESSED_DATA_DIR, 'test_processed_full.csv')
+SUBMISSION_PATH = os.path.join(SUBMISSION_DIR, "submission_final_prediction.csv")
+ZIPPED_SUBMISSION_PATH = os.path.join(SUBMISSION_DIR, "submission.zip")
+
+print(f"✔️ Configuration loaded. Model will be loaded from: {BEST_CHECKPOINT_PATH}")
+
+"""# 3. MODEL AND DATASET CLASS DEFINITIONS"""
+
+# =====================================================================================
+# 3. MODEL AND DATASET CLASS DEFINITIONS
+# =====================================================================================
+
+class ReadabilityDataset(TorchDataset):
+    """
+    Custom PyTorch Dataset for readability prediction.
+    It takes text, extra numerical features, and optional labels.
+    """
+    def __init__(self, texts, features, labels=None, tokenizer_obj=None, max_len=256):
+        self.texts = texts
+        self.features = features
+        self.labels = labels
+        self.tokenizer = tokenizer_obj
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        text = str(self.texts[idx])
+        feature_vec = torch.tensor(self.features[idx], dtype=torch.float)
+
+        encoding = self.tokenizer.encode_plus(
+            text,
+            add_special_tokens=True,
+            max_length=self.max_len,
+            return_token_type_ids=False,
+            padding='max_length',
+            truncation=True,
+            return_attention_mask=True,
+            return_tensors='pt',
+        )
+
+        item = {
+            'input_ids': encoding['input_ids'].flatten(),
+            'attention_mask': encoding['attention_mask'].flatten(),
+            'extra_features': feature_vec
+        }
+
+        if self.labels is not None:
+            item['labels'] = torch.tensor(self.labels[idx], dtype=torch.float)
+
+        return item
+
+class HybridRegressionModel(nn.Module):
+    """
+    A hybrid model that combines a transformer base with additional numerical features.
+    The output is a single regression value.
+    """
+    def __init__(self, model_name, num_extra_features):
+        super(HybridRegressionModel, self).__init__()
+        self.transformer = AutoModel.from_pretrained(model_name)
+        self.regressor = nn.Linear(self.transformer.config.hidden_size + num_extra_features, 1)
+
+    def forward(self, input_ids, attention_mask, extra_features, labels=None):
+        outputs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)
+        pooler_output = outputs.pooler_output
+        combined_features = torch.cat((pooler_output, extra_features), dim=1)
+        logits = self.regressor(combined_features)
+
+        loss = None
+        if labels is not None:
+            # Using MSELoss for regression
+            loss_fct = nn.MSELoss()
+            loss = loss_fct(logits.squeeze(), labels.squeeze())
+
+        return (loss, logits) if loss is not None else logits
+
+"""# 4. PREDICTION AND SUBMISSION SCRIPT"""
+
+# =====================================================================================
+# 4. PREDICTION AND SUBMISSION SCRIPT
+# =====================================================================================
+
+from safetensors.torch import load_file # Import load_file
+
+def generate_predictions():
+    """
+    Loads the trained model from the specified checkpoint, runs predictions on the
+    test set, and saves the formatted submission file.
+    """
+    print("\n===== 🏆 STARTING PREDICTION PIPELINE =====\n")
+    try:
+        # --- Validate Checkpoint Path ---
+        model_weights_path = os.path.join(BEST_CHECKPOINT_PATH, 'model.safetensors')
+        if not os.path.exists(model_weights_path):
+            print(f"❌ ERROR: 'model.safetensors' not found at the specified path: {model_weights_path}")
+            print("Please ensure the BEST_CHECKPOINT_PATH variable is set correctly to the directory containing the model weights.")
+            return
+
+        # --- Initialize Tokenizer ---
+        print("1. Initializing tokenizer...")
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+        # --- Load Test Data ---
+        print(f"2. Loading preprocessed test data from: {TEST_PROCESSED_PATH}")
+        test_df = pd.read_csv(TEST_PROCESSED_PATH)
+        test_df['features'] = test_df['features'].apply(ast.literal_eval)
+        print(f"   Loaded {len(test_df)} test records.")
+
+        # --- Load Model from Checkpoint ---
+        print(f"3. Loading model from checkpoint: {BEST_CHECKPOINT_PATH}")
+        model = HybridRegressionModel(MODEL_NAME, num_extra_features=NUM_FEATURES)
+
+        # Load the state dictionary from the .safetensors file
+        state_dict = load_file(model_weights_path)
+
+        # --- Inspect the state dictionary keys ---
+        print("\n--- State Dictionary Keys ---")
+        for key in state_dict.keys():
+            print(key)
+        print("-----------------------------\n")
+        # --- End Inspection ---
+
+        # Load the state dictionary into the model
+        model.load_state_dict(state_dict)
+        print("   Model weights loaded successfully from model.safetensors.")
+
+        # --- Initialize Trainer ---
+        # A minimal Trainer is sufficient for making predictions
+        trainer = Trainer(model=model, args=TrainingArguments(output_dir="./temp_results"))
+
+        # --- Create Test Dataset ---
+        print("4. Creating test dataset...")
+        test_dataset = ReadabilityDataset(
+            texts=test_df['d3tok_text'].tolist(),
+            features=test_df['features'].tolist(),
+            tokenizer_obj=tokenizer
+        )
+
+        # --- Generate Predictions ---
+        print("5. Generating predictions on the test set...")
+        raw_predictions = trainer.predict(test_dataset)
+
+        # Predictions are in the .predictions attribute; flatten them to a 1D array
+        predictions_logits = raw_predictions.predictions.flatten()
+
+        # --- Process Predictions ---
+        # Round to the nearest integer, clip to the valid label range [0, 18],
+        # and convert back to original class labels [1, 19]
+        clipped_preds = np.clip(np.round(predictions_logits), 0, TARGET_CLASSES - 1)
+        final_predictions = (clipped_preds + 1).astype(int)
+        test_df['Prediction'] = final_predictions
+        print("   Predictions generated and processed.")
+
+        # --- Create and Save Submission File ---
+        submission_df = test_df.rename(columns={'ID': 'id'})[['id', 'Prediction']]
+        print(f"\n6. Saving final prediction CSV to: {SUBMISSION_PATH}")
+        submission_df.to_csv(SUBMISSION_PATH, index=False)
+
+        print(f"7. Compressing submission file into: {ZIPPED_SUBMISSION_PATH}")
+        with zipfile.ZipFile(ZIPPED_SUBMISSION_PATH, 'w', zipfile.ZIP_DEFLATED) as zipf:
+            zipf.write(SUBMISSION_PATH, arcname=os.path.basename(SUBMISSION_PATH))
+
+        print("\n--- ✅ All Done! ---")
+        print(f"Submission file '{os.path.basename(ZIPPED_SUBMISSION_PATH)}' has been saved to your Google Drive.")
+        print(f"Location: {SUBMISSION_DIR}")
+
+    except FileNotFoundError as e:
+        print(f"❌ ERROR: A required file was not found: {e}")
+        print("   Please ensure all paths in the configuration section are correct.")
+    except Exception as e:
+        print(f"❌ An unexpected error occurred: {e}")
+
+"""# 5. EXECUTE SCRIPT"""
+
+# =====================================================================================
+# 5. EXECUTE SCRIPT
+# =====================================================================================
+
+if __name__ == '__main__':
+    generate_predictions()
+
+"""# Results of Sentence-level Readability Assessment - Constrained on The Blind Test
+{'accuracy': 42.1, 'accuracy+-1': 71.6, 'avg_abs_dist': 1.2, 'qwk': 82.1, 'accuracy_7': 59.9, 'accuracy_5': 65.4, 'accuracy_3': 73.4}
+"""

samer_preprocessing_feature_extraction.py

@@ -0,0 +1,205 @@
+# =====================================================================================
+# 0. INSTALLATIONS & IMPORTS
+# =====================================================================================
+# This will install all necessary libraries quietly.
+# !pip install transformers[torch] datasets pandas scikit-learn arabert accelerate pyarrow -q
+
+import pandas as pd
+import numpy as np
+import os
+from transformers import AutoTokenizer
+from arabert.preprocess import ArabertPreprocessor
+
+print("✅ Libraries imported successfully.")
+
+# =====================================================================================
+# 1. CONFIGURATION
+# =====================================================================================
+# --- Model & Tokenizer ---
+# Using the model from your original script for consistency
+MODEL_NAME = "aubmindlab/bert-large-arabertv2"
+MAX_LENGTH = 256  # Max sequence length for tokenizer
+
+# --- File Paths ---
+# Assumes your initial dataset is in the default /kaggle/input/sentses directory
+RAW_DATA_DIR = '/kaggle/input/sentses/'
+BAREC_TRAIN_PATH = os.path.join(RAW_DATA_DIR, 'train.csv')
+BAREC_DEV_PATH = os.path.join(RAW_DATA_DIR, 'dev.csv')
+BLIND_TEST_PATH = os.path.join(RAW_DATA_DIR, 'blind_test_data.csv')
+# The SAMER files are assumed to be in the same directory for this example
+SAMER_CORPUS_PATH = os.path.join(RAW_DATA_DIR, 'samer_train.tsv')
+SAMER_LEXICON_PATH = os.path.join(RAW_DATA_DIR, 'SAMER-Readability-Lexicon-v2.tsv')
+
+# --- Output Path ---
+# Processed files will be saved here, ready for output
+OUTPUT_DIR = '/kaggle/working/'
+
+# --- Initialize Processors ---
+try:
+    arabert_preprocessor = ArabertPreprocessor(model_name=MODEL_NAME)
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+    print("✅ AraBERT Preprocessor and Tokenizer initialized.")
+except Exception as e:
+    print(f"❗️ Error initializing processors: {e}")
+    # Exit if the core components can't be loaded
+    exit()
+
+# =====================================================================================
+# 2. DATA LOADING FUNCTIONS
+# =====================================================================================
+
+def load_samer_lexicon(file_path):
+    """Loads the SAMER Lexicon for feature engineering."""
+    print("\n--- Loading SAMER Lexicon ---")
+    try:
+        df = pd.read_csv(file_path, sep='\t')
+        df[['lemma', 'pos']] = df['lemma#pos'].str.split('#', expand=True)
+        lexicon_dict = df.set_index('lemma')['readability (rounded average)'].to_dict()
+        print(f"Loaded {len(lexicon_dict)} lemmas into lexicon dictionary.")
+        return lexicon_dict
+    except FileNotFoundError:
+        print("❗️ SAMER Lexicon not found. Lexical features will be disabled.")
+        return {}
+
+def load_training_and_validation_data(lexicon):
+    """Loads and augments training/validation data."""
+    print("\n--- Loading BAREC Training & Validation Data ---")
+    try:
+        train_df = pd.read_csv(BAREC_TRAIN_PATH)[['Sentence', 'Readability_Level_19']].rename(columns={'Sentence': 'text', 'Readability_Level_19': 'label'})
+        val_df = pd.read_csv(BAREC_DEV_PATH)[['Sentence', 'Readability_Level_19']].rename(columns={'Sentence': 'text', 'Readability_Level_19': 'label'})
+        train_df.dropna(subset=['text', 'label'], inplace=True)
+        val_df.dropna(subset=['text', 'label'], inplace=True)
+        print(f"Loaded {len(train_df)} BAREC training sentences and {len(val_df)} validation sentences.")
+    except Exception as e:
+        print(f"❗️ ERROR loading BAREC CSVs: {e}")
+        return None, None
+
+    print("\n--- Loading SAMER Corpus for Augmentation ---")
+    try:
+        samer_level_map = {'L3': 4, 'L4': 10, 'L5': 16}
+        samer_df = pd.read_csv(SAMER_CORPUS_PATH, sep='\t')
+        samer_records = []
+        for level_name, barec_level in samer_level_map.items():
+            samer_subset = samer_df[[level_name]].dropna().rename(columns={level_name: 'text'})
+            samer_subset['label'] = barec_level
+            samer_records.append(samer_subset)
+        samer_augmentation_df = pd.concat(samer_records, ignore_index=True)
+        print(f"Loaded {len(samer_augmentation_df)} sentences from SAMER.")
+        
+        full_train_df = pd.concat([train_df, samer_augmentation_df], ignore_index=True)
+        full_train_df.dropna(subset=['text', 'label'], inplace=True)
+        full_train_df['label'] = full_train_df['label'].astype(int)
+        full_train_df = full_train_df.sample(frac=1, random_state=42).reset_index(drop=True)
+        print(f"Total unified training sentences: {len(full_train_df)}")
+        return full_train_df, val_df
+    except FileNotFoundError:
+        print("❗️ SAMER Corpus not found, proceeding with BAREC data only.")
+        return train_df, val_df
+
+def load_blind_test_data(file_path):
+    """Loads and prepares the blind test set."""
+    print(f"\n--- Loading Blind Test Data ---")
+    try:
+        doc_test_df = pd.read_csv(file_path).rename(columns={'ID': 'doc_id', 'Sentence': 'text'})
+        sentence_test_df = doc_test_df.assign(text=doc_test_df['text'].str.split('\n')).explode('text').reset_index(drop=True)
+        sentence_test_df.dropna(subset=['text'], inplace=True)
+        print(f"Loaded and exploded {len(sentence_test_df)} sentences for prediction.")
+        return sentence_test_df
+    except Exception as e:
+        print(f"❗️ ERROR loading blind test file: {e}")
+        return None
+
+# =====================================================================================
+# 3. FEATURE ENGINEERING & PREPROCESSING FUNCTION
+# =====================================================================================
+
+def get_lexical_features(text, lexicon):
+    """Calculates lexical features based on the SAMER lexicon."""
+    if not lexicon or not isinstance(text, str):
+        return [0.0] * 7
+
+    words = text.split()
+    if not words: return [0.0] * 7
+
+    word_difficulties = [lexicon.get(word, 3.0) for word in words]
+    
+    # Use float() to ensure type consistency for pyarrow
+    features = [
+        float(len(text)),
+        float(len(words)),
+        float(np.mean([len(w) for w in words]) if words else 0.0),
+        float(np.mean(word_difficulties)),
+        float(np.max(word_difficulties)),
+        float(np.sum(np.array(word_difficulties) > 4)),
+        float(len([w for w in words if w not in lexicon]) / len(words))
+    ]
+    return features
+
+def process_dataframe(df, lexicon, is_test=False):
+    """Applies all preprocessing steps to a dataframe."""
+    print(f"\n--- Starting processing for {'Test' if is_test else 'Train/Val'} dataframe ---")
+    
+    # 1. Clean and preprocess text
+    print("Step 1: Applying AraBERT preprocessor...")
+    df['text_preprocessed'] = df['text'].apply(arabert_preprocessor.preprocess)
+    
+    # 2. Engineer lexical features
+    print("Step 2: Engineering lexical features...")
+    features = np.array(df['text_preprocessed'].apply(lambda x: get_lexical_features(x, lexicon)).tolist())
+    df['features'] = list(features)
+
+    # 3. Tokenize text
+    print("Step 3: Tokenizing text...")
+    encodings = tokenizer(
+        df['text_preprocessed'].tolist(),
+        truncation=True,
+        padding="max_length",
+        max_length=MAX_LENGTH
+    )
+    df['input_ids'] = encodings['input_ids']
+    df['attention_mask'] = encodings['attention_mask']
+    
+    # 4. Select final columns
+    if is_test:
+        final_cols = ['doc_id', 'input_ids', 'attention_mask', 'features']
+    else:
+        final_cols = ['label', 'input_ids', 'attention_mask', 'features']
+    
+    print("✅ Processing complete.")
+    return df[final_cols]
+
+# =====================================================================================
+# 4. EXECUTION
+# =====================================================================================
+
+# Load all data first
+samer_lexicon = load_samer_lexicon(SAMER_LEXICON_PATH)
+train_df, val_df = load_training_and_validation_data(samer_lexicon)
+test_df = load_blind_test_data(BLIND_TEST_PATH)
+
+# Check if data loading was successful before proceeding
+if train_df is not None and val_df is not None and test_df is not None:
+    # Process each dataframe
+    processed_train_df = process_dataframe(train_df, samer_lexicon)
+    processed_val_df = process_dataframe(val_df, samer_lexicon)
+    processed_test_df = process_dataframe(test_df, samer_lexicon, is_test=True)
+
+    # Save the processed dataframes to Feather files
+    print("\n--- Saving processed dataframes to Feather files ---")
+    
+    train_save_path = os.path.join(OUTPUT_DIR, 'train_processed.feather')
+    val_save_path = os.path.join(OUTPUT_DIR, 'val_processed.feather')
+    test_save_path = os.path.join(OUTPUT_DIR, 'test_processed.feather')
+
+    processed_train_df.to_feather(train_save_path)
+    print(f"✅ Training data saved to {train_save_path}")
+    
+    processed_val_df.to_feather(val_save_path)
+    print(f"✅ Validation data saved to {val_save_path}")
+    
+    processed_test_df.to_feather(test_save_path)
+    print(f"✅ Test data saved to {test_save_path}")
+
+    print("\n🎉 All preprocessing is complete. You can now save this notebook's output as a new dataset.")
+else:
+    print("\n❗️ Script aborted due to data loading errors.")