app.py

import gradio as gr
from codecarbon import EmissionsTracker
from datasets import load_dataset
import numpy as np
from sklearn.metrics import accuracy_score
import random
import os
import json
from datetime import datetime
from huggingface_hub import HfApi
from huggingface_hub import upload_file
import tempfile
from dotenv import load_dotenv
import spaces

# Use dotenv to load the environment variables
load_dotenv()

# Get HF token from environment variable
HF_TOKEN = os.getenv("HF_TOKEN_TEXT")
print(HF_TOKEN)
if not HF_TOKEN:
    print("Warning: HF_TOKEN not found in environment variables. Submissions will not work.")

tracker = EmissionsTracker(allow_multiple_runs=True)

# Function to get space username and URL
def get_space_info():
    space_name = os.getenv("SPACE_ID", "")
    if space_name:
        try:
            username = space_name.split("/")[0]
            space_url = f"https://huggingface.co/spaces/{space_name}"
            return username, space_url
        except Exception as e:
            print(f"Error getting space info: {e}")
    return "local-user", "local-development"

def clean_emissions_data(emissions_data):
    """Remove unwanted fields from emissions data"""
    data_dict = emissions_data.__dict__
    fields_to_remove = ['timestamp', 'project_name', 'experiment_id', 'latitude', 'longitude']
    return {k: v for k, v in data_dict.items() if k not in fields_to_remove}

@spaces.GPU
def evaluate():
    # Get space info
    username, space_url = get_space_info()
    
    # Initialize tracker
    tracker.start()
    tracker.start_task("inference")
    
    # Make random predictions
    true_labels = test_dataset["label"]
    predictions = [random.randint(0, 7) for _ in range(len(true_labels))]
    
    # Calculate accuracy
    accuracy = accuracy_score(true_labels, predictions)
    
    # Stop tracking emissions
    emissions_data = tracker.stop_task()
    
    # Prepare complete results
    results = {
        "username": username,
        "space_url": space_url,
        "submission_timestamp": datetime.now().isoformat(),
        "accuracy": float(accuracy),
        "energy_consumed_wh": emissions_data.energy_consumed * 1000,
        "emissions_gco2eq": emissions_data.emissions * 1000,
        "emissions_data": clean_emissions_data(emissions_data)
    }
    
    # Return both summary and detailed results
    return [
        accuracy,
        emissions_data.emissions * 1000,
        emissions_data.energy_consumed * 1000,
        json.dumps(results, indent=2)
    ]

def submit_results(results_json):
    if not results_json:
        return gr.Warning("No results to submit")
    
    if not HF_TOKEN:
        return gr.Warning("HF_TOKEN not found. Please set up your Hugging Face token.")
    
    # try:
    # results_json is already a string, no need to load it
    results_str = json.dumps(results_json)  # Parse the JSON string to get the data
    
    # Create a temporary file with the results
    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f:
        # Write the original JSON string to file
        f.write(results_str)
        temp_path = f.name
    
    # Upload to the dataset
    api = HfApi(token=HF_TOKEN)
    path_in_repo = f"submissions/{results_json['username']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
    api.upload_file(
        path_or_fileobj=temp_path,
        path_in_repo=path_in_repo,
        repo_id="frugal-ai-challenge/public-leaderboard-text",
        repo_type="dataset",
        token=HF_TOKEN
    )
    
    # Clean up
    os.unlink(temp_path)
    
    return gr.Info("Results submitted successfully to the leaderboard! 🎉")
    # except Exception as e:
        # return gr.Warning(f"Error submitting results: {str(e)}")

# Define the label mapping
LABEL_MAPPING = {
    "0_not_relevant": 0,  # No relevant claim detected
    "1_not_happening": 1,  # Global warming is not happening
    "2_not_human": 2,  # Not caused by humans
    "3_not_bad": 3,  # Not bad or beneficial
    "4_solutions_harmful_unnecessary": 4,  # Solutions harmful/unnecessary
    "5_science_unreliable": 5,  # Science is unreliable
    "6_proponents_biased": 6,  # Proponents are biased
    "7_fossil_fuels_needed": 7  # Fossil fuels are needed
}

# Reverse mapping for display purposes
LABEL_DESCRIPTIONS = {
    0: "No relevant claim detected",
    1: "Global warming is not happening",
    2: "Not caused by humans",
    3: "Not bad or beneficial",
    4: "Solutions harmful/unnecessary",
    5: "Science is unreliable",
    6: "Proponents are biased",
    7: "Fossil fuels are needed"
}

# Load and prepare the dataset
print("Loading dataset...")
dataset = load_dataset("QuotaClimat/frugalaichallenge-text-train")

# Convert string labels to integers
dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})

# Split dataset
train_test = dataset["train"].train_test_split(test_size=0.2, seed=42)
train_dataset = train_test["train"]
test_dataset = train_test["test"]

# Display preview
print("\nFirst 5 rows of test set:")
for i, example in enumerate(test_dataset.select(range(5))):
    print(f"\nExample {i+1}:")
    print(f"Text: {example['quote'][:100]}...")
    print(f"Label: {example['label']} - {LABEL_DESCRIPTIONS[example['label']]}")

# Create the demo interface
with gr.Blocks() as demo:


    gr.Markdown("""
# Frugal AI Challenge - Text task - Submission portal
## Climate Disinformation Classification
""")

    with gr.Tabs():

        with gr.Tab("Instructions"):

            gr.Markdown("""
To submit your results, please follow the steps below:
                        
## Prepare your model submission
1. Clone the space of this portal on your own Hugging Face account.
2. Modify the ``evaluate`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked.
3. Eventually complete the requirements and/or any necessaries dependencies in your space.
4. Write down your model card in the ``modelcard.md`` file.
5. Deploy your space and verify that it works.
6. (Optional) You can change the Space hardware to use any GPU directly on Hugging Face.
                        
## Submit your model to the leaderboard in the ``Model Submission`` tab
7. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions.
8. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard.
9. You can see the leaderboard at https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text
""")

        with gr.Tab("Model Submission"):
            gr.Markdown("## Random Baseline Model")
            
            with gr.Row():
                with gr.Column(scale=1):
                    evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
                with gr.Column(scale=1):
                    submit_btn = gr.Button("2. Submit to leaderboard", variant="primary", size="lg")
                    
            with gr.Row():
                accuracy_output = gr.Number(label="Accuracy", precision=4)
                emissions_output = gr.Number(label="Emissions (gCO2eq)", precision=12)
                energy_output = gr.Number(label="Energy Consumed (Wh)", precision=12)
            
            with gr.Row():
                results_json = gr.JSON(label="Detailed Results", visible=True)
        
            evaluate_btn.click(
                evaluate,
                inputs=None,
                outputs=[accuracy_output, emissions_output, energy_output, results_json]
            )
            
            submit_btn.click(
                submit_results,
                inputs=[results_json],
                outputs=None  # No need for output component with popups
            )
        
        with gr.Tab("Model Card"):
            with open("modelcard.md", "r") as f:
                model_card_content = f.read()
            gr.Markdown(model_card_content)

if __name__ == "__main__":
    demo.launch()