Upload folder using huggingface_hub

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.gitignore

@@ -0,0 +1,4 @@
+#macosfile 
+**/.DS_Store
+
+venv/

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

FuelInFranceData/full_dataset.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9c34c7529b8d83ce451fd486b9d23e7775906331c227abf448b1cd123a315ec
+size 1187133604

README.md

@@ -1,12 +1,7 @@
 ---
-title: Fuelprediction
-emoji: 🌖
-colorFrom: indigo
-colorTo: red
+title: fuelprediction
+app_file: runningscript.py
 sdk: gradio
 sdk_version: 5.0.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+predict fuel price in france

downloadandconvertdataset.py

@@ -0,0 +1,14 @@
+from datasets import load_dataset, concatenate_datasets
+
+# Charger le dataset depuis Hugging Face
+dataset_dict = load_dataset("VincentGOURBIN/FuelInFranceData")
+
+# Récupérer toutes les partitions du dataset
+datasets = [dataset for dataset in dataset_dict.values()]
+
+# Concaténer toutes les partitions en un seul dataset
+full_dataset = concatenate_datasets(datasets)
+
+# Sauvegarder le dataset concaténé au format Parquet
+full_dataset.to_parquet("full_dataset.parquet")
+print("Le dataset complet a été sauvegardé en 'full_dataset.parquet'")

fuel_price_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb289ddde1c8f47da78c132c31a7e85886021d6b2f24fb253fc8cff34f195a5e
+size 501316

fuel_price_model_E10.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317842c5746adeda2c4d33a3d771dc6dca0dbc5fa01d9b6cabd884974c0f162b
+size 501177

fuel_price_model_Gazole.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e510361c8a8671e89ef42321ef12df7b8ea38178266afe55fd06e7f7fde19074
+size 501587

fuel_price_model_SP95.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:756bce1166a7b448d342decc8fbc8cf4eedf96cec30c698d433078c257ab05c3
+size 500839

fuel_price_model_SP98.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a2a5fb48757eaf190fa686dc9ed2feb435279c6889f2b3b9ca7e8de95c0e492
+size 501519

full_dataset.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9c34c7529b8d83ce451fd486b9d23e7775906331c227abf448b1cd123a315ec
+size 1187133604

mainscript.py

@@ -0,0 +1,395 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.init as init
+from torch.utils.data import Dataset, DataLoader
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+import os
+from datetime import datetime, timedelta
+import argparse
+import json
+import matplotlib.pyplot as plt
+
+# Vérifier si MPS est disponible
+device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
+print(f"Utilisation de l'appareil: {device}")
+
+def load_brent_data(file_path):
+    print(f"Chargement des données Brent depuis {file_path}")
+    brent_data = pd.read_csv(file_path)
+    brent_data['brent_date'] = pd.to_datetime(brent_data['brent_date'])
+    
+    # Filtrer les données à partir de 2024
+    brent_data = brent_data[brent_data['brent_date'].dt.year >= 2024]
+    
+    brent_data = brent_data.sort_values('brent_date')
+    print(f"Données Brent chargées, triées et filtrées à partir de 2024. Shape: {brent_data.shape}")
+    return brent_data
+
+def load_fuel_data(folder_path):
+    print(f"Chargement des données de carburant depuis {folder_path}")
+    all_data = []
+    for filename in os.listdir(folder_path):
+        if filename.endswith('.csv'):
+            file_path = os.path.join(folder_path, filename)
+            df = pd.read_csv(file_path)
+            df['rate_date'] = pd.to_datetime(df['rate_date'])
+            all_data.append(df)
+    fuel_data = pd.concat(all_data, ignore_index=True)
+    fuel_data = fuel_data[~fuel_data['fuel_name'].isin(['GPLc', 'E85'])]
+    
+    # Filtrer les données à partir de 2024
+    fuel_data = fuel_data[fuel_data['rate_date'].dt.year >= 2024]
+    
+    print(f"Données de carburant chargées et filtrées à partir de 2024. Shape: {fuel_data.shape}")
+    return fuel_data
+
+
+def classify_stations(fuel_data):
+    print("Classification des stations par gamme de prix")
+    station_classifications = {}
+    fuel_types = fuel_data['fuel_name'].unique()
+
+    for fuel_type in fuel_types:
+        fuel_type_data = fuel_data[fuel_data['fuel_name'] == fuel_type]
+        station_avg_prices = fuel_type_data.groupby('id')['price'].mean().reset_index()
+        
+        thresholds = np.percentile(station_avg_prices['price'], [33, 66])
+        
+        def classify_price(price):
+            if price <= thresholds[0]:
+                return 'low-cost'
+            elif price <= thresholds[1]:
+                return 'normal'
+            else:
+                return 'premium'
+        
+        station_classifications[fuel_type] = station_avg_prices.set_index('id')['price'].apply(classify_price).to_dict()
+
+    return station_classifications
+
+def save_station_classifications(station_classifications, output_dir):
+    classification_df = pd.DataFrame(station_classifications)
+    classification_df.index.name = 'station_id'
+    classification_df.reset_index(inplace=True)
+    
+    classification_file = os.path.join(output_dir, 'station_classifications.csv')
+    classification_df.to_csv(classification_file, index=False)
+    print(f"Classifications des stations sauvegardées dans {classification_file}")
+
+class FuelPriceDataset(Dataset):
+    def __init__(self, data, sequence_length, target_days):
+        self.data = data
+        self.sequence_length = sequence_length
+        self.target_days = target_days
+        print(f"Shape of data in FuelPriceDataset: {self.data.shape}")
+
+    def __len__(self):
+        return len(self.data) - self.sequence_length - max(self.target_days)
+
+    def __getitem__(self, idx):
+        x = self.data.iloc[idx:idx+self.sequence_length].values
+        y = self.data.iloc[idx+self.sequence_length:idx+self.sequence_length+max(self.target_days)+1]['price'].values
+        y = [y[day] for day in self.target_days]
+        
+        if idx == 0:  # Print only for the first item   
+            print(f"Sample input (X) at index 0:")
+            print(x)
+            print(f"Sample output (y) at index 0:")
+            print(y)
+        
+        return torch.FloatTensor(x), torch.FloatTensor(y)
+
+class LSTMModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size):
+        super(LSTMModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size, output_size)
+        
+        # Initialisation des poids
+        for name, param in self.lstm.named_parameters():
+            if 'weight' in name:
+                init.xavier_uniform_(param)
+            elif 'bias' in name:
+                init.constant_(param, 0.0)
+        init.xavier_uniform_(self.fc.weight)
+        init.constant_(self.fc.bias, 0.0)
+
+    def forward(self, x):
+        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+        out, _ = self.lstm(x, (h0, c0))
+        out = self.fc(out[:, -1, :])
+        return out
+
+def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs, patience, output_dir, fuel_type, price_range, scaler):
+    train_losses = []
+    val_losses = []
+    best_val_loss = float('inf')
+    epochs_no_improve = 0
+
+    for epoch in range(num_epochs):
+        model.train()
+        train_loss = 0
+        for batch_x, batch_y in train_loader:
+            batch_x, batch_y = batch_x.to(device), batch_y.to(device)
+            optimizer.zero_grad()
+            outputs = model(batch_x)
+            loss = criterion(outputs, batch_y)
+            loss.backward()
+            optimizer.step()
+            train_loss += loss.item()
+        
+        model.eval()
+        val_loss = 0
+        with torch.no_grad():
+            for batch_x, batch_y in val_loader:
+                batch_x, batch_y = batch_x.to(device), batch_y.to(device)
+                outputs = model(batch_x)
+                loss = criterion(outputs, batch_y)
+                val_loss += loss.item()
+        
+        train_loss /= len(train_loader)
+        val_loss /= len(val_loader)
+        train_losses.append(train_loss)
+        val_losses.append(val_loss)
+        
+        print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}")
+
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            epochs_no_improve = 0
+            # Sauvegarder le meilleur modèle
+            torch.save(model.state_dict(), os.path.join(output_dir, f'best_model_{fuel_type}_{price_range}.pth'))
+        else:
+            epochs_no_improve += 1
+
+        if epochs_no_improve == patience:
+            print(f"Early stopping triggered after {epoch + 1} epochs")
+            break
+
+    # Charger le meilleur modèle avant de faire les prédictions finales
+    model.load_state_dict(torch.load(os.path.join(output_dir, f'best_model_{fuel_type}_{price_range}.pth')))
+    
+    # Générer le graphique et calculer les métriques
+    mse, mae, r2 = plot_predictions_vs_actual(model, val_loader, scaler, output_dir, fuel_type, price_range)
+    
+    return train_losses, val_losses, mse, mae, r2
+
+def plot_learning_curves(train_losses, val_losses, output_dir, fuel_type, price_range):
+    plt.figure(figsize=(10, 6))
+    plt.plot(train_losses, label='Train Loss')
+    plt.plot(val_losses, label='Validation Loss')
+    plt.title(f'Learning Curves - {fuel_type} - {price_range}')
+    plt.xlabel('Epochs')
+    plt.ylabel('Loss')
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+    plt.savefig(os.path.join(output_dir, f'learning_curves_{fuel_type}_{price_range}.png'))
+    plt.close()
+
+def plot_predictions_vs_actual(model, val_loader, scaler, output_dir, fuel_type, price_range):
+    model.eval()
+    predictions = []
+    actual_values = []
+
+    with torch.no_grad():
+        for batch_x, batch_y in val_loader:
+            batch_x = batch_x.to(device)
+            outputs = model(batch_x)
+            predictions.extend(outputs.cpu().numpy())
+            actual_values.extend(batch_y.numpy())
+
+    predictions = np.array(predictions)
+    actual_values = np.array(actual_values)
+
+    plt.figure(figsize=(12, 6))
+    plt.scatter(actual_values[:, 0], predictions[:, 0], alpha=0.5)
+    plt.plot([actual_values[:, 0].min(), actual_values[:, 0].max()], 
+             [actual_values[:, 0].min(), actual_values[:, 0].max()], 
+             'r--', lw=2)
+    plt.xlabel('Valeurs réelles')
+    plt.ylabel('Prédictions')
+    plt.title(f'Prédictions vs Valeurs réelles - {fuel_type} - {price_range}')
+    plt.tight_layout()
+    plt.savefig(os.path.join(output_dir, f'predictions_vs_actual_{fuel_type}_{price_range}.png'))
+    plt.close()
+
+    # Calcul des métriques
+    mse = np.mean((predictions[:, 0] - actual_values[:, 0])**2)
+    mae = np.mean(np.abs(predictions[:, 0] - actual_values[:, 0]))
+    r2 = 1 - (np.sum((actual_values[:, 0] - predictions[:, 0])**2) / 
+              np.sum((actual_values[:, 0] - np.mean(actual_values[:, 0]))**2))
+
+    print(f"MSE: {mse:.4f}")
+    print(f"MAE: {mae:.4f}")
+    print(f"R2 Score: {r2:.4f}")
+
+    return mse, mae, r2
+
+
+def prepare_data_for_fuel_type_and_range(merged_data, fuel_type, price_range, station_classifications, sequence_length, target_days):
+    print(f"Préparation des données pour {fuel_type} - {price_range}")
+    stations_in_range = [station for station, range_ in station_classifications[fuel_type].items() if range_ == price_range]
+    fuel_data = merged_data[(merged_data['fuel_name'] == fuel_type) & (merged_data['id'].isin(stations_in_range))].copy()
+    
+    # Traitement des variables temporelles
+    fuel_data['day_of_week'] = fuel_data['rate_date'].dt.dayofweek
+    fuel_data['month'] = fuel_data['rate_date'].dt.month
+    
+    # Encodage cyclique pour le jour de la semaine et le mois
+    fuel_data['day_of_week_sin'] = np.sin(2 * np.pi * fuel_data['day_of_week'] / 7)
+    fuel_data['day_of_week_cos'] = np.cos(2 * np.pi * fuel_data['day_of_week'] / 7)
+    fuel_data['month_sin'] = np.sin(2 * np.pi * fuel_data['month'] / 12)
+    fuel_data['month_cos'] = np.cos(2 * np.pi * fuel_data['month'] / 12)
+    
+    # Standardisation du prix du Brent (au lieu de normaliser)
+    scaler = StandardScaler()
+    fuel_data['brent_rate_eur_scaled'] = scaler.fit_transform(fuel_data[['brent_rate_eur']])
+    
+    # Sélection des colonnes finales
+    columns_to_use = ['price', 'brent_rate_eur_scaled', 'day_of_week_sin', 'day_of_week_cos', 'month_sin', 'month_cos']
+    fuel_data_prepared = fuel_data[columns_to_use]
+    
+    print("Statistiques des données préparées:")
+    print(fuel_data_prepared.describe())
+    
+    print("\nNombre de valeurs uniques par colonne:")
+    print(fuel_data_prepared.nunique())
+    
+    print("\nVérification des valeurs nulles:")
+    print(fuel_data_prepared.isnull().sum())
+
+    dataset = FuelPriceDataset(fuel_data_prepared, sequence_length, target_days)
+    
+    train_size = int(0.8 * len(dataset))
+    train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, len(dataset) - train_size])
+    
+    return train_dataset, val_dataset, scaler
+
+def main(args):
+    print("Début du processus principal")
+    
+    brent_data = load_brent_data(args.brent_data)
+    fuel_data = load_fuel_data(args.fuel_data)
+
+    print("Fusion des données Brent et carburant")
+    merged_data = pd.merge_asof(fuel_data.sort_values('rate_date'), 
+                                brent_data.reset_index().sort_values('brent_date'),
+                                left_on='rate_date', 
+                                right_on='brent_date',
+                                direction='nearest')
+    print(f"Données fusionnées. Shape: {merged_data.shape}")
+
+    station_classifications = classify_stations(fuel_data)
+    save_station_classifications(station_classifications, args.output_dir)
+
+    price_ranges = ['low-cost', 'normal', 'premium']
+    fuel_types = merged_data['fuel_name'].unique()
+
+    for fuel_type in fuel_types:
+        for price_range in price_ranges:
+            print(f"\nTraitement de {fuel_type} - {price_range}")
+            
+            output_dir = os.path.join(args.output_dir, fuel_type, price_range)
+            os.makedirs(output_dir, exist_ok=True)
+            
+            try:
+                train_dataset, val_dataset, scaler = prepare_data_for_fuel_type_and_range(
+                    merged_data, fuel_type, price_range, station_classifications, args.sequence_length, args.target_days
+                )
+                
+                if len(train_dataset) < args.min_train_samples:
+                    print(f"Pas assez de données pour {fuel_type} - {price_range}. Ignoré.")
+                    continue
+                
+                train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
+                val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
+                
+                print(f"Taille du dataset d'entraînement : {len(train_dataset)}")
+                print(f"Taille du dataset de validation : {len(val_dataset)}")
+                print(f"Nombre de batchs d'entraînement : {len(train_loader)}")
+                print(f"Nombre de batchs de validation : {len(val_loader)}")
+                
+                sample_x, sample_y = next(iter(train_loader))
+                input_size = sample_x.shape[2]
+                model = LSTMModel(input_size, args.hidden_size, args.num_layers, len(args.target_days)).to(device)
+                
+                criterion = nn.MSELoss()
+                optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
+
+                train_losses, val_losses, mse, mae, r2 = train_model(
+                    model, train_loader, val_loader, criterion, optimizer, 
+                    args.num_epochs, args.patience, output_dir, fuel_type, price_range, scaler
+                )
+
+                # Sauvegarder le modèle final
+                model_filename = os.path.join(output_dir, f'final_model_{fuel_type}_{price_range}.pth')
+                torch.save(model.state_dict(), model_filename)
+
+                # Sauvegarder le scaler
+                scaler_filename = os.path.join(output_dir, f'scaler_{fuel_type}_{price_range}.pkl')
+                pd.to_pickle(scaler, scaler_filename)
+
+                # Sauvegarder les paramètres du modèle
+                params = {
+                    'input_size': input_size,
+                    'hidden_size': args.hidden_size,
+                    'num_layers': args.num_layers,
+                    'output_size': len(args.target_days),
+                    'sequence_length': args.sequence_length,
+                    'target_days': args.target_days
+                }
+                params_filename = os.path.join(output_dir, f'model_params_{fuel_type}_{price_range}.json')
+                with open(params_filename, 'w') as f:
+                    json.dump(params, f)
+
+                # Sauvegarder les métriques
+                metrics = {
+                    'mse': mse,
+                    'mae': mae,
+                    'r2': r2
+                }
+                metrics_filename = os.path.join(output_dir, f'metrics_{fuel_type}_{price_range}.json')
+                with open(metrics_filename, 'w') as f:
+                    json.dump(metrics, f)
+
+                # Tracer et sauvegarder les courbes d'apprentissage
+                plot_learning_curves(train_losses, val_losses, output_dir, fuel_type, price_range)
+
+                print(f"Modèle, paramètres, métriques et graphiques pour {fuel_type} - {price_range} sauvegardés dans {output_dir}")
+
+            except Exception as e:
+                print(f"Erreur lors du traitement de {fuel_type} - {price_range}: {str(e)}")
+
+    print("Processus terminé pour tous les types de carburant et gammes de prix.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Entraînement du modèle de prédiction des prix du carburant")
+    parser.add_argument("--brent_data", type=str, required=True, help="Chemin vers le fichier de données Brent")
+    parser.add_argument("--fuel_data", type=str, required=True, help="Chemin vers le dossier contenant les données de carburant")
+    parser.add_argument("--output_dir", type=str, default="./output", help="Dossier de sortie pour les modèles et les paramètres")
+    parser.add_argument("--hidden_size", type=int, default=64, help="Taille de la couche cachée LSTM")
+    parser.add_argument("--num_layers", type=int, default=2, help="Nombre de couches LSTM")
+    parser.add_argument("--sequence_length", type=int, default=30, help="Longueur de la séquence d'entrée")
+    parser.add_argument("--target_days", nargs='+', type=int, default=[3, 7, 15, 30], help="Jours cibles pour la prédiction")
+    parser.add_argument("--batch_size", type=int, default=32, help="Taille du batch pour l'entraînement")
+    parser.add_argument("--num_epochs", type=int, default=50, help="Nombre d'époques d'entraînement")
+    parser.add_argument("--learning_rate", type=float, default=0.001, help="Taux d'apprentissage")
+    parser.add_argument("--min_train_samples", type=int, default=50, help="Nombre minimum d'échantillons d'entraînement")
+    parser.add_argument("--patience", type=int, default=5, help="Nombre d'époques sans amélioration avant l'arrêt précoce")
+
+    args = parser.parse_args()
+    
+    print(f"Arguments reçus: {args}")
+    
+    os.makedirs(args.output_dir, exist_ok=True)
+    print(f"Dossier de sortie principal créé/vérifié: {args.output_dir}")
+
+    main(args)

runningscript.py

@@ -0,0 +1,273 @@
+# Importation des bibliothèques nécessaires
+import pandas as pd
+import numpy as np
+import glob
+import os
+import joblib
+import gradio as gr
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from xgboost import XGBRegressor
+
+# 1. Chargement des données
+print("Chargement des données...")
+parquet_files = glob.glob('FuelInFranceData/*.parquet')
+
+if not parquet_files:
+    raise FileNotFoundError("Aucun fichier Parquet trouvé dans le répertoire spécifié.")
+
+df_list = []
+for f in parquet_files:
+    print(f"Chargement du fichier {f}")
+    df_list.append(pd.read_parquet(f))
+df = pd.concat(df_list, ignore_index=True)
+del df_list  # Libération de la mémoire
+print(f"Nombre total d'enregistrements: {len(df)}")
+
+# 2. Prétraitement des données
+print("Prétraitement des données...")
+df['rate_date'] = pd.to_datetime(df['rate_date'])
+df['brent_date'] = pd.to_datetime(df['brent_date'])
+df = df.sort_values('rate_date')
+df = df.dropna()
+
+# Exclure les carburants E85 et GPLc
+df = df[~df['fuel_name'].isin(['E85', 'GPLc'])]
+
+# Sélection des colonnes pertinentes
+cols_to_use = ['station_id', 'commune', 'marque', 'departement', 'regioncode',
+               'coordlatitude', 'coordlongitude', 'fuel_name', 'price',
+               'rate_date', 'brent_rate_eur', 'brent_date']
+df = df[cols_to_use]
+
+# Encodage des variables catégorielles
+print("Encodage des variables catégorielles...")
+label_encoders = {}
+categorical_cols = ['station_id', 'commune', 'marque', 'departement',
+                    'regioncode', 'fuel_name']
+
+for col in categorical_cols:
+    le = LabelEncoder()
+    df[col] = le.fit_transform(df[col].astype(str))
+    label_encoders[col] = le
+
+# Création des mappings pour les communes et les départements
+commune_mapping = pd.DataFrame({
+    'commune_encoded': np.arange(len(label_encoders['commune'].classes_)),
+    'commune_decoded': label_encoders['commune'].classes_
+})
+
+departement_mapping = pd.DataFrame({
+    'departement_encoded': np.arange(len(label_encoders['departement'].classes_)),
+    'departement_decoded': label_encoders['departement'].classes_
+})
+
+# Obtenir les types de carburant uniques
+fuel_types = label_encoders['fuel_name'].classes_.tolist()
+
+# Obtenir les départements uniques
+departments = label_encoders['departement'].classes_.tolist()
+
+# Fonction pour mettre à jour la liste des stations
+def update_stations(commune_input, departments):
+    if commune_input:
+        # Recherche insensible à la casse avec correspondance partielle
+        matching_communes = commune_mapping[commune_mapping['commune_decoded'].str.contains(commune_input, case=False, na=False)]
+        if matching_communes.empty:
+            return gr.update(choices=[], value=None)
+        commune_encoded_values = matching_communes['commune_encoded'].values
+        # Filtrer les stations par les communes correspondantes
+        filtered_df = df[df['commune'].isin(commune_encoded_values)]
+    elif departments:
+        # Vérifier si les départements existent
+        valid_departments = [dept for dept in departments if dept in label_encoders['departement'].classes_]
+        if not valid_departments:
+            return gr.update(choices=[], value=None)
+        # Filtrer les stations par départements
+        departments_encoded = label_encoders['departement'].transform(valid_departments)
+        filtered_df = df[df['departement'].isin(departments_encoded)]
+    else:
+        # Si aucun filtre, afficher toutes les stations
+        filtered_df = df.copy()
+    
+    if filtered_df.empty:
+        return gr.update(choices=[], value=None)
+    
+    # Obtenir les informations des stations uniques
+    station_info = filtered_df[['station_id', 'commune', 'marque']].drop_duplicates()
+    
+    # Décoder les valeurs encodées
+    station_info['station_id_decoded'] = label_encoders['station_id'].inverse_transform(station_info['station_id'])
+    station_info['commune_decoded'] = label_encoders['commune'].inverse_transform(station_info['commune'])
+    station_info['marque_decoded'] = label_encoders['marque'].inverse_transform(station_info['marque'])
+    
+    # Construire les chaînes d'affichage
+    station_info['station_display'] = station_info.apply(
+        lambda row: f"{row['commune_decoded']} - {row['marque_decoded']} ({row['station_id_decoded']})",
+        axis=1
+    )
+    
+    # Construire les choix sous forme de tuples (affichage, valeur)
+    station_choices = list(zip(station_info['station_display'], station_info['station_id_decoded']))
+    
+    return gr.update(choices=station_choices, value=None)
+
+# Fonction pour effectuer les prévisions
+def forecast_prices(model, last_known_data, scaler, required_columns, brent_price, horizons=[3, 7, 15, 30]):
+    forecasts = {}
+    for horizon in horizons:
+        future_date = last_known_data['rate_date'] + pd.Timedelta(days=horizon)
+        input_data = last_known_data.copy()
+        input_data['rate_date'] = future_date
+        input_data['day_of_week'] = future_date.dayofweek
+        input_data['month'] = future_date.month
+        input_data['year'] = future_date.year
+
+        # Mise à jour des variables de décalage du Brent
+        for lag in [1, 3, 7, 15, 30]:
+            input_data[f'brent_rate_eur_lag_{lag}'] = brent_price
+
+        # Préparation des features
+        input_features = input_data.drop(['price', 'rate_date', 'brent_date'])
+        input_features = input_features.to_frame().T
+
+        # S'assurer que toutes les colonnes sont présentes
+        missing_cols = set(required_columns) - set(input_features.columns)
+        for col in missing_cols:
+            input_features[col] = 0
+
+        input_features = input_features[required_columns]
+
+        # Mise à l'échelle des features
+        input_features_scaled = scaler.transform(input_features)
+        predicted_price = model.predict(input_features_scaled)
+        forecasts[horizon] = predicted_price[0]
+    return forecasts
+
+# Fonction principale pour obtenir les prédictions
+def get_predictions(station_selection, fuel_types_selected, brent_price, commune_input, departments):
+    if not station_selection or not fuel_types_selected:
+        return "Veuillez sélectionner une station et au moins un type de carburant."
+    
+    results = ""
+    
+    # station_selection est l'ID décodé de la station
+    station_id = station_selection
+    if station_id not in label_encoders['station_id'].classes_:
+        return f"Station ID {station_id} non trouvé dans les données."
+    station_id_encoded = label_encoders['station_id'].transform([station_id])[0]
+    
+    for fuel_type in fuel_types_selected:
+        # Charger le modèle et le scaler pour le type de carburant
+        model_filename = f'fuel_price_model_{fuel_type}.pkl'
+        scaler_filename = f'scaler_{fuel_type}.pkl'
+        
+        if not os.path.exists(model_filename) or not os.path.exists(scaler_filename):
+            results += f"\nModèle ou scaler pour le carburant {fuel_type} non trouvé."
+            continue
+        
+        model = joblib.load(model_filename)
+        scaler = joblib.load(scaler_filename)
+        
+        # Obtenir les 5 derniers prix
+        fuel_name_encoded = label_encoders['fuel_name'].transform([fuel_type])[0]
+        
+        df_station_fuel = df[(df['station_id'] == station_id_encoded) & (df['fuel_name'] == fuel_name_encoded)]
+        df_station_fuel = df_station_fuel.sort_values('rate_date', ascending=False)
+        
+        if df_station_fuel.empty:
+            results += f"\nAucune donnée trouvée pour la station {station_id} et le carburant {fuel_type}."
+            continue
+        
+        last_5_prices = df_station_fuel.head(5)[['rate_date', 'price']]
+        last_5_prices['rate_date'] = last_5_prices['rate_date'].dt.strftime('%Y-%m-%d %H:%M:%S')
+        results += f"\n\nType de carburant : {fuel_type}\nLes 5 derniers prix :\n{last_5_prices.to_string(index=False)}"
+        
+        # Préparation des données pour la prédiction
+        last_known_data = df_station_fuel.iloc[0].copy()
+        last_known_data['brent_rate_eur'] = brent_price
+        
+        # Recréer les features utilisées lors de l'entraînement
+        df_fuel = df[df['fuel_name'] == fuel_name_encoded].copy()
+        
+        # Ingénierie des caractéristiques
+        df_fuel['day_of_week'] = df_fuel['rate_date'].dt.dayofweek
+        df_fuel['month'] = df_fuel['rate_date'].dt.month
+        df_fuel['year'] = df_fuel['rate_date'].dt.year
+        
+        for lag in [1, 3, 7, 15, 30]:
+            df_fuel[f'brent_rate_eur_lag_{lag}'] = df_fuel['brent_rate_eur'].shift(lag)
+        df_fuel = df_fuel.dropna()
+        
+        X = df_fuel.drop(['price', 'rate_date', 'brent_date'], axis=1)
+        required_columns = X.columns.tolist()
+        
+        # Prévisions
+        forecasts = forecast_prices(model, last_known_data, scaler, required_columns, brent_price)
+        
+        results += "\nPrévisions :\n"
+        for horizon, price in forecasts.items():
+            results += f"Dans {horizon} jours : {price:.4f} €\n"
+    
+    return results
+
+# 7. Construction de l'Interface Gradio
+with gr.Blocks() as demo:
+    gr.Markdown("# Prédiction du Prix des Carburants")
+    
+    with gr.Row():
+        fuel_type_checkbox = gr.CheckboxGroup(
+            choices=fuel_types,
+            label="Sélectionnez les types de carburant",
+            value=fuel_types  # Tous sélectionnés par défaut
+        )
+    
+    with gr.Row():
+        commune_input = gr.Textbox(
+            label="Entrez la commune",
+            placeholder="Tapez le nom de la commune..."
+        )
+        department_dropdown = gr.Dropdown(
+            choices=departments,
+            label="Sélectionnez le(s) département(s)",
+            multiselect=True
+        )
+    
+    station_dropdown = gr.Dropdown(
+        choices=[],
+        label="Sélectionnez la station"
+    )
+    
+    # Mettre à jour la liste des stations lorsque la commune ou le département change
+    def update_stations_wrapper(commune, departments):
+        return update_stations(commune, departments)
+    
+    commune_input.change(
+        fn=update_stations_wrapper,
+        inputs=[commune_input, department_dropdown],
+        outputs=station_dropdown
+    )
+    department_dropdown.change(
+        fn=update_stations_wrapper,
+        inputs=[commune_input, department_dropdown],
+        outputs=station_dropdown
+    )
+    
+    brent_price_input = gr.Number(
+        label="Entrez le cours du Brent (€)",
+        value=70.0
+    )
+    
+    predict_button = gr.Button("Prédire")
+    
+    output = gr.Textbox(label="Résultats")
+    
+    def on_predict_click(station_selection, fuel_types_selected, brent_price, commune_input, departments):
+        return get_predictions(station_selection, fuel_types_selected, brent_price, commune_input, departments)
+    
+    predict_button.click(
+        fn=on_predict_click,
+        inputs=[station_dropdown, fuel_type_checkbox, brent_price_input, commune_input, department_dropdown],
+        outputs=output
+    )
+
+demo.launch(share=True)

scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30e5e37b6791b83db7bc3ec6ce5385ec4b85bd5c1d8545e465c89e4a1caa414a
+size 1959

scaler_E10.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:163f18a61e413e08dae4c57a509f14caadf11dab483b94b139abede236e27c64
+size 1631

scaler_Gazole.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1947942b4c27af30944d8e4b0e5f9dd3e7837b4a1289d62f635d35e5f99d365a
+size 1631

scaler_SP95.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4af1fa4538632ee67d0cd4e5fab37ae7e60458950b023dc0e0f7273a14afcc
+size 1631

scaler_SP98.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97859428b468552eee47ec5ca923c790ffa0fd50e83d0cb0529e351b10aba9f
+size 1631

trainingscript.py

@@ -0,0 +1,224 @@
+# Importation des bibliothèques nécessaires
+import pandas as pd
+import numpy as np
+import glob
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.metrics import mean_absolute_error, mean_squared_error
+from sklearn.model_selection import TimeSeriesSplit
+from xgboost import XGBRegressor
+import joblib
+import os
+
+# Ajustement du paramètre agg.path.chunksize pour éviter l'OverflowError
+mpl.rcParams['agg.path.chunksize'] = 10000  # Vous pouvez ajuster la valeur si nécessaire
+
+# 1. Chargement des données
+print("Chargement des données...")
+parquet_files = glob.glob('FuelInFranceData/*.parquet')
+
+if not parquet_files:
+    raise FileNotFoundError("Aucun fichier Parquet trouvé dans le répertoire spécifié.")
+
+df_list = []
+for f in parquet_files:
+    print(f"Chargement du fichier {f}")
+    df_list.append(pd.read_parquet(f))
+df = pd.concat(df_list, ignore_index=True)
+del df_list  # Libération de la mémoire
+print(f"Nombre total d'enregistrements: {len(df)}")
+
+# 2. Prétraitement des données
+print("Prétraitement des données...")
+df['rate_date'] = pd.to_datetime(df['rate_date'])
+df['brent_date'] = pd.to_datetime(df['brent_date'])
+df = df.sort_values('rate_date')
+df = df.dropna()
+
+# Exclure les carburants E85 et GPLc
+df = df[~df['fuel_name'].isin(['E85', 'GPLc'])]
+
+# Sélection des colonnes pertinentes (inclure 'brent_date')
+cols_to_use = ['station_id', 'commune', 'marque', 'departement', 'regioncode',
+               'coordlatitude', 'coordlongitude', 'fuel_name', 'price',
+               'rate_date', 'brent_rate_eur', 'brent_date']
+
+df = df[cols_to_use]
+
+# Encodage des variables catégorielles
+print("Encodage des variables catégorielles...")
+label_encoders = {}
+categorical_cols = ['station_id', 'commune', 'marque', 'departement',
+                    'regioncode', 'fuel_name']
+
+for col in categorical_cols:
+    le = LabelEncoder()
+    df[col] = le.fit_transform(df[col].astype(str))
+    label_encoders[col] = le
+
+# 3. Nettoyage des valeurs aberrantes (outliers)
+print("Nettoyage des valeurs aberrantes...")
+# Suppression des outliers en utilisant l'IQR (Interquartile Range)
+def remove_outliers_iqr(data, column):
+    Q1 = data[column].quantile(0.25)
+    Q3 = data[column].quantile(0.75)
+    IQR = Q3 - Q1
+    lower_bound = Q1 - 1.5 * IQR
+    upper_bound = Q3 + 1.5 * IQR
+    data_clean = data[(data[column] >= lower_bound) & (data[column] <= upper_bound)]
+    return data_clean
+
+df = remove_outliers_iqr(df, 'price')
+
+# 4. Entraînement de modèles séparés pour chaque type de carburant avec validation croisée temporelle
+print("Entraînement de modèles séparés pour chaque type de carburant avec validation croisée temporelle...")
+fuel_types = df['fuel_name'].unique()
+models = {}
+scalers = {}
+results = {}
+
+for fuel in fuel_types:
+    fuel_name_decoded = label_encoders['fuel_name'].inverse_transform([fuel])[0]
+    print(f"\nTraitement du carburant: {fuel_name_decoded}")
+    df_fuel = df[df['fuel_name'] == fuel].copy()
+
+    # Ingénierie des caractéristiques
+    df_fuel['day_of_week'] = df_fuel['rate_date'].dt.dayofweek
+    df_fuel['month'] = df_fuel['rate_date'].dt.month
+    df_fuel['year'] = df_fuel['rate_date'].dt.year
+
+    # Création des variables de décalage (lags) pour le prix du Brent
+    for lag in [1, 3, 7, 15, 30]:
+        df_fuel[f'brent_rate_eur_lag_{lag}'] = df_fuel['brent_rate_eur'].shift(lag)
+    df_fuel = df_fuel.dropna()
+
+    # Variables features et target
+    X = df_fuel.drop(['price', 'rate_date', 'brent_date'], axis=1)
+    y = df_fuel['price']
+    dates = df_fuel['rate_date']
+
+    # Normalisation des données
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X)
+
+    # Validation croisée temporelle
+    tscv = TimeSeriesSplit(n_splits=5)
+    y_tests = []
+    y_preds = []
+    dates_list = []
+
+    for fold, (train_index, test_index) in enumerate(tscv.split(X_scaled)):
+        print(f"  Fold {fold+1}/{tscv.get_n_splits()}")
+        X_train, X_test = X_scaled[train_index], X_scaled[test_index]
+        y_train, y_test = y.iloc[train_index], y.iloc[test_index]
+        dates_test = dates.iloc[test_index]
+
+        # Entraînement du modèle
+        model = XGBRegressor(objective='reg:squarederror', n_estimators=100, learning_rate=0.1)
+        model.fit(X_train, y_train)
+
+        # Prédiction sur l'ensemble de test
+        y_pred = model.predict(X_test)
+
+        # Stockage des résultats
+        y_tests.append(y_test)
+        y_preds.append(y_pred)
+        dates_list.append(dates_test)
+
+    # Concaténation des résultats
+    y_test_total = pd.concat(y_tests)
+    y_pred_total = np.concatenate(y_preds)
+    dates_total = pd.concat(dates_list)
+
+    # Évaluation du modèle
+    mae = mean_absolute_error(y_test_total, y_pred_total)
+    rmse = mean_squared_error(y_test_total, y_pred_total, squared=False)
+    print(f"Erreur Absolue Moyenne (MAE): {mae:.4f}")
+    print(f"Racine de l'Erreur Quadratique Moyenne (RMSE): {rmse:.4f}")
+
+    # Entraînement final sur l'ensemble des données pour la prévision future
+    model_final = XGBRegressor(objective='reg:squarederror', n_estimators=100, learning_rate=0.1)
+    model_final.fit(X_scaled, y)
+
+    # Sauvegarde du modèle final et du scaler
+    models[fuel] = model_final
+    scalers[fuel] = scaler
+
+    # Stockage des résultats pour l'analyse
+    results[fuel] = {
+        'y_test': y_test_total,
+        'y_pred': y_pred_total,
+        'dates': dates_total
+    }
+
+    # Sous-échantillonnage des données pour le tracé
+    downsample_rate = max(1, len(dates_total) // 1000)  # Limiter à 1000 points
+    dates_sampled = dates_total.iloc[::downsample_rate]
+    y_test_sampled = y_test_total.iloc[::downsample_rate]
+    y_pred_sampled = y_pred_total[::downsample_rate]
+
+    # Tri des données pour le tracé
+    sorted_indices = np.argsort(dates_sampled)
+    dates_sampled = dates_sampled.iloc[sorted_indices]
+    y_test_sampled = y_test_sampled.iloc[sorted_indices]
+    y_pred_sampled = y_pred_sampled[sorted_indices]
+
+    # Graphique de validation (Prix prédit vs Prix réel)
+    plt.figure(figsize=(12, 6))
+    plt.plot(dates_sampled, y_test_sampled, label='Prix Réel', marker='o', linestyle='None', markersize=4)
+    plt.plot(dates_sampled, y_pred_sampled, label='Prix Prédit', marker='x', linestyle='None', markersize=4)
+    plt.xlabel('Date')
+    plt.ylabel('Prix (€)')
+    plt.title(f"Comparaison du Prix Réel et Prédit pour le Carburant {fuel_name_decoded}")
+    plt.legend()
+    plt.tight_layout()
+    # Enregistrer le graphique
+    plt.savefig(f'validation_{fuel_name_decoded}.png')
+    plt.close()
+
+# 5. Prévision pour les 3, 7, 15 et 30 prochains jours pour chaque carburant
+print("\nPrévision pour les prochains jours pour chaque carburant...")
+for fuel in fuel_types:
+    fuel_name_decoded = label_encoders['fuel_name'].inverse_transform([fuel])[0]
+    print(f"\nPrévisions pour le carburant: {fuel_name_decoded}")
+    df_fuel = df[df['fuel_name'] == fuel]
+    last_known_data = df_fuel.iloc[-1]
+    model = models[fuel]
+    scaler = scalers[fuel]
+
+    def forecast_prices(model, last_known_data, scaler, horizons=[3, 7, 15, 30]):
+        forecasts = {}
+        for horizon in horizons:
+            future_date = last_known_data['rate_date'] + pd.Timedelta(days=horizon)
+            input_data = last_known_data.to_frame().T.copy()
+            input_data['rate_date'] = future_date
+            input_data['day_of_week'] = future_date.dayofweek
+            input_data['month'] = future_date.month
+            input_data['year'] = future_date.year
+
+            # Mise à jour des variables de décalage du Brent
+            for lag in [1, 3, 7, 15, 30]:
+                input_data[f'brent_rate_eur_lag_{lag}'] = last_known_data['brent_rate_eur']
+            input_data = input_data.dropna(axis=1, how='all')
+
+            # Préparation des données pour la prédiction
+            input_features = input_data.drop(['price', 'rate_date', 'brent_date'], axis=1)
+            input_features_scaled = scaler.transform(input_features)
+            predicted_price = model.predict(input_features_scaled)
+            forecasts[horizon] = predicted_price[0]
+        return forecasts
+
+    # Prévision des prix
+    forecasts = forecast_prices(model, last_known_data, scaler)
+    for horizon, price in forecasts.items():
+        print(f"Dans {horizon} jours: {price:.4f} €")
+
+# 6. Sauvegarde des modèles et des scalers pour une utilisation future
+print("\nSauvegarde des modèles et des scalers...")
+for fuel in fuel_types:
+    fuel_name_decoded = label_encoders['fuel_name'].inverse_transform([fuel])[0]
+    joblib.dump(models[fuel], f'fuel_price_model_{fuel_name_decoded}.pkl')
+    joblib.dump(scalers[fuel], f'scaler_{fuel_name_decoded}.pkl')
+
+print("Script terminé avec succès.")