vorrei che mi spiegassi in che modo e dove dovrei aggiungere questa sezione di codice:
import shap

# Dopo aver addestrato il tuo modello
explainer = shap.LinearExplainer(adapted_model, embeddings_tensor.cpu().numpy())
shap_values = explainer.shap_values(embeddings_tensor.cpu().numpy())

# Visualizza l'importanza media dei concetti
shap.summary_plot(shap_values, features=embeddings_tensor.cpu().numpy(), feature_names=concept_labels)
poiché nel codice in cui utilizzo quello di cui abbiamo parlato, l'unico modello che ho creato è una piccola rete neurale, come puoi leggere dal codice aggiornato con i consigli che mi hai dato
import torch
from torch import nn
import numpy as np
from FlagEmbedding import BGEM3FlagModel
import pandas as pd
from sklearn.model_selection import train_test_split
import os
from collections import Counter
import re
import os
import nltk
from nltk.corpus import stopwords
from tqdm import tqdm
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
import argparse
import importlib
import conceptSHAP.conceptNet
importlib.reload(conceptSHAP.conceptNet)
from conceptSHAP.ConceptNet_correzioni import ConceptNet
import random

plot_shap = True

def ensure_output_folder(folder_name='output_graphs'):
    if not os.path.exists(folder_name):
        os.makedirs(folder_name)
    return folder_name
output_folder = ensure_output_folder()
            
            
def analyze_word_distribution(df):
    all_words = ' '.join(df['cleaned_description']).split()
    word_counts = Counter(all_words)
    print("Distribuzione delle parole nell'intero dataset:")
    print(word_counts.most_common(20))
    print("\nNumero totale di parole uniche:", len(word_counts))
    

def visualize_embeddings_and_concepts(embeddings, concepts, output_folder):
    pca = PCA(n_components=2)
    embeddings_2d = pca.fit_transform(embeddings)
    concepts_2d = pca.transform(concepts.T)

    plt.figure(figsize=(12, 8))
    plt.scatter(embeddings_2d[:, 0], embeddings_2d[:, 1], alpha=0.1)
    plt.scatter(concepts_2d[:, 0], concepts_2d[:, 1], c='red', s=100, marker='x')
    for i, concept in enumerate(concepts_2d):
        plt.annotate(f'C{i+1}', (concept[0], concept[1]))
    plt.title('Embeddings and Concepts Visualization')
    plt.savefig(os.path.join(output_folder, 'embeddings_and_concepts.png'))
    plt.show()

# Scaricare il pacchetto stopwords di NLTK
nltk.download('stopwords', quiet=True)

# Definire le lingue desiderate
languages = ['italian', 'english', 'french', 'german', 'spanish']

# Creare un set di stopwords multilingue
stop_words_multi = set()
for lang in languages:
    try:
        stop_words_multi.update(stopwords.words(lang))
        print(f'Abbiamo caricato le stopwords della lingua {lang}')
    except OSError:
        print(f"Avviso: stopwords non disponibili per la lingua '{lang}'")

# Aggiungere stopwords personalizzate
custom_stopwords = {
    'di', 'e', 'in', 'a', 'per', 'del', 'il', 'la', 'con', 'un', 'al', 'i', 'dei', 'alla', 'all','nel','all','all\'','nel\'',
    'nel', 'della', 'delle', 'su', 'ed', 'd', 'ai', 'da', 'dal', 'ad', 'degli', 'alle', 'sul', 'and', 
    'the', 'of', 'to', 'us','l\'','a','d','n', 'linkedin', 'etjca','conoscenza',
    'italia', 'it', 'ue','candidatura','tempo determinato','tempo indeterminato','determinato','indeterminato',
    'lgs', 'informativa', 'dati', 'privacy', 'occuperà', 'sensi', 'aut', 'dlgs','esperienza','ricerca','tempo'
    'orario', 'lunedì', 'time', 'turni','martedì','mercoledì','giovedì','venerdì','sabato',
    'www', 'span', 'p', 'n', 'linkedin', 'etjca', 's', 'spa', 'br', 'job','candidati',
    'randstad', 'gi group','manpower','lavoropiù','gigroup', 'adecco','synergie', 'gi', 'group','lavoro','azienda','settore',
    'esperieza','contratto'
}
stop_words_multi.update(custom_stopwords)

def remove_stopwords(text):
    text = text.lower()
    text = re.sub(r'\d+','',text)
    text = re.sub(r'[^\w\s]', ' ', text)
    words = text.split()
    cleaned_words = [word for word in words if word not in stop_words_multi and len(word) > 2]
    return ' '.join(cleaned_words)

def generate_concept_labels(concept_words):
    labels = []
    for concept in concept_words:
        top_words = [word for word, _ in concept if word not in stop_words_multi][:3]
        label = " ".join(top_words)
        labels.append(label)
    return labels

def save_concepts(concept_model):
    concepts = concept_model.concept.detach().cpu().numpy()
    os.makedirs('conceptSHAP', exist_ok=True)
    np.save('conceptSHAP/concepts.npy', concepts)

def concept_analysis(train_embeddings, train_data):
    concepts = np.load('conceptSHAP/concepts.npy')
    concepts = torch.from_numpy(concepts.T)
    train_embeddings = torch.from_numpy(train_embeddings)

    concept_words = []
    for i, concept in enumerate(concepts):
        distance = torch.norm(train_embeddings - concept, dim=1)
        k = min(150, len(distance))
        knn = distance.topk(k, largest=False).indices

        words = []
        for idx in knn:
            words += train_data.iloc[int(idx)]['cleaned_description'].split()

        cx = Counter(words)
        most_occur = cx.most_common(25)
        concept_words.append(most_occur)
        print(f"Concept {i+1} most common words:")
        print(most_occur)
        
        print(f"\nNumero di parole uniche in questo concetto: {len(cx)}")
        print(f"Indici dei documenti più vicini a questo concetto: {knn.tolist()[:10]}")
        print("\nPrimi 3 documenti più vicini a questo concetto:")
        for j in range(3):
            print(f"Documento {j+1}: {train_data.iloc[int(knn[j])]['cleaned_description'][:100]}...")
        print("\n" + "="*50 + "\n")

    return concept_words

# Carica il dataset
dfi = pd.read_csv("/kaggle/input/jobs-10-labels/10_professions.csv")
train, df = train_test_split(dfi, test_size=0.01, random_state=15)

# Preprocess del testo
df['cleaned_description'] = df['description'].apply(remove_stopwords)
print('abbiamo fatto pulizia (di stopwords)')

device = ("cuda" if torch.cuda.is_available() else 'cpu')
print(device)

# Inizializza il modello BGE-M3
bge_model = BGEM3FlagModel('BAAI/bge-m3', use_fp16=True)
print('abbiamo inizializzato il modello')
# Genera gli embedding
descriptions = df['cleaned_description'].tolist()
print('embeddiamo')
embeddings = bge_model.encode(descriptions, batch_size=16, max_length=512)['dense_vecs']

# Standandizza gli embeddings
scaler = StandardScaler()
nomalizized_embeddings = scaler.fit_transform(embeddings)

# Estrai le etichette dal DataFrame
unique_labels = df['found_label_5d'].unique()
label_to_index = {label: index for index, label in enumerate(unique_labels)}
label_indices = df['found_label_5d'].map(label_to_index).values
labels_tensor = torch.tensor(label_indices).long().to(device)

# Converti gli embedding in un tensor PyTorch
embeddings_tensor = torch.tensor(nomalizized_embeddings).float().to(device)

# Definisci una semplice rete neurale come modello adattato
class AdaptedModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super().__init__()
        self.fc = nn.Linear(input_dim, output_dim)

    def forward(self, x):
        return self.fc(x)

    @property
    def weight(self):
        return self.fc.weight

    @property
    def bias(self):
        return self.fc.bias

# Inizializza il modello adattato
input_dim = embeddings_tensor.shape[1]
output_dim = len(df['found_label_5d'].unique())
adapted_model = AdaptedModel(input_dim, output_dim).to(device)

# Inizializza ConceptNet
n_concepts = 10  # o il numero desiderato di concetti
num_classes = len(unique_labels)
concept_model = ConceptNet(n_concepts, embeddings_tensor, num_classes, bge_model, descriptions).to(device)

# Ottimizzatore
optimizer = torch.optim.Adam(concept_model.parameters(), lr=0.001)

l_1 = 0.001  # puoi regolare questo valore
l_2 = 0.002 

# Training loop
topk = 10
for epoch in range(100):
    optimizer.zero_grad()
    completeness, conceptSHAP, loss, pred_loss, L_sparse_1_new, L_sparse_2_new,metrics, concept_diversity = concept_model.loss(
        embeddings_tensor, labels_tensor, adapted_model, True, False, l_1, l_2, topk)

    loss.backward()
    optimizer.step()

    if epoch % 10 == 0:
        print(f"Epoch {epoch}, Loss: {loss.item()}, Concept Diversity: {concept_diversity}")

# Salva i concetti
save_concepts(concept_model)

# Dopo aver caricato e preprocessato il dataset
analyze_word_distribution(df)

# Dopo aver generato gli embedding e addestrato il modello
concept_words = concept_analysis(embeddings, df)

# Genera le etichette dei concetti
concept_labels = generate_concept_labels(concept_words)

# Estrai i concetti
concepts = concept_model.concept.detach().cpu().numpy()

# Analizza i risultati
if isinstance(conceptSHAP, torch.Tensor):
    conceptSHAP_scores = conceptSHAP.detach().cpu().numpy()
    for i, (concept, score, label) in enumerate(zip(concepts, conceptSHAP_scores, concept_labels)):
        print(f"Concept {i+1}: {label}")
        #print(f"ConceptSHAP score = {score}")
        print()
        
# Usa questa funzione dopo aver addestrato il modello
visualize_embeddings_and_concepts(nomalizized_embeddings, concept_model.concept.detach().cpu().numpy(), output_folder)

# questa sezione dovrebbe aggiungere dei grafici SHAP-like per i concetti
import shap
import matplotlib.pyplot as plt

def plot_concept_importance(conceptSHAP, concept_labels, output_folder):
    # Crea un DataFrame con i concetti e i loro valori SHAP
    concept_df = pd.DataFrame({
        'Concept': concept_labels,
        'SHAP Value': conceptSHAP
    })

    # Ordina i concetti per valore SHAP
    concept_df = concept_df.sort_values('SHAP Value', ascending=True)

    # Crea il grafico a barre orizzontali
    plt.figure(figsize=(10, 6))
    plt.barh(concept_df['Concept'], concept_df['SHAP Value'], color='skyblue')
    plt.xlabel('SHAP Value')
    plt.title('ConceptSHAP Values for Concepts')
    plt.tight_layout()
    plt.savefig(os.path.join(output_folder, 'concept_shap_values.png'))
    plt.show()    
    
if plot_shap:  # Aggiungi un argomento per controllare se eseguire questa parte
    plot_concept_importance(concept_model, embeddings, label_indices, output_folder)
tuttavia non ho aggiungo il codice che hai scritto nel punto 3 poiché non sapevo dove aggiungerlo. Puoi mostrarmi il codice come per dovrebbe essere aggiornato per intero e non solo le sezioni che mi hai consigliato di aggingere?
Ora ti mostro anche la nuova versione di conceptNet, dimmi se sono necessarie modifiche:
import torch
import torch.nn as nn
from itertools import chain, combinations
import numpy as np
import math
import torch.nn.functional as F
import itertools

class ConceptNet(nn.Module):
    def __init__(self, n_concepts, train_embeddings, num_classes, bge_model, original_texts):
        super(ConceptNet, self).__init__()
        embedding_dim = train_embeddings.shape[1]
        self.concept = nn.Parameter(torch.randn(embedding_dim, n_concepts))
        self.n_concepts = n_concepts
        self.train_embeddings = train_embeddings.T
        self.num_classes = num_classes
        self.bge_model = bge_model
        self.original_texts = original_texts
        self.linear = nn.Linear(embedding_dim, num_classes)

    def init_concept(self, embedding_dim, n_concepts):
        r_1 = -0.5
        r_2 = 0.5
        concept = (r_2 - r_1) * torch.rand(embedding_dim, n_concepts) + r_1
        return concept
    def get_top_words_for_concept(self, concept_idx, train_embedding, topk):
        concept = self.concept[:, concept_idx]
        
        # Controlla se train_embedding è un singolo embedding o un batch
        if train_embedding.dim() == 1:
            train_embedding = train_embedding.unsqueeze(0)
        
        similarities = torch.mm(train_embedding, concept.unsqueeze(1)).squeeze()
        
        # Se abbiamo un batch, prendiamo la media delle similarità
        if similarities.dim() > 1:
            similarities = similarities.mean(dim=0)
        
        top_indices = torch.topk(similarities, min(topk, len(similarities))).indices
        
        # Usa i testi originali invece degli embedding
        top_texts = [self.original_texts[idx] for idx in top_indices]
        
        # Tokenizza i testi più simili
        tokenizer = self.bge_model.tokenizer
        all_tokens = [token for text in top_texts for token in tokenizer.tokenize(text)]
        
        # Prendi le prime topk parole uniche
        unique_tokens = list(dict.fromkeys(all_tokens))
        return unique_tokens[:topk]

    def forward(self, train_embedding, h_x, topk):
        print(f"train_embedding shape: {train_embedding.shape}")
        print(f"self.train_embeddings shape: {self.train_embeddings.shape}")
        print(f"self.concept shape: {self.concept.shape}")
        # calculating projection of train_embedding onto the concept vector space
        proj_matrix = (self.concept @ torch.inverse((self.concept.T @ self.concept))) \
                      @ self.concept.T # (embedding_dim x embedding_dim)
        proj = proj_matrix @ train_embedding.T  # (embedding_dim x batch_size)
    
        # passing projected activations through rest of model
        y_pred = self.linear(proj.T)
    
        orig_pred = self.linear(train_embedding)
    
        # Calculate the regularization terms as in new version of paper
        k = topk # this is a tunable parameter
    
        ### calculate first regularization term, to be maximized
        # 1. find the top k nearest neighbour
        all_concept_knns = []
        for concept_idx in range(self.n_concepts):
            c = self.concept[:, concept_idx].unsqueeze(dim=1) # (activation_dim, 1)
    
            # euc dist
            distance = torch.norm(self.train_embeddings - c, dim=0) # (num_total_activations)
            knn = distance.topk(k, largest=False)
            indices = knn.indices # (k)
            knn_activations = self.train_embeddings[:, indices] # (activation_dim, k)
            all_concept_knns.append(knn_activations)


        # 2. calculate the avg dot product for each concept with each of its knn
        L_sparse_1_new = 0.0
        for concept_idx in range(self.n_concepts):
            c = self.concept[:, concept_idx].unsqueeze(dim=1) # (activation_dim, 1)
            c_knn = all_concept_knns[concept_idx] # knn for c
            dot_prod = torch.sum(c * c_knn) / k # avg dot product on knn
            L_sparse_1_new += dot_prod
        L_sparse_1_new = L_sparse_1_new / self.n_concepts

        ### calculate Second regularization term, to be minimized
        all_concept_dot = self.concept.T @ self.concept
        mask = torch.eye(self.n_concepts).cuda() * -1 + 1 # mask the i==j positions
        L_sparse_2_new = torch.mean(all_concept_dot * mask)

        norm_metrics = torch.mean(all_concept_dot * torch.eye(self.n_concepts).cuda())
        similarity_penality = torch.mean(torch.abs(torch.matmul(self.concept.T, self.concept) - torch.eye(self.n_concepts).to(self.concept.device)))

        metrics = [norm_metrics, similarity_penality]
        
        return orig_pred, y_pred, L_sparse_1_new, L_sparse_2_new, metrics

    
    def calculate_concept_diversity(self):
        all_words = set()
        concept_words = []
    
        for i in range(self.n_concepts):
            top_words = self.get_top_words_for_concept(i, self.train_embeddings.T, 25)
            concept_words.append([(word, 1) for word in top_words])  # Simuliamo il conteggio
            all_words.update(top_words)
        
        unique_words = 0
        for concept in concept_words:
            concept_words_set = set([word for word, _ in concept])
            unique_words += len(concept_words_set - (all_words - concept_words_set))
        
        diversity = unique_words / self.n_concepts
        return diversity
    def powerset(self, iterable):
        "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"
        s = list(iterable)
        return list(itertools.chain.from_iterable(itertools.combinations(s, r) for r in range(len(s)+1)))
        
    def proj_and_predict(self, concept, train_embedding):
        proj_matrix = (concept @ torch.inverse((concept.T @ concept))) @ concept.T
        proj = proj_matrix @ train_embedding.T
        y_pred = self.linear(proj.T)
        return y_pred
    
    def loss(self, train_embedding, train_y_true, h_x, regularize, doConceptSHAP, l_1, l_2, topk):
        orig_pred, y_pred, L_sparse_1_new, L_sparse_2_new, metrics = self.forward(train_embedding, h_x, topk)
        
        ce_loss = nn.CrossEntropyLoss()
        loss_new = ce_loss(y_pred, train_y_true)
        pred_loss = torch.mean(loss_new)
    
        # completeness score
        def n(y_pred):
            orig_correct = torch.sum(train_y_true == torch.argmax(orig_pred, axis=1))
            new_correct = torch.sum(train_y_true == torch.argmax(y_pred, axis=1))
            return torch.div(new_correct - (1/self.n_concepts), orig_correct - (1/self.n_concepts))
    
        completeness = n(y_pred)
        conceptSHAP = []
    
        if doConceptSHAP:
            # Approximate SHAP values using sampling
            num_samples = 100  # Numero di campioni da usare per l'approssimazione
            c_id = list(range(len(self.concept.T)))
    
            for idx in c_id:
                shap_value = 0.0
                for _ in range(num_samples):
                    # Genera una coalizione casuale che non include il concetto corrente
                    subset = [i for i in c_id if i != idx and np.random.rand() > 0.5]
                    # Calcola la previsione con e senza il concetto corrente
                    c_with = subset + [idx]
                    c_without = subset
    
                    # Costruisci il concetto con e senza il concetto corrente
                    concept_with = self.concept[:, c_with]
                    concept_without = self.concept[:, c_without] if c_without else None
    
                    # Previsione con il concetto corrente
                    y_pred_with = self.proj_and_predict(concept_with, train_embedding)
                    score_with = n(y_pred_with)
    
                    # Previsione senza il concetto corrente
                    if concept_without is not None:
                        y_pred_without = self.proj_and_predict(concept_without, train_embedding)
                        score_without = n(y_pred_without)
                    else:
                        # Se non ci sono concetti nella coalizione, score_without è 0
                        score_without = torch.tensor(0.0).to(self.concept.device)
    
                    # Aggiorna il valore SHAP per il concetto corrente
                    shap_value += (score_with - score_without).item()
    
                # Calcola il valore medio SHAP
                shap_value /= num_samples
                conceptSHAP.append(shap_value)    
        concept_diversity = self.calculate_concept_diversity()
    
        if regularize:
            diversity_weight = 0.1  # Puoi regolare questo peso
            final_loss = pred_loss + (l_1 * L_sparse_1_new * -1) + (l_2 * L_sparse_2_new) - (diversity_weight * concept_diversity)
        else:
            final_loss = pred_loss
    
        return completeness, conceptSHAP, final_loss, pred_loss, L_sparse_1_new, L_sparse_2_new, metrics, concept_diversity