Joey0619/zzh_fduAImodle

from datasets import load_dataset import torch import numpy as np from transformers import AutoTokenizer, AutoModelForSequenceClassification from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn.metrics import classification_report, accuracy_score from sklearn.preprocessing import MinMaxScaler import umap import matplotlib.pyplot as plt from sklearn.model_selection import GridSearchCV

1. 加载数据集

emotions = load_dataset("emotion")

2. 数据预处理

train_ds = emotions["train"] emotions.set_format(type="pandas") df = emotions["train"][:] df["label_name"] = df["label"].apply(lambda row: emotions["train"].features["label"].int2str(row)) print(df.head())

3. 创建 token2idx 字典

all_text = " ".join(train_ds["text"]) # 将所有推文拼接成一个大字符串 unique_chars = set(all_text) # 获取所有唯一字符

创建一个 token2idx 字典，将字符映射到整数

token2idx = {char: idx for idx, char in enumerate(unique_chars)}

输出前10个映射

print(dict(list(token2idx.items())[:10]))

4. 加载DistilBERT模型与Tokenizer

model_ckpt = "distilbert-base-uncased" tokenizer = AutoTokenizer.from_pretrained(model_ckpt) model = AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=6)

5. 提取隐藏层 [CLS] 特征

def extract_hidden_states(batch, model, tokenizer): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") batch = {key: value.to(device) for key, value in batch.items()}

with torch.no_grad():
    outputs = model(**batch, output_hidden_states=True)
    hidden_states = outputs.hidden_states  # 获取所有隐藏状态

cls_hidden_state = hidden_states[-1][:, 0, :]  # 获取[CLS]特征
return {'hidden_state': cls_hidden_state.cpu().numpy()}  # 返回numpy格式的特征

6. 数据标记化

emotions_encoded = emotions.map(lambda batch: tokenizer(batch['text'], padding=True, truncation=True, max_length=512), batched=True) emotions_encoded.set_format(type="torch", columns=["input_ids", "attention_mask", "label"])

提取训练集的[CLS]特征

emotions_hidden = emotions_encoded.map(lambda batch: extract_hidden_states(batch, model, tokenizer), batched=True)

7. 准备训练数据

X_train = np.array(emotions_hidden["train"]["hidden_state"]) X_valid = np.array(emotions_hidden["validation"]["hidden_state"]) y_train = np.array(emotions_hidden["train"]["label"]) y_valid = np.array(emotions_hidden["validation"]["label"])

8. 训练并评估不同的分类模型

models = { 'Logistic Regression': LogisticRegression(max_iter=200), 'Random Forest': RandomForestClassifier(), 'SVM': SVC(), }

results = {} for model_name, model in models.items(): model.fit(X_train, y_train) y_pred = model.predict(X_valid)

accuracy = accuracy_score(y_valid, y_pred)
results[model_name] = accuracy

print(f"Model: {model_name}")
print(classification_report(y_valid, y_pred))
print("-" * 50)

输出模型对比结果

print("\nModel Comparison (Accuracy Scores):") for model_name, accuracy in results.items(): print(f"{model_name}: {accuracy:.4f}")

9. 超参数调整（GridSearch）

param_grid = { 'C': [0.1, 1, 10], 'solver': ['liblinear', 'saga'] }

grid_search_lr = GridSearchCV(LogisticRegression(max_iter=200), param_grid, cv=5) grid_search_lr.fit(X_train, y_train)

print("Best parameters for Logistic Regression:", grid_search_lr.best_params_)

对其他模型（如SVM、RandomForest）可以进行类似的调优

10. 使用UMAP进行特征降维与可视化

X_scaled = MinMaxScaler().fit_transform(X_train)

umap_model = umap.UMAP(n_components=2, metric='cosine') X_umap = umap_model.fit_transform(X_scaled)

df_emb = pd.DataFrame(X_umap, columns=["X", "Y"]) df_emb["label"] = y_train

plt.figure(figsize=(8, 6)) for label in range(6): # 6种情感标签 df_label = df_emb[df_emb["label"] == label] plt.scatter(df_label["X"], df_label["Y"], label=f"Class {label}")

plt.legend() plt.title("UMAP projection of the hidden state features") plt.show()

11. 保存模型并上传到Hugging Face

model.save_pretrained("./distilbert_finetuned_emotion") tokenizer.save_pretrained("./distilbert_finetuned_emotion")

上传到Hugging Face

model.push_to_hub("distilbert_finetuned_emotion") tokenizer.push_to_hub("distilbert_finetuned_emotion")