Epoch 10/10

-------------------------------
Validation: 0%| | 0/6 [00:00<?, ?it/s]Training loss: 0.30
Validation Accuracy: 0.86

dataset/CustomDataset.py

@@ -0,0 +1,40 @@
+import torch
+from torch.utils.data import Dataset
+
+n_classes = 2
+
+class CustomDataset(Dataset):
+    def __init__(self, data, tokenizer, max_length=512):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        item = self.data[idx]
+        # 假设我们的数据是一个字典，包含"text"和"label"
+        text = item['Description']
+        label = item['label']
+
+        # 对文本进行编码
+        encoded = self.tokenizer.encode_plus(
+            text,
+            add_special_tokens=True,
+            max_length=self.max_length,
+            padding='max_length',
+            truncation=True,
+            return_attention_mask=True,
+            return_tensors='pt',
+        )
+        # 在这里添加任何需要的数据检查逻辑
+        # 例如，检查标签是否在预期的范围内
+        if label < 0 or label > n_classes:  # 假设n_classes是标签的数量
+            raise ValueError("Found an invalid label")
+
+        return {
+            'input_ids': encoded['input_ids'].flatten(),
+            'attention_mask': encoded['attention_mask'].flatten(),
+            'labels': torch.tensor(label, dtype=torch.long)
+        }

dataset/load_dataset.py

@@ -10,8 +10,11 @@ def prepare_dataset(df, tokenizer, max_length=512):
     input_ids = []
     attention_masks = []
     labels = []
-
     for _, row in df.iterrows():
+        # 检查标签是否有效（例如，不是NaN）
+        if pd.isna(row['SAS_Class']) or pd.isna(row['SDS_Class']):
+            continue  # 跳过这个样本
+
         encoded = tokenizer.encode_plus(
             row['Description'],
             add_special_tokens=True,
@@ -23,18 +26,19 @@ def prepare_dataset(df, tokenizer, max_length=512):
         )
         input_ids.append(encoded['input_ids'])
         attention_masks.append(encoded['attention_mask'])
-        labels.append([row['SAS_Class'], row['SDS_Class']])
+        # labels.append([row['SAS_Class'], row['SDS_Class']])
+        # 将SAS_Class和SDS_Class转换为one-hot编码
+        sas_label = [0] * 4  # 初始化4个元素为0的列表
+        sds_label = [0] * 4  # 同上
+        sas_label[int(row['SAS_Class'])] = 1  # 将对应的位置设为1
+        sds_label[int(row['SDS_Class'])] = 1  # 同上
+        combined_label = sas_label + sds_label  # 组合两个标签
+
+        labels.append(combined_label)
 
     input_ids = torch.cat(input_ids, dim=0)
     attention_masks = torch.cat(attention_masks, dim=0)
     labels = torch.tensor(labels, dtype=torch.float)
-
     return TensorDataset(input_ids, attention_masks, labels)
 
-# 分割数据集
-train_df, val_df = train_test_split(df, test_size=0.1)  # 以90%训练，10%验证的比例分割数据集
-
-
-# 创建DataLoader
-
 

new.py

@@ -1,10 +1,13 @@
 from transformers import AdamW, get_linear_schedule_with_warmup, AutoTokenizer, AutoModelForSequenceClassification
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
-from torch.nn import CrossEntropyLoss
 import torch
 from sklearn.model_selection import train_test_split
-
 from dataset.load_dataset import df, prepare_dataset
+from torch.nn import BCEWithLogitsLoss
+from transformers import BertForSequenceClassification, BertConfig
+from tqdm.auto import tqdm
+from torch.cuda.amp import GradScaler, autocast
+
 
 epochs = 10
 tokenizer = AutoTokenizer.from_pretrained(
@@ -18,37 +21,88 @@ train_df, val_df = train_test_split(df, test_size=0.1)  # 以90%训练，10%验
 train_dataset = prepare_dataset(train_df, tokenizer)
 val_dataset = prepare_dataset(val_df, tokenizer)
 # 现在train_dataloader和validation_dataloader已准备好，可用于模型训练和验证
-train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=64)
-validation_dataloader = DataLoader(val_dataset, sampler=SequentialSampler(val_dataset), batch_size=64)
+train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=16)
+validation_dataloader = DataLoader(val_dataset, sampler=SequentialSampler(val_dataset), batch_size=16)
+
+
+# 加载配置
+config = BertConfig.from_pretrained("pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition")
+config.num_labels = 8  # 调整为你的标签数量
 
 model = AutoModelForSequenceClassification.from_pretrained(
-    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition").to(device)
-input = tokenizer("I love using transformers for natural language processing.", return_tensors="pt")
-# 使用模型进行预测
-# with torch.no_grad():
-#     logits = model(**input).logits
-# 解析预测结果
-# predicted_class_id = logits.argmax().item()
-# print(f"Predicted class id: {predicted_class_id}")
+    "pretrained_models/Bio_ClinicalBERT-finetuned-medicalcondition",config=config,ignore_mismatched_sizes=True).to(device)
 # 准备优化器和学习率调度器
-optimizer = AdamW(model.parameters(), lr=2e-5, eps=1e-8)
+optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-8)
 total_steps = len(train_dataloader) * epochs  # epochs是您想要训练的轮数
 scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)
-
+loss_fn = BCEWithLogitsLoss()
 # 微调模型
-model.train()
+scaler = GradScaler()
+
 for epoch in range(epochs):  # 迭代多个epoch
-    for step, batch in enumerate(train_dataloader):
+    print(f"\nEpoch {epoch + 1}/{epochs}")
+    print('-------------------------------')
+    model.train()
+    total_loss = 0
+    train_progress_bar = tqdm(train_dataloader, desc="Training", leave=False)
+    for step, batch in enumerate(train_progress_bar):
         # 将数据加载到GPU
         batch = tuple(t.to(device) for t in batch)
         b_input_ids, b_input_mask, b_labels = batch
         model.zero_grad()
         # 前向传播
-        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
-        loss = outputs.loss
+        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
         logits = outputs.logits
         # 反向传播
-        loss.backward()
-        optimizer.step()
-        scheduler.step()
-# 评估阶段省略，但在实际应用中非常重要
+        loss = loss_fn(logits, b_labels)
+        total_loss += loss.item()
+        # loss.backward()
+        # optimizer.step()
+        # scheduler.step()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+        # 检查loss是否为nan
+        if torch.isnan(loss).any():
+            print(f"Loss is nan in epoch {epoch + 1}, step {step}.")
+            # 可选：打印出问题数据的更多信息或采取其他措施
+            # 注意：直接跳过可能不解决根本问题，最好检查为何loss会是nan
+            continue  # 跳过当前批次的反向传播和优化器步骤
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+        train_progress_bar.set_postfix({'loss': f"{loss.item():.2f}"})
+
+    # 评估阶段
+    avg_train_loss = total_loss / len(train_dataloader)
+    print(f"Training loss: {avg_train_loss:.2f}")
+
+    # 验证阶段
+    model.eval()
+    total_eval_accuracy = 0
+    eval_progress_bar = tqdm(validation_dataloader, desc="Validation", leave=False)
+
+    for batch in eval_progress_bar:
+        batch = tuple(t.to(device) for t in batch)
+        b_input_ids, b_input_mask, b_labels = batch
+        with torch.no_grad():
+            outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
+
+        logits = outputs.logits
+        # predictions = torch.argmax(logits, dim=1).flatten()
+        # labels = b_labels.flatten()
+        # accuracy = (predictions == labels).cpu().numpy().mean()
+        # 使用sigmoid函数将logits转换为概率值
+        probs = torch.sigmoid(logits)
+        # 将概率高于0.5的预测为正类（1），低于0.5的预测为负类（0）
+        predictions = (probs > 0.5).int()
+
+        # 比较预测和真实标签
+        correct_predictions = (predictions == b_labels.int()).float()  # 确保标签也是整数类型
+        # 计算每个样本的正确预测的平均数，然后计算整个批次的平均值
+        accuracy_per_sample = correct_predictions.mean(dim=1)
+        accuracy = accuracy_per_sample.mean().item()
+        total_eval_accuracy += accuracy
+        # 更新进度条
+        eval_progress_bar.set_postfix({'accuracy': f"{accuracy:.2f}"})
+
+    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
+    print(f"Validation Accuracy: {avg_val_accuracy:.2f}")