from transformers import BertTokenizer, BertModel
import torch


# 選擇模型，例如'bert-base-uncased'（英文），'bert-base-chinese'（中文）
model_name = 'bert-base-chinese'

# 加載分詞器
tokenizer = BertTokenizer.from_pretrained(model_name)

# 加載模型
model = BertModel.from_pretrained(model_name)
model.eval()  # 設置為評估模式


# 示例文本
text = "今天天氣很好，我打算去公園散步。"

# 分詞並轉換為ID
inputs = tokenizer(text, return_tensors='pt')

# 查看分詞結果
print(inputs)




with torch.no_grad():
    outputs = model(**inputs)

# outputs是包含多個層的信息，例如last_hidden_state和pooler_output
last_hidden_state = outputs.last_hidden_state  # (batch_size, sequence_length, hidden_size)
pooler_output = outputs.pooler_output  # (batch_size, hidden_size)

print("Last hidden state shape:", last_hidden_state.shape)
print("Pooler output shape:", pooler_output.shape)




from sklearn.model_selection import train_test_split
from torch.utils.data import Dataset, DataLoader

# 示例數據
texts = ["我喜歡這本書", "這個產品很差", "服務態度很好", "質量不好"]
labels = [1, 0, 1, 0]  # 1表示正面，0表示負面

# 分割數據集
train_texts, val_texts, train_labels, val_labels = train_test_split(texts, labels, test_size=0.2)



class TextDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len=128):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, idx):
        text = self.texts[idx]
        label = self.labels[idx]
        encoding = self.tokenizer(
            text,
            add_special_tokens=True,
            max_length=self.max_len,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'labels': torch.tensor(label, dtype=torch.long)
        }

# 創建數據集對象
train_dataset = TextDataset(train_texts, train_labels, tokenizer)
val_dataset = TextDataset(val_texts, val_labels, tokenizer)



batch_size = 8

train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size)


import torch.nn as nn
import torch.optim as optim

class BertClassifier(nn.Module):
    def __init__(self, bert_model, num_classes):
        super(BertClassifier, self).__init__()
        self.bert = bert_model
        self.dropout = nn.Dropout(p=0.3)
        self.classifier = nn.Linear(self.bert.config.hidden_size, num_classes)
    
    def forward(self, input_ids, attention_mask):
        outputs = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask
        )
        pooled_output = outputs.pooler_output
        dropout_output = self.dropout(pooled_output)
        logits = self.classifier(dropout_output)
        return logits

# 初始化模型
num_classes = 2
model = BertClassifier(model, num_classes)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)



optimizer = optim.AdamW(model.parameters(), lr=2e-5)
criterion = nn.CrossEntropyLoss()



def train_epoch(model, data_loader, optimizer, criterion, device):
    model.train()
    total_loss = 0
    correct_predictions = 0
    
    for batch in data_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        
        optimizer.zero_grad()
        outputs = model(input_ids, attention_mask)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        
        total_loss += loss.item()
        _, preds = torch.max(outputs, dim=1)
        correct_predictions += torch.sum(preds == labels)
    
    avg_loss = total_loss / len(data_loader)
    accuracy = correct_predictions.double() / len(data_loader.dataset)
    return avg_loss, accuracy

# 簡單訓練迴圈
epochs = 3
for epoch in range(epochs):
    train_loss, train_acc = train_epoch(model, train_loader, optimizer, criterion, device)
    print(f"Epoch {epoch+1}/{epochs}")
    print(f"Train loss: {train_loss:.4f}, Train accuracy: {train_acc:.4f}")



def eval_model(model, data_loader, criterion, device):
    model.eval()
    total_loss = 0
    correct_predictions = 0
    
    with torch.no_grad():
        for batch in data_loader:
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            labels = batch['labels'].to(device)
            
            outputs = model(input_ids, attention_mask)
            loss = criterion(outputs, labels)
            total_loss += loss.item()
            
            _, preds = torch.max(outputs, dim=1)
            correct_predictions += torch.sum(preds == labels)
    
    avg_loss = total_loss / len(data_loader)
    accuracy = correct_predictions.double() / len(data_loader.dataset)
    return avg_loss, accuracy

val_loss, val_acc = eval_model(model, val_loader, criterion, device)
print(f"Validation loss: {val_loss:.4f}, Validation accuracy: {val_acc:.4f}")



Last hidden state shape: torch.Size([1, 18, 768])
Pooler output shape: torch.Size([1, 768])
{'input_ids': tensor([[ 101,  791, 1921, 1921, 3706, 2523, 1962, 8024, 2769, 2802, 5050, 1343,
         1062, 1754, 3141, 3635,  511,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}
BertModel(
  (embeddings): BertEmbeddings(
    (word_embeddings): Embedding(21128, 768, padding_idx=0)
    (position_embeddings): Embedding(512, 768)
    (token_type_embeddings): Embedding(2, 768)
    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
    (dropout): Dropout(p=0.1, inplace=False)
  )
  (encoder): BertEncoder(
    (layer): ModuleList(
      (0-11): 12 x BertLayer(
        (attention): BertAttention(
          (self): BertSdpaSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
...
  (pooler): BertPooler(
    (dense): Linear(in_features=768, out_features=768, bias=True)
    (activation): Tanh()
  )
)
Output is truncated. View as a scrollable element or open in a text editor. Adjust cell output settings...
Epoch 1/3
Train loss: 0.8123, Train accuracy: 0.3333
Epoch 2/3
Train loss: 0.9957, Train accuracy: 0.3333
Epoch 3/3
Train loss: 0.4933, Train accuracy: 0.6667
Validation loss: 0.5164, Validation accuracy: 1.0000