Upload 21 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+macbert/dataset/datagame_sms_stage1(in).csv filter=lfs diff=lfs merge=lfs -text

macbert/checkpoints/saved_checkpoints/name_checkpoint15_train8000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9c216795384fe75c774e640eb2929f46b8dbc409ffd9d961e74627d6224204
+size 1222778219

macbert/checkpoints/saved_checkpoints/name_checkpoint17_train9000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1abe8fd9b5099381277eb432db9c95f7467bb362bf7244a979bcdd691492770
+size 1222778219

macbert/checkpoints/saved_checkpoints/travel_checkpoint15_train8000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:716b58feddc2bcaae611858581b72725e1592aca1d0645dc2abf38b9f499e791
+size 1222778219

macbert/checkpoints/saved_checkpoints/travel_checkpoint_17_train9000.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:027e43ef9081d786a967ffa95ca95a52a1487c8a76f2f6b4409e796dab657b53
+size 1222778219

macbert/dataset.py

@@ -0,0 +1,89 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+# from transformers import AutoModel
+# from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+
+import numpy as np
+import random
+import os
+import copy
+import pandas as pd
+
+
+class MyDataset(Dataset):
+    def __init__(
+        self,
+        ann_file,
+        cfg,
+        mode='tra',
+    ):
+        super().__init__()
+
+        data = np.array(pd.read_csv(ann_file))
+        self.data = data
+        self.mode = mode
+        self.cfg = cfg
+
+    def __getitem__(self, index):
+        if self.mode == 'test':
+            d = self.data[index]
+            context = d[1]
+            sms_id = d[0]
+            return context, sms_id
+        else :
+            d = self.data[index]
+            context = d[1]
+            label = d[2]
+            label = int(label)
+            return context, label
+
+    def __len__(self):
+        return len(self.data)
+
+
+if __name__ == '__main__':
+    d = 'C:/Users/u/Desktop/workspace/bs/myr/data/val.csv'
+    D = MyDataset(d, cfg={})
+    nb_1 = 0
+    for i, d in enumerate(D):
+        _, l = d
+        if l==1:
+            nb_1 += 1
+    print(nb_1/len(D))
+
+
+    np.random.seed(666)
+
+    ann_file1 = 'C:/Users/u/Desktop/workspace/bs/myr/data/test_samples.csv'
+    ann_file2 = 'C:/Users/u/Desktop/workspace/bs/myr/data/train_samples.csv'
+    data1 = pd.read_csv(ann_file1)
+    data2 = pd.read_csv(ann_file2)
+    data = pd.concat([data1, data2])
+
+    data = np.array(data)
+    np.random.shuffle(data)
+
+    data_tra = data[:int(len(data)*0.7)]
+    data_val = data[int(len(data)*0.7):]
+
+    data_tra = pd.DataFrame(data_tra, columns=['content', 'label'])
+    data_val = pd.DataFrame(data_val, columns=['content', 'label'])
+
+    data_tra.to_csv('C:/Users/u/Desktop/workspace/bs/myr/data/tra.csv', index=False)
+    data_val.to_csv('C:/Users/u/Desktop/workspace/bs/myr/data/val.csv', index=False)
+
+
+
+
+
+

macbert/dataset/datagame_sms_stage1(in).csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f83bdff3455421e19a75bb1fe947b752751046189fec28a2f865bdef32ae2e9
+size 47370252

macbert/infer.py

@@ -0,0 +1,138 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+from transformers import AutoModel
+from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+
+from tqdm import tqdm
+import random
+import numpy as np
+from collections import OrderedDict
+from rich import print
+import time
+import cv2
+from glob import glob
+import string
+from torch.optim import AdamW
+from transformers import get_linear_schedule_with_warmup
+from models import get_model
+from dataset import MyDataset
+from utils import save_checkpoint, AverageMeter, ProgressMeter
+
+
+def test_epoch(model, epoch, dataloader, tokenizer):
+    print(f"\n\n=> val")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+    model.to(device)
+    model.eval()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+    predictions = []
+
+    for batch_index, data_batch in enumerate(tqdm(dataloader)):
+        context_str_batch = data_batch
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+
+        # forward
+        data_input_batch = context_token_batch
+        output_batch = model(**data_input_batch)
+
+        pred_batch = output_batch.softmax(dim=-1)
+        pred = torch.argmax(pred_batch, dim=-1)
+        predictions.extend(pred.cpu().numpy())
+
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    return predictions
+
+
+def infer20221212():
+    checkpoint_file = '/home/elaine/Desktop/macbert_code/checkpoints_name/checkpoint_epoch015_acc1.0000.pth.tar'
+    output_file = r'/home/elaine/Desktop/macbert_code/output.csv'
+
+    cache_dir = '/home/elaine/Desktop/macbert_code/code/cache'
+    ann_file_test = r'/home/elaine/Desktop/macbert_code/dataset/name_test_8000.csv'
+
+    model_cfg = {
+        "pretrained_transformers": "hfl/chinese-macbert-base",
+        "cache_dir": cache_dir
+    }
+    # 模型
+    model_dict = get_model(model_cfg, mode='base')
+    model = model_dict['model']
+    tokenizer = model_dict['tokenizer']
+    print(model)
+
+
+    data_loader_cfg = {}
+    test_dataset = MyDataset(ann_file_test, data_loader_cfg, mode='test')
+    test_loader = DataLoader(test_dataset, batch_size=8, num_workers=4, pin_memory=True)
+
+    # resume
+    assert checkpoint_file is not None and os.path.exists(checkpoint_file)
+    checkpoint = torch.load(checkpoint_file, map_location='cpu')
+    # model.load_state_dict(checkpoint['state_dict'])
+    model.load_state_dict({k.replace('module.', ''): v for k, v in checkpoint['state_dict'].items()})
+    print(f"=> Resume: loaded checkpoint {checkpoint_file} (epoch {checkpoint['epoch']})")
+
+    #model = model.cuda()
+    pred_res = test_epoch(model, 1, test_loader, tokenizer)
+    with open(output_file, 'w') as f:
+        for pred in pred_res:
+            f.write(f"{pred}\n")
+
+    # 讀取val.csv的label
+    import csv
+    true_labels = []
+    with open(ann_file_test, 'r') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for row in reader:
+            true_labels.append(int(row[3]))
+
+    # 計算confusion matrix
+    from sklearn.metrics import confusion_matrix
+    cm = confusion_matrix(true_labels, pred_res)
+    print('Confusion Matrix:')
+    print(cm)
+
+    # 印出預測錯誤的內容、預測值和正確答案
+    with open(ann_file_test) as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for idx, row in enumerate(reader):
+            sms, label = row[1], int(row[3])
+            pred = pred_res[idx]
+            if pred != label:
+                print(f"錯誤: sms='{sms}'，預測={pred}，正確={label}")
+
+
+if __name__ == '__main__':
+    infer20221212()

macbert/infer_all.py

@@ -0,0 +1,167 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+from transformers import AutoModel
+from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+
+from tqdm import tqdm
+import random
+import numpy as np
+from collections import OrderedDict
+from rich import print
+import time
+from glob import glob
+import string
+from torch.optim import AdamW
+from transformers import get_linear_schedule_with_warmup
+from models import get_model
+from dataset import MyDataset
+from utils import save_checkpoint, AverageMeter, ProgressMeter
+
+
+def test_epoch(travel_model, name_model, epoch, dataloader, tokenizer):
+    print(f"\n\n=> val")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+    travel_model.to(device)
+    travel_model.eval()
+
+    name_model.to(device)
+    name_model.eval()
+
+    sms_ids = []
+    travel_probs = []
+    travel_predictions = []
+    name_probs = []
+    name_predictions = []
+
+    for batch_index, data_batch in enumerate(tqdm(dataloader)):
+        
+        context_str_batch, sms_id = data_batch
+        sms_ids.append(sms_id.detach().cpu().numpy()[0])
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+
+        # forward travel
+        data_input_batch = context_token_batch
+        travel_output_batch = travel_model(**data_input_batch)
+        name_output_batch = name_model(**data_input_batch)
+
+        travel_pred_batch = travel_output_batch.softmax(dim=-1)
+        travel_probs.append(travel_pred_batch.detach().cpu().numpy()[0][1])
+        travel_pred = torch.argmax(travel_pred_batch, dim=-1)
+        travel_predictions.extend(travel_pred.cpu().numpy())
+
+        # forward name
+        name_pred_batch = name_output_batch.softmax(dim=-1)
+        name_probs.append(name_pred_batch.detach().cpu().numpy()[0][1])
+        name_pred = torch.argmax(name_pred_batch, dim=-1)
+        name_predictions.extend(name_pred.cpu().numpy())
+
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    return travel_predictions, travel_probs, name_predictions, name_probs, sms_ids
+
+def inference():
+    travel_checkpoint_file = '/home/jchsiao/Desktop/macbert/checkpoints/saved_checkpoints/travel_checkpoint15_train8000.pth.tar'
+    name_checkpoint_file = '/home/jchsiao/Desktop/macbert/checkpoints/saved_checkpoints/name_checkpoint17_train9000.pth.tar'
+    ann_file_test = r'/home/jchsiao/Desktop/macbert/dataset/datagame_sms_stage1(in).csv'
+    output_file = r'/home/jchsiao/Desktop/macbert/both_macbertBase_20250731_2.csv'
+    cache_dir = 'cache'
+
+    model_cfg = {
+        "pretrained_transformers": "hfl/chinese-macbert-base",
+        "cache_dir": cache_dir
+    }
+    # 模型
+    travel_model_dict = get_model(model_cfg, mode='base')
+    travel_model = travel_model_dict['model']
+
+    name_model_dict = get_model(model_cfg, mode='base')
+    name_model = name_model_dict['model']
+
+
+    tokenizer = travel_model_dict['tokenizer']
+    # print(model)
+
+
+    data_loader_cfg = {}
+    test_dataset = MyDataset(ann_file_test, data_loader_cfg, mode='test')
+    test_loader = DataLoader(test_dataset, batch_size=1, pin_memory=True, shuffle=False)
+
+    # resume
+    assert travel_checkpoint_file is not None and os.path.exists(travel_checkpoint_file)
+    assert name_checkpoint_file is not None and os.path.exists(name_checkpoint_file)
+
+    travel_checkpoint = torch.load(travel_checkpoint_file, map_location='cpu')
+    name_checkpoint = torch.load(name_checkpoint_file, map_location='cpu')
+    # model.load_state_dict(checkpoint['state_dict'])
+    travel_model.load_state_dict({k.replace('module.', ''): v for k, v in travel_checkpoint['state_dict'].items()})
+    print(f"=> Resume: loaded travel checkpoint {travel_checkpoint_file} (epoch {travel_checkpoint['epoch']})")
+
+    name_model.load_state_dict({k.replace('module.', ''): v for k, v in name_checkpoint['state_dict'].items()})
+    print(f"=> Resume: loaded name checkpoint {name_checkpoint_file} (epoch {name_checkpoint['epoch']})")
+
+    #model = model.cuda()
+    travel_predictions, travel_probs, name_predictions, name_probs, sms_ids = test_epoch(travel_model, name_model, 1, test_loader, tokenizer)
+    with open(output_file, 'w') as f:
+        f.write("sms_id,travel_prob,label,name_prob,name_flg\n")
+        for travel_pred, travel_prob, name_pred, name_prob, sms_id in zip(travel_predictions, travel_probs, name_predictions, name_probs, sms_ids):
+            f.write(f"{sms_id},{travel_prob},{travel_pred},{name_prob},{name_pred}\n")
+    print('Output file saved!')
+
+"""
+
+    # 讀取val.csv的label
+    import csv
+    true_labels = []
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for row in reader:
+            true_labels.append(int(row[2]))
+
+    # 計算confusion matrix
+    from sklearn.metrics import confusion_matrix
+    cm = confusion_matrix(true_labels, pred_res)
+    print('Confusion Matrix:')
+    print(cm)
+
+    # 印出預測錯誤的內容、預測值和正確答案
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for idx, row in enumerate(reader):
+            id, sms, label = int(row[0]), row[1], int(row[2])
+            pred = pred_res[idx]
+            if pred != label:
+                print(f"錯誤: sms_id={id}，sms='{sms}'，預測={pred}，正確={label}")
+"""
+
+if __name__ == '__main__':
+    inference()

macbert/infer_name.py

@@ -0,0 +1,144 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+from transformers import AutoModel
+from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+
+from tqdm import tqdm
+import random
+import numpy as np
+from collections import OrderedDict
+from rich import print
+import time
+import cv2
+from glob import glob
+import string
+from torch.optim import AdamW
+from transformers import get_linear_schedule_with_warmup
+from models import get_model
+from dataset import MyDataset
+from utils import save_checkpoint, AverageMeter, ProgressMeter
+
+
+def test_epoch(model, epoch, dataloader, tokenizer):
+    print(f"\n\n=> val")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+    model.to(device)
+    model.eval()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+    sms_ids = []
+    probs = []
+    predictions = []
+
+    for batch_index, data_batch in enumerate(tqdm(dataloader)):
+        
+        context_str_batch, sms_id = data_batch
+        sms_ids.append(sms_id.detach().cpu().numpy()[0])
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+
+        # forward
+        data_input_batch = context_token_batch
+        output_batch = model(**data_input_batch)
+
+        pred_batch = output_batch.softmax(dim=-1)
+        probs.append(pred_batch.detach().cpu().numpy()[0][1])
+        pred = torch.argmax(pred_batch, dim=-1)
+        predictions.extend(pred.cpu().numpy())
+
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    return predictions, probs, sms_ids
+
+
+def infer20221212():
+    checkpoint_file = '/home/elaine/Desktop/macbert_code/checkpoints_name/checkpoint_epoch017_acc0.9965.pth.tar'
+    output_file = r'/home/elaine/Desktop/macbert_code/name_v2_output.csv'
+
+    cache_dir = '/home/elaine/Desktop/macbert_code/cache'
+    ann_file_test = r'/home/elaine/Desktop/macbert_code/dataset/name_test_9000.csv'
+
+    model_cfg = {
+        "pretrained_transformers": "hfl/chinese-macbert-base",
+        "cache_dir": cache_dir
+    }
+    # 模型
+    model_dict = get_model(model_cfg, mode='base')
+    model = model_dict['model']
+    tokenizer = model_dict['tokenizer']
+    print(model)
+
+
+    data_loader_cfg = {}
+    test_dataset = MyDataset(ann_file_test, data_loader_cfg, mode='test')
+    test_loader = DataLoader(test_dataset, batch_size=1, pin_memory=True, shuffle=False)
+
+    # resume
+    assert checkpoint_file is not None and os.path.exists(checkpoint_file)
+    checkpoint = torch.load(checkpoint_file, map_location='cpu')
+    # model.load_state_dict(checkpoint['state_dict'])
+    model.load_state_dict({k.replace('module.', ''): v for k, v in checkpoint['state_dict'].items()})
+    print(f"=> Resume: loaded checkpoint {checkpoint_file} (epoch {checkpoint['epoch']})")
+
+    #model = model.cuda()
+    pred_res, probs, sms_ids = test_epoch(model, 1, test_loader, tokenizer)
+    with open(output_file, 'w') as f:
+        f.write("sms_id,prob,name_flg\n")
+        for pred, prob, sms_id in zip(pred_res, probs, sms_ids):
+            f.write(f"{sms_id},{prob},{pred}\n")
+
+    # 讀取val.csv的label
+    import csv
+    true_labels = []
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for row in reader:
+            true_labels.append(int(row[3]))
+
+    # 計算confusion matrix
+    from sklearn.metrics import confusion_matrix
+    cm = confusion_matrix(true_labels, pred_res)
+    print('Confusion Matrix:')
+    print(cm)
+
+    # 印出預測錯誤的內容、預測值和正確答案
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for idx, row in enumerate(reader):
+            id, sms, label = int(row[0]), row[1], int(row[3])
+            pred = pred_res[idx]
+            if pred != label:
+                print(f"錯誤: sms_id={id}，sms='{sms}'，預測={pred}，正確={label}")
+
+
+if __name__ == '__main__':
+    infer20221212()

macbert/infer_travel.py

@@ -0,0 +1,144 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+from transformers import AutoModel
+from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+
+from tqdm import tqdm
+import random
+import numpy as np
+from collections import OrderedDict
+from rich import print
+import time
+import cv2
+from glob import glob
+import string
+from torch.optim import AdamW
+from transformers import get_linear_schedule_with_warmup
+from models import get_model
+from dataset import MyDataset
+from utils import save_checkpoint, AverageMeter, ProgressMeter
+
+
+def test_epoch(model, epoch, dataloader, tokenizer):
+    print(f"\n\n=> val")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+    model.to(device)
+    model.eval()
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+    sms_ids = []
+    probs = []
+    predictions = []
+
+    for batch_index, data_batch in enumerate(tqdm(dataloader)):
+        
+        context_str_batch, sms_id = data_batch
+        sms_ids.append(sms_id.detach().cpu().numpy()[0])
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+
+        # forward
+        data_input_batch = context_token_batch
+        output_batch = model(**data_input_batch)
+
+        pred_batch = output_batch.softmax(dim=-1)
+        probs.append(pred_batch.detach().cpu().numpy()[0][1])
+        pred = torch.argmax(pred_batch, dim=-1)
+        predictions.extend(pred.cpu().numpy())
+
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    return predictions, probs, sms_ids
+
+
+def infer20221212():
+    checkpoint_file = '/home/elaine/Desktop/macbert_code/checkpoints_travel/checkpoint_epoch017_acc0.9988.pth.tar'
+    output_file = r'/home/elaine/Desktop/macbert_code/travel_v2_output.csv'
+
+    cache_dir = '/home/elaine/Desktop/macbert_code/cache'
+    ann_file_test = r'/home/elaine/Desktop/macbert_code/dataset/travel_test_9000.csv'
+
+    model_cfg = {
+        "pretrained_transformers": "hfl/chinese-macbert-base",
+        "cache_dir": cache_dir
+    }
+    # 模型
+    model_dict = get_model(model_cfg, mode='base')
+    model = model_dict['model']
+    tokenizer = model_dict['tokenizer']
+    print(model)
+
+
+    data_loader_cfg = {}
+    test_dataset = MyDataset(ann_file_test, data_loader_cfg, mode='test')
+    test_loader = DataLoader(test_dataset, batch_size=1, pin_memory=True, shuffle=False)
+
+    # resume
+    assert checkpoint_file is not None and os.path.exists(checkpoint_file)
+    checkpoint = torch.load(checkpoint_file, map_location='cpu')
+    # model.load_state_dict(checkpoint['state_dict'])
+    model.load_state_dict({k.replace('module.', ''): v for k, v in checkpoint['state_dict'].items()})
+    print(f"=> Resume: loaded checkpoint {checkpoint_file} (epoch {checkpoint['epoch']})")
+
+    #model = model.cuda()
+    pred_res, probs, sms_ids = test_epoch(model, 1, test_loader, tokenizer)
+    with open(output_file, 'w') as f:
+        f.write("sms_id,prob,label\n")
+        for pred, prob, sms_id in zip(pred_res, probs, sms_ids):
+            f.write(f"{sms_id},{prob},{pred}\n")
+
+    # 讀取val.csv的label
+    import csv
+    true_labels = []
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for row in reader:
+            true_labels.append(int(row[2]))
+
+    # 計算confusion matrix
+    from sklearn.metrics import confusion_matrix
+    cm = confusion_matrix(true_labels, pred_res)
+    print('Confusion Matrix:')
+    print(cm)
+
+    # 印出預測錯誤的內容、預測值和正確答案
+    with open(ann_file_test, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # skip header
+        for idx, row in enumerate(reader):
+            id, sms, label = int(row[0]), row[1], int(row[2])
+            pred = pred_res[idx]
+            if pred != label:
+                print(f"錯誤: sms_id={id}，sms='{sms}'，預測={pred}，正確={label}")
+
+
+if __name__ == '__main__':
+    infer20221212()

macbert/main.py

@@ -0,0 +1,285 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms.functional import to_tensor, to_pil_image
+import torchvision.transforms as transforms
+from transformers import AutoModel
+from transformers import AutoTokenizer, AutoConfig
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+from torch.cuda.amp import autocast, GradScaler
+from torch.utils.data.distributed import DistributedSampler
+from torch.utils.data import RandomSampler, SequentialSampler
+
+from tqdm import tqdm
+import random
+import numpy as np
+# from collections import OrderedDict
+from rich import print
+import time
+import cv2
+# from glob import glob
+import string
+from torch.optim import AdamW
+from transformers import get_linear_schedule_with_warmup
+
+
+from models import get_model
+from dataset import MyDataset
+from utils import save_checkpoint, AverageMeter, ProgressMeter
+
+# if __name__ == '__main__':
+#     torch.distributed.init_process_group(backend="nccl")
+#     local_rank = torch.distributed.get_rank()
+#     torch.cuda.set_device(local_rank)
+#     device = torch.device("cuda", local_rank)
+#     scaler = GradScaler()
+# else:
+#     device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+if __name__ == '__main__':
+    # 檢查是否為分散式訓練模式（例如 torchrun 啟動）
+    if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
+        torch.distributed.init_process_group(backend="nccl")
+        local_rank = int(os.environ["LOCAL_RANK"])
+        torch.cuda.set_device(local_rank)
+        device = torch.device("cuda", local_rank)
+        print(f"[Distributed] Rank {os.environ['RANK']} using device {local_rank}")
+    else:
+        # 單機單卡訓練模式
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        print(f"[Single] Using device: {device}")
+
+    # AMP scaler 建議新版寫法
+    scaler = torch.amp.GradScaler(device='cuda' if torch.cuda.is_available() else 'cpu')
+
+local_rank = int(os.environ.get("LOCAL_RANK", 0))
+print_raw = print
+def print(*info):
+    if local_rank == 0:
+        print_raw(*info)
+
+
+
+
+def crossentropy(y_true, y_pred):
+    return F.cross_entropy(y_pred, y_true, label_smoothing=0.2)
+
+
+def evaluate(predictions, labels):
+    nb_all = len(predictions)
+    acc = sum([int(p==l) for p, l in zip(predictions, labels)]) / (nb_all + 1e-8)
+
+    eval_results = {'acc': acc}
+    
+    return eval_results
+
+
+def train_epoch(model, optimizer, epoch, dataloader, sampler, tokenizer, scheduler):
+    print(f"\n\n=> train")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    losses = AverageMeter('- loss', ':.4e')
+    acces = AverageMeter('- acc', ':.4f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, losses, acces, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    model.train()
+    if hasattr(sampler, "set_epoch"):
+        sampler.set_epoch(epoch)
+
+    predictions, labels = [], []
+
+    for batch_index, data_batch in enumerate(dataloader):
+        optimizer.zero_grad()
+
+        context_str_batch, target_batch = data_batch
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+        target_batch = target_batch.to(device)
+
+        # forward
+        data_input_batch = context_token_batch
+        output_batch = model(**data_input_batch)
+
+        pred_batch = output_batch.softmax(dim=-1)
+
+        loss_batch = crossentropy(target_batch, output_batch)
+        loss = torch.mean(loss_batch)
+        # print(loss)
+        loss.backward()
+        optimizer.step()
+        if scheduler is not None:
+            scheduler.step()
+
+        loss_value = loss.item()
+        losses.update(loss_value, len(target_batch))
+        pred = torch.argmax(pred_batch, dim=-1)
+        predictions.extend(pred.cpu().numpy())
+        labels.extend(target_batch.cpu().numpy())
+        acc_batch = (target_batch==pred).sum().cpu().numpy() / (len(target_batch) + 1e-8)
+        acces.update(acc_batch, len(target_batch))
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    results = evaluate(predictions, labels)
+    print(results)
+    return results
+
+
+def val_epoch(model, optimizer, epoch, dataloader, sampler, tokenizer):
+    print(f"\n\n=> val")
+    data_time = AverageMeter('- data', ':4.3f')
+    batch_time = AverageMeter('- batch', ':6.3f')
+    losses = AverageMeter('- loss', ':.4e')
+    acces = AverageMeter('- acc', ':.4f')
+    progress = ProgressMeter(
+        len(dataloader), data_time, batch_time, losses, acces, prefix=f"Epoch: [{epoch}]")
+
+    end = time.time()
+    model.train()
+    if hasattr(sampler, "set_epoch"):
+        sampler.set_epoch(epoch)
+
+    predictions, labels = [], []
+
+    for batch_index, data_batch in enumerate(dataloader):
+        optimizer.zero_grad()
+
+        context_str_batch, target_batch = data_batch
+
+        # data tokenizer
+        context_token_batch = tokenizer(context_str_batch, padding=True, truncation=True, max_length=500, return_tensors='pt')
+        
+        # to gpu
+        context_token_batch = {k:v.to(device) for k,v in context_token_batch.items()}
+        target_batch = target_batch.to(device)
+
+        # forward
+        data_input_batch = context_token_batch
+        output_batch = model(**data_input_batch)
+
+        pred_batch = output_batch.softmax(dim=-1)
+
+        loss_batch = crossentropy(target_batch, output_batch)
+        loss = torch.mean(loss_batch)
+        # print(pred_batch)
+        # print(target_batch)
+        # print(loss)
+
+        loss_value = loss.item()
+        losses.update(loss_value, len(target_batch))
+        pred = torch.argmax(pred_batch, dim=-1)
+        predictions.extend(pred.cpu().numpy())
+        labels.extend(target_batch.cpu().numpy())
+        acc_batch = (target_batch==pred).sum().cpu().numpy() / (len(target_batch) + 1e-8)
+        acces.update(acc_batch, len(target_batch))
+        batch_time.update(time.time() - end)
+        end = time.time()
+
+        if batch_index % 50 == 0:
+            progress.print(batch_index)
+
+    results = evaluate(predictions, labels)
+    print(results)
+    return results
+
+
+
+def gogogo():
+
+    output_dir = '/home/elaine/Desktop/macbert_code/checkpoints_travel'
+    ann_file_tra = '/home/elaine/Desktop/macbert_code/dataset/travel_train_9000.csv'
+    ann_file_val = '/home/elaine/Desktop/macbert_code/dataset/travel_val_9000.csv'
+    checkpoint_file = None
+
+    batch_size = 4
+    epochs = 20
+    cache_dir = ' /home/elaine/Desktop/macbert_code/cache'
+    
+    model_cfg = {
+        "pretrained_transformers": "hfl/chinese-macbert-base",
+        "cache_dir": cache_dir
+    }
+
+    # 模型與 tokenizer
+    model_dict = get_model(model_cfg, mode='base')
+    model = model_dict['model']
+    tokenizer = model_dict['tokenizer']
+    print(model)
+
+    # 優化器參數設計
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+         'weight_decay': 0.01},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
+         'weight_decay': 0.0}
+    ]
+    optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-8)
+    scheduler = None  # 如果你需要可啟用
+
+    # Dataset 與 DataLoader（單卡不使用 DistributedSampler）
+    data_loader_cfg = {}
+    tra_dataset = MyDataset(ann_file_tra, data_loader_cfg, mode='tra')
+    val_dataset = MyDataset(ann_file_val, {}, mode='val')
+
+    # Sampler（單卡用 RandomSampler / SequentialSampler）
+    sampler_tra = RandomSampler(tra_dataset)
+    sampler_val = SequentialSampler(val_dataset)
+
+    tra_loader = DataLoader(tra_dataset, batch_size=batch_size, num_workers=8, pin_memory=True, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=batch_size, num_workers=8, pin_memory=True, shuffle=False)
+
+    # checkpoint resume
+    if checkpoint_file is not None and os.path.exists(checkpoint_file):
+        checkpoint = torch.load(checkpoint_file, map_location='cpu')
+        init_epoch = checkpoint['epoch'] + 1
+        model.load_state_dict({k.replace('module.', ''): v for k, v in checkpoint['state_dict'].items()})
+        optimizer.load_state_dict(checkpoint['optimizer'])
+        if torch.cuda.is_available():
+            for state in optimizer.state.values():
+                for k, v in state.items():
+                    if torch.is_tensor(v):
+                        state[k] = v.cuda()
+        print(f"=> Resume: loaded checkpoint {checkpoint_file} (epoch {checkpoint['epoch']})")
+    else:
+        init_epoch = 1
+        print(f"=> No checkpoint. ")
+
+    # 將模型送上 GPU
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = model.to(device)
+
+    # 開始訓練
+    acc = 0.
+    for epoch in range(init_epoch, epochs + 1):
+        results_tra = train_epoch(model, optimizer, epoch, tra_loader, sampler_tra, tokenizer, scheduler)
+        results_val = val_epoch(model, optimizer, epoch, val_loader, sampler_val, tokenizer)
+        acc_val = results_val['acc']
+        if acc_val >= acc:
+            acc = acc_val
+            save_checkpoint({
+                'epoch': epoch,
+                'state_dict': model.state_dict(),
+                'best_acc': acc,
+                'optimizer': optimizer.state_dict(),
+            }, outname=f'{output_dir}/checkpoint_epoch{epoch:03d}_acc{acc:.4f}.pth.tar', local_rank=0)
+
+
+if __name__ == '__main__':
+    gogogo()

macbert/models.py

@@ -0,0 +1,42 @@
+import torch.nn.functional as F
+from transformers import AutoTokenizer, AutoModel, AutoConfig
+from transformers import BertModel, BertTokenizer
+import torch.nn as nn
+import torch
+import os
+from collections import OrderedDict
+import copy
+
+
+class MyModel(nn.Module):
+    def __init__(self, nlp_model, mode):
+        super().__init__()
+        self.nlp_model = nlp_model
+        if mode == 'large':
+            nb_feature = 1024
+        else:
+            nb_feature = 768
+
+        self.cls = nn.Linear(nb_feature, 2)
+
+
+    def forward(self, **data):
+        x = self.nlp_model(**data)
+        y = self.cls(x['last_hidden_state'][:, 0])
+        return y
+
+
+def get_model(cfg, mode='large'):
+    # tokenizer = AutoTokenizer.from_pretrained(cfg['common_encoder_pretrained_transformers'], additional_special_tokens=added_token)
+    # tokenizer = BertTokenizer.from_pretrained(cfg['common_encoder_pretrained_transformers'], use_fast=True, cache_dir='/share/wangqixun/workspace/bs/tx_mm/code/cache', additional_special_tokens=added_token)
+    tokenizer = AutoTokenizer.from_pretrained(cfg['pretrained_transformers'], use_fast=True, cache_dir=cfg['cache_dir'])
+
+    nlp_model = AutoModel.from_pretrained(cfg['pretrained_transformers'], cache_dir=cfg['cache_dir'])
+    model = MyModel(nlp_model=nlp_model, mode=mode)
+
+    return_dict = {
+        'model': model,
+        'tokenizer': tokenizer,
+    }
+    return return_dict
+

macbert/requirements.txt

@@ -0,0 +1,5 @@
+torch
+torchvision
+transformers
+rich
+pandas

macbert/test.py

@@ -0,0 +1,6 @@
+import pandas as pd
+import numpy as np
+
+ann_file = '/home/jchsiao/Desktop/macbert_code/dataset/travel_train_8000.csv'
+data = np.array(pd.read_csv(ann_file))
+print(data[1, 1])

macbert/utils.py

@@ -0,0 +1,70 @@
+import torch
+import os
+import json
+import numpy as np
+from rich import print
+
+
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self, name, fmt=':f'):
+        self.name = name
+        self.fmt = fmt
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+    def __str__(self):
+        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
+        return fmtstr.format(**self.__dict__)
+
+
+class ProgressMeter(object):
+    def __init__(self, num_batches, *meters, prefix=""):
+        self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
+        self.meters = meters
+        self.prefix = prefix
+
+    def print(self, batch):
+        entries = [self.prefix + self.batch_fmtstr.format(batch)]
+        entries += [str(meter) for meter in self.meters]
+        print('\t'.join(entries))
+
+    def _get_batch_fmtstr(self, num_batches):
+        num_digits = len(str(num_batches // 1))
+        fmt = '{:' + str(num_digits) + 'd}'
+        return '[' + fmt + '/' + fmt.format(num_batches) + ']'
+
+
+def save_checkpoint(state, outname, local_rank):
+    if local_rank == 0:
+        # best_acc = state['best_acc']
+        # epoch = state['epoch']
+        # filename = 'checkpoint_acc_%.4f_epoch_%02d.pth.tar' % (best_acc, epoch)
+        filename = outname
+        # filename = 'checkpoint_best_%d.pth.tar'
+        # filename = os.path.join('output/', filename)
+        dir_name = os.path.dirname(filename)
+        os.makedirs(dir_name, exist_ok=True)
+        torch.save(state, filename)
+
+        # best_filename = os.path.join(model_dir, 'checkpoint_best_%d.pth.tar' % name_no)
+        # best_filename = filename
+        # shutil.copyfile(filename, best_filename)
+        print('=> Save model to %s' % filename)