Upload 6 files

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "albert-base-v2",
+  "architectures": [
+    "AlbertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0,
+  "bos_token_id": 2,
+  "classifier_dropout_prob": 0.1,
+  "down_scale_factor": 1,
+  "embedding_size": 128,
+  "eos_token_id": 3,
+  "gap_size": 0,
+  "hidden_act": "gelu_new",
+  "hidden_dropout_prob": 0,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "inner_group_num": 1,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "albert",
+  "net_structure_type": 0,
+  "num_attention_heads": 12,
+  "num_hidden_groups": 1,
+  "num_hidden_layers": 12,
+  "num_memory_blocks": 0,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.0.dev0",
+  "type_vocab_size": 2,
+  "vocab_size": 30000
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8000a8d12b39ac9177adf5dbf2b532020795034f652684803573c766f94f2621
+size 46746988

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fefb02b667a6c5c2fe27602d28e5fb3428f66ab89c7d6f388e7c8d44a02d0336
+size 760289

tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 512}

train.py

@@ -0,0 +1,93 @@
+import torch
+from transformers import AlbertTokenizer, AlbertForSequenceClassification, Trainer, TrainingArguments
+from datasets import load_dataset
+import evaluate
+import wandb
+import numpy as np
+
+# Initialize WandB
+wandb.init(entity="dejan", project="good-vibes")
+
+# Adjustable parameters
+model_name = "albert-base-v2"
+batch_size = 32
+epochs = 10
+learning_rate = 2e-5
+gradient_clip_value = 1.0
+warmup_steps = 500
+
+# Load tokenizer and model
+tokenizer = AlbertTokenizer.from_pretrained(model_name)
+model = AlbertForSequenceClassification.from_pretrained(model_name, num_labels=3)
+
+# Load dataset
+dataset = load_dataset('csv', data_files={'train': 'sentences.csv'})
+dataset = dataset['train'].train_test_split(test_size=0.1)
+
+# Preprocess the data
+def preprocess_function(examples):
+    return tokenizer(examples['text'], padding='max_length', truncation=True)
+
+encoded_dataset = dataset.map(preprocess_function, batched=True)
+encoded_dataset = encoded_dataset.rename_column("label", "labels")
+
+# Define metrics
+accuracy_metric = evaluate.load("accuracy")
+f1_metric = evaluate.load("f1")
+precision_metric = evaluate.load("precision")
+recall_metric = evaluate.load("recall")
+
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    accuracy = accuracy_metric.compute(predictions=predictions, references=labels)
+    f1 = f1_metric.compute(predictions=predictions, references=labels, average='weighted')
+    precision = precision_metric.compute(predictions=predictions, references=labels, average='weighted')
+    recall = recall_metric.compute(predictions=predictions, references=labels, average='weighted')
+    return {
+        "accuracy": accuracy["accuracy"],
+        "f1": f1["f1"],
+        "precision": precision["precision"],
+        "recall": recall["recall"]
+    }
+
+# Training arguments
+training_args = TrainingArguments(
+    output_dir="./results",
+    evaluation_strategy="epoch",
+    save_strategy="epoch",
+    learning_rate=learning_rate,
+    per_device_train_batch_size=batch_size,
+    per_device_eval_batch_size=batch_size,
+    num_train_epochs=epochs,
+    weight_decay=0.01,
+    logging_dir="./logs",
+    logging_steps=10,
+    load_best_model_at_end=True,
+    metric_for_best_model="accuracy",  # Use accuracy to define the best model
+    greater_is_better=True,  # Set to True if higher metric value is better
+    gradient_accumulation_steps=2,
+    fp16=True,
+    report_to="wandb",
+    run_name="albert-finetuning",
+    warmup_steps=warmup_steps,
+    max_grad_norm=gradient_clip_value  # Correct parameter for gradient clipping
+)
+
+# Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=encoded_dataset['train'],
+    eval_dataset=encoded_dataset['test'],
+    compute_metrics=compute_metrics
+)
+
+# Train the model
+trainer.train()
+
+# Save the model
+trainer.save_model("fine-tuned-albert-base-v2")
+
+# Finish WandB run
+wandb.finish()