Upload tokenize_dataset.py with huggingface_hub

tokenize_dataset.py

@@ -1,7 +1,6 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
-#     "torch>=2.0.0",
 #     "transformers>=4.50.0",
 #     "datasets>=2.14.0",
 #     "huggingface_hub",
@@ -218,14 +217,14 @@ def main():
     train_dataset = train_dataset.map(
         lambda x: tokenize_conversation(x, tokenizer, MAX_SEQ_LENGTH),
         remove_columns=["messages"],
-        num_proc=4,
+        num_proc=1,  # Use single process to reduce memory
         desc="Tokenizing train",
     )
 
     eval_dataset = eval_dataset.map(
         lambda x: tokenize_conversation(x, tokenizer, MAX_SEQ_LENGTH),
         remove_columns=["messages"],
-        num_proc=4,
+        num_proc=1,  # Use single process to reduce memory
         desc="Tokenizing eval",
     )