fdsafdas

hugging_face/dataset.py

@@ -4,20 +4,13 @@ from pandas import DataFrame, read_csv
 REPO_ID = "kompiangg/twitter_hate_speech_classification"
 
 def load_dataset(filename) -> DataFrame:
-  try:
-    df = read_csv(filename, encoding='latin-1')
-  except:
-    datasets = hf_load_dataset(
-      REPO_ID, 
-      data_files=filename, 
-      encoding='latin-1',
-      cache_dir='./.cache'
-    )
+  datasets = hf_load_dataset(
+    REPO_ID, 
+    data_files=filename, 
+    encoding='latin-1',
+    cache_dir='./.cache'
+  )
 
-    df = DataFrame(data=datasets['train'])
-    df.to_csv(filename, sep=',', index=False)
-  
-  df = df.astype({ 'HS': 'category' })
-  df = df.astype({ 'Tweet': 'string' })
+  df = DataFrame(data=datasets['train'])
 
   return df