Update utilities/setup.py

utilities/setup.py

@@ -1,10 +1,39 @@
 import json
 import os
+from datasets import load_dataset
 
+class get_files:
 
-def get_json_cfg():
-    """Retrieve configuration file"""
-    config_path = os.getenv('CONFIG_PATH')
-    with open(config_path, 'r') as file:
-        config = json.load(file)
-    return config
+    def predefined_dataset(dataset_name):
+        global dataset # bad practice, I know... But just bear with me. Will later update to state dict.
+        dataset = load_dataset(dataset_name, split = "train")
+        return 'Successfully loaded dataset'
+    
+    def uploaded_dataset(file):
+        global dataset # bad practice, I know... But just bear with me. Will later update to state dict.
+        dataset = []
+        if file is None:
+            return "File not found. Please upload the file again."
+        try:
+            with open(file,'r') as file:
+                for line in file:
+                    dataset.append(json.loads(line.strip()))
+            return "File retrieved."
+        except FileNotFoundError:
+            return "File not found. Please upload the file again."
+
+    def load_markdown_file(file_path):
+        try:
+            with open(file_path, 'r') as f:
+                return f.read()
+        except FileNotFoundError:
+            return "File not found. Please check the file path."
+        except Exception as e:
+            return f"Error loading file: {str(e)}"
+
+    def json_cfg():
+        """Retrieve configuration file"""
+        config_path = os.getenv('CONFIG_PATH')
+        with open(config_path, 'r') as file:
+            config = json.load(file)
+        return config