WIP: adding functions for loading and tokenization

LLM_middleware.py

@@ -5,10 +5,24 @@ from typing import List, Optional
 
 import openai
 from transformers import AutoTokenizer, AutoModelForCausalLM
-
+from datasets import load_datasets
 
 class LLM_Middleware():
     hf_key: str
+    dataset
+
     def __init__(self, openai_key, hf) -> None:
         openai.key = openai_key
         self.hf_key = hf
+'''
+function for loading the dataset using hf trainer.
+'''
+    def loadDataset(self,datasetName: str):
+        self.dataset = load_datasets(datasetName)
+        return self.dataset
+
+
+    def TokenizerFunction(modelName: str, dataset):
+        tokenizer = AutoTokenizer.from_pretrained(modelName)
+        ## as its the JSON function, we need to specify other function in order to be specific.
+        tokenizer(dataset["text"], padding="max_length", truncation=True)

README.md

@@ -15,3 +15,21 @@ tags: ["python", "transformers", "devops", "bioinformatics", ]
 ## BioML hackathon submission: building search engine for ressurecting archieved / old packages 🪦
 
 This repo will integrate the current promonent packages being deployed as transformers to develop the builds for the software and resolve any issues in the current package.
+
+
+
+## How to use
+
+- **Install dependencies:**
+
+```bash
+pip install -r requirements.txt
+```
+
+- **Run the app:**
+
+```bash
+gradio app.py
+```
+
+Also you can do the settings in order to setup on hf spaces.