Delete procedure/corpus_QA_book.py

procedure/corpus_QA_book.py

@@ -1,108 +0,0 @@
-# Import os library
-import os
-import json
-import random
-# Import requests library
-import requests
-
-# Define the file name
-file_name = "raw-book-cleaned.txt"
-
-preprompt = """
-A chat. 
-USER: Below is an excerpt from a book. Based on this excerpt, please write 5 Questions and answers about the text. Make sure that answers follow the same style as the excerpt.
-Every question should start with "Reader:" and every answer should start with "Thomas:"
-
-BOOK EXCERPT START
-"""
-
-afterprompt = """
-BOOK EXCERPT STOP
-
-ASSISTANT:
-Sure, below are 5 questions and answers that can be inferred based on the content of the BOOK EXCERPT, the person asking the question is "Reader:" and the person who responds is called "Thomas:".
-I made sure that the questions are created are in context to the BOOK EXCERPT.
-
-Reader:
-"""
-
-
-def call_api(prompt, config):
-    url = "http://127.0.0.1:5001/api/v1/generate"
- 
-    with open(config, "r", encoding="utf-8") as config_file:
-        config_data = json.load(config_file)
- 
-    data = {
-        "prompt": f"{prompt}",
-        **config_data,
-    }
-    response = requests.post(url, json=data)
- 
-    try:
-        response_json = response.json()
-        response_text = response_json.get("results", [{}])[0].get("text", "")
-        return response_text
-    except json.JSONDecodeError:
-        print("API response could not be decoded as JSON.")
-        return ""
-
-
-# Check if the file exists
-if os.path.exists(file_name):
-    # Open the file in read mode
-    with open(file_name, "r") as f:
-        # Read the file content
-        text = f.read()
-        # Get the length of the text
-        length = len(text)
-        # Define an empty list to store the chunks
-        chunks = []
-        # Loop through the text with a step of 1000
-        for i in range(0, length, 10000):
-            # Get a slice of 1000 characters from the text
-            chunk = text[i:i+10000]
-            # Append the chunk to the list
-            chunks.append(chunk)
-        # Store the list in a variable
-        output = chunks
-        chunkcount = str(len(output))
-        # Define the url of the koboldcpp api
-        url = "http://127.0.0.1:5001/api/v1/generate"
-        # Define an empty list to store the responses from the koboldcpp api
-        responses = []
-        # Loop through the output list
-        file_size_limit = 50 * 1024 * 1024  # 50 megabytes
-        corpus_file = open("autocorpus4.txt", "a", encoding="utf-8")
-        k = 0
-        while True:
-            for chunk in output:
-                k = k + 1
-                ki = str(k)
-                progress = "\nProcessing chunk " + ki + " out of " + chunkcount + " chunks\n"
-                print(progress)
-                data1 = preprompt + chunk + afterprompt
-                data = data1.encode("utf-8")
-                header = {"Content-Type": "text/plain; charset=utf-8"}
-                # Send a post request with the chunk as data
-                response = response = call_api(data, "config.json")
-                # Check if the response is successful
-                if response:
-                    # Store the response in a variable
-                    result = "Reader: " + response
-                
-                    # Append the result to the responses list
-                    responses.append(result)
-                    # Print the result with a newline
-                    print(result + "\n")
-                    corpus_file.write(result + "\n\n\n")
-                    corpus_file.flush()  # Ensure data is written immediately
-                    #Check if the file size exceeds the limit
-                    if os.path.getsize("autocorpus4.txt") > file_size_limit:
-                        break
-            else:
-                # Print an error message
-                print("Something went wrong. Please check the url and the chunk.")
-else:
-    # Print an error message
-    print("The file does not exist. Please check the file name and location.")