الگوریتم کد زیر رو به طور کامل توضیح بده. لازم نیست در توضیحات کد منبع رو بیاری.
import json
import os
import re

import spacy

from auto_gptq import exllama_set_max_input_length
from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer

# model_name_or_path = "TheBloke/neural-chat-7B-v3-3-GPTQ"
model_name_or_path = "TheBloke/Mixtral_7Bx2_MoE-GPTQ"

model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="cuda:0", trust_remote_code=False,
                                             revision="main")  # load_in_4bit=True,)  # gptq-8bit-32g-actorder_True

model = exllama_set_max_input_length(model, max_input_length=10000)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True, trust_remote_code=True)

nlp = spacy.load("en_core_web_sm")

# book_path = './Rice Breeding in Iran, Curren.txt'
# book_path = './ITP_Animal Nutrition_1_230417_152729 - Copy.txt'
# book_path = './ITP_Animal Nutrition_1_230417_152729 - Copy - Copy.txt'
book_path = './#1# General rice.txt'
# book_path = './ITP_Animal-Nutrition_1_230417_152729_chapter2.txt'
with open(book_path, encoding='utf-8') as f:
    docs = f.readlines()


# page_pattern = r'(-+ Page \d+-+\n)(.*?)(?=\n-+ Page|\Z)'
# pages = re.findall(page_pattern, docs, flags=re.DOTALL)
# # pages = pages[22:323]
# pages = pages[22:35]


def text_communicator(text):  # communicate text with ','
    text = re.findall(r'(.+)\.\n', text, re.MULTILINE)
    t = ''
    l_text = len(text)
    cnt = 0
    for sent in text:  # doc:
        cnt += 1
        # t += str(sent) + (", " if l_text != cnt else '.')  # for making a long text
        t += str(sent) + (". " if l_text != cnt else '.')  # for saving text style
    return t


def txt_cleaner(text):  # deleting whitespace from the text:
    t = re.findall(r'\S+', text, re.DOTALL)
    txt = ""
    for tt in t:
        txt += " " + tt
    tx = nlp(txt)
    text = ''
    for sent in tx.sents:  # doc:
        text += str(sent) + "\n"
    return text, list(tx.sents)


class MakeQA:
    __summary_content = []
    __error_num = 0
    __json_file_path = 'Data_list/modern_Q_&_A.json'  # result file path.
    __error_file_path = 'Data_list/error.txt'  # result path which have error in regex fitting.
    __mentioned_text_file_path = 'Data_list/mentioned.json'  # file path which mentioned to the text
    __patterns = [
        r"(\spage|\(page)\s",
        r"(\sfigure|\(figure)\s",
        r"(\sfig|\(fig)\s",
        r"(\stable|\(table)\s",
        r"(\sbox|\(box)\s\d",
        r"(\schapter|\(chapter)\s\d",
        r"(\schapters|\(chapters)\s\d",
        r"(\stext|\(text)\W",
        r"(\sbook|\(book)\W",
        r"(\sauthor|\(author)\W",
    ]

    def __init__(self, merge_content: list, table_content: list):
        self.merge_content = merge_content
        self.table_content = table_content
        self.__max_len_key = max([int(x[0]) for x in merge_content])
        self.book_name = input("enter the book name: ")
        self.number_of_supergroup = int(input('How many super title need?(1,2,3,4): '))

        # Check if the directory exists
        if not os.path.exists(self.__json_file_path.split('/')[0]):
            # If it does not exist, create it
            os.makedirs(self.__json_file_path.split('/')[0])
        # Check if the file exists
        if os.path.isfile(self.__mentioned_text_file_path):
            print(f"The file 'mentioned json file path' exists in '{self.__mentioned_text_file_path}'.")
        else:
            with open(self.__mentioned_text_file_path, 'w') as file:
                # Save the updated JSON data
                json.dump([], file, indent=2)
        if os.path.isfile(self.__json_file_path):
            print(f"The file 'data file path' exists in '{self.__json_file_path}'.")
        else:
            with open(self.__json_file_path, 'w') as file:
                # Save the updated JSON data
                json.dump([], file, indent=2)

    def __make_source_content(self, key: tuple):
        source_txt = key[2]
        index_key = self.merge_content.index(key)
        supergroup = list()
        super_key_id = 1
        for k in self.merge_content[:index_key][::-1]:  # find supergroup of each group.
            if len(supergroup) == self.number_of_supergroup:  # save last two supergroup
                break
            # We will separate the subgroups of each branch.
            if int(k[0]) + super_key_id == int(key[0]):
                supergroup.append(k[1])
                super_key_id += 1
        # print(f'{key[1]} : {supergroup}')

        title = ''
        for t in supergroup[::-1]:
            title += t + ':'
        # print(f'{title}{key[1]}')
        return title.upper() + key[1].upper() + '\n' + source_txt, title[:len(title) - 1] if len(title) != 0 else key[1]

    def run(self):
        # for table
        for iter_table in self.table_content:
            table_str = iter_table[0]
            question_num = iter_table[1]
            self._create_table_q_a(table_str, question_num)
        # for non table
        for key in self.merge_content[::-1]:
            if len(key[2]) != 0:
                # Examining the text with its important topics.
                source_txt, topic = self.__make_source_content(key)
                # # topic = self._topic(source_txt_with_out_topic, key)
                print(f'\"{topic}\" ', source_txt)
                print(len(key[-1]))
                if len(key[-1]) <= 30:  # the maximum q&a the model can make is 30, so we divided into 3 sections.
                    self._merge_q_a(source_txt, key[1], len(key[-1]), topic=topic)
                else:
                    self._merge_q_a(source_txt, key[1], round(len(key[-1]) / 2), topic=topic)
                    self._merge_q_a_v2(source_txt, key, round(len(key[-1]) / 3), 'middel', topic=topic)
                    self._merge_q_a_v2(source_txt, key, round(len(key[-1]) / 3), 'last', topic=topic)

                for t in self.__topic_list(source_txt, round(len(key[-1]) / 2), False):
                    self.__long_q_a(source_txt, key[1], t)
                # Making summary from text.
                self._make_summary(source_txt, (key[0], key[1]))
            else:
                self.__summary_content.append((key[0], key[1], ''))
            key_id = int(key[0])
            if key_id < self.__max_len_key:  # check is group or subgroup.
                subgroup = list()
                for k in self.merge_content[self.merge_content.index(key) + 1:]:
                    if int(k[0]) == int(key[0]):
                        break
                    # We will separate the subgroups of each branch.
                    if int(k[0]) - 1 == int(key[0]):
                        subgroup.append((k[0], k[1]))
                print('This is subGroup: ', subgroup)
                if len(subgroup) != 0:  # Check for the presence or absence of a subgroup.
                    t = self.__summary_content[-1][2]
                    self.__summary_content.pop()
                    sbg = []
                    for temp in self.__summary_content[::-1]:
                        for temp2 in subgroup:
                            if temp[0] == temp2[0] and temp[1] == temp2[1]:
                                sbg.append(temp)
                    for k in sbg:  # delete subgroup values from summary list.
                        self.__summary_content.remove(k)
                        t += '\n' + k[1].upper() + '\n' + k[2]
                    # self.__saver(key, t, key)
                    print(t)
                    ##########################
                    #
                    # Adding sort options
                    #
                    print('***********Iam in the supergroup***********')
                    print(key[1])
                    for t in self.__topic_list(key[1].upper() + '\n\n' + t, round(len(key[-1]) / 3), False):
                        self.__long_q_a(key[1].upper() + '\n\n' + t, key[1], t)
                    ##########################
                    self._make_summary(t, (key[0], key[1]))
            print("()()()()()()()()()()()()()()()()()()")

    def _create_table_q_a(self, txt: str, num: int, reload=False):
        sys_message = "You are an AI assistant."
        txt = f'Extract {num} questions and answers from the table below.\n' + txt
        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, temperature=0.1,
                                    top_k=10)
        output += f"\n\n{num + 1}. "
        if self.__exits_check(output, num, '', 'table'):
            return self._create_table_q_a(txt, num, reload=True)
        for n in range(1, num + 1):
            q = re.findall(rf'^\s*{n}\.(.+)', output, re.MULTILINE)
            a = re.findall(rf'{n}\..+answer:\s+(.+)\n{n + 1}\.', output.lower(), re.DOTALL)
            if '
' in a[0]:
                a = re.findall(r'(.+)<\/s>', a[0], re.DOTALL)
            if re.findall(r'\s+', q[0]) != 0:
                q = re.findall(r'\s+(.+)', q[0])
            a = re.sub(r'\n+$', '', a[0])
            q = re.sub(r'\n+$', '', q[0])
            # a = 'In short, ' + a
            # q = 'In short, ' + q
            q, a = self.__filter(q, a, 'table')
            self.__saver(q, a, 'table')
            print(f'Table: ', q, '\n', a, '\n###########\n')

    def _merge_q_a_v2(self, txt: str, key: tuple, len_no_merge: int, partition: str, reload=False, topic=None):
        output_format = "\n"
        for _ in range(1, len_no_merge + 1 + 2):
            output_format += f"Q.{_}:\nA.{_}:\n\n"
        if partition == 'last':
            s_txt = key[-1][len(key[-1]) - len_no_merge:]
        else:
            s_txt = key[-1][len_no_merge:len(key[-1]) - len_no_merge]
        source_txt = ''
        for tex in s_txt:
            source_txt = source_txt + str(tex)
        print('this is source_txt in _merge_q_a_v2: ', source_txt)
        sys_message = ((f'You are an AI assistant. Your task is to create the {len_no_merge + 2} questions with the'
                        f' answers from the user text, which is a part of the text below.'
                        f' The main topic of the text must be mentioned in the title of the question.')
                       # f' Design the topic of the question so that it has the meaning of the main title of the text.')
                       + f'\n{txt}' + "\n***OutPut format:\n" + output_format)
        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=source_txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=source_txt, max_new_token=11000,
                                    temperature=0.1, top_k=10)
        output += f"\n\nQ.{len_no_merge + 1 + 2}:"
        if self.__exits_check(output, len_no_merge, key[1], 'q_a'):
            return self._merge_q_a_v2(txt, key, len_no_merge, partition, reload=True, topic=topic)
        for n in range(1, len_no_merge + 1):
            q = re.findall(rf'Q\.{n}:\s+(.+)A\.{n}:', output, re.DOTALL | re.IGNORECASE)
            a = re.findall(rf'A\.{n}:\s+(.+)Q\.{n + 1}:', output, re.DOTALL | re.IGNORECASE)
            if '
' in a[0]:
                a = re.findall(r'(.+)<\/s>', a[0], re.DOTALL)
            # if len(re.findall(r'(.+)\n+$', a[0], re.DOTALL)) != 0:
            #     a = re.findall(r'(.+)\n+$', a[0], re.DOTALL)
            # if len(re.findall(r'(.+)\n+$', q[0], re.DOTALL)) != 0:
            #     q = re.findall(r'(.+)\n+$', q[0], re.DOTALL)
            a = re.sub(r'\n+$', '', a[0])
            q = re.sub(r'\n+$', '', q[0])
            if topic:
                q = q + f'(in the context of {topic})'
            a = 'In short, ' + a
            q = 'In short, ' + q
            q, a = self.__filter(q, a, key[1])
            self.__saver(q, a, key[1])
            print(f'With Merge {partition} {key[1]}: ', q, '\n', a, '\n###########\n')

    def _merge_q_a(self, txt: str, key: str, len_no_merge: int, reload=False, topic=None):
        output_format = "\n"
        for _ in range(1, len_no_merge + 1 + 2):
            output_format += f"Q.{_}:\nA.{_}:\n\n"
        sys_message = (f"Extract {len_no_merge + 2} creative questions and answers from the text."
                       # f"Design the topic of the question so that it has the meaning of the main title of the text."
                       f"The main topic of the text must be mentioned in the title of the questions."
                       f" \n***OutPut format:\n" + output_format)
        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, temperature=0.1,
                                    top_k=10)
        output += f"\n\nQ.{len_no_merge + 1 + 2}:"
        if self.__exits_check(output, len_no_merge, key, 'q_a'):
            return self._merge_q_a(txt, key, len_no_merge, reload=True, topic=topic)
        for n in range(1, len_no_merge + 1):
            q = re.findall(rf'Q\.{n}:\s+(.+)A\.{n}:', output, re.DOTALL | re.IGNORECASE)
            a = re.findall(rf'A\.{n}:\s+(.+)Q\.{n + 1}:', output, re.DOTALL | re.IGNORECASE)
            if '
' in a[0]:
                a = re.findall(r'(.+)<\/s>', a[0], re.DOTALL)
            a = re.sub(r'\n+$', '', a[0])
            q = re.sub(r'\n+$', '', q[0])
            if topic:
                q = q + f'(in the context of {topic})'
            a = 'In short, ' + a
            q = 'In short, ' + q
            q, a = self.__filter(q, a, key)
            # if len(re.findall(r'(.+)\n+$', a[0], re.DOTALL)) != 0:
            #     a = re.findall(r'(.+)\n+$', a[0], re.DOTALL)
            # if len(re.findall(r'(.+)\n+$', q[0], re.DOTALL)) != 0:
            #     q = re.findall(r'(.+)\n+$', q[0], re.DOTALL)
            # q, a = self.__filter(q[0], a[0], key)
            self.__saver(q, a, key)
            print(f'With Merge {key}: ', q, '\n', a, '\n###########\n')
        # """
        # All Pattern needed for extracting output result.
        # """

    def _topic(self, txt: str, key: str, reload=False):
        # sys_message = (f'You are an AI assistant. Your task is to find the short general topic of the text.'
        sys_message = (f'You are an AI assistant. Your task is to find general topic of the text.'
                       + "\n***OutPut format:\nTOPIC:")
        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, temperature=0.1,
                                    top_k=5)
        if self.__exits_check(output, 1, key, 'topic'):
            return self._topic(txt, key, True)
        if len(re.findall(r'topic:(.+)', output.lower())) != 0:
            topic = re.findall(r'topic:(.+)', output.lower())[0]
        else:
            topic = output
        if '
' in topic:
            topic = re.findall(r'(.+)<\/s>', topic, re.DOTALL)
        if len(re.findall(r'\s+(.+)', topic[0])) != 0:
            topic = re.findall(r'\s+(.+)', topic[0])
        print('topic: ', topic[0])
        return topic[0]

    def __long_q_a(self, txt: str, key: str, t: str, reload=False) -> None:

        sys_message = ((f"You are an AI assistant. Your task is to create important question with answer"
                        f" about this topic '{t}' from the user's text which have long answers.")
                       + "\n***OutPut format:" + "\nQuestion:\nAnswer:\n")

        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, temperature=0.1,
                                    top_k=10)

        # print(output, '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
        a = re.findall(r'Answer:\s+(.+)', output, re.DOTALL | re.IGNORECASE)
        q = re.findall(r'Question:\s+(.+)', output, re.MULTILINE | re.IGNORECASE)
        if len(a) == 0 or len(q) == 0:
            self.__error_saver(key, output)
            # a = q = ['error']
            return self.__long_q_a(txt, key, t, reload=True)
        if '
' in a[0]:
            a = re.findall(r'(.+)<\/s>', a[0], re.DOTALL)
        # if len(re.findall(r'(.+)\n+$', a[0], re.DOTALL)) != 0:
        #     a = re.findall(r'(.+)\n+$', a[0], re.DOTALL)
        # if len(re.findall(r'(.+)\n+$', q[0], re.DOTALL)) != 0:
        #     q = re.findall(r'(.+)\n+$', q[0], re.DOTALL)
        # q, a = self.__filter(q[0], a[0], key)
        a = re.sub(r'\n+$', '', a[0])
        q = re.sub(r'\n+$', '', q[0])
        q, a = self.__filter(q, a, key)
        self.__saver(q, a, key)
        print(f'long question answer: {key} topic={t}: ', q, '\n', a, '\n###########\n')

    def __topic_list(self, txt: str, len_no_merge: int, reload=False) -> list:
        """
         Extract all Important topic.
        :param txt: str
        :param len_no_merge: int
        :param reload: bool
        :return: list
        """
        output_format = "\n"
        for _ in range(1, len_no_merge + 1):
            output_format += f"{_}. \n\n"
        sys_message = (f"You are an AI assistant. Your task is to extract {len_no_merge} important topic from the"
                       f" user's text.") + "\n***OutPut format:" + output_format
        if reload:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, top_k=60,
                                    temperature=0.3)
        else:
            output = self.__run_llm(sys_message=sys_message, user_message=txt, max_new_token=11000, temperature=0.2,
                                    top_k=40)
        output += f"\n{len_no_merge + 1}. \n"
        if self.__exits_check(output, len_no_merge, 'nothing', 'topic_list'):
            return self.__topic_list(txt, len_no_merge, reload=True)
        topic_list = []
        for n in range(1, len_no_merge + 1):
            t = re.findall(rf'{n}\.\s+(.+)\n\s*{n + 1}\.\s+', output, re.DOTALL)
            if len(re.findall(r'(.+)\n+$', t[0], re.DOTALL)) != 0:
                t = re.findall(r'(.+)\n+$', t[0], re.DOTALL)
            if '
' in t[0]:
                t = re.findall(r'(.+)<\/s>', t[0], re.DOTALL)
            print(f'In topic list : ', t, '\n###########\n')
            topic_list.append(t[0])
        return topic_list

    def __exits_check(self, output: str, len_no_merge: int, key: str, style: str):
        """
         check all questions exists.
        :param output:
        :param len_no_merge:
        :param key:
        :return:
        """
        if style == 'topic_list':
            for n in range(1, len_no_merge + 1):
                t = re.findall(rf'{n}\.\s+(.+)\n\s*{n + 1}\.\s+', output, re.DOTALL)
                if len(t) == 0:
                    self.__error_num += 1
                    if self.__error_num == 3:
                        pass
                    print('Error in Merge.++++')
                    self.__error_saver(key, output)
                    # a = q = ['error in Merge']
                    return 1
        elif style == 'q_a':
            for n in range(1, len_no_merge + 1):
                q = re.findall(rf'Q\.{n}:\s+(.+)A\.{n}:', output, re.DOTALL | re.IGNORECASE)
                a = re.findall(rf'A\.{n}:\s+(.+)Q\.{n + 1}:', output, re.DOTALL | re.IGNORECASE)
                if len(a) == 0 or len(q) == 0:
                    self.__error_num += 1
                    if self.__error_num == 3:
                        pass
                    print('Error in Merge.++++')
                    self.__error_saver(key, output)
                    # a = q = ['error in Merge']
                    return 1
        elif style == 'qs':
            for n in range(1, len_no_merge + 1):
                q = re.findall(rf'Q\.{n}:\s*', output, re.DOTALL | re.IGNORECASE)
                if len(q) == 0:
                    self.__error_num += 1
                    if self.__error_num == 3:
                        pass
                    print('Error in Merge.++++')
                    self.__error_saver(key, output)
                    # a = q = ['error in Merge']
                    return 1
        elif style == 'as':
            for n in range(1, len_no_merge + 1):
                q = re.findall(rf'A\.{n}:\s*', output, re.DOTALL | re.IGNORECASE)
                if len(q) == 0:
                    self.__error_num += 1
                    if self.__error_num == 3:
                        pass
                    print('Error in Merge.++++')
                    self.__error_saver(key, output)
                    # a = q = ['error in Merge']
                    return 1
        elif style == 'topic':
            topic = re.findall(r'topic:(.+)', output.lower())
            if len(topic) == 0:
                print('error in topic', output)
                self.__error_saver(key, output)
                return 1
            if len(topic[0]) >= 100:
                return 1
        elif style == 'table':
            for n in range(1, len_no_merge + 1):
                if len(re.findall(rf'^\s*{n}\.', output, re.MULTILINE)) == 0:
                    print('error in Table+++')
                    self.__error_saver(key, output)
                    return 1
                if len(re.findall(rf'{n}\..+answer:\s+(.+)\n{n + 1}\.', output.lower(), re.DOTALL)) == 0:
                    print('error in Table+++')
                    self.__error_saver(key, output)
                    return 1
        return 0

    def _make_summary(self, txt: str, key: tuple):  # create summary.
        sys_message = 'You are an AI assistant. Your task is to summarize.'
        output = self.__run_llm(sys_message=sys_message, user_message=txt)
        if '
' in output:
            output = re.findall(r'(.+)<\/s>', output, re.DOTALL)[0]
        x = (output,)
        self.__summary_content.append(key + x)
        while 1:
            topic = self._title(txt)
            if topic is not None:
                break
        if '
' in topic:
            topic = re.findall(r'(.+)<\/s>', topic, re.DOTALL)[0]
        # self.__saver(topic, output, key)  # save summery content in file.
        topic, txt = self.__filter(topic, txt, key[1])
        self.__saver(topic, txt, key[1])  # save txt content in file.
        print(f'{key[1]} topic: {topic}', '\n', f'summary: {output}', '\n###########\n')
        return output

    def _title(self, txt):  # create topic.
        sys_message = f'You are an AI assistant. Your task is to suggest a title.' + "\n***OutPut format:\nTITLE:"
        output = self.__run_llm(sys_message=sys_message, user_message=txt, temperature=0.1, top_k=10)
        output = re.findall(r'title:(.+)', output.lower())
        if '
' in output:
            output = re.findall(r'(.+)<\/s>', output, re.DOTALL)
        if len(output) == 0:
            return None
        return output[0]

    def __run_llm(self, sys_message: str, user_message: str, temperature=0.1, top_p=0.95, top_k=10, max_new_token=2000):
        # Format the input using the provided template
        prompt = f"### System:\n{sys_message}\n### User:\n{user_message}\n### Assistant:\n"
        streamer = TextStreamer(tokenizer)
        # Tokenize and encode the prompt
        inputs = tokenizer.encode(prompt, return_tensors="pt", add_special_tokens=False)
        input_ids = tokenizer(prompt, return_tensors='pt').input_ids.cuda()
        response = model.generate(inputs=input_ids, temperature=temperature, do_sample=True, top_p=top_p, top_k=top_k,
                                  max_new_tokens=max_new_token, )  # streamer=streamer)
        response = tokenizer.decode(response[0])
        return response.split("### Assistant:\n")[-1]

    def __saver(self, question: str, answer: str, key: str):
        # Open the JSON file for reading
        with open(self.__json_file_path, 'r') as file:
            existing_data = json.load(file)
        # Add a new object to the existing data
        new_object = {'id': key, 'question': question, 'answer': answer}
        existing_data.append(new_object)
        # Open the JSON file for writing
        with open(self.__json_file_path, 'w') as file:
            # Save the updated JSON data
            json.dump(existing_data, file, indent=2)

    def __error_saver(self, key: str, output: str):
        with open(self.__error_file_path, 'a', encoding='utf-8') as file:
            file.write(key + ' ********\n' + output + '\n()()()()()()()()()()()()()()()()()()\n')

    def __filter(self, q: str, a: str, key: str):
        error_list_output = []
        error_list_instruction = []
        for pt in self.__patterns:
            pattern = re.compile(pt)
            searcher_output = pattern.search(a.lower(), re.DOTALL)
            searcher_instruction = pattern.search(q.lower(), re.DOTALL)
            if searcher_output:
                error_list_output.append(searcher_output.start())
            if searcher_instruction:
                error_list_instruction.append(searcher_instruction.start())
        if len(error_list_output) != 0 or len(error_list_instruction) != 0:  # check if q or a mention to the text.
            with open(self.__mentioned_text_file_path, 'r') as file:
                existing_data = json.load(file)
            # Add a new object to the existing data
            new_object = {'id': key, 'question': a, 'answer': q}
            existing_data.append(new_object)
            # Open the JSON file for writing
            with open(self.__mentioned_text_file_path, 'w') as file:
                # Save the updated JSON data
                json.dump(existing_data, file, indent=2)
        for error in error_list_output:
            a = f"{a[:error]} ({self.book_name}) {a[error:]}"
        for error in error_list_instruction:
            q = f"{q[:error]} ({self.book_name}) {q[error:]}"
        return q, a


content = []  # content_with_out_merge is dict with list of text. this table use for extract small answer.
texts = ''
start_fetch = False
book_mark = []
table = ''
start_fetch_table = False
tables = []

check_txt_cleaning = True if 'yes' in input("do you need cleaning text?(yes/no): ").lower() else False

for i in docs:
    if re.search(r'#\d+#\s+.+', i):
        i = re.findall(r'#(\d+)#\s+(.+)', i)  # i = Bookmark
        print(i[0])
        # add text by bookmark
        add_book_now = True
        if len(book_mark) == 0:
            add_book_now = False
            book_mark.append(i)
        # if len(texts) != 0:
        # merge is string but no_merge is list
        merge, no_merge = txt_cleaner(texts)
        # content is dict with merged text. in this data, table related togather with ',' \
        # so when can extract long answer
        # content[f'{book_mark[-1][0][0]} {book_mark[-1][0][1]}'] = text_communicator(merge)
        if check_txt_cleaning:
            content.append((book_mark[-1][0][0], book_mark[-1][0][1], text_communicator(merge), no_merge))
        else:
            content.append((book_mark[-1][0][0], book_mark[-1][0][1], texts, list(nlp(texts).sents)))

        if add_book_now:
            book_mark.append(i)

        if i == [('00', 'END')]:
            # print(content)
            # print(tables)
            create_q_a = MakeQA(merge_content=content, table_content=tables)
            create_q_a.run()  # run for Extracting Q&A
            # clearing the variable for next chapter.
            content.clear()
            book_mark.clear()
            tables.clear()
            start_fetch = False
            start_fetch_table = False
        # clean the text for next bookmark
        texts = ''
        continue
    # spite txt.
    if re.search(r'@@@+', i):
        start_fetch = True if start_fetch is False else False
        continue
    if start_fetch:
        texts += i

    if re.search(r'#table#', i.lower()) and start_fetch_table is True:
        if len(table) != 0:
            number_of_questions = int(re.findall(r'#number\s+=\s*(\d+)', table, re.MULTILINE | re.IGNORECASE)[0])
            table = re.sub(r'#number\s+=\s*\d+', '', table)
            print(number_of_questions)
            print(table)
            tables.append((table, number_of_questions))
        table = ''
    # spite txt.
    if re.search(r'#table#', i.lower()):
        start_fetch_table = True if start_fetch_table is False else False
        continue
    if start_fetch_table:
        table += i

print(len(content))

# for i in content[::-1]:
#     print(i, '\n#####\n')

# print(content_with_out_merge['1 The animal and its food'])
# print(book_mark)
#
# for i in content_with_out_merge['1 The animal and its food']:
#     print(i, '\n()()()()\n')