import argparse
import hashlib
import json
import random

import numpy as np
from langdetect import detect, LangDetectException
from pymongo import MongoClient
from tqdm import tqdm

from app.ner_loader import ner


class InstructConverter:
    CONF_PROMPTS_PER_PRODUCT = 4
    CONF_DIGITS_PRECISION = 2

    CONF_NUTRIENT_DICT = {'proteins': 'prots', 'fats': 'fats', 'carbohydrates': 'carbs'}

    CONF_WEIGHT_MIN = 50.0
    CONF_WEIGHT_MAX = 200.0
    CONF_NUTRIENT_ADD_CHANCE = 20

    CONF_WEIGHT_TEMPLATES = [
        ', {weight} g',
        ', {weight} grams',
        ', {weight} grams of product',
        ', {weight} grams of it',
        ', weight {weight} g'
    ]

    CONF_NUTRIENT_TEMPLATES = [
        ', {nutrient_weight} g of {nutrient}',
        ', {nutrient} {nutrient_weight} g',
        ', {nutrient_weight} grams of {nutrient}',
        ', {nutrient} in {nutrient_weight} g'
    ]

    def __init__(self):
        CONNECTION_STRING = 'mongodb+srv://doadmin:7pfg65eH480k2wl3@db-mongodb-fibert-data-lake-1a57fff3.mongo.ondigitalocean.com/admin?tls=true&authSource=admin&replicaSet=db-mongodb-fibert-data-lake'
        client = MongoClient(CONNECTION_STRING)
        self.db = client['nutrition_database']['products']

        self.user_prompts = []
        with open('./resources/static/data/food_requests.txt', 'r', encoding="utf-8") as file:
            for line in file:
                self.user_prompts.append(line.strip())

        self.cache = {}

    def replace_entities(self, input_string, product_names):
        ner_entities = self.ner_cached(input_string)
        if len(product_names) < len(ner_entities):
            return input_string

        generated_string = input_string[:]
        for i, entity in enumerate(ner_entities):
            generated_string = generated_string.replace(entity, product_names[i])

        return generated_string

    def count_entities(self, input_string):
        ner_entities = self.ner_cached(input_string)
        return len(ner_entities)

    def ner_cached(self, input_string):
        input_tag = hashlib.md5(input_string.encode()).hexdigest()
        if input_tag in self.cache:
            return self.cache[input_tag]
        else:
            result = ner(input_string)
            self.cache[input_tag] = result
            return result

    def convert_ss(self):
        with open('./resources/static/data/train1_instruct_ss.jsonl', 'w') as f:
            for i, product in tqdm(enumerate(self.db.find()), total=self.db.count_documents({})):
                random_user_prompts = np.random.choice(self.user_prompts, self.CONF_PROMPTS_PER_PRODUCT)
                for j, user_prompt in enumerate(random_user_prompts):

                    try:
                        if detect(user_prompt) != "en":
                            continue
                        if self.count_entities(user_prompt) != 1:
                            continue
                    except LangDetectException:
                        continue

                    product_name = product['product_name']
                    weight_rand = round(
                        random.uniform(self.CONF_WEIGHT_MIN, self.CONF_WEIGHT_MAX),
                        self.CONF_DIGITS_PRECISION
                    )
                    weight_template = random.choice(self.CONF_WEIGHT_TEMPLATES)
                    product_name += weight_template.format(weight=weight_rand)

                    product_entry = {
                        "name": product['product_name'],
                        "measure": "g",
                        "weight": {
                            "min": weight_rand,
                            "max": weight_rand,
                        },
                        "prots_100g": {
                            "min": product['macronutra']['prots'],
                            "max": product['macronutra']['prots'],
                        },
                        "fats_100g": {
                            "min": product['macronutra']['fats'],
                            "max": product['macronutra']['fats'],
                        },
                        "carbs_100g": {
                            "min": product['macronutra']['carbs'],
                            "max": product['macronutra']['carbs'],
                        },
                        "pols_100g": {
                            "min": product['macronutra']['polyols'],
                            "max": product['macronutra']['polyols'],
                        },
                        "acid_100g": {
                            "min": product['macronutra']['acids'],
                            "max": product['macronutra']['acids'],
                        },
                        "fiber_100g": {
                            "min": product['macronutra']['fiber'],
                            "max": product['macronutra']['fiber'],
                        },
                        "alco_100g": {
                            "min": product['macronutra']['alcohol'],
                            "max": product['macronutra']['alcohol'],
                        },
                        "kcal_100g": {
                            "min": product['custom_extra']['estimated_mean_energy_kcal'],
                            "max": product['custom_extra']['estimated_mean_energy_kcal'],
                        }
                    }

                    if (i * 4 + j) % self.CONF_NUTRIENT_ADD_CHANCE == 0:
                        nutrient = np.random.choice(list(self.CONF_NUTRIENT_DICT.keys()))
                        nutrient_key = self.CONF_NUTRIENT_DICT[nutrient]
                        nutr_weight = product['macronutra'][nutrient_key]

                        nutrient_template = random.choice(self.CONF_NUTRIENT_TEMPLATES)
                        nutrient_str = nutrient_template.format(nutrient_weight=nutr_weight, nutrient=nutrient)
                        product_name += f"{nutrient_str}"

                        product_entry['name'] = nutrient_str
                        product_entry[nutrient_key + "_100g"]['min'] = nutr_weight
                        product_entry[nutrient_key + "_100g"]['max'] = nutr_weight

                    generated_user_prompt = self.replace_entities(user_prompt, [product_name])

                    product_json = {
                        "messages": [
                            {
                                "role": "user",
                                "content": generated_user_prompt,
                            },
                            {
                                "role": "assistant",
                                "content": json.dumps([product_entry])
                            }
                        ]
                    }
                    f.write(json.dumps(product_json))
                    f.write("\n")

    def convert_bs(self, w=0.95, k=0.99):

        cur_user_prompt_idx = 0
        cur_products_count = 0
        product_names = []
        product_data = []

        with open('./resources/static/data/train1_instruct_bs.jsonl', 'w') as f:
            for i, product in tqdm(enumerate(self.db.find()), total=self.db.count_documents({})):
                if cur_products_count == 0:
                    current_prompt = self.user_prompts[cur_user_prompt_idx % len(self.user_prompts)]

                    try:
                        if detect(current_prompt) != "en":
                            cur_user_prompt_idx += 1
                            continue
                    except LangDetectException:
                        cur_user_prompt_idx += 1
                        continue

                    num_entities = self.count_entities(current_prompt)
                    if num_entities == 0:
                        cur_user_prompt_idx += 1
                        continue

                    cur_user_prompt_idx += 1

                product_name = product['product_name']
                weight_rand = round(
                    random.uniform(self.CONF_WEIGHT_MIN, self.CONF_WEIGHT_MAX),
                    self.CONF_DIGITS_PRECISION
                )
                weight_min = weight_rand * w
                weight_max = weight_rand / w

                weight_template = random.choice(self.CONF_WEIGHT_TEMPLATES)
                product_name += weight_template.format(weight=weight_rand)

                product_entry = {
                    "name": product['product_name'],
                    "measure": "g",
                    "weight": {
                        "min": weight_min,
                        "max": weight_max,
                    },
                    "prots_100g": {
                        "min": product['macronutra']['prots'] * k,
                        "max": product['macronutra']['prots'] / k,
                    },
                    "fats_100g": {
                        "min": product['macronutra']['fats'] * k,
                        "max": product['macronutra']['fats'] / k,
                    },
                    "carbs_100g": {
                        "min": product['macronutra']['carbs'] * k,
                        "max": product['macronutra']['carbs'] / k,
                    },
                    "pols_100g": {
                        "min": product['macronutra']['polyols'] * k,
                        "max": product['macronutra']['polyols'] / k,
                    },
                    "acid_100g": {
                        "min": product['macronutra']['acids'] * k,
                        "max": product['macronutra']['acids'] / k,
                    },
                    "fiber_100g": {
                        "min": product['macronutra']['fiber'] * k,
                        "max": product['macronutra']['fiber'] / k,
                    },
                    "alco_100g": {
                        "min": product['macronutra']['alcohol'] * k,
                        "max": product['macronutra']['alcohol'] / k,
                    },
                    "kcal_100g": {
                        "min": min(
                            product['custom_extra']['energy_kcal'],
                            product['custom_extra']['estimated_mean_energy_kcal']
                        ) * k,
                        "max": max(
                            product['custom_extra']['energy_kcal'],
                            product['custom_extra']['estimated_mean_energy_kcal']
                        ) / k,
                    },
                    "prots": {
                        "min": product['macronutra']['prots'] * (weight_min / 100) * k,
                        "max": product['macronutra']['prots'] * (weight_max / 100) / k,
                    },
                    "fats": {
                        "min": product['macronutra']['fats'] * (weight_min / 100) * k,
                        "max": product['macronutra']['fats'] * (weight_max / 100) / k,
                    },
                    "carbs": {
                        "min": product['macronutra']['carbs'] * (weight_min / 100) * k,
                        "max": product['macronutra']['carbs'] * (weight_max / 100) / k,
                    },
                    "pols": {
                        "min": product['macronutra']['polyols'] * (weight_min / 100) * k,
                        "max": product['macronutra']['polyols'] * (weight_max / 100) / k,
                    },
                    "acid": {
                        "min": product['macronutra']['acids'] * (weight_min / 100) * k,
                        "max": product['macronutra']['acids'] * (weight_max / 100) / k,
                    },
                    "fiber": {
                        "min": product['macronutra']['fiber'] * (weight_min / 100) * k,
                        "max": product['macronutra']['fiber'] * (weight_max / 100) / k,
                    },
                    "alco": {
                        "min": product['macronutra']['alcohol'] * (weight_min / 100) * k,
                        "max": product['macronutra']['alcohol'] * (weight_max / 100) / k,
                    },
                    "kcal": {
                        "min": min(
                            product['custom_extra']['energy_kcal'],
                            product['custom_extra']['estimated_mean_energy_kcal']
                        ) * (weight_min / 100) * k,
                        "max": max(
                            product['custom_extra']['energy_kcal'],
                            product['custom_extra']['estimated_mean_energy_kcal']
                        ) * (weight_max / 100) / k,
                    }
                }

                if i % self.CONF_NUTRIENT_ADD_CHANCE == 0:
                    nutrient = np.random.choice(list(self.CONF_NUTRIENT_DICT.keys()))
                    nutrient_key = self.CONF_NUTRIENT_DICT[nutrient]
                    nutr_weight = product['macronutra'][nutrient_key]

                    nutrient_template = random.choice(self.CONF_NUTRIENT_TEMPLATES)
                    nutrient_str = nutrient_template.format(nutrient_weight=nutr_weight, nutrient=nutrient)
                    product_name += f"{nutrient_str}"

                    product_entry['name'] = nutrient_str
                    product_entry[nutrient_key+"_100g"]['min'] = nutr_weight
                    product_entry[nutrient_key+"_100g"]['max'] = nutr_weight
                    product_entry[nutrient_key]['min'] = nutr_weight * (weight_rand / 100) * k
                    product_entry[nutrient_key]['max'] = nutr_weight * (weight_rand / 100) / k

                product_names.append(product_name)
                product_data.append(product_entry)
                cur_products_count += 1

                if cur_products_count == num_entities:
                    generated_user_prompt = self.replace_entities(current_prompt, product_names)

                    product_json = {
                        "messages": [
                            {
                                "role": "user",
                                "content": generated_user_prompt
                            },
                            {
                                "role": "assistant",
                                "content": json.dumps(product_data)
                            }
                        ]
                    }

                    f.write(json.dumps(product_json))
                    f.write('\n')

                    cur_products_count = 0
                    product_names = []
                    product_data = []


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('conversion_type', type=str)
    args = parser.parse_args()

    converter = InstructConverter()
    if args.conversion_type == 'ss':
        converter.convert_ss()
    elif args.conversion_type == 'bs':
        converter.convert_bs()


Этот код работает шикарно но очень очень долго (200+ часов на полный процесс)
Переработай этот код на асинхронные задачи Paralell и не забудь оставить tqdm!