Structure and optimize the code, do a complete refactoring
import json

from data_sources.adapters import ConstructionAdapter
from data_sources.gateways import NeosintezGateway
from domain.repositories import ConstructionRepository
from domain.entities import Construction, Filter

from data_sources.adapters import ManufacturingDataAdapter
from domain.repositories import ManufacturingDataRepository
from domain.entities import ManufacturingData
from application.modules import ManufactureUpload

from data_sources.adapters import SetDataAdapter
from domain.repositories import SetDataRepository
from domain.entities import SetData
from application.modules import SetUpload

import os
import pandas as pd
import logging
from datetime import datetime

def load_json(file_path):
    with open(file_path, 'r', encoding='utf-8-sig') as file:
        return json.load(file, strict=False)

def load_xlsx (file_path: str) -> pd.DataFrame:
    return pd.read_excel(file_path)

if __name__ == "__main__":
    start_time = datetime.now()
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler(f"C:\python\logs\{datetime.now().strftime('%Y-%m-%d')}_milestone_and_set_uploader_log.txt", encoding='utf-8')
    file_handler.setFormatter(formatter)
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    logger.addHandler(console_handler)

    auth_data = load_json(r"C:\python\appius-neosintez_milestones_uploader\auth_data.json")
    config = load_json(r"C:\python\appius-neosintez_milestones_uploader\config.json")
    gateway = NeosintezGateway(**auth_data["neosintez"])

    try:
        manufacturing_data = load_xlsx(config['path']['milestone_path'])
        os.rename(config["path"]["milestone_path"], f"prev_{datetime.now().strftime('%Y-%m-%d')}_{config['path']['milestone_path']}")
        os.replace(f"prev_{datetime.now().strftime('%Y-%m-%d')}_{config['path']['milestone_path']}", config["path"]["prev_path"])
        logger.info('Manufacturing data file is copied in prev folder')
    except:
        logger.warning('Manufacturing data file is not found')

    try:
        set_rd_data = load_xlsx(config['path']['set_path']) 
        os.rename(set_rd_data, f"prev_{datetime.now().strftime('%Y-%m-%d')}_{config['path']['set_path']}")
        os.replace(f"prev_{datetime.now().strftime('%Y-%m-%d')}_{set_rd_data}", config["path"]["prev_path"])
        logger.info('Set data file is copied in prev folder')
    except:
        logger.warning('Set data file is not found')
    
    construction_adapter = ConstructionAdapter(gateway)
    construction_repository = ConstructionRepository(construction_adapter)
    
    df_id_exists = Filter("appius_ir_flag", "exist",)
    constructions: list[Construction] = construction_repository.list(df_id_exists)
    if constructions:
        manufacturing_adapter = ManufacturingDataAdapter(gateway, manufacturing_data)
        manufacturing_data_repository = ManufacturingDataRepository(manufacturing_adapter)

        set_data_adapter = SetDataAdapter(gateway, set_rd_data)
        set_data_repository = SetDataRepository(set_data_adapter)
        
        for construction in constructions:
            try:
                id_di_check = Filter("mvz", "mvz", construction.mvz)
                manufacturing_data_list: list[ManufacturingData] = manufacturing_data_repository.list(id_di_check)
                if len(manufacturing_data_list) <= 0:
                    logging.warning('Manufacturing data file is empty')
                id_df_count_by_mvz = manufacturing_adapter.count_id_df_per_mvz()
                print(id_df_count_by_mvz)
                manufacturing_upload = ManufactureUpload(gateway, construction, logger)
                manufacturing_upload.execute(manufacturing_data, manufacturing_data_list, id_df_count_by_mvz)

                id_di_check = Filter("mvz", "mvz", construction.mvz)
                set_data_list: list[SetData] = set_data_repository.list(id_di_check)
                uniq_df= sorted(set([item.df_code for item in set_data_list if item.df_code]))
                uniq_stage = sorted(set([item.stage for item in set_data_list if item.stage]))
                if len(set_data_list) <= 0:
                    logging.warning('Set data file is empty')
                set_upload = SetUpload(gateway, construction, logger)
                set_upload.execute(set_data_list)
            except Exception as e:
                logger.error(e)
    logger.info(f"Время работы: {datetime.now() - start_time}")

import requests
import json

from domain.repositories import ManufacturingDataRepository
from domain.entities import Construction
from data_sources.gateways import NeosintezGateway
from data_sources.adapters import ManufacturingDataAdapter
from pprint import pprint

class ManufactureUpload:
    manufacture_folder_class = "f581e310-d747-ee11-917e-005056b6948b" 
    mielstone_class = "f9232b4a-4046-ee11-917e-005056b6948b"
    id_attr = "8f3be7b2-ff59-e911-8115-817c3f53a992"
    guide_folder_id = "f8120300-1f59-ee11-9182-005056b6948b"
    guide_mielstone_id = "64b97688-1c59-ee11-9182-005056b6948b"
    base_folder_class = "0dd7d9f9-d259-e911-8115-817c3f53a992"

    UPDATED_MILESTONE = 0
    CREATED_MILESTONE = 0
    MARK_AS_DELETED = 0

    def __init__(self, gateway: NeosintezGateway, construction: Construction, logger) -> None:
        self._gateway = gateway
        self._construction = construction
        self._construction_entity_id = construction.ns_entity_id
        self._manufacture_folder_id = None
        self._data_folder_id = None
        self.logger = logger
        self._id_df_folder_id = None
        
    @property
    def manufacture_folder_id(self):
        if not self._manufacture_folder_id:
            response: requests.Response = (
                self._gateway.get_items_by_class_and_parent_folder(
                    class_id=self.manufacture_folder_class,
                    parent_folder_id=self._construction_entity_id,
                )
            )
            if response.status_code == 200:
                response_data = json.loads(response.text)
                total = response_data["Total"]
                if total == 1:
                    self._manufacture_folder_id = response_data["Result"][0]["Object"]["Id"]
        return self._manufacture_folder_id
 
    def id_df_folder_id(self, id_df):
        if not self._id_df_folder_id:
            self._id_df_folder_id = self._gateway.get_item_by_name_and_parent_folder(id_df ,self.base_folder_class, self.manufacture_folder_id)
        return self._id_df_folder_id
    
    def upload_data_to_folder(self, data, id_df_count_by_mvz):
        if not self.manufacture_folder_id:
            self.create_manufacture_folder()
            self.set_id_attr_to_ns_entity(self.id_attr, self.manufacture_folder_id)

        search_payload = {
                "Filters": [
                    {"Type": 4, "Value": self.manufacture_folder_id},  # Фильтр по parent_folder_id
                    
                    {"Type": 5, "Value": self.mielstone_class},
                ],
                "Conditions": [
            ]}
                
        for manufacturing_data in data:
            
            search_response = self._gateway.search(search_payload)
            if search_response.status_code != 200:
                raise self.logger.error((f"Search error: {search_response.status_code} - {search_response.text}"))

            search_data = json.loads(search_response.text)
            
            match_found = False  # флаг для отслеживания наличия совпадений
            for item in search_data["Result"]:
                if len(id_df_count_by_mvz[self._construction.mvz]) != 1:
                    parent = self._gateway.get_parent(item["Object"]["Id"]).split(" ")[3]
                else:
                    parent = manufacturing_data.id_df
                if item["Object"]["Attributes"]["fb0ba9a2-9653-ee11-9181-005056b6948b"]["Value"] == manufacturing_data.guid and manufacturing_data.id_df == int(parent):
                    match_found = True  # нашли совпадение
                    guid = manufacturing_data.guid
                    end_index = guid.rindex("-0")
                    clean_guid = guid[:end_index]
                    item_id = item["Object"]["Id"]
                    self.update_milestone(item_id, manufacturing_data, update_payload=[])
                    self.UPDATED_MILESTONE += 1
                    if manufacturing_data.req_type == "Фактическая дата" and guid.endswith("-0"):
                        for item in search_data["Result"]:
                            if item["Object"]["Attributes"]["fb0ba9a2-9653-ee11-9181-005056b6948b"]["Value"] == clean_guid + "-1":
                                plan_date = item["Object"]["Attributes"]["78c6b7f2-4046-ee11-917e-005056b6948b"]["Value"]
                                update_payload = [{
                                "Name": 'forvalidation',
                                "Value": plan_date,
                                "Type": 3,
                                'Id': "78c6b7f2-4046-ee11-917e-005056b6948b"
                                }]
                                plan_number = item["Object"]["Attributes"]["87b8ab6c-4546-ee11-917e-005056b6948b"]["Value"]
                                update_payload.append({
                                    "Name": 'forvalidation',
                                    "Value": plan_number,
                                    "Type": 1,
                                    'Id': "87b8ab6c-4546-ee11-917e-005056b6948b"
                                })
                                self.update_milestone(item_id, manufacturing_data, update_payload=update_payload)
                                
            
            if not match_found:
                guid = manufacturing_data.guid
                if manufacturing_data.req_type == "Планируемая дата":
                    created_id = self.create_new_milestone(manufacturing_data, id_df_count_by_mvz)
                    self.update_milestone(created_id, manufacturing_data, update_payload=[])
                    self.CREATED_MILESTONE += 1
                if manufacturing_data.req_type == "Фактическая дата" and guid.endswith("-1") or guid.endswith("-0"):
                    created_id = self.create_new_milestone(manufacturing_data, id_df_count_by_mvz)
                    self.update_milestone(created_id, manufacturing_data, update_payload=[])
                    self.CREATED_MILESTONE += 1
                if manufacturing_data.req_type == "Фактическая дата":
                    self.check_last_fact(manufacturing_data, id_df_count_by_mvz)
                
                    
    def check_last_fact(self, manufacturing_data, id_df_count_by_mvz):
        if manufacturing_data.guid:
            search_payload = {
                "Filters": [
                    {"Type": 4, "Value": self.guide_folder_id},  # Фильтр по parent_folder_id
                    
                    {"Type": 5, "Value": self.guide_mielstone_id},  # Фильтр по class_id
                ],
                "Conditions": [
                ]
            }
            search_response = self._gateway.search(search_payload)
            if search_response.status_code != 200:
                raise self.logger.warning(f"Search error: {search_response.status_code} - {search_response.text}")
    
            search_data = json.loads(search_response.text)
            for i, data in enumerate(search_data["Result"]):
                try:
                    if data["Object"]["Attributes"]['cb78acd9-ddcb-ee11-9199-005056b6948b']["Value"] == manufacturing_data.guid:
                        manufacturing_data.guid = data["Object"]["Attributes"]['fb0ba9a2-9653-ee11-9181-005056b6948b']["Value"]
                        self.upload_data_to_folder([manufacturing_data], id_df_count_by_mvz)
                except KeyError:
                    print("Error updating")
                    
    def update_milestone(self, item_id, manufacturing_data, update_payload=[]):      
        # Обновление существующей Вехи ПП
        update_payload = update_payload
        if manufacturing_data.guid:
            update_payload.append({
                "Name": 'forvalidation',
                "Value": manufacturing_data.guid,
                "Type": 2,
                'Id': "fb0ba9a2-9653-ee11-9181-005056b6948b"
            })
        if manufacturing_data.req_type == "Планируемая дата":
            update_payload.append({
                "Name": 'forvalidation',
                "Value": manufacturing_data.value,
                "Type": 3,
                'Id': "78c6b7f2-4046-ee11-917e-005056b6948b"
            }
            )
        if manufacturing_data.req_type == "Планируемая дата":
            if manufacturing_data.number_of_items:
                update_payload.append({
                    "Name": 'forvalidation',
                    "Value": manufacturing_data.number_of_items,
                    "Type": 1,
                    'Id': "87b8ab6c-4546-ee11-917e-005056b6948b"
                })
        if manufacturing_data.req_type == "Прогнозная дата":
            update_payload.append({
                "Name": 'forvalidation',
                "Value": manufacturing_data.value,
                "Type": 3,
                'Id': "2bbacfcf-4046-ee11-917e-005056b6948b"
            })
        if manufacturing_data.req_type == "Фактическая дата":
            update_payload.append({
                "Name": 'forvalidation',
                "Value": manufacturing_data.value,
                "Type": 3,
                'Id': "b07782b8-4046-ee11-917e-005056b6948b"
            })
        if manufacturing_data.req_type == "Фактическая дата":
            if manufacturing_data.number_of_items:
                update_payload.append({
                    "Name": 'forvalidation',
                    "Value": manufacturing_data.number_of_items,
                    "Type": 1,
                    'Id': "58fd5c74-4546-ee11-917e-005056b6948b"
                })
        update_payload.append({
                "Name": 'forvalidation',
                "Value": {"Name": " "},
                "Type": 8,
                'Id': "7d39b7fb-b9eb-ec11-9131-005056b6948b"
            })
        self.update_milestone_by_guid(manufacturing_data, update_payload)
        update_response = self._gateway.set_attr(item_id, update_payload)
        if update_response.status_code != 200:
            raise self.logger.warning(f"Update error: {update_response.status_code} - {update_response.text}")
                
    def update_milestone_by_guid(self, manufacturing_data, update_payload):
        if manufacturing_data.guid:
            search_payload = {
                "Filters": [
                    {"Type": 4, "Value": self.guide_folder_id},  # Фильтр по parent_folder_id
                    
                    {"Type": 5, "Value": self.guide_mielstone_id},  # Фильтр по class_id
                ],
                "Conditions": [
                ]
            }
            search_response = self._gateway.search(search_payload)
            if search_response.status_code != 200:
                raise self.logger.warning(f"Search error: {search_response.status_code} - {search_response.text}")
    
            search_data = json.loads(search_response.text)
            for i, data in enumerate(search_data["Result"]):
                try:
                    
                    if  data["Object"]["Attributes"]['fb0ba9a2-9653-ee11-9181-005056b6948b']["Value"] == manufacturing_data.guid:
                        update_item_id = data["Object"]["Id"]
                        name = data["Object"]["Name"]
                        update_payload.append({
                            "Name": 'forvalidation',
                            "Type": 8,
                            'Id': "0303ada5-525b-ee11-9183-005056b6948b",
                            "Value": {"Id":update_item_id, "Name": name}
                        })
                except KeyError:
                    print("Error updating")
        return update_payload


    def create_new_milestone(self, manufacturing_data, id_df_count_by_mvz):
        if len(id_df_count_by_mvz[manufacturing_data.mvz]) == 1:
            create_payload = {
                        "Id": "00000000-0000-0000-0000-000000000000",
                        "Name": manufacturing_data.section_pp,
                        "Entity": {"Id": self.mielstone_class, "Name": "forvalidation", },
                    }
            create_response = self._gateway.create_item(self.manufacture_folder_id, create_payload)
            if create_response.status_code != 200:
                raise self.logger.warning(f"Creation error: {create_response.status_code} - {create_response.text}")
            created_item_data = json.loads(create_response.text)
            item_id = created_item_data["Id"]
            return item_id
        
        elif id_df_count_by_mvz[manufacturing_data.mvz]:
            if not self.id_df_folder_id("ID Дф-16 - " + str(manufacturing_data.id_df)):
                parent_id = self.create_id_df_folder("ID Дф-16 - " + str(manufacturing_data.id_df))
            else: 
                parent_id = self.id_df_folder_id("ID Дф-16 - " + str(manufacturing_data.id_df))
            create_payload = {
                    "Id": "00000000-0000-0000-0000-000000000000",
                    "Name": manufacturing_data.section_pp,
                    "Entity": {"Id": self.mielstone_class, "Name": "forvalidation", },
                }
            create_response = self._gateway.create_item(parent_id, create_payload)
            if create_response.status_code != 200:
                raise self.logger.warning(f"Creation error: {create_response.status_code} - {create_response.text}")
            created_item_data = json.loads(create_response.text)
            item_id = created_item_data["Id"]
            self._id_df_folder_id = None
            return item_id
                

    def create_manufacture_folder(self):
        data_folder_id = self._gateway.get_item_by_name_and_parent_folder("!Данные из информационных систем", self.base_folder_class, self._construction_entity_id)
        create_payload = {
            "Id": "00000000-0000-0000-0000-000000000000",
            "Name": "Вехи ПП",
            "Entity": {"Id": self.manufacture_folder_class, "Name": "forvalidation"},
        }
        create_response = self._gateway.create_item(data_folder_id, create_payload)
        if create_response.status_code != 200:
            self.logger.warning(f"Creation error: {create_response.status_code} - {create_response.text}")

        created_folder_data = json.loads(create_response.text)
        self._manufacture_folder_id = created_folder_data["Id"]

        self.logger.info("Folder 'Вехи ПП' successfully created")

    def create_id_df_folder(self, id_df):
        create_payload = {
            "Id": "00000000-0000-0000-0000-000000000000",
            "Name": id_df,
            "Entity": {"Id": self.base_folder_class, "Name": "forvalidation"},
        }
        create_response = self._gateway.create_item(self.manufacture_folder_id, create_payload)
        if create_response.status_code != 200:
            self.logger.warning(f"Creation error: {create_response.status_code} - {create_response.text}")

        created_folder_data = json.loads(create_response.text)

        self.logger.info(f"Folder {id_df} successfully created")
        return created_folder_data["Id"]

    def check_actual_data(self, manufacturing_data, id_df_count_by_mvz):
        search_payload = {
                    "Filters": [
                        {"Type": 4, "Value": self.manufacture_folder_id},
                        
                        {"Type": 5, "Value": self.mielstone_class},
                    ],
                    "Conditions": [
                        {"Attribute": "7d39b7fb-b9eb-ec11-9131-005056b6948b", "Type": 1, 'Operator':8}
                    ]
                }
        search_response = self._gateway.search(search_payload)
        if search_response.status_code!= 200:
                raise self.logger.warning(f"Search error: {search_response.status_code} - {search_response.text}")
                
        search_data = json.loads(search_response.text)
        
        manufacturing_data_for_delete = manufacturing_data.copy()
        for item in reversed(manufacturing_data):
            for i, data in enumerate(reversed(search_data["Result"])):
                if len(id_df_count_by_mvz[self._construction.mvz]) != 1:
                    parent = self._gateway.get_parent(data["Object"]["Id"]).split(" ")[3]
                else:
                    parent = item.id_df
                if  data["Object"]["Attributes"]['fb0ba9a2-9653-ee11-9181-005056b6948b']["Value"] == item.guid and item.id_df == int(parent):
                    search_data["Result"].pop(len(search_data["Result"]) - 1 - i)
                    for del_item in manufacturing_data_for_delete:
                        if del_item.guid == data["Object"]["Attributes"]['fb0ba9a2-9653-ee11-9181-005056b6948b']["Value"]:
                            manufacturing_data_for_delete.remove(del_item)

        if len(search_data["Result"]) > 0:
            for item in search_data["Result"]:
                update_payload = [{
                    "Id": "7d39b7fb-b9eb-ec11-9131-005056b6948b",
                    "Name": "forvalidation",
                    "Type": 8,
                    "Value": {"Id" : "d5fa86ec-b9eb-ec11-9131-005056b6948b", "Name": "Удалить"}
                }]
                update_response = self._gateway.set_attr(item["Object"]["Id"], update_payload)
                self.MARK_AS_DELETED += 1
                if update_response.status_code!= 200:
                    raise self.logger.warning(f"Update error: {update_response.status_code} - {update_response.text}")
    
    def set_id_attr_to_ns_entity(self, id_attr_value: str, entity_id: str,) -> requests.Response:
        payload = [
            {
                "Id": self._manufacture_folder_id,
                "Name": "forvalidation",
                "Type": 2,
                "Value": f"{self._construction.name}",
                "Constraints": [],
            }
        ]
        return self._gateway.set_attr(entity_id, payload)


    def execute(self, adapter: ManufacturingDataAdapter, data, id_df_count_by_mvz):
        self.logger.info("-"*50)
        self.logger.info(f"Manufacture Upload: Старт обработки объекта {self._construction.name}")
        manufacturing_data_list = data
        self.upload_data_to_folder(manufacturing_data_list, id_df_count_by_mvz)
        self.check_actual_data(manufacturing_data_list, id_df_count_by_mvz)

        self.logger.info(f"Updated {self.UPDATED_MILESTONE} milestones")
        self.logger.info(f"Created {self.CREATED_MILESTONE} milestones")
        self.logger.info(f"Mark as deleted {self.MARK_AS_DELETED} milestones")
        self.logger.info("-"*50)