explain theoritically 
import subprocess
import os
import pwinput
import logging
import re
from sqlalchemy import create_engine
from sqlalchemy.engine import url
import pandas as pd
import numpy as np
import warnings
import os.path
import time
import numpy as np
import warnings
import os.path
import csv
import string
from datetime import datetime
import time
from datetime import timedelta
from dateutil import relativedelta
import multiprocessing
import json
import zipfile
import shutil
import teradatasql

#warnings.filterwarnings('ignore')

home='/home/ec2-user/POC/QS_Validator'
data_path=home + '/data'
output_path=home + '/output'
query_path=home + '/queries'
ds_query_path=home + '/ds_query'
scripts_path=home + '/scripts'
temp_path=home + '/temp'
log_path=home + '/logs'
config_file=home + '/config/config.json'
connection_file=home + '/config/connection.json'
archive_dir=home + '/archive/'
partition_template_query = query_path + '/partition_logic.sql'
partition_object_list = temp_path + '/deep_scan_partition_flag.lst'
partition_obj_list_file = temp_path + '/loop_object_list_file.txt'
partition_qry_filter_params = temp_path + '/prtn_query_param_input.txt'
part_final_out_hdr = "Database_Name~Table_Name~ColumnName~Seq_No~Partition_Type~Partition_Filter\n"
size_info_file = temp_path + '/DV_3007_partition_object.lst'
#currentDT = datetime.now()
#log_filename = "qsv_log_multiprocess" + currentDT.strftime("%Y%m%d%H%M%S") + ".log"
#logging.basicConfig(filename = log_path + '/' + log_filename, level = logging.INFO, format = '%(message)s')

g = open(connection_file)
conn = json.load(g)

#======Multiprocessing===============
def exe_query_w(file_list):
    try:
        sub_module_num = "DV_3008"
        query_file_path=ds_query_path+'/'+str(file_list)
        process_id=os.getpid()
        output_file=home + '/output/'+'output_'+str(process_id)+'_'+str(datetime.now())+'.txt'
        partition_number = file_list.split("MINUS_")[-1].replace(".sql","")
        tablename = file_list.split("_MINUS")[0] + "_" + partition_number
        #print(f"Start Time of process  with ID {process_id} :{datetime.now()}.")
        #logging.info (f"Start Time of process with ID {process_id}:{datetime.now()}."
        with open(query_file_path, 'r') as file:
            for line in file:
                read_query = line.strip()
                print(f"Query: {tablename}.sql started with process ID:{process_id} at {datetime.now()}.")
                logging.info (f"Query: {tablename}.sql started with process ID:{process_id} at {datetime.now()}.")#eng_connection=create_engine(url="teradatasql://{0}:{1}@{2}".format(conn["connection"][0]["username"],conn["connection"][0]["pwd"],conn["connection"][0]["host"]))
                with teradatasql.connect(host = conn["connection"][0]["host"], user = conn["connection"][0]["username"], password = conn["connection"][0]["pwd"], ignoreErrors = 3807) as eng_connection:
                    diff_file_name = output_path + '/' + sub_module_num + '_' +tablename +  "_" + conn["connection"][0]["name"] + '_vs_' + conn["connection"][1]["name"] + '_compare.csv'
                    df_diff = pd.read_sql(read_query,eng_connection)
                    check_flag = 0
       
        if df_diff.shape[0] > 0:
            df_diff.to_csv(diff_file_name,index=False)
            return_value='Fail'
            print (f"Query: {tablename}.sql: Fail process ID:{process_id} ended at {datetime.now()}")
            logging.info (f"Query: {tablename}.sql: Fail process ID:{process_id} ended at {datetime.now()}")
            print (f"Difference File:  {diff_file_name}")
            logging.info (f"Difference File:  {diff_file_name}")
            print(" ")
            logging.info (" ")
            check_flag=1
            return (tablename,"Fail")
            
        if check_flag == 0:
            print (f"Query: {tablename}.sql: Pass process ID:{process_id} ended at {datetime.now()}")
            logging.info (f"Query: {tablename}.sql: Pass process ID:{process_id} ended at {datetime.now()}")
            return (tablename,"Pass")
    except Exception as Argument:
        logging.error(str(Argument))

def multi_exe_query_t(file_list):
    try:    
        #print('file_list :',file_list)
        number_of_processes = 5 #multiprocessing.cpu_count()
        #print(len(file_list))
        if(len(file_list)==number_of_processes):
            #print("multi_exe_query_t inside else")
            with multiprocessing.Pool(number_of_processes) as pool:
                results = pool.map(exe_query_w, file_list)
        else:
            #print("multi_exe_query_t inside if ")
            new_file_list=[]
            #new_file_list.append(file_list)
            new_file_list=new_file_list+file_list
            #print(new_file_list)
            proc=int(len(new_file_list))
            #print(proc)
            with multiprocessing.Pool(proc) as pool:
                results  = pool.map(exe_query_w, new_file_list)
        return results
    except Exception as Argument:
        logging.error(str(Argument))
#====================================


def data_export_multiprocess(sub_module_num,sub_module_name):
    try:
        return_value='Pass'
        outfilename=output_path+'/'+sub_module_num+'_compare.csv'
        df_row_compare = pd.DataFrame()
        df_row_compare.insert(loc = 0,column = 'Sub Module No',value = '')
        df_row_compare.insert(loc = 1,column = 'Sub Module Name',value = '')
        df_row_compare.insert(loc = 2,column = 'Tablename',value = '')
        df_row_compare.insert(loc = 3,column = 'Compare_Result',value = '')        
        query_file_path=ds_query_path+'/'
        query_file_list=[]
        query_file_list=sorted(os.listdir(query_file_path))
        final_op = []
        print ("==============MULTI-PROCESSING STARTED===============")
        logging.info ("==============MULTI-PROCESSING STARTED===============")
        #print('query_file_list ',str(query_file_list))
        count_seq = multiprocessing.Value('i')
        count_seq.value=0  
        #multi_exe_query_t(query_file_list)
        new_file_list=[]
        
        number_of_processes = 5 #multiprocessing.cpu_count()
        #while(i<=number_of_processes):
        count_files=len(query_file_list)
        #print('count_files',count_files)
        quo=count_files//number_of_processes        
        
        mod=count_files%number_of_processes        
        
        i=1
        j=0
        #if(i<=quo):                               
        while(i<=quo):
            #print("value of i in iterations",i,'j= ',j)
            new_file_list=query_file_list[j:j+number_of_processes]  
            print("New file list: ", new_file_list)
            final_op.extend(multi_exe_query_t(new_file_list))
            #print(final_op)
            i=i+1 #number_of_processess #2
            j=j+number_of_processes #5
        #print("value of i after coming out of while loop",i)    
        if(i>quo):
            new_list=[]
            mod=count_files%number_of_processes
            if(mod==1):
                new_list=new_list+[query_file_list[-mod:]]
                print("New file list: ", new_file_list)
                final_op.extend(multi_exe_query_t(new_list))
                #multi_exe_query_t(new_list)
            elif(mod>1):
                new_list=query_file_list[-mod:] #[-3:]
                print("New file list: ", new_file_list)
                final_op.extend(multi_exe_query_t(new_list))
        row_compare_op = [(sub_module_num, sub_module_name,x,y) for x,y in final_op]
        df_row_compare = pd.DataFrame(row_compare_op, columns = ['Sub Module No','Sub Module Name','Tablename','Compare_result'])
        df_row_compare.to_csv(outfilename,index=False)
        if 'Fail' in df_row_compare['Compare_result'].values:
            return "Fail"
        else:
            return "Pass"
    except Exception as Argument:
        logging.error(str(Argument))

def append_row_count():
    try:
        outfilename=output_path+'/DV_3008_compare.csv'
        df_rcount_out = pd.read_csv(temp_path + '/rcount_out.csv', sep=",")
        df_rcount_out = df_rcount_out.fillna('')
        df_DV3008_out = pd.read_csv(output_path + '/DV_3008_compare.csv', sep=",")
        df_DV3008_out = df_DV3008_out.fillna('')
        df_col_add = pd.merge(df_DV3008_out, df_rcount_out[['Tablename', 'RowCount_source', 'RowCount_target', 'Data_Size_Bytes_source', 'Data_Size_Bytes_target']], how='left', on=['Tablename'])
        df_col_add.to_csv(outfilename,index=False)
    except Exception as Argument:
        logging.error(str(Argument))

def archive_query_list():
    try:
        # Define the new directory path with the current date in 'ddmmyyyy' format
        current_date = datetime.now().strftime('%d%m%Y')
        new_directory = archive_dir + 'ds_query_' + current_date
        print (new_directory)
        os.makedirs(new_directory, exist_ok=True)  # Create the new directory if it doesn't exist
        # Move files from current directory to new directory
        for file_name in os.listdir(ds_query_path):
            if os.path.isfile(os.path.join(ds_query_path, file_name)):
                shutil.move(os.path.join(ds_query_path, file_name), os.path.join(new_directory, file_name))
        # Zip files in the new directory
        zip_file_name = archive_dir + 'ds_query_' + current_date + '.zip'
        with zipfile.ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, _, files in os.walk(new_directory):
                for file in files:
                    zipf.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), archive_dir))
        shutil.rmtree(new_directory)
        print(f"Files zipped into '{zip_file_name}'.")
    except Exception as Argument:
        logging.error(str(Argument))