python

check  the algoritm. It's very slow because of all_combinations = product(*[get_param_data(pid) for pid in param_ids]). Try to make it faster


import pandas as pd
import re
from itertools import product

# Load the CSV files
url_base_infm = pd.read_csv('url_base_infm.csv')
url_base_infm.drop_duplicates(subset='url', inplace=True)
#####

DF2 = pd.read_csv('parameters_all.csv')
DF3 = pd.read_csv('parameter_child_parent.csv')
DF1 = pd.read_csv('formula0.csv')

pattern = r'\{([^{}]*)\}'

# Initialize an empty list to store the extracted values
control_values_list = []

# Iterate through each row in the DataFrame
for index, row in DF1.iterrows():
    formula = row['formula']
    
    # Find all matches of the pattern in the formula
    matches = re.findall(pattern, formula)
    
    # Append the matched values to the list
    control_values_list.extend(matches)

#display(control_values_list)

# Transform the extracted values to integers
control_values_list = [int(value) for value in control_values_list]
 
#display(DF2.loc[DF2['param_id']==720])
#display(DF2.loc[DF2['param_id'].isin(control_values_list)])
DF2 = (DF2.loc[DF2['param_id'].isin(control_values_list)])


# Initialize a list to store the results
result_list = []

# Creating dictionaries for quick lookup
param_id_to_data = DF2.groupby('param_id').apply(lambda x: x.to_dict('records')).to_dict()

# Group by category for quick lookup of parent-child value pairs
category_to_pair_values = DF3.groupby('category').apply(lambda x: list(zip(x['parent_value'], x['value']))).to_dict()
print('Data loaded:')
# Function to get all relevant data for a parameter id
def get_param_data(param_id):
    return param_id_to_data.get(param_id, [])


def get_param_data_id(param_id):
    return param_id_to_data.get(id, [])
 
#id	param_id


# Function to append the result
def append_result(idf, category, formula_pattern, param_ids, combination):
    formula = formula_pattern
    formula_pattern_start = formula_pattern
    slugs = []
    values = []
    add_item_labels = []
    
    for pid, data in zip(param_ids, combination):
        slug = data['slug']
        value = data['value']
        add_item_label = data['addItemLabel']
        id0  = data['id']
        param_id0  = data['param_id']   
        
        # Replace the parameter id in the formula with its slug
        formula = formula.replace(f'{{{pid}}}', slug)
        
        slugs.append(slug)
        values. append(value)
        add_item_labels.append(add_item_label)
  # id	param_id	 
    result_list.append({
        'formula_pattern_start': formula_pattern,
        'param_ids': param_ids,
         
        'idf': idf,
        'category': category,
        'formula': formula,
        'slug': '; '.join(slugs),
        'value': '; '.join(values),
        'addItemLabel': '; '.join(add_item_labels),
        #'id': '; '.join(id0),
        #'param_id': '; '.join(param_id0),        
        'parent_value': 'N/A'  # We can loop through the possible pairs to match the actual parent value if needed
    })

# Process each row in DF1
for index, row in DF1.iterrows():
    formula_pattern = row['formula']
    category = row['category']
    idf = row['idf']
    
    # Find all the parameter ids in the formula using regex
    param_ids = re.findall(r'{(\d+)}', formula_pattern)
    param_ids = [int(pid) for pid in param_ids]

    # Get all combinations of parameter values for the current category
    all_combinations = product(*[get_param_data(pid) for pid in param_ids])
    
    # Validate each combination
    valid_combinations = []
    for combination in all_combinations:
        valid = True
        for i in range(len(param_ids) - 1):
            for j in range(i + 1, len(param_ids)):
                value_i = combination[i]['value']
                value_j = combination[j]['value']
                
                # Check if these pairs are valid according to DF3
                if (value_i, value_j) not in category_to_pair_values.get(category, []):
                    valid = False
                    break
            if not valid:
                break
        
        if valid:
            valid_combinations.append(combination)
    
    # Append all valid combinations
    for combination in valid_combinations:
        append_result(idf, category, formula_pattern, param_ids, combination)

# Convert the result list to a DataFrame
result_df = pd.DataFrame(result_list)

# Print the resultant DataFrame
print('Final Results:')
result_df.reset_index(drop=True) 

result_df['param_ids'] = result_df['param_ids'].astype(str)
#result_df['param_ids'] = result_df['param_ids'].astype(str)
#result_df = pd.merge(result_df, url_base_infm, how='left', left_on='formula_pattern_start' , right_on='rewrite_template')
# url_base_infm[['baseurl','infmbase'	,'url','template count','category_id','rewrite_templateId','rewrite_template']]
result_df.drop_duplicates(inplace=True) 
display(result_df)

if not result_df.empty:
    print(result_df.iloc[0, 2])