Please update this script so that for each participant it select the 20 images to repeat that is shown most rarely on a group level. Adjust the part relating to # Loop through each group and create a separate CSV file:

import pandas as pd
import random
import numpy as np
import matplotlib.pyplot as plt
import os


def generate_images(num_samples, num_images):
    # Take list (with 3 equal chunks)
    # Give each chunk the same number of distortions
    chunk_size = num_samples // 3

    imgs_chunk1 = random.sample(range(1, num_images // 3 + 1), chunk_size)
    imgs_chunk2 = random.sample(range(num_images // 3 + 1, 2 * num_images // 3 + 1), chunk_size)
    imgs_chunk3 = random.sample(range(2 * num_images // 3 + 1, num_images + 1), chunk_size)
    imgs = imgs_chunk1 + imgs_chunk2 + imgs_chunk3

    # Create letters for each chunk
    distortions = [f'{d}{l}' for d in ['_jpeg', '_blur', '_contrast'] for l in ['_70', '_80', '_90']] + ['_original']
    num_distortions = len(distortions)
    num_chunks = 3

    let = pd.Series(distortions * (num_samples // (num_distortions * num_chunks)))
    # Combine chunks, each randomized
    test = pd.concat([let.sample(frac=1) for _ in range(num_chunks)], ignore_index=True)

    # Check that each chunk has the same number of letters
    test = test.astype('category')  # Convert to category for overview

    # Combine and shuffle
    images = pd.DataFrame({'imgs': imgs, 'test': test})

    return images

num_samples = 420  # Default 420
num_images = 540  # Default 540
num_observers = 40  # Number of observers # 10000
num_runs = 10000 # Number of runs to perform

min_variance = float('inf')
min_variance_df = None
min_variance_seed = None

for run in range(1, num_runs + 1):
    # Set the seed for reproducibility
    seed = run
    random.seed(seed)
    np.random.seed(seed)

    # Create an empty list to store the DataFrames for each observer
    observer_dfs = []

    # Generate images for each observer and store the DataFrames in the list
    for observer in range(1, num_observers + 1):
        observer_images = generate_images(num_samples, num_images)
        observer_images['observer_nr'] = observer  # Add observer number column
        observer_dfs.append(observer_images)

    # Concatenate the DataFrames from each observer into a single result DataFrame
    result_df = pd.concat(observer_dfs, ignore_index=True)

    # Convert 'imgs' and 'test' columns to category data type for overview
    result_df['imgs'] = result_df['imgs'].astype('category')
    result_df['test'] = result_df['test'].astype('category')

    # Count the occurrences of each combination
    combination_counts = result_df.groupby(['imgs', 'test']).size().reset_index(name='count')

    # Measure the variance in the count
    variance = np.var(combination_counts['count'])
    #print(f"Run {run}: Variance in count: {variance}")

    # Check if this run has the lowest variance so far
    if variance < min_variance:
        min_variance = variance
        min_variance_df = result_df.copy()
        min_variance_seed = seed
        print(f"Seed {seed}: Variance in count: {variance}")
        plt.hist(combination_counts['count'], bins=np.arange(18),align='left')
        plt.xlabel('Count')
        plt.ylabel('Frequency')
        plt.title(f'Histogram of Counts\nVariance: {variance:.4f} | Seed: {min_variance_seed}')
        plt.xlim(0, 16)
        plt.ylim(0, 1100)
        plt.xticks(range(17))
        plt.show()
      
# Count the occurrences of each combination
combination_counts = min_variance_df.groupby(['imgs', 'test']).size().reset_index(name='count')

# Output to csvs
min_variance_df['image'] = 'images/'+min_variance_df['imgs'].astype(str).str.cat(min_variance_df['test'], sep='') + '.jpg'
# Group the dataframe by observer_nr
groups = min_variance_df.groupby('observer_nr')

# Loop through each group and create a separate CSV file
for name, group in groups:
    # Shuffle the rows of the group
    shuffled_group = group.sample(frac=1)

    # Select 20 random images to repeat
    repeat_images = shuffled_group.sample(n=20)

    # Add the repeated images to the shuffled group
    shuffled_group = pd.concat([shuffled_group, repeat_images])

    # Shuffle the group again to distribute the repeated images
    shuffled_group = shuffled_group.sample(frac=1)
    # Create a filename for the CSV file
    filename = f"observer_{name}.csv"
    # Create a path to the CSV file in the current working directory
    path = os.path.join(os.getcwd(), filename)
    # Select the 'filename' column and save it to a CSV file
    shuffled_group[['image']].to_csv(path, index=False)
# -*- coding: utf-8 -*-
"""
Created on Tue May 23 09:06:44 2023

@author: Del Pin
"""

import pandas as pd
import random
import numpy as np
import matplotlib.pyplot as plt
import os


def generate_images(num_samples, num_images):
    # Take list (with 3 equal chunks)
    # Give each chunk the same number of distortions
    chunk_size = num_samples // 3

    imgs_chunk1 = random.sample(range(1, num_images // 3 + 1), chunk_size)
    imgs_chunk2 = random.sample(range(num_images // 3 + 1, 2 * num_images // 3 + 1), chunk_size)
    imgs_chunk3 = random.sample(range(2 * num_images // 3 + 1, num_images + 1), chunk_size)
    imgs = imgs_chunk1 + imgs_chunk2 + imgs_chunk3

    # Create letters for each chunk
    distortions = [f'{d}{l}' for d in ['_jpeg', '_blur', '_contrast'] for l in ['_70', '_80', '_90']] + ['_original']
    num_distortions = len(distortions)
    num_chunks = 3

    let = pd.Series(distortions * (num_samples // (num_distortions * num_chunks)))
    # Combine chunks, each randomized
    test = pd.concat([let.sample(frac=1) for _ in range(num_chunks)], ignore_index=True)

    # Check that each chunk has the same number of letters
    test = test.astype('category')  # Convert to category for overview

    # Combine and shuffle
    images = pd.DataFrame({'imgs': imgs, 'test': test})

    return images

num_samples = 420  # Default 420
num_images = 540  # Default 540
num_observers = 40  # Number of observers # 10000
num_runs = 10000 # Number of runs to perform

min_variance = float('inf')
min_variance_df = None
min_variance_seed = None

for run in range(1, num_runs + 1):
    # Set the seed for reproducibility
    seed = run
    random.seed(seed)
    np.random.seed(seed)

    # Create an empty list to store the DataFrames for each observer
    observer_dfs = []

    # Generate images for each observer and store the DataFrames in the list
    for observer in range(1, num_observers + 1):
        observer_images = generate_images(num_samples, num_images)
        observer_images['observer_nr'] = observer  # Add observer number column
        observer_dfs.append(observer_images)

    # Concatenate the DataFrames from each observer into a single result DataFrame
    result_df = pd.concat(observer_dfs, ignore_index=True)

    # Convert 'imgs' and 'test' columns to category data type for overview
    result_df['imgs'] = result_df['imgs'].astype('category')
    result_df['test'] = result_df['test'].astype('category')

    # Count the occurrences of each combination
    combination_counts = result_df.groupby(['imgs', 'test']).size().reset_index(name='count')

    # Measure the variance in the count
    variance = np.var(combination_counts['count'])
    #print(f"Run {run}: Variance in count: {variance}")

    # Check if this run has the lowest variance so far
    if variance < min_variance:
        min_variance = variance
        min_variance_df = result_df.copy()
        min_variance_seed = seed
        print(f"Seed {seed}: Variance in count: {variance}")
        plt.hist(combination_counts['count'], bins=np.arange(18),align='left')
        plt.xlabel('Count')
        plt.ylabel('Frequency')
        plt.title(f'Histogram of Counts\nVariance: {variance:.4f} | Seed: {min_variance_seed}')
        plt.xlim(0, 16)
        plt.ylim(0, 1100)
        plt.xticks(range(17))
        plt.show()
      
# Count the occurrences of each combination
combination_counts = min_variance_df.groupby(['imgs', 'test']).size().reset_index(name='count')

# Output to csvs
min_variance_df['image'] = 'images/'+min_variance_df['imgs'].astype(str).str.cat(min_variance_df['test'], sep='') + '.jpg'
# Group the dataframe by observer_nr
groups = min_variance_df.groupby('observer_nr')

# Loop through each group and create a separate CSV file
for name, group in groups:
    # Shuffle the rows of the group
    shuffled_group = group.sample(frac=1)

    # Select 20 random images to repeat
    repeat_images = shuffled_group.sample(n=20)

    # Add the repeated images to the shuffled group
    shuffled_group = pd.concat([shuffled_group, repeat_images])

    # Shuffle the group again to distribute the repeated images
    shuffled_group = shuffled_group.sample(frac=1)
    # Create a filename for the CSV file
    filename = f"observer_{name}.csv"
    # Create a path to the CSV file in the current working directory
    path = os.path.join(os.getcwd(), filename)
    # Select the 'filename' column and save it to a CSV file
    shuffled_group[['image']].to_csv(path, index=False)