//  [python]
def compress_system_based_on_structural_diversity(
        self, 
        frames,
        number = 320,
        energy_shift = 0.0
    ):
        """ Strategy for compressing large dataset
            1. if atomic energy is way large, skip them
            2. random selection based on (
                cur_scores * bin_prop * boltzmann
                # geometric, phase volume, energy-wise
            )
        """

        print("\n\n")
        print("!!! number of frames to compress: ", len(frames))

        #print(frames[0].info)
        #write("fuccc.xyz", frames)
        #print(frames[0].get_potential_energy())

        # find converged structures TODO: move this outside
        converged_frames, converged_indices = self.find_converged(self.name, frames, self.prefix)
        nconverged = len(converged_indices)
        print("Number of converged: ", nconverged)

        # cur in each bin
        print("\n\n--- start histogram ---")
        energies = np.array([a.get_potential_energy() for a in frames])
        # convert to atomic energies
        energies = energies / np.sum(list(self.type_numbers.values()))

        en_width = self.compress_params["en_width"]  # energy width
        en_min = np.floor(np.min(energies))
        en_max = np.ceil(np.max(energies))
        bin_edges = np.arange(en_min, en_max+en_width, en_width)
        nbins = len(bin_edges) - 1  # number of bins

        print(f"AtomicEnergy Range: {en_min} _ {en_max}")
        print(f"number of bins: {nbins}")

        _, bin_edges, binnumber = sp.stats.binned_statistic(
            energies, energies, statistic="mean", bins=bin_edges,
        )

        # add boltzmann coefficient
        beta = self.compress_params["beta"]  # eV
        shifts = self.compress_params["shifts"]
        boltz_prob = np.zeros(len(bin_edges)-1)
        for es in shifts:
            boltz_prob += np.exp(-beta*(bin_edges[:-1]-(en_min+es)))
        norm_boltz = boltz_prob / np.sum(boltz_prob)
        print(f"Boltzmann {boltz_prob}, norm: {norm_boltz}")

        # get frame index in each bin
        fidx_list = []  # frame index list
        for x in range(nbins):
            fidx_list.append([])
        for k, g in groupby(enumerate(binnumber), lambda x: x[1]):
            # k starts with 1 since histogram uses right boundary
            fidx_list[k-1].extend([ig[0] for ig in g])
        num_array = np.zeros(nbins, dtype=int)
        for x in range(nbins):
            num_array[x] = len(fidx_list[x])
        num_density = num_array / np.sum(num_array)
        print(f"NumDensity: {num_array}, norm: {num_density}")

        assert len(frames) == np.sum(
            num_array), "all frames must be in the hist."

        # choose number of structures in eacn bin
        merged_probs = num_density*norm_boltz
        merged_probs = merged_probs / np.sum(merged_probs)
        print("BoltzDensity: ", merged_probs)
        print("random choice: ", number-nconverged)
        if number > nconverged:
            selected_indices = np.random.choice(
                nbins, number-nconverged, replace=True, p=merged_probs)

            final_num_array = np.zeros(nbins, dtype=int)
            for k, g in groupby(selected_indices):
                final_num_array[k] += len(list(g))
            print("Numbers selected in each bin: ", final_num_array)

            # select structures in each bin
            print("\n\n--- start cur decompostion ---")
            # check if selected numbers are valid
            new_final_selected_num = []
            rest_num = 0
            for bin_i, (bin_num, bin_idx) in enumerate(zip(final_num_array, fidx_list)):
                nframes_bin = len(bin_idx) # number of structure in the bin
                #if bin_num == 0:
                #    new_final_selected_num.append(bin_num)
                #    continue
                bin_num += rest_num
                if bin_num > nframes_bin:
                    rest_num = bin_num - nframes_bin
                    bin_num = nframes_bin
                else:
                    rest_num = 0
                #if rest_num < 0:
                #    bin_num -= rest_num
                #rest_num = nframes_bin - bin_num # all minus selected
                #if rest_num < 0:
                #    bin_num = nframes_bin
                new_final_selected_num.append(bin_num)
            print("NEW: Numbers selected in each bin: ", new_final_selected_num)
            all_indices = []
            for bin_i, (bin_num, bin_idx) in enumerate(zip(new_final_selected_num, fidx_list)):
                nframes_bin = len(bin_idx)
                if bin_num == 0 or nframes_bin == 0:
                    continue
                print(
                    "{} candidates are selected from {} structures in bin {}...".format(
                        bin_num, nframes_bin, bin_i
                    )
                )
                #bin_frames = []
                #for i in bin_idx:
                #    bin_frames.append(frames[i])
                bin_frames = [frames[i] for i in bin_idx]
                #features = self.selector.calc_desc(bin_frames)
                #if len(bin_frames) == 1:
                #    features = features[:, np.newaxis]
                #    selected_indices = [0]
                #else:
                #    selected_indices = self.selector.select_structures(
                #        features, bin_num
                #    )
                selected_indices = self.selector.select(bin_frames, ret_indices=True)
                # NOTE: map selected into global indices
                selected_indices = [bin_idx[s] for s in selected_indices]
                    # TODO: sometims the converged one will be selected... so duplicate...
                all_indices.extend(selected_indices)
            all_indices.extend(converged_indices)
        else:
            print("nconverged is greater than number...")
            converged_energies = np.array([a.get_potential_energy() for a in converged_frames])
            converged_probs = -converged_energies + np.max(converged_energies) + 0.1
            converged_probs = converged_probs / np.sum(converged_probs)
            all_indices = np.random.choice(
                converged_indices, number, replace=False, p=converged_probs
            )
        print("number selected: ", len(all_indices))
        print("number no-duplicated: ", len(set(all_indices)))

        selected_frames = []
        for i in all_indices:
            selected_frames.append(frames[i])

        # output final candidates
        if energy_shift > 0:
            es = energy_shift
            print(f"add {es} [eV] energy correction to each structure!!!")
            for atoms in selected_frames:
                new_energy = atoms.get_potential_energy() + es
                new_forces = atoms.get_forces(apply_constraint=False).copy()
                calc = SinglePointCalculator(
                    atoms, energy=new_energy, forces=new_forces)
                atoms.calc = calc

        print("Writing structure file... ")
        selected_frames.sort(key=lambda a:a.get_potential_energy()) # sort energy
        write(self.prefix / ("miaow-sel.xyz"), selected_frames)
        print("")

        return len(all_indices), nconverged