Project: s41467-024-46614-z | Function: truvari_overlap (python)

system: You are an expert Python programmer implementing algorithms from scientific papers.
Complete the masked function based on the algorithm description.
Follow Python best practices and maintain consistent code style.
Return **only** the complete Python function code.

user: Complete this Python function:

**DESCRIPTION:** Implements structural variant overlap analysis using Truvari benchmarking with grid search parameter optimization for comprehensive SV caller evaluation

**KEY TERMS:** {
  "pctstim": "minimum allele sequence similarity (p parameter)",
  "pctsize": "minimum allele size similarity (P parameter)",
  "pctovl": "minimum reciprocal overlap ratio (O parameter)",
  "refdist": "maximum reference location difference (r parameter)",
  "benchmark_vcf": "GIAB gold standard SV callset",
  "comp_vcf": "SV caller output for comparison"
}

**CONTEXT CODE:**     os.popen('vcf-sort '+input_vcf+' > '+input_vcf.rstrip('.vcf')+'_sorted.vcf').read()
    os.popen('bgzip -c '+input_vcf.rstrip('.vcf')+'_sorted.vcf > '+input_vcf.rstrip('.vcf')+'_sorted.vcf.gz').read()
    os.popen('tabix -p vcf '+input_vcf.rstrip('.vcf')+'_sorted.vcf.gz').read()
    os.popen('rm '+input_vcf.rstrip('.vcf')+'_sorted.vcf').read()

    output_vcfgz = input_vcf.rstrip('.vcf')+'_sorted.vcf.gz'

    return output_vcfgz

def truvari_eval(benchmark_vcf,comp_vcf,comp_sv_caller,out_dir,truvari_parameters,out_vcf_name='Multi_SV_Caller_Merged'):

    #if not os.path.isdir(out_dir+'/truvari_tmp'):
    #    os.mkdir(out_dir+'/truvari_tmp')

    bmk_vcfgz = gz_sort_and_tabix(benchmark_vcf)
    cmp_vcfgz = gz_sort_and_tabix(comp_vcf)

    os.popen('truvari bench -b '+bmk_vcfgz+' -c '+cmp_vcfgz+' -o '+out_dir+'/truvari_tmp '+truvari_parameters.rstrip('\n')).read()

    bmk_only = out_dir+'/truvari_tmp/fn.vcf'
    cmp_only = out_dir+'/truvari_tmp/fp.vcf'
    overlap  = out_dir+'/truvari_tmp/tp-base.vcf'

    with open(bmk_only,'r') as bmk_file:
        with open(cmp_only,'r') as cmp_file:
            with open(overlap,'r') as ovlp_file:
                with open(out_dir+'/'+out_vcf_name+'.vcf','w') as merged_file:
                    for ovlp_line in ovlp_file:
                        if ovlp_line[0] == '#':
                            merged_file.write(ovlp_line.replace('##source=', '##source='+comp_sv_caller+'/'))
                        else:
                            merged_file.write(ovlp_line.replace('SC=', 'SC='+comp_sv_caller+','))

                    for cmp_line in cmp_file:
                        if cmp_line[0] !='#':
                            merged_file.write(cmp_line)

                    for bmk_line in bmk_file:
                        if bmk_line[0] !='#':
                            merged_file.write(bmk_line)

    #clean up
    os.popen('rm -r '+out_dir+'/truvari_tmp').read()
    os.popen('rm '+bmk_vcfgz+' '+bmk_vcfgz+'.tbi').read()
    os.popen('rm '+cmp_vcfgz+' '+cmp_vcfgz+'.tbi').read()
    
    new_benchmark_vcf = out_dir+'/Multi_SV_Caller_Merged.vcf'

    return new_benchmark_vcf

def truvari_overlap(reg_vcfs,out_dir,tcf): #reg_vcfs is OrderedDict()
    # MASKED: Implements structural variant overlap analysis using Truvari benchmarking with grid search parameter optimization for comprehensive SV caller evaluation
    raise NotImplementedError("This function is masked for testing purposes.")

def filter_subset_id(merged_vcf, subset_sv, out_dir, comp):
    subset_ids = dict()
    with open(subset_sv, "r") as sf:
        for line in sf:
            if line[0]!="#":
                subset_ids[line.rstrip("\n").split("\t")[2]] = 0
    out_vcf = out_dir+"/IDfiltered_Merged_SV.vcf"
    with open(merged_vcf, "r") as mf:
        with open(out_vcf, "w") as otf:
            for line in mf:
                if line[0] == "#":
                    otf.write(line)
                else:
                    curid = line.rstrip("\n").split("\t")[2]
                    if curid in subset_ids and not comp:
                        otf.write(line)
                    elif curid not in subset_ids and comp:
                        otf.write(line)

    return out_vcf

def get_overlap_dict(vcf_file, sv_types=['ALL','DEL','INS']):
    #overlap_dict = OrderedDict()

    overlap_dict = dict()
    for sv_type in sv_types:
        overlap_dict[sv_type] = OrderedDict()
    
    with open(vcf_file,'r') as vcf:
        for line in vcf:
            if '##source=' in line:
                all_sv_callers = line.rstrip('\n')[9:].split('/')
                all_sv_callers.reverse()
                total_sv_caller_num = len(all_sv_callers)
                break
        
        #for i in range(total_sv_caller_num,0,-1):
        #    overlap_dict[i] = [0]*total_sv_caller_num
        for sv_type in sv_types:
            for i in range(total_sv_caller_num,0,-1):
                overlap_dict[sv_type][i] = [0]*total_sv_caller_num

        for line in vcf:
            if line[0] != '#':
                sv_callers=re.findall("SC=([^;\t]+)",line)[0].split(',')
                sv_caller_num = len(sv_callers)
                sv_type = re.findall("SVTYPE=(\w+)",line)[0]

                if 'ALL' in overlap_dict.keys():
                    for sv_caller in sv_callers:
                        overlap_dict['ALL'][sv_caller_num][all_sv_callers.index(sv_caller)] += 1

                if sv_type in overlap_dict.keys():
                    for sv_caller in sv_callers:
                        overlap_dict[sv_type][sv_caller_num][all_sv_callers.index(sv_caller)] += 1
    
    return overlap_dict, all_sv_callers

def plot_overlap_by_callers(vcf_file, out_dir, sv_types=['ALL','DEL','INS'], vcf_file2=None):

    overlap_dict, all_sv_callers = get_overlap_dict(vcf_file, sv_types)
    if vcf_file2 is not None:
        overlap_dict2, all_sv_callers2 = get_overlap_dict(vcf_file2, sv_types)
        if all_sv_callers != all_sv_callers2:
            raise ValueError("SV callers in vcf list file 2 should be the same as vcf list file 1 (including their order in the list)")
    else:
        overlap_dict2 = None

    for sv_type in sv_types:
        overlap_df = pd.DataFrame(overlap_dict[sv_type], index=all_sv_callers,)
        if overlap_dict2 is None:
            overlap_df.to_excel(out_dir+'/'+sv_type+'_SV_overlap_by_callers.xlsx')
            fig, ax = plt.subplots(figsize=(len(all_sv_callers)*0.5,3))
            overlap_df.plot.bar(stacked=True,ax=ax,colormap="Blues_r", edgecolor='gray', linewidth=0.2)
            ax.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
        else:
            overlap_df2 = pd.DataFrame(overlap_dict2[sv_type], index=all_sv_callers,)
            overlap_df.to_excel(out_dir+'/overlap1/'+sv_type+'_SV_overlap_by_callers.xlsx')
            overlap_df2.to_excel(out_dir+'/overlap2/'+sv_type+'_SV_overlap_by_callers.xlsx')
            fig, axes = plt.subplots(2, 1, figsize=(len(all_sv_callers)*0.5,3))
            overlap_df.plot.bar(stacked=True,ax=axes[0],colormap="Blues_r", edgecolor='gray', linewidth=0.2)
            overlap_df2.plot.bar(stacked=True,ax=axes[1],colormap="Oranges_r", edgecolor='gray', linewidth=0.2)
            if len(all_sv_callers) > 5:
                axes[0].legend(bbox_to_anchor=(-0.15, 1), loc=1, borderaxespad=0)
            else:
                axes[0].legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0)
            axes[1].legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
            axes[1].invert_yaxis()
            axes[0].set_xticks([])
            plt.subplots_adjust(wspace=0, hspace=0)
        #overlap_df.plot.bar(stacked=True,ax=ax, edgecolor='gray', linewidth=0.2)
        matplotlib.rcParams['pdf.fonttype'] = 42
        matplotlib.rcParams['ps.fonttype'] = 42
        plt.savefig(out_dir+'/'+sv_type+'_SV_overlap_by_callers.pdf',bbox_inches='tight')
        plt.close(fig)



Complete the function implementation replacing "raise NotImplementedError".