// File: /workspace/github_c_makefile_star2_O2/kkanellis[P]matvec-sparse/matvec_mpi_bcast.c
// Function: mat_vec_mult_parallel
double* mat_vec_mult_parallel(int rank, int nprocs, proc_info_t *all_proc_info,
                            int *i_idx, int *j_idx, double *values, double *x)
{
    proc_info_t proc_info;  /* info about submatrix; different per process */
    double *res;            /* result of multiplication res = A*x */

    /***** MPI MASTER (root) process only ******/
    int *count, *offset;    /* auxilliary arrays used for Scatterv/Gatherv */

    /* scatter to processors all info that will be needed */
    MPI_Scatter(all_proc_info, 1, proc_info_type, &proc_info,
                    1, proc_info_type, MASTER, MPI_COMM_WORLD);

    /* allocate memory for vectors and submatrixes */
    double *y = (double *)calloc_or_exit( proc_info.row_count, sizeof(double) );
    if (rank != MASTER) {
        x = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
        i_idx = (int *)malloc_or_exit( proc_info.nz_count * sizeof(int) );
        j_idx = (int *)malloc_or_exit( proc_info.nz_count * sizeof(int) );
        values = (double *)malloc_or_exit( proc_info.nz_count * sizeof(double) );
    }
    else {
        res = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
    }

    //debug("[%d] %d %d %d\n", rank, proc_info.N, proc_info.NZ, proc_info.nz_count);
    
    /* broadcast x vector */
    /* TODO: change late to Point-2-Point communication */
    MPI_Bcast(x, proc_info.N, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* scatter matrix elements to processes */
    if (rank == MASTER) {
        count = (int *)malloc_or_exit( nprocs * sizeof(int) );
        offset = (int *)malloc_or_exit( nprocs * sizeof(int) );
        for (int p = 0; p < nprocs; p++) {
            count[p] = all_proc_info[p].nz_count;
            offset[p] = all_proc_info[p].nz_start_idx;
        }
    }
    MPI_Scatterv(i_idx, count, offset, MPI_INT, i_idx, 
                    proc_info.nz_count, MPI_INT, MASTER, MPI_COMM_WORLD);
    MPI_Scatterv(j_idx, count, offset, MPI_INT, j_idx, 
                    proc_info.nz_count, MPI_INT, MASTER, MPI_COMM_WORLD);
    MPI_Scatterv(values, count, offset, MPI_DOUBLE, values, 
                    proc_info.nz_count, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* validation check */
    /*for (int i = 0; i < proc_info.nz_count; i++) {
        debug("[%d] %d %d %lf\n", rank, i_idx[i], j_idx[i], values[i]);
    }*/

    /* multiplication kernel */
    for (int k = 0 ; k < proc_info.nz_count; k++) {
        y[ i_idx[k] - proc_info.row_start_idx ] += values[k] * x[ j_idx[k] ];
    }

    /* gather y elements from processes and save it to res */
    if (rank == MASTER) {
        res = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
        for (int p = 0; p < nprocs; p++) {
            count[p] = all_proc_info[p].row_count;
            offset[p] = all_proc_info[p].row_start_idx;
        }
    }
    MPI_Gatherv(y, proc_info.row_count, MPI_DOUBLE, res, count, 
                offset, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* return final result */
    return res;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %r9, %rbp
movq %r8, %r12
movq %rcx, 0x28(%rsp)
movq %rdx, %rbx
movq %rsi, 0x20(%rsp)
movl %edi, %r13d
movq 0x2bed(%rip), %rdx      # 0x5138
movq 0x2a8e(%rip), %rax      # 0x4fe0
leaq 0x30(%rsp), %r14
pushq $0x1
popq %rsi
movq %rbx, %rdi
movq %r14, %rcx
movl %esi, %r8d
movq %rdx, %r9
pushq %rax
pushq $0x0
callq 0x1180
popq %rcx
popq %rdx
movslq 0x10(%r14), %rdi
pushq $0x8
popq %rsi
callq 0x24dd
movq %rax, %r15
movslq (%r14), %rdi
shlq $0x3, %rdi
callq 0x24ac
movl %r13d, 0x1c(%rsp)
testl %r13d, %r13d
movq %rax, 0x50(%rsp)
je 0x25d4
movq %rax, %r14
movslq 0x38(%rsp), %rdi
shlq $0x2, %rdi
callq 0x24ac
movq %rax, 0x28(%rsp)
movslq 0x38(%rsp), %rdi
shlq $0x2, %rdi
callq 0x24ac
movq %rax, %r12
movslq 0x38(%rsp), %rdi
shlq $0x3, %rdi
callq 0x24ac
movq %rax, %rbp
jmp 0x25dc
movq 0x90(%rsp), %r14
movl 0x30(%rsp), %esi
movq 0x29f1(%rip), %rdx      # 0x4fd8
movq 0x29f2(%rip), %r8       # 0x4fe0
xorl %r13d, %r13d
movq %r14, %rdi
xorl %ecx, %ecx
callq 0x1150
cmpl $0x0, 0x1c(%rsp)
jne 0x2667
movslq 0x20(%rsp), %rax
movq %rax, 0x8(%rsp)
leaq (,%rax,4), %rdi
movq %rdi, 0x48(%rsp)
callq 0x24ac
movq %rax, 0x10(%rsp)
movq 0x48(%rsp), %rdi
callq 0x24ac
movq 0x10(%rsp), %rdi
movq %rax, %r8
cmpl $0x0, 0x8(%rsp)
movl $0x0, %eax
cmovgl 0x20(%rsp), %eax
leaq 0xc(%rbx), %rcx
xorl %edx, %edx
cmpq %rdx, %rax
je 0x2664
movl -0x4(%rcx), %esi
movl %esi, (%rdi,%rdx,4)
movl (%rcx), %esi
movl %esi, (%r8,%rdx,4)
incq %rdx
addq $0x18, %rcx
jmp 0x264a
movq %rdi, %rax
movl 0x38(%rsp), %r9d
subq $0x8, %rsp
movq 0x2939(%rip), %rcx      # 0x4fb0
movq 0x30(%rsp), %rdi
movq %rax, %rsi
movq %rax, 0x18(%rsp)
movq %r8, 0x10(%rsp)
movq %r8, %rdx
movq %rdi, %r8
movq 0x294a(%rip), %rax      # 0x4fe0
pushq %rax
pushq $0x0
pushq %rcx
callq 0x10c0
addq $0x20, %rsp
movl 0x38(%rsp), %r9d
subq $0x8, %rsp
movq %r12, %rdi
movq 0x18(%rsp), %rsi
movq 0x10(%rsp), %rdx
movq 0x28f0(%rip), %rcx      # 0x4fb0
movq %r12, %r8
pushq 0x2917(%rip)            # 0x4fe0
pushq $0x0
pushq %rcx
callq 0x10c0
addq $0x20, %rsp
movl 0x38(%rsp), %r9d
subq $0x8, %rsp
movq %rbp, %rdi
movq 0x18(%rsp), %rsi
movq 0x10(%rsp), %rdx
movq 0x28e6(%rip), %rcx      # 0x4fd8
movq %rbp, %r8
pushq 0x28e5(%rip)            # 0x4fe0
pushq $0x0
pushq %rcx
callq 0x10c0
movq 0x48(%rsp), %rdi
addq $0x20, %rsp
movl 0x38(%rsp), %eax
movslq 0x44(%rsp), %rcx
testl %eax, %eax
cmovlel %r13d, %eax
cmpq %r13, %rax
je 0x2749
movsd (%rbp,%r13,8), %xmm0
movslq (%r12,%r13,4), %rdx
movslq (%rdi,%r13,4), %rsi
subq %rcx, %rsi
mulsd (%r14,%rdx,8), %xmm0
addsd (%r15,%rsi,8), %xmm0
movsd %xmm0, (%r15,%rsi,8)
incq %r13
jmp 0x271b
cmpl $0x0, 0x1c(%rsp)
movq 0x50(%rsp), %r14
movq 0x287e(%rip), %r12      # 0x4fd8
movq 0x287f(%rip), %r13      # 0x4fe0
je 0x279f
movl 0x40(%rsp), %esi
subq $0x8, %rsp
movq %r15, %rdi
movq %r12, %rdx
movq %r14, %rcx
movq 0x18(%rsp), %r8
movq 0x10(%rsp), %r9
pushq %r13
pushq $0x0
pushq %r12
callq 0x1210
addq $0x20, %rsp
movq %r14, %rax
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movslq 0x30(%rsp), %rdi
shlq $0x3, %rdi
callq 0x24ac
movq %rax, %r14
xorl %eax, %eax
movq 0x20(%rsp), %rdx
testl %edx, %edx
cmovlel %eax, %edx
addq $0x14, %rbx
movq 0x10(%rsp), %rsi
movq 0x8(%rsp), %rdi
cmpq %rax, %rdx
je 0x2763
movl -0x4(%rbx), %ecx
movl %ecx, (%rsi,%rax,4)
movl (%rbx), %ecx
movl %ecx, (%rdi,%rax,4)
incq %rax
addq $0x18, %rbx
jmp 0x27ca