// File: /workspace/github_c_makefile_star2_O0/kkanellis[P]matvec-sparse/matvec_mpi_bcast.c
// Function: mat_vec_mult_parallel
double* mat_vec_mult_parallel(int rank, int nprocs, proc_info_t *all_proc_info,
                            int *i_idx, int *j_idx, double *values, double *x)
{
    proc_info_t proc_info;  /* info about submatrix; different per process */
    double *res;            /* result of multiplication res = A*x */

    /***** MPI MASTER (root) process only ******/
    int *count, *offset;    /* auxilliary arrays used for Scatterv/Gatherv */

    /* scatter to processors all info that will be needed */
    MPI_Scatter(all_proc_info, 1, proc_info_type, &proc_info,
                    1, proc_info_type, MASTER, MPI_COMM_WORLD);

    /* allocate memory for vectors and submatrixes */
    double *y = (double *)calloc_or_exit( proc_info.row_count, sizeof(double) );
    if (rank != MASTER) {
        x = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
        i_idx = (int *)malloc_or_exit( proc_info.nz_count * sizeof(int) );
        j_idx = (int *)malloc_or_exit( proc_info.nz_count * sizeof(int) );
        values = (double *)malloc_or_exit( proc_info.nz_count * sizeof(double) );
    }
    else {
        res = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
    }

    //debug("[%d] %d %d %d\n", rank, proc_info.N, proc_info.NZ, proc_info.nz_count);
    
    /* broadcast x vector */
    /* TODO: change late to Point-2-Point communication */
    MPI_Bcast(x, proc_info.N, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* scatter matrix elements to processes */
    if (rank == MASTER) {
        count = (int *)malloc_or_exit( nprocs * sizeof(int) );
        offset = (int *)malloc_or_exit( nprocs * sizeof(int) );
        for (int p = 0; p < nprocs; p++) {
            count[p] = all_proc_info[p].nz_count;
            offset[p] = all_proc_info[p].nz_start_idx;
        }
    }
    MPI_Scatterv(i_idx, count, offset, MPI_INT, i_idx, 
                    proc_info.nz_count, MPI_INT, MASTER, MPI_COMM_WORLD);
    MPI_Scatterv(j_idx, count, offset, MPI_INT, j_idx, 
                    proc_info.nz_count, MPI_INT, MASTER, MPI_COMM_WORLD);
    MPI_Scatterv(values, count, offset, MPI_DOUBLE, values, 
                    proc_info.nz_count, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* validation check */
    /*for (int i = 0; i < proc_info.nz_count; i++) {
        debug("[%d] %d %d %lf\n", rank, i_idx[i], j_idx[i], values[i]);
    }*/

    /* multiplication kernel */
    for (int k = 0 ; k < proc_info.nz_count; k++) {
        y[ i_idx[k] - proc_info.row_start_idx ] += values[k] * x[ j_idx[k] ];
    }

    /* gather y elements from processes and save it to res */
    if (rank == MASTER) {
        res = (double *)malloc_or_exit( proc_info.N * sizeof(double) );
        for (int p = 0; p < nprocs; p++) {
            count[p] = all_proc_info[p].row_count;
            offset[p] = all_proc_info[p].row_start_idx;
        }
    }
    MPI_Gatherv(y, proc_info.row_count, MPI_DOUBLE, res, count, 
                offset, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);

    /* return final result */
    return res;
}

// Assembly:
subq $0x88, %rsp
movq 0x90(%rsp), %rax
movl %edi, 0x84(%rsp)
movl %esi, 0x80(%rsp)
movq %rdx, 0x78(%rsp)
movq %rcx, 0x70(%rsp)
movq %r8, 0x68(%rsp)
movq %r9, 0x60(%rsp)
movq 0x78(%rsp), %rdi
movq 0x30cb(%rip), %rdx      # 0x6128
movq 0x30c4(%rip), %r9       # 0x6128
movl $0x1, %r8d
leaq 0x48(%rsp), %rcx
xorl %eax, %eax
movq 0x2f68(%rip), %rax      # 0x5fe0
movl %r8d, %esi
movl $0x0, (%rsp)
movq %rax, 0x8(%rsp)
callq 0x1180
movslq 0x58(%rsp), %rdi
movl $0x8, %esi
callq 0x2f80
movq %rax, 0x28(%rsp)
cmpl $0x0, 0x84(%rsp)
je 0x30fb
movslq 0x48(%rsp), %rdi
shlq $0x3, %rdi
callq 0x2f30
movq %rax, 0x90(%rsp)
movslq 0x50(%rsp), %rdi
shlq $0x2, %rdi
callq 0x2f30
movq %rax, 0x70(%rsp)
movslq 0x50(%rsp), %rdi
shlq $0x2, %rdi
callq 0x2f30
movq %rax, 0x68(%rsp)
movslq 0x50(%rsp), %rdi
shlq $0x3, %rdi
callq 0x2f30
movq %rax, 0x60(%rsp)
jmp 0x310e
movslq 0x48(%rsp), %rdi
shlq $0x3, %rdi
callq 0x2f30
movq %rax, 0x40(%rsp)
movq 0x90(%rsp), %rdi
movl 0x48(%rsp), %esi
movq 0x2eb7(%rip), %rdx      # 0x5fd8
xorl %ecx, %ecx
movq 0x2eb6(%rip), %r8       # 0x5fe0
callq 0x1140
cmpl $0x0, 0x84(%rsp)
jne 0x31cf
movslq 0x80(%rsp), %rdi
shlq $0x2, %rdi
callq 0x2f30
movq %rax, 0x38(%rsp)
movslq 0x80(%rsp), %rdi
shlq $0x2, %rdi
callq 0x2f30
movq %rax, 0x30(%rsp)
movl $0x0, 0x24(%rsp)
movl 0x24(%rsp), %eax
cmpl 0x80(%rsp), %eax
jge 0x31cd
movq 0x78(%rsp), %rax
movslq 0x24(%rsp), %rcx
imulq $0x18, %rcx, %rcx
addq %rcx, %rax
movl 0x8(%rax), %edx
movq 0x38(%rsp), %rax
movslq 0x24(%rsp), %rcx
movl %edx, (%rax,%rcx,4)
movq 0x78(%rsp), %rax
movslq 0x24(%rsp), %rcx
imulq $0x18, %rcx, %rcx
addq %rcx, %rax
movl 0xc(%rax), %edx
movq 0x30(%rsp), %rax
movslq 0x24(%rsp), %rcx
movl %edx, (%rax,%rcx,4)
movl 0x24(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x24(%rsp)
jmp 0x3171
jmp 0x31cf
movq 0x70(%rsp), %rdi
movq 0x38(%rsp), %rsi
movq 0x30(%rsp), %rdx
movq 0x70(%rsp), %r8
movl 0x50(%rsp), %r9d
movq 0x2dc1(%rip), %r10      # 0x5fb0
xorl %eax, %eax
movq 0x2de8(%rip), %rax      # 0x5fe0
movq %r10, %rcx
movq %r10, (%rsp)
movl $0x0, 0x8(%rsp)
movq %rax, 0x10(%rsp)
callq 0x10c0
movq 0x68(%rsp), %rdi
movq 0x38(%rsp), %rsi
movq 0x30(%rsp), %rdx
movq 0x68(%rsp), %r8
movl 0x50(%rsp), %r9d
movq 0x2d7f(%rip), %r10      # 0x5fb0
xorl %eax, %eax
movq 0x2da6(%rip), %rax      # 0x5fe0
movq %r10, %rcx
movq %r10, (%rsp)
movl $0x0, 0x8(%rsp)
movq %rax, 0x10(%rsp)
callq 0x10c0
movq 0x60(%rsp), %rdi
movq 0x38(%rsp), %rsi
movq 0x30(%rsp), %rdx
movq 0x60(%rsp), %r8
movl 0x50(%rsp), %r9d
movq 0x2d65(%rip), %r10      # 0x5fd8
xorl %eax, %eax
movq 0x2d64(%rip), %rax      # 0x5fe0
movq %r10, %rcx
movq %r10, (%rsp)
movl $0x0, 0x8(%rsp)
movq %rax, 0x10(%rsp)
callq 0x10c0
movl $0x0, 0x20(%rsp)
movl 0x20(%rsp), %eax
cmpl 0x50(%rsp), %eax
jge 0x3309
movq 0x60(%rsp), %rax
movslq 0x20(%rsp), %rcx
movsd (%rax,%rcx,8), %xmm0
movq 0x90(%rsp), %rax
movq 0x68(%rsp), %rcx
movslq 0x20(%rsp), %rdx
movslq (%rcx,%rdx,4), %rcx
movsd (%rax,%rcx,8), %xmm2
movq 0x28(%rsp), %rax
movq 0x70(%rsp), %rcx
movslq 0x20(%rsp), %rdx
movl (%rcx,%rdx,4), %ecx
subl 0x5c(%rsp), %ecx
movslq %ecx, %rcx
movsd (%rax,%rcx,8), %xmm1
mulsd %xmm2, %xmm0
addsd %xmm1, %xmm0
movsd %xmm0, (%rax,%rcx,8)
movl 0x20(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x20(%rsp)
jmp 0x329d
cmpl $0x0, 0x84(%rsp)
jne 0x338c
movslq 0x48(%rsp), %rdi
shlq $0x3, %rdi
callq 0x2f30
movq %rax, 0x40(%rsp)
movl $0x0, 0x1c(%rsp)
movl 0x1c(%rsp), %eax
cmpl 0x80(%rsp), %eax
jge 0x338a
movq 0x78(%rsp), %rax
movslq 0x1c(%rsp), %rcx
imulq $0x18, %rcx, %rcx
addq %rcx, %rax
movl 0x10(%rax), %edx
movq 0x38(%rsp), %rax
movslq 0x1c(%rsp), %rcx
movl %edx, (%rax,%rcx,4)
movq 0x78(%rsp), %rax
movslq 0x1c(%rsp), %rcx
imulq $0x18, %rcx, %rcx
addq %rcx, %rax
movl 0x14(%rax), %edx
movq 0x30(%rsp), %rax
movslq 0x1c(%rsp), %rcx
movl %edx, (%rax,%rcx,4)
movl 0x1c(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x1c(%rsp)
jmp 0x332e
jmp 0x338c
movq 0x28(%rsp), %rdi
movl 0x58(%rsp), %esi
movq 0x40(%rsp), %rcx
movq 0x38(%rsp), %r8
movq 0x30(%rsp), %r9
movq 0x2c2d(%rip), %r10      # 0x5fd8
xorl %eax, %eax
movq 0x2c2c(%rip), %rax      # 0x5fe0
movq %r10, %rdx
movq %r10, (%rsp)
movl $0x0, 0x8(%rsp)
movq %rax, 0x10(%rsp)
callq 0x11f0
movq 0x40(%rsp), %rax
addq $0x88, %rsp
retq
nopw (%rax,%rax)