// File: /workspace/github_cpp_makefile_star2_O3/WeihuaZhu[P]hpc_prog2/mpi_jacobi.cpp
// Function: distributed_matrix_vector_mult(int, double*, double*, double*, ompi_communicator_t*)
void distributed_matrix_vector_mult(const int n, double* local_A, double* local_x, double* local_y, MPI_Comm comm)
{
    // obtain the necessary parameters required in thie method
    int rank;
    MPI_Comm_rank(comm, &rank);
    int p;
    MPI_Comm_size(comm, &p);
    int q = sqrt(p);
    int dims[2];
    int coords[2];
    int periods[2];
    MPI_Cart_get(comm, 2, dims, periods, coords);

    //distribute x using transpose method
    double* local_sol = (double*) malloc(sizeof(double) * block_decompose(n, q, coords[1]));
    transpose_bcast_vector(n, local_x, local_sol, comm);

    //calculate locally on each processor
    int rowBound = block_decompose(n, q, coords[0]);
    int colBound = block_decompose(n, q, coords[1]);
    double* temp_sol = (double*) malloc(sizeof(double) * rowBound);
    for (int i = 0; i < rowBound; i++) {
        temp_sol[i] = 0;
    }
    for(int i = 0; i < rowBound; i++){
        for(int j = 0; j < colBound;j++){
            temp_sol[i] += local_A[i * colBound + j] * local_sol[j];
        }
    }

    //reduce sum of rows to get the solution
    MPI_Comm n_comm;
    MPI_Comm_split(comm, coords[0], rank, &n_comm);
    MPI_Reduce(temp_sol, local_y, rowBound, MPI_DOUBLE, MPI_SUM, 0, n_comm);

    // free the useless memory
    MPI_Comm_free(&n_comm);
    free(temp_sol);
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %r8, %r14
movq %rcx, 0x30(%rsp)
movq %rdx, 0x10(%rsp)
movq %rsi, %r15
movl %edi, %r13d
leaq 0x24(%rsp), %rsi
movq %r8, %rdi
callq 0x6790
leaq 0x3c(%rsp), %rbx
movq %r14, %rdi
movq %rbx, %rsi
callq 0x65e0
cvtsi2sdl (%rbx), %xmm0
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x27a53
sqrtsd %xmm0, %xmm0
jmp 0x27a58
callq 0x63a0
cvttsd2si %xmm0, %ebx
leaq 0x50(%rsp), %rdx
leaq 0x48(%rsp), %rcx
leaq 0x28(%rsp), %r8
movq %r14, %rdi
movl $0x2, %esi
callq 0x6420
movl %r13d, %eax
cltd
idivl %ebx
movl %eax, %r12d
movl %edx, %ebp
xorl %eax, %eax
leaq 0x28(%rsp), %rcx
cmpl 0x4(%rcx), %edx
setg %al
addl %r12d, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x6720
movq %rax, %rbx
movl %r13d, %edi
movq 0x10(%rsp), %rsi
movq %rax, %rdx
movq %r14, 0x10(%rsp)
movq %r14, %rcx
callq 0x278ba
leaq 0x28(%rsp), %rcx
movl (%rcx), %eax
movl 0x4(%rcx), %ecx
movl %ecx, 0x18(%rsp)
xorl %r13d, %r13d
movl %eax, 0x20(%rsp)
movl %ebp, 0x1c(%rsp)
cmpl %eax, %ebp
setg %r13b
addl %r12d, %r13d
movslq %r13d, %r14
leaq (,%r14,8), %rdi
callq 0x6720
movq %rax, %rbp
testl %r14d, %r14d
jle 0x27b5f
xorl %eax, %eax
movl 0x18(%rsp), %ecx
cmpl %ecx, 0x1c(%rsp)
setg %al
addl %eax, %r12d
leal -0x1(%r13), %eax
leaq 0x8(,%rax,8), %rdx
xorl %r14d, %r14d
movq %rbp, %rdi
xorl %esi, %esi
callq 0x6280
movl %r13d, %eax
leaq (,%r12,8), %rcx
testl %r12d, %r12d
jle 0x27b54
movsd (%rbp,%r14,8), %xmm0
xorl %edx, %edx
movsd (%r15,%rdx,8), %xmm1
mulsd (%rbx,%rdx,8), %xmm1
addsd %xmm1, %xmm0
incq %rdx
cmpq %rdx, %r12
jne 0x27b36
movsd %xmm0, (%rbp,%r14,8)
incq %r14
addq %rcx, %r15
cmpq %rax, %r14
jne 0x27b28
movl 0x24(%rsp), %edx
leaq 0x40(%rsp), %rbx
movq 0x10(%rsp), %rdi
movl 0x20(%rsp), %esi
movq %rbx, %rcx
callq 0x68d0
movq (%rbx), %rax
movq %rax, (%rsp)
movq 0x11419(%rip), %rcx     # 0x38fa0
movq 0x113da(%rip), %r8      # 0x38f68
movq %rbp, %rdi
movq 0x30(%rsp), %rsi
movl %r13d, %edx
xorl %r9d, %r9d
callq 0x6710
movq %rbx, %rdi
callq 0x68b0
movq %rbp, %rdi
callq 0x6630
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq