// File: /workspace/github_cpp_makefile_star2_O1/WeihuaZhu[P]hpc_prog2/mpi_jacobi.cpp
// Function: distributed_matrix_vector_mult(int, double*, double*, double*, ompi_communicator_t*)
void distributed_matrix_vector_mult(const int n, double* local_A, double* local_x, double* local_y, MPI_Comm comm)
{
    // obtain the necessary parameters required in thie method
    int rank;
    MPI_Comm_rank(comm, &rank);
    int p;
    MPI_Comm_size(comm, &p);
    int q = sqrt(p);
    int dims[2];
    int coords[2];
    int periods[2];
    MPI_Cart_get(comm, 2, dims, periods, coords);

    //distribute x using transpose method
    double* local_sol = (double*) malloc(sizeof(double) * block_decompose(n, q, coords[1]));
    transpose_bcast_vector(n, local_x, local_sol, comm);

    //calculate locally on each processor
    int rowBound = block_decompose(n, q, coords[0]);
    int colBound = block_decompose(n, q, coords[1]);
    double* temp_sol = (double*) malloc(sizeof(double) * rowBound);
    for (int i = 0; i < rowBound; i++) {
        temp_sol[i] = 0;
    }
    for(int i = 0; i < rowBound; i++){
        for(int j = 0; j < colBound;j++){
            temp_sol[i] += local_A[i * colBound + j] * local_sol[j];
        }
    }

    //reduce sum of rows to get the solution
    MPI_Comm n_comm;
    MPI_Comm_split(comm, coords[0], rank, &n_comm);
    MPI_Reduce(temp_sol, local_y, rowBound, MPI_DOUBLE, MPI_SUM, 0, n_comm);

    // free the useless memory
    MPI_Comm_free(&n_comm);
    free(temp_sol);
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %r8, %r14
movq %rcx, 0x30(%rsp)
movq %rdx, 0x10(%rsp)
movq %rsi, %r15
movl %edi, %ebp
leaq 0x1c(%rsp), %rsi
movq %r8, %rdi
callq 0x6790
leaq 0x3c(%rsp), %rbx
movq %r14, 0x20(%rsp)
movq %r14, %rdi
movq %rbx, %rsi
callq 0x65e0
cvtsi2sdl (%rbx), %xmm0
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x28025
sqrtsd %xmm0, %xmm0
jmp 0x2802a
callq 0x63a0
cvttsd2si %xmm0, %ebx
leaq 0x50(%rsp), %rdx
leaq 0x48(%rsp), %rcx
leaq 0x28(%rsp), %r8
movq 0x20(%rsp), %r13
movq %r13, %rdi
movl $0x2, %esi
callq 0x6420
movl %ebp, %eax
cltd
idivl %ebx
movl %eax, %r12d
movl %edx, %r14d
xorl %eax, %eax
leaq 0x28(%rsp), %rcx
cmpl 0x4(%rcx), %edx
setg %al
addl %r12d, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x6720
movl %ebp, %edi
movq %rax, %rbp
movq 0x10(%rsp), %rsi
movq %rax, %rdx
movq %r13, %rcx
callq 0x27e94
leaq 0x28(%rsp), %rcx
movl (%rcx), %eax
xorl %ebx, %ebx
movl %eax, 0x18(%rsp)
cmpl %eax, %r14d
setg %bl
cmpl 0x4(%rcx), %r14d
leal (%r12,%rbx), %eax
setg 0xf(%rsp)
movl %eax, 0x10(%rsp)
movslq %eax, %r14
leaq (,%r14,8), %rdi
callq 0x6720
movq %rax, %r13
testl %r14d, %r14d
jle 0x280df
leal (%r12,%rbx), %eax
decl %eax
leaq 0x8(,%rax,8), %rdx
movq %r13, %rdi
xorl %esi, %esi
callq 0x6280
movl 0x10(%rsp), %ebx
testl %ebx, %ebx
jle 0x28139
xorl %eax, %eax
movb 0xf(%rsp), %cl
movb %cl, %al
addl %eax, %r12d
movl %r12d, %eax
movl %ebx, %ecx
leaq (,%rax,8), %rdx
xorl %esi, %esi
testl %r12d, %r12d
jle 0x2812e
movsd (%r13,%rsi,8), %xmm0
xorl %edi, %edi
movsd (%r15,%rdi,8), %xmm1
mulsd (%rbp,%rdi,8), %xmm1
addsd %xmm1, %xmm0
incq %rdi
cmpq %rdi, %rax
jne 0x2810f
movsd %xmm0, (%r13,%rsi,8)
incq %rsi
addq %rdx, %r15
cmpq %rcx, %rsi
jne 0x28101
movl 0x1c(%rsp), %edx
leaq 0x40(%rsp), %r14
movq 0x20(%rsp), %rdi
movl 0x18(%rsp), %esi
movq %r14, %rcx
callq 0x68d0
movq (%r14), %rax
movq %rax, (%rsp)
movq 0x10e3f(%rip), %rcx     # 0x38fa0
movq 0x10e00(%rip), %r8      # 0x38f68
movq %r13, %rdi
movq 0x30(%rsp), %rsi
movl %ebx, %edx
xorl %r9d, %r9d
callq 0x6710
movq %r14, %rdi
callq 0x68b0
movq %r13, %rdi
callq 0x6630
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq