// File: /workspace/github_cpp_makefile_star2_O2/WeihuaZhu[P]hpc_prog2/mpi_jacobi.cpp
// Function: distributed_matrix_vector_mult(int, double*, double*, double*, ompi_communicator_t*)
void distributed_matrix_vector_mult(const int n, double* local_A, double* local_x, double* local_y, MPI_Comm comm)
{
    // obtain the necessary parameters required in thie method
    int rank;
    MPI_Comm_rank(comm, &rank);
    int p;
    MPI_Comm_size(comm, &p);
    int q = sqrt(p);
    int dims[2];
    int coords[2];
    int periods[2];
    MPI_Cart_get(comm, 2, dims, periods, coords);

    //distribute x using transpose method
    double* local_sol = (double*) malloc(sizeof(double) * block_decompose(n, q, coords[1]));
    transpose_bcast_vector(n, local_x, local_sol, comm);

    //calculate locally on each processor
    int rowBound = block_decompose(n, q, coords[0]);
    int colBound = block_decompose(n, q, coords[1]);
    double* temp_sol = (double*) malloc(sizeof(double) * rowBound);
    for (int i = 0; i < rowBound; i++) {
        temp_sol[i] = 0;
    }
    for(int i = 0; i < rowBound; i++){
        for(int j = 0; j < colBound;j++){
            temp_sol[i] += local_A[i * colBound + j] * local_sol[j];
        }
    }

    //reduce sum of rows to get the solution
    MPI_Comm n_comm;
    MPI_Comm_split(comm, coords[0], rank, &n_comm);
    MPI_Reduce(temp_sol, local_y, rowBound, MPI_DOUBLE, MPI_SUM, 0, n_comm);

    // free the useless memory
    MPI_Comm_free(&n_comm);
    free(temp_sol);
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %r8, %r14
movq %rcx, 0x28(%rsp)
movq %rdx, 0x10(%rsp)
movq %rsi, %r15
movl %edi, %r12d
leaq 0x1c(%rsp), %rsi
movq %r8, %rdi
callq 0x67d0
leaq 0x3c(%rsp), %rbx
movq %r14, 0x20(%rsp)
movq %r14, %rdi
movq %rbx, %rsi
callq 0x6600
cvtsi2sdl (%rbx), %xmm0
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x1dc79
sqrtsd %xmm0, %xmm0
jmp 0x1dc7e
callq 0x63b0
cvttsd2si %xmm0, %ebx
pushq $0x2
popq %rsi
leaq 0x50(%rsp), %rdx
leaq 0x48(%rsp), %rcx
leaq 0x34(%rsp), %r8
movq 0x20(%rsp), %rbp
movq %rbp, %rdi
movq %r8, %r13
callq 0x6430
movl %r12d, %eax
cltd
idivl %ebx
movl %eax, %ebx
movl %r12d, %r14d
movl %edx, %r12d
xorl %eax, %eax
cmpl 0x4(%r13), %edx
setg %al
addl %ebx, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x6750
movq %rax, %r13
movl %r14d, %edi
movq 0x10(%rsp), %rsi
movq %rax, %rdx
movq %rbp, %rcx
callq 0x1dadd
leaq 0x34(%rsp), %rcx
movl (%rcx), %eax
xorl %r14d, %r14d
movl %eax, 0x10(%rsp)
cmpl %eax, %r12d
setg %r14b
addl %ebx, %r14d
xorl %eax, %eax
cmpl 0x4(%rcx), %r12d
setg %al
movl %eax, 0x18(%rsp)
movslq %r14d, %rbp
leaq (,%rbp,8), %rdi
callq 0x6750
movq %rax, %r12
xorl %ecx, %ecx
testl %ebp, %ebp
movl $0x0, %eax
cmovgl %r14d, %eax
cmpq %rcx, %rax
je 0x1dd33
andq $0x0, (%r12,%rcx,8)
incq %rcx
jmp 0x1dd24
addl 0x18(%rsp), %ebx
xorl %ecx, %ecx
testl %ebx, %ebx
movl $0x0, %edx
cmovgl %ebx, %edx
movslq %ebx, %rsi
shlq $0x3, %rsi
cmpq %rax, %rcx
je 0x1dd7c
xorl %edi, %edi
cmpq %rdi, %rdx
je 0x1dd74
movsd (%r15,%rdi,8), %xmm0
mulsd (%r13,%rdi,8), %xmm0
addsd (%r12,%rcx,8), %xmm0
movsd %xmm0, (%r12,%rcx,8)
incq %rdi
jmp 0x1dd51
incq %rcx
addq %rsi, %r15
jmp 0x1dd4a
movl 0x1c(%rsp), %edx
leaq 0x40(%rsp), %rbx
movq 0x20(%rsp), %rdi
movl 0x10(%rsp), %esi
movq %rbx, %rcx
callq 0x6920
movq (%rbx), %rax
movq %rax, (%rsp)
movq 0x13204(%rip), %rcx     # 0x30fa8
movq 0x131cd(%rip), %r8      # 0x30f78
movq %r12, %rdi
movq 0x28(%rsp), %rsi
movl %r14d, %edx
xorl %r9d, %r9d
callq 0x6740
movq %rbx, %rdi
callq 0x6900
movq %r12, %rdi
callq 0x6650
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq