// File: WeihuaZhu[P]hpc_prog2/mpi_jacobi.cpp
// Function: distributed_matrix_vector_mult(int, double*, double*, double*, ompi_communicator_t*)
void distributed_matrix_vector_mult(const int n, double* local_A, double* local_x, double* local_y, MPI_Comm comm)
{
    // obtain the necessary parameters required in thie method
    int rank;
    MPI_Comm_rank(comm, &rank);
    int p;
    MPI_Comm_size(comm, &p);
    int q = sqrt(p);
    int dims[2];
    int coords[2];
    int periods[2];
    MPI_Cart_get(comm, 2, dims, periods, coords);

    //distribute x using transpose method
    double* local_sol = (double*) malloc(sizeof(double) * block_decompose(n, q, coords[1]));
    transpose_bcast_vector(n, local_x, local_sol, comm);

    //calculate locally on each processor
    int rowBound = block_decompose(n, q, coords[0]);
    int colBound = block_decompose(n, q, coords[1]);
    double* temp_sol = (double*) malloc(sizeof(double) * rowBound);
    for (int i = 0; i < rowBound; i++) {
        temp_sol[i] = 0;
    }
    for(int i = 0; i < rowBound; i++){
        for(int j = 0; j < colBound;j++){
            temp_sol[i] += local_A[i * colBound + j] * local_sol[j];
        }
    }

    //reduce sum of rows to get the solution
    MPI_Comm n_comm;
    MPI_Comm_split(comm, coords[0], rank, &n_comm);
    MPI_Reduce(temp_sol, local_y, rowBound, MPI_DOUBLE, MPI_SUM, 0, n_comm);

    // free the useless memory
    MPI_Comm_free(&n_comm);
    free(temp_sol);
}

// Assembly:
subq $0x88, %rsp
movl %edi, 0x84(%rsp)
movq %rsi, 0x78(%rsp)
movq %rdx, 0x70(%rsp)
movq %rcx, 0x68(%rsp)
movq %r8, 0x60(%rsp)
movq 0x60(%rsp), %rdi
leaq 0x5c(%rsp), %rsi
callq 0xf050
movq 0x60(%rsp), %rdi
leaq 0x58(%rsp), %rsi
callq 0xece0
movl 0x58(%rsp), %edi
callq 0x119d0
cvttsd2si %xmm0, %eax
movl %eax, 0x54(%rsp)
movq 0x60(%rsp), %rdi
leaq 0x4c(%rsp), %rdx
leaq 0x3c(%rsp), %rcx
leaq 0x44(%rsp), %r8
movl $0x2, %esi
callq 0xe8f0
movl 0x84(%rsp), %edi
movl 0x54(%rsp), %esi
movl 0x48(%rsp), %edx
callq 0x4e0e0
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0xef20
movq %rax, 0x30(%rsp)
movl 0x84(%rsp), %edi
movq 0x70(%rsp), %rsi
movq 0x30(%rsp), %rdx
movq 0x60(%rsp), %rcx
callq 0x4d630
movl 0x84(%rsp), %edi
movl 0x54(%rsp), %esi
movl 0x44(%rsp), %edx
callq 0x4e0e0
movl %eax, 0x2c(%rsp)
movl 0x84(%rsp), %edi
movl 0x54(%rsp), %esi
movl 0x48(%rsp), %edx
callq 0x4e0e0
movl %eax, 0x28(%rsp)
movslq 0x2c(%rsp), %rdi
shlq $0x3, %rdi
callq 0xef20
movq %rax, 0x20(%rsp)
movl $0x0, 0x1c(%rsp)
movl 0x1c(%rsp), %eax
cmpl 0x2c(%rsp), %eax
jge 0x4d8a3
movq 0x20(%rsp), %rax
movslq 0x1c(%rsp), %rcx
xorps %xmm0, %xmm0
movsd %xmm0, (%rax,%rcx,8)
movl 0x1c(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x1c(%rsp)
jmp 0x4d87a
movl $0x0, 0x18(%rsp)
movl 0x18(%rsp), %eax
cmpl 0x2c(%rsp), %eax
jge 0x4d928
movl $0x0, 0x14(%rsp)
movl 0x14(%rsp), %eax
cmpl 0x28(%rsp), %eax
jge 0x4d919
movq 0x78(%rsp), %rax
movl 0x18(%rsp), %ecx
imull 0x28(%rsp), %ecx
addl 0x14(%rsp), %ecx
movslq %ecx, %rcx
movsd (%rax,%rcx,8), %xmm0
movq 0x30(%rsp), %rax
movslq 0x14(%rsp), %rcx
movsd (%rax,%rcx,8), %xmm2
movq 0x20(%rsp), %rax
movslq 0x18(%rsp), %rcx
movsd (%rax,%rcx,8), %xmm1
mulsd %xmm2, %xmm0
addsd %xmm1, %xmm0
movsd %xmm0, (%rax,%rcx,8)
movl 0x14(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x14(%rsp)
jmp 0x4d8bd
jmp 0x4d91b
movl 0x18(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x18(%rsp)
jmp 0x4d8ab
movq 0x60(%rsp), %rdi
movl 0x44(%rsp), %esi
movl 0x5c(%rsp), %edx
leaq 0x8(%rsp), %rcx
callq 0xf260
movq 0x20(%rsp), %rdi
movq 0x68(%rsp), %rsi
movl 0x2c(%rsp), %edx
movq 0x8(%rsp), %rax
movq 0x1c64f(%rip), %rcx     # 0x69fa8
movq 0x1c610(%rip), %r8      # 0x69f70
xorl %r9d, %r9d
movq %rax, (%rsp)
callq 0xef00
leaq 0x8(%rsp), %rdi
callq 0xf240
movq 0x20(%rsp), %rdi
callq 0xed30
addq $0x88, %rsp
retq
nopl (%rax,%rax)