// File: /workspace/github_c_makefile_star2_O3/baileywjohnson[P]mpi-matrix-mult/mm-parallel.c
// Function: main
int main(int argc, char *argv[]){

	MPI_Init(NULL, NULL);

	double io_start_1 = 0;
	GET_TIME(io_start_1);

	int rank, size;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	MPI_Comm_size(MPI_COMM_WORLD, &size);

	char *a_in;
	char *b_in;
	char *c_out;

	int c;
	if(argc != 7){
		printf("USAGE: mm-serial -A <matrix 1 filename> -B <matrix 2 filename> -C <output filename>\n");
		exit(1);
	}
	while((c = getopt(argc, argv, "A:B:C:")) != -1){
		switch(c){
			case 'A':
				a_in = optarg;
				break;
			case 'B':
				b_in = optarg;
				break;
			case 'C':
				c_out = optarg;
				break;
			case '?':
				exit(1);
				break;
			default:
				break;
		}
	}

	int m_A, n_A;
	double **subs_A;
	double *storage_A;
	read_row_striped_matrix(a_in, (void***)&subs_A, (void**)&storage_A, MPI_DOUBLE, &m_A, &n_A, MPI_COMM_WORLD);

	int m_B, n_B;
	double **subs_B;
	double *storage_B;
	read_row_striped_matrix(b_in, (void***)&subs_B, (void**)&storage_B, MPI_DOUBLE, &m_B, &n_B, MPI_COMM_WORLD);

	if(n_A != m_B){
		if(rank == 0){
			printf("Matrices Incompatible for Multiplication\n");
		}
		MPI_Finalize();
		exit(1);
	}

	int m_C, n_C;
	double *C;
	m_C = m_A;
	n_C = n_B;
	C = calloc((n_C * (m_C/size)), sizeof(double));

	double io_finish_1 = 0;
	GET_TIME(io_finish_1);

	double flops = 0;

	int i, j, k;
	for(i=0; i<size; i++){

		for(j=0; j<((m_A/size)*(m_A/size)); j++){

			int a_index = 0;
			//not first row in block
			if(j >= m_A/size){
				//one processor
				if(size == 1){
					a_index = j;
				}
				//more than one processor
				else{
					a_index = j + ((j/(m_A/size)) * (m_A - (m_A/size))) + (rank * (m_A/size));
				}

				//adjust for phase, handle wraparound if necessary
				if(((a_index - (i*(m_A/size)))/m_A) != (a_index/m_A)){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}
			//first row
			else{
				a_index = j+(rank*(m_A/size));

				//adjust for phase, handle wraparound if necessary
				if(a_index - (i*(m_A/size)) < 0){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}

			for(k=0; k<n_B; k++){
				int b_index = 0;
				if(a_index % m_A < m_A/size){
					if(a_index >= m_A){
						b_index = k+(m_A * (a_index % m_A));
					}
					else{
						b_index = k+(m_A * a_index);
					}
				}
				else{
					if(a_index >= m_A){
						b_index = k+(m_A * ((a_index % (m_A/size)) % m_A));
					}
					else{
						b_index = k+(m_A * (a_index % (m_A/size)));
					}
				}
				double result = storage_A[a_index] * storage_B[b_index];
				C[(k+(m_A*(a_index/m_A)))] += result;
				flops += 1;
			}
		}
		MPI_Sendrecv(storage_B, n_B*(n_B/size), MPI_DOUBLE, ((rank + 1) % size), 1, storage_B, n_B*(n_B/size), MPI_DOUBLE,
		 (((rank - 1) + size) % size), 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
	}

	MPI_Barrier(MPI_COMM_WORLD);

	double io_start_2 = 0;
	GET_TIME(io_start_2);

	double *C_final = malloc(sizeof(double) * (m_C * n_C));

	MPI_Gather(C, n_C * (m_C/size), MPI_DOUBLE, C_final, n_C * (m_C/size), MPI_DOUBLE, 0, MPI_COMM_WORLD);

	if(rank == 0){
		write_matrix(c_out, m_C, n_C, C_final);
	}

	free(storage_A);
	free(storage_B);
	free(C);
	free(C_final);

	double io_finish_2 = 0;
	GET_TIME(io_finish_2);

	if(rank == 0){
		double io_time = (io_finish_2 - io_start_2) + (io_finish_1 - io_start_1);
		double mm_time = (io_start_2 - io_finish_1);
		double mflop_count = (flops * size)/1000000;
		printf("M: %i, N: %i\n", m_A, n_A);
		printf("Time Doing I/O: %f sec.\n", io_time);
		printf("Time Doing MM: %f sec.\n", mm_time);
		printf("Total Time: %f sec.\n", io_time + mm_time);
		printf("MFlops (w/o IO): %e, MFlops (w/ IO): %e\n", mflop_count/mm_time, mflop_count/(io_time + mm_time));
	}

	MPI_Finalize();

	return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xc8, %rsp
movq %rsi, %r15
movl %edi, %ebp
xorl %edi, %edi
xorl %esi, %esi
callq 0x10d0
leaq 0x78(%rsp), %rbx
movq %rbx, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%rbx), %xmm0
movsd %xmm0, 0x10(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdq 0x8(%rbx), %xmm0
movsd %xmm0, 0x40(%rsp)
movq 0x4c3d(%rip), %rbx      # 0x5fd8
leaq 0x18(%rsp), %rsi
movq %rbx, %rdi
callq 0x1170
leaq 0xc(%rsp), %rsi
movq %rbx, %rdi
callq 0x1100
cmpl $0x7, %ebp
jne 0x1905
movsd 0x40(%rsp), %xmm0
divsd 0x2c3c(%rip), %xmm0     # 0x4008
addsd 0x10(%rsp), %xmm0
movsd %xmm0, 0x40(%rsp)
leaq 0x2c4d(%rip), %r13      # 0x402c
leaq 0x2c32(%rip), %rbx      # 0x4018
movq 0x4bdb(%rip), %r14      # 0x5fc8
movl $0x7, %edi
movq %r15, %rsi
movq %r13, %rdx
callq 0x10a0
leal -0x3f(%rax), %ecx
cmpl $0x4, %ecx
ja 0x1413
movslq (%rbx,%rcx,4), %rax
addq %rbx, %rax
jmpq *%rax
movq (%r14), %rbp
jmp 0x13ed
cmpl $-0x1, %eax
jne 0x13ed
jmp 0x142c
movq (%r14), %r12
jmp 0x13ed
movq (%r14), %rax
movq %rax, 0x90(%rsp)
jmp 0x13ed
subq $0x8, %rsp
movq 0x4b99(%rip), %r14      # 0x5fd0
leaq 0xc8(%rsp), %rsi
leaq 0x78(%rsp), %rdx
leaq 0x24(%rsp), %r8
leaq 0x64(%rsp), %r15
movq %rbp, %rdi
movq %r14, %rcx
movq %r15, %r9
movq 0x4b7a(%rip), %rbx      # 0x5fd8
pushq %rbx
callq 0x24b6
addq $0x8, %rsp
leaq 0xc0(%rsp), %rsi
leaq 0x70(%rsp), %rdx
leaq 0xbc(%rsp), %rbp
leaq 0x3c(%rsp), %r9
movq %r12, %rdi
movq %r14, %rcx
movq %rbp, %r8
pushq %rbx
callq 0x24b6
addq $0x10, %rsp
movl (%r15), %eax
cmpl (%rbp), %eax
jne 0x1913
movl 0x1c(%rsp), %eax
movl 0x34(%rsp), %ecx
movl %eax, 0x4c(%rsp)
cltd
idivl 0xc(%rsp)
movl %ecx, 0x48(%rsp)
imull %ecx, %eax
movslq %eax, %rdi
movl $0x8, %esi
callq 0x11b0
movq %rax, %r15
leaq 0x78(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%r14), %xmm1
cvtsi2sdq 0x8(%r14), %xmm0
movsd %xmm0, 0x20(%rsp)
movl 0xc(%rsp), %edi
testl %edi, %edi
movsd %xmm1, 0x88(%rsp)
xorpd %xmm0, %xmm0
jle 0x16fe
movsd %xmm0, 0x28(%rsp)
movsd 0x2b00(%rip), %xmm2     # 0x4010
xorl %ecx, %ecx
movl 0x1c(%rsp), %esi
movl %esi, %eax
cltd
movq %rdi, 0x60(%rsp)
idivl %edi
movl %eax, %r8d
movq 0x68(%rsp), %rax
movq %rax, 0x38(%rsp)
movl 0x34(%rsp), %ebx
movl 0x18(%rsp), %eax
testl %r8d, %r8d
movsd 0x28(%rsp), %xmm1
movl %ecx, 0x50(%rsp)
movq %rax, 0x98(%rsp)
je 0x1686
movl %r8d, %edx
imull %r8d, %edx
movl %esi, %edi
subl %r8d, %edi
movl %edi, 0x54(%rsp)
movl %eax, %edi
imull %r8d, %edi
movq %rdi, 0xa0(%rsp)
movl %r8d, %edi
imull %ecx, %edi
movl %edi, 0x58(%rsp)
subl %ecx, %eax
imull %r8d, %eax
movq %rax, 0xa8(%rsp)
movq 0x70(%rsp), %r13
cmpl $0x1, %edx
adcl $0x0, %edx
movl %edx, 0x10(%rsp)
xorl %r9d, %r9d
cmpl %r8d, %r9d
jge 0x15b7
movq 0xa8(%rsp), %rax
addl %r9d, %eax
movl %eax, %r10d
sarl $0x1f, %r10d
andl %esi, %r10d
addl %eax, %r10d
jmp 0x1608
movl %r9d, %r10d
cmpl $0x1, 0x60(%rsp)
je 0x15df
movl %r9d, %eax
cltd
idivl %r8d
movl %eax, %r10d
imull 0x54(%rsp), %r10d
movq 0xa0(%rsp), %rax
addl %r9d, %eax
addl %eax, %r10d
movl %r10d, %r11d
subl 0x58(%rsp), %r11d
movl %r11d, %eax
cltd
idivl %esi
movl %eax, %r14d
movl %r10d, %eax
cltd
idivl %esi
cmpl %eax, %r14d
movl %esi, %r10d
movl $0x0, %eax
cmovel %eax, %r10d
addl %r11d, %r10d
testl %ebx, %ebx
jle 0x1678
movl %r10d, %eax
cltd
idivl %esi
movl %edx, %r11d
cmpl %esi, %r10d
movl %edx, %r14d
cmovll %r10d, %r14d
movslq %r10d, %rdi
movl %edi, %ebp
subl %edx, %ebp
movq %rbx, %rcx
movq 0x38(%rsp), %r12
movl %r14d, %edx
cmpl %r8d, %r11d
jl 0x1647
movl %r10d, %eax
cltd
idivl %r8d
cmpl %esi, %r10d
jl 0x1647
movl %edx, %eax
cltd
idivl %esi
imull %esi, %edx
movslq %edx, %rax
movsd (%r13,%rdi,8), %xmm0
mulsd (%r12,%rax,8), %xmm0
movslq %ebp, %rbp
addsd (%r15,%rbp,8), %xmm0
movsd %xmm0, (%r15,%rbp,8)
addsd %xmm2, %xmm1
addq $0x8, %r12
incl %ebp
decq %rcx
jne 0x162e
incl %r9d
cmpl 0x10(%rsp), %r9d
jne 0x1598
movsd %xmm1, 0x28(%rsp)
movl %ebx, %eax
cltd
movq 0x60(%rsp), %rdi
idivl %edi
movl %eax, %esi
imull %ebx, %esi
movq 0x98(%rsp), %r8
leal 0x1(%r8), %eax
cltd
idivl %edi
movl %edx, %ecx
leal (%rdi,%r8), %eax
decl %eax
cltd
idivl %edi
movl %edx, %eax
movq 0x38(%rsp), %rdi
movq 0x490d(%rip), %rdx      # 0x5fd0
movl $0x1, %r8d
movq %rdi, %r9
pushq $0x0
pushq 0x4904(%rip)            # 0x5fd8
pushq $0x1
pushq %rax
pushq %rdx
pushq %rsi
callq 0x1150
movsd 0x292a(%rip), %xmm2     # 0x4010
addq $0x30, %rsp
movl 0x50(%rsp), %ecx
incl %ecx
movl 0xc(%rsp), %edi
cmpl %edi, %ecx
jl 0x1512
jmp 0x1704
movsd %xmm0, 0x28(%rsp)
movq 0x48cd(%rip), %rbx      # 0x5fd8
movq %rbx, %rdi
callq 0x1030
leaq 0x78(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
xorps %xmm0, %xmm0
cvtsi2sdq (%r14), %xmm0
movsd %xmm0, 0x38(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdq 0x8(%r14), %xmm0
movsd %xmm0, 0x10(%rsp)
movl 0x48(%rsp), %r14d
movl %r14d, %eax
movl 0x4c(%rsp), %ebp
imull %ebp, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x10b0
movq %rax, %r13
movl %ebp, %eax
cltd
idivl 0xc(%rsp)
imull %r14d, %eax
movq 0x4861(%rip), %rdx      # 0x5fd0
movq %r15, %rdi
movl %eax, %esi
movq %r13, %rcx
movl %eax, %r8d
movq %rdx, %r9
pushq %rbx
pushq $0x0
callq 0x1130
addq $0x10, %rsp
cmpl $0x0, 0x18(%rsp)
jne 0x17a5
movq 0x90(%rsp), %rdi
movl %ebp, %esi
movl %r14d, %edx
movq %r13, %rcx
callq 0x1a5a
movq 0x70(%rsp), %rdi
callq 0x10f0
movq 0x68(%rsp), %rdi
callq 0x10f0
movq %r15, %rdi
callq 0x10f0
movq %r13, %rdi
callq 0x10f0
leaq 0x78(%rsp), %rdi
xorl %esi, %esi
callq 0x1080
cmpl $0x0, 0x18(%rsp)
jne 0x18ec
movsd 0x20(%rsp), %xmm3
divsd 0x281a(%rip), %xmm3     # 0x4008
addsd 0x88(%rsp), %xmm3
movsd 0x10(%rsp), %xmm4
divsd 0x2803(%rip), %xmm4     # 0x4008
xorps %xmm0, %xmm0
cvtsi2sdq 0x78(%rsp), %xmm0
addsd 0x38(%rsp), %xmm4
cvtsi2sdq 0x80(%rsp), %xmm1
movsd 0x27e1(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm0, %xmm1
subsd %xmm4, %xmm1
subsd %xmm3, %xmm4
movsd %xmm4, 0x10(%rsp)
subsd 0x40(%rsp), %xmm3
addsd %xmm1, %xmm3
movsd %xmm3, 0x20(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdl 0xc(%rsp), %xmm0
movsd 0x28(%rsp), %xmm1
mulsd %xmm0, %xmm1
divsd %xmm2, %xmm1
movsd %xmm1, 0x28(%rsp)
movl 0x1c(%rsp), %esi
movl 0x5c(%rsp), %edx
leaq 0x27ba(%rip), %rdi      # 0x4033
xorl %eax, %eax
callq 0x1050
leaq 0x27ba(%rip), %rdi      # 0x4041
movsd 0x20(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
leaq 0x27bf(%rip), %rdi      # 0x405a
movsd 0x10(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
movsd 0x20(%rsp), %xmm0
addsd 0x10(%rsp), %xmm0
movsd %xmm0, 0x20(%rsp)
leaq 0x27b1(%rip), %rdi      # 0x4072
movb $0x1, %al
callq 0x1050
movsd 0x28(%rsp), %xmm1
movapd %xmm1, %xmm0
divsd 0x10(%rsp), %xmm0
divsd 0x20(%rsp), %xmm1
leaq 0x27a2(%rip), %rdi      # 0x4087
movb $0x2, %al
callq 0x1050
callq 0x1190
xorl %eax, %eax
addq $0xc8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x27cd(%rip), %rdi      # 0x40d9
callq 0x1060
jmp 0x192b
cmpl $0x0, 0x18(%rsp)
jne 0x1926
leaq 0x278f(%rip), %rdi      # 0x40b0
callq 0x1060
callq 0x1190
movl $0x1, %edi
callq 0x1070
nopl (%rax)