// File: /workspace/github_c_makefile_star2_O1/baileywjohnson[P]mpi-matrix-mult/mm-parallel.c
// Function: main
int main(int argc, char *argv[]){

	MPI_Init(NULL, NULL);

	double io_start_1 = 0;
	GET_TIME(io_start_1);

	int rank, size;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	MPI_Comm_size(MPI_COMM_WORLD, &size);

	char *a_in;
	char *b_in;
	char *c_out;

	int c;
	if(argc != 7){
		printf("USAGE: mm-serial -A <matrix 1 filename> -B <matrix 2 filename> -C <output filename>\n");
		exit(1);
	}
	while((c = getopt(argc, argv, "A:B:C:")) != -1){
		switch(c){
			case 'A':
				a_in = optarg;
				break;
			case 'B':
				b_in = optarg;
				break;
			case 'C':
				c_out = optarg;
				break;
			case '?':
				exit(1);
				break;
			default:
				break;
		}
	}

	int m_A, n_A;
	double **subs_A;
	double *storage_A;
	read_row_striped_matrix(a_in, (void***)&subs_A, (void**)&storage_A, MPI_DOUBLE, &m_A, &n_A, MPI_COMM_WORLD);

	int m_B, n_B;
	double **subs_B;
	double *storage_B;
	read_row_striped_matrix(b_in, (void***)&subs_B, (void**)&storage_B, MPI_DOUBLE, &m_B, &n_B, MPI_COMM_WORLD);

	if(n_A != m_B){
		if(rank == 0){
			printf("Matrices Incompatible for Multiplication\n");
		}
		MPI_Finalize();
		exit(1);
	}

	int m_C, n_C;
	double *C;
	m_C = m_A;
	n_C = n_B;
	C = calloc((n_C * (m_C/size)), sizeof(double));

	double io_finish_1 = 0;
	GET_TIME(io_finish_1);

	double flops = 0;

	int i, j, k;
	for(i=0; i<size; i++){

		for(j=0; j<((m_A/size)*(m_A/size)); j++){

			int a_index = 0;
			//not first row in block
			if(j >= m_A/size){
				//one processor
				if(size == 1){
					a_index = j;
				}
				//more than one processor
				else{
					a_index = j + ((j/(m_A/size)) * (m_A - (m_A/size))) + (rank * (m_A/size));
				}

				//adjust for phase, handle wraparound if necessary
				if(((a_index - (i*(m_A/size)))/m_A) != (a_index/m_A)){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}
			//first row
			else{
				a_index = j+(rank*(m_A/size));

				//adjust for phase, handle wraparound if necessary
				if(a_index - (i*(m_A/size)) < 0){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}

			for(k=0; k<n_B; k++){
				int b_index = 0;
				if(a_index % m_A < m_A/size){
					if(a_index >= m_A){
						b_index = k+(m_A * (a_index % m_A));
					}
					else{
						b_index = k+(m_A * a_index);
					}
				}
				else{
					if(a_index >= m_A){
						b_index = k+(m_A * ((a_index % (m_A/size)) % m_A));
					}
					else{
						b_index = k+(m_A * (a_index % (m_A/size)));
					}
				}
				double result = storage_A[a_index] * storage_B[b_index];
				C[(k+(m_A*(a_index/m_A)))] += result;
				flops += 1;
			}
		}
		MPI_Sendrecv(storage_B, n_B*(n_B/size), MPI_DOUBLE, ((rank + 1) % size), 1, storage_B, n_B*(n_B/size), MPI_DOUBLE,
		 (((rank - 1) + size) % size), 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
	}

	MPI_Barrier(MPI_COMM_WORLD);

	double io_start_2 = 0;
	GET_TIME(io_start_2);

	double *C_final = malloc(sizeof(double) * (m_C * n_C));

	MPI_Gather(C, n_C * (m_C/size), MPI_DOUBLE, C_final, n_C * (m_C/size), MPI_DOUBLE, 0, MPI_COMM_WORLD);

	if(rank == 0){
		write_matrix(c_out, m_C, n_C, C_final);
	}

	free(storage_A);
	free(storage_B);
	free(C);
	free(C_final);

	double io_finish_2 = 0;
	GET_TIME(io_finish_2);

	if(rank == 0){
		double io_time = (io_finish_2 - io_start_2) + (io_finish_1 - io_start_1);
		double mm_time = (io_start_2 - io_finish_1);
		double mflop_count = (flops * size)/1000000;
		printf("M: %i, N: %i\n", m_A, n_A);
		printf("Time Doing I/O: %f sec.\n", io_time);
		printf("Time Doing MM: %f sec.\n", mm_time);
		printf("Total Time: %f sec.\n", io_time + mm_time);
		printf("MFlops (w/o IO): %e, MFlops (w/ IO): %e\n", mflop_count/mm_time, mflop_count/(io_time + mm_time));
	}

	MPI_Finalize();

	return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xb8, %rsp
movq %rsi, %r15
movl %edi, %ebp
xorl %edi, %edi
xorl %esi, %esi
callq 0x10d0
leaq 0x70(%rsp), %rbx
movq %rbx, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%rbx), %xmm0
movsd %xmm0, 0x8(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdq 0x8(%rbx), %xmm0
movsd %xmm0, 0x30(%rsp)
movq 0x4c3d(%rip), %rbx      # 0x5fd8
leaq 0x14(%rsp), %rsi
movq %rbx, %rdi
callq 0x1170
leaq 0x10(%rsp), %rsi
movq %rbx, %rdi
callq 0x1100
cmpl $0x7, %ebp
jne 0x18fb
movsd 0x30(%rsp), %xmm0
divsd 0x2c3c(%rip), %xmm0     # 0x4008
addsd 0x8(%rsp), %xmm0
movsd %xmm0, 0x30(%rsp)
leaq 0x2c4d(%rip), %r13      # 0x402c
leaq 0x2c32(%rip), %rbx      # 0x4018
movq 0x4bdb(%rip), %r12      # 0x5fc8
movl %ebp, %edi
movq %r15, %rsi
movq %r13, %rdx
callq 0x10a0
leal -0x3f(%rax), %ecx
cmpl $0x4, %ecx
ja 0x1411
movslq (%rbx,%rcx,4), %rax
addq %rbx, %rax
jmpq *%rax
movq (%r12), %r14
jmp 0x13ed
cmpl $-0x1, %eax
jne 0x13ed
jmp 0x1431
movq (%r12), %rax
movq %rax, 0x8(%rsp)
jmp 0x13ed
movq (%r12), %rax
movq %rax, 0x88(%rsp)
jmp 0x13ed
subq $0x8, %rsp
movq 0x4b94(%rip), %r12      # 0x5fd0
leaq 0xb8(%rsp), %rsi
leaq 0x70(%rsp), %rdx
leaq 0x24(%rsp), %r8
leaq 0x64(%rsp), %r15
movq %r14, %rdi
movq %r12, %rcx
movq %r15, %r9
movq 0x4b75(%rip), %rbx      # 0x5fd8
pushq %rbx
callq 0x24ae
addq $0x8, %rsp
leaq 0xb0(%rsp), %rsi
leaq 0x48(%rsp), %rdx
leaq 0xac(%rsp), %r14
leaq 0x20(%rsp), %r9
movq 0x10(%rsp), %rdi
movq %r12, %rcx
movq %r14, %r8
pushq %rbx
callq 0x24ae
addq $0x10, %rsp
movl (%r15), %eax
cmpl (%r14), %eax
jne 0x1909
movl 0x1c(%rsp), %eax
movl 0x18(%rsp), %ecx
movl %eax, 0x4c(%rsp)
cltd
idivl 0x10(%rsp)
movl %ecx, 0x48(%rsp)
imull %ecx, %eax
movslq %eax, %rdi
movl $0x8, %esi
callq 0x11b0
movq %rax, %r15
leaq 0x70(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%r14), %xmm1
cvtsi2sdq 0x8(%r14), %xmm0
movsd %xmm0, 0x20(%rsp)
movl 0x10(%rsp), %esi
testl %esi, %esi
movsd %xmm1, 0x80(%rsp)
xorpd %xmm0, %xmm0
jle 0x16f7
movsd %xmm0, 0x28(%rsp)
movsd 0x2af9(%rip), %xmm2     # 0x4010
xorl %edi, %edi
movl 0x1c(%rsp), %ecx
movl %ecx, %eax
cltd
movq %rsi, 0x60(%rsp)
idivl %esi
testl %eax, %eax
movsd 0x28(%rsp), %xmm1
movl %edi, 0x50(%rsp)
je 0x167b
movl %eax, %esi
imull %esi, %eax
movl %ecx, %edx
subl %esi, %edx
movl %edx, 0x54(%rsp)
movl 0x14(%rsp), %edx
movl %edx, %r8d
imull %esi, %r8d
movq %r8, 0x90(%rsp)
movl %esi, %r8d
imull %edi, %r8d
movl %r8d, 0x58(%rsp)
subl %edi, %edx
imull %esi, %edx
movq %rdx, 0x98(%rsp)
movl 0x18(%rsp), %r11d
movq 0x68(%rsp), %r12
movq 0x40(%rsp), %rdx
movq %rdx, 0x38(%rsp)
cmpl $0x1, %eax
adcl $0x0, %eax
movl %eax, 0x8(%rsp)
xorl %edi, %edi
cmpl %esi, %edi
jge 0x15af
movq 0x98(%rsp), %rax
addl %edi, %eax
movl %eax, %r8d
sarl $0x1f, %r8d
andl %ecx, %r8d
addl %eax, %r8d
jmp 0x15fd
movl %edi, %r8d
cmpl $0x1, 0x60(%rsp)
je 0x15d4
movl %edi, %eax
cltd
idivl %esi
movl %eax, %r8d
imull 0x54(%rsp), %r8d
movq 0x90(%rsp), %rax
addl %edi, %eax
addl %eax, %r8d
movl %r8d, %r9d
subl 0x58(%rsp), %r9d
movl %r9d, %eax
cltd
idivl %ecx
movl %eax, %r10d
movl %r8d, %eax
cltd
idivl %ecx
cmpl %eax, %r10d
movl %ecx, %r8d
movl $0x0, %eax
cmovel %eax, %r8d
addl %r9d, %r8d
testl %r11d, %r11d
jle 0x166f
movl %r8d, %eax
cltd
idivl %ecx
movl %edx, %r9d
cmpl %ecx, %r8d
movl %edx, %r10d
cmovll %r8d, %r10d
movslq %r8d, %r14
movl %r14d, %r13d
subl %edx, %r13d
movq %r11, %rbx
movq 0x38(%rsp), %rbp
movl %r10d, %edx
cmpl %esi, %r9d
jl 0x163e
movl %r8d, %eax
cltd
idivl %esi
cmpl %ecx, %r8d
jl 0x163e
movl %edx, %eax
cltd
idivl %ecx
imull %ecx, %edx
movslq %edx, %rax
movsd (%r12,%r14,8), %xmm0
mulsd (%rbp,%rax,8), %xmm0
movslq %r13d, %r13
addsd (%r15,%r13,8), %xmm0
movsd %xmm0, (%r15,%r13,8)
addsd %xmm2, %xmm1
addq $0x8, %rbp
incl %r13d
decq %rbx
jne 0x1626
incl %edi
cmpl 0x8(%rsp), %edi
jne 0x1592
movsd %xmm1, 0x28(%rsp)
movq 0x40(%rsp), %rdi
movl 0x18(%rsp), %ecx
movl %ecx, %eax
cltd
movq 0x60(%rsp), %r9
idivl %r9d
movl %eax, %esi
imull %ecx, %esi
movl 0x14(%rsp), %r8d
leal 0x1(%r8), %eax
cltd
idivl %r9d
movl %edx, %ecx
leal (%r9,%r8), %eax
decl %eax
cltd
idivl %r9d
movl %edx, %eax
movq 0x4914(%rip), %rdx      # 0x5fd0
movl $0x1, %r8d
movq %rdi, %r9
pushq $0x0
pushq 0x490b(%rip)            # 0x5fd8
pushq $0x1
pushq %rax
pushq %rdx
pushq %rsi
callq 0x1150
movsd 0x2931(%rip), %xmm2     # 0x4010
addq $0x30, %rsp
movl 0x50(%rsp), %edi
incl %edi
movl 0x10(%rsp), %esi
cmpl %esi, %edi
jl 0x1519
jmp 0x16fd
movsd %xmm0, 0x28(%rsp)
movq 0x48d4(%rip), %rbx      # 0x5fd8
movq %rbx, %rdi
callq 0x1030
leaq 0x70(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
xorps %xmm0, %xmm0
cvtsi2sdq (%r14), %xmm0
movsd %xmm0, 0x38(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdq 0x8(%r14), %xmm0
movsd %xmm0, 0x8(%rsp)
movl 0x48(%rsp), %r12d
movl %r12d, %eax
movl 0x4c(%rsp), %ebp
imull %ebp, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x10b0
movq %rax, %r14
movl %ebp, %eax
cltd
idivl 0x10(%rsp)
imull %r12d, %eax
movq 0x4868(%rip), %rdx      # 0x5fd0
movq %r15, %rdi
movl %eax, %esi
movq %r14, %rcx
movl %eax, %r8d
movq %rdx, %r9
pushq %rbx
pushq $0x0
callq 0x1130
addq $0x10, %rsp
cmpl $0x0, 0x14(%rsp)
jne 0x179e
movq 0x88(%rsp), %rdi
movl %ebp, %esi
movl %r12d, %edx
movq %r14, %rcx
callq 0x1a4e
movq 0x68(%rsp), %rdi
callq 0x10f0
movq 0x40(%rsp), %rdi
callq 0x10f0
movq %r15, %rdi
callq 0x10f0
movq %r14, %rdi
callq 0x10f0
leaq 0x70(%rsp), %rdi
xorl %esi, %esi
callq 0x1080
cmpl $0x0, 0x14(%rsp)
jne 0x18e2
movsd 0x20(%rsp), %xmm3
divsd 0x2821(%rip), %xmm3     # 0x4008
addsd 0x80(%rsp), %xmm3
movsd 0x8(%rsp), %xmm4
divsd 0x280a(%rip), %xmm4     # 0x4008
xorps %xmm0, %xmm0
cvtsi2sdq 0x70(%rsp), %xmm0
addsd 0x38(%rsp), %xmm4
cvtsi2sdq 0x78(%rsp), %xmm1
movsd 0x27eb(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm0, %xmm1
subsd %xmm4, %xmm1
subsd %xmm3, %xmm4
movsd %xmm4, 0x8(%rsp)
subsd 0x30(%rsp), %xmm3
addsd %xmm1, %xmm3
movsd %xmm3, 0x20(%rsp)
xorps %xmm0, %xmm0
cvtsi2sdl 0x10(%rsp), %xmm0
movsd 0x28(%rsp), %xmm1
mulsd %xmm0, %xmm1
divsd %xmm2, %xmm1
movsd %xmm1, 0x28(%rsp)
movl 0x1c(%rsp), %esi
movl 0x5c(%rsp), %edx
leaq 0x27c4(%rip), %rdi      # 0x4033
xorl %eax, %eax
callq 0x1050
leaq 0x27c4(%rip), %rdi      # 0x4041
movsd 0x20(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
leaq 0x27c9(%rip), %rdi      # 0x405a
movsd 0x8(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
movsd 0x20(%rsp), %xmm0
addsd 0x8(%rsp), %xmm0
movsd %xmm0, 0x20(%rsp)
leaq 0x27bb(%rip), %rdi      # 0x4072
movb $0x1, %al
callq 0x1050
movsd 0x28(%rsp), %xmm1
movapd %xmm1, %xmm0
divsd 0x8(%rsp), %xmm0
divsd 0x20(%rsp), %xmm1
leaq 0x27ac(%rip), %rdi      # 0x4087
movb $0x2, %al
callq 0x1050
callq 0x1190
xorl %eax, %eax
addq $0xb8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x27d7(%rip), %rdi      # 0x40d9
callq 0x1060
jmp 0x1921
cmpl $0x0, 0x14(%rsp)
jne 0x191c
leaq 0x2799(%rip), %rdi      # 0x40b0
callq 0x1060
callq 0x1190
movl $0x1, %edi
callq 0x1070
nop