// File: /workspace/github_c_makefile_star2_O2/baileywjohnson[P]mpi-matrix-mult/mm-parallel.c
// Function: main
int main(int argc, char *argv[]){

	MPI_Init(NULL, NULL);

	double io_start_1 = 0;
	GET_TIME(io_start_1);

	int rank, size;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	MPI_Comm_size(MPI_COMM_WORLD, &size);

	char *a_in;
	char *b_in;
	char *c_out;

	int c;
	if(argc != 7){
		printf("USAGE: mm-serial -A <matrix 1 filename> -B <matrix 2 filename> -C <output filename>\n");
		exit(1);
	}
	while((c = getopt(argc, argv, "A:B:C:")) != -1){
		switch(c){
			case 'A':
				a_in = optarg;
				break;
			case 'B':
				b_in = optarg;
				break;
			case 'C':
				c_out = optarg;
				break;
			case '?':
				exit(1);
				break;
			default:
				break;
		}
	}

	int m_A, n_A;
	double **subs_A;
	double *storage_A;
	read_row_striped_matrix(a_in, (void***)&subs_A, (void**)&storage_A, MPI_DOUBLE, &m_A, &n_A, MPI_COMM_WORLD);

	int m_B, n_B;
	double **subs_B;
	double *storage_B;
	read_row_striped_matrix(b_in, (void***)&subs_B, (void**)&storage_B, MPI_DOUBLE, &m_B, &n_B, MPI_COMM_WORLD);

	if(n_A != m_B){
		if(rank == 0){
			printf("Matrices Incompatible for Multiplication\n");
		}
		MPI_Finalize();
		exit(1);
	}

	int m_C, n_C;
	double *C;
	m_C = m_A;
	n_C = n_B;
	C = calloc((n_C * (m_C/size)), sizeof(double));

	double io_finish_1 = 0;
	GET_TIME(io_finish_1);

	double flops = 0;

	int i, j, k;
	for(i=0; i<size; i++){

		for(j=0; j<((m_A/size)*(m_A/size)); j++){

			int a_index = 0;
			//not first row in block
			if(j >= m_A/size){
				//one processor
				if(size == 1){
					a_index = j;
				}
				//more than one processor
				else{
					a_index = j + ((j/(m_A/size)) * (m_A - (m_A/size))) + (rank * (m_A/size));
				}

				//adjust for phase, handle wraparound if necessary
				if(((a_index - (i*(m_A/size)))/m_A) != (a_index/m_A)){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}
			//first row
			else{
				a_index = j+(rank*(m_A/size));

				//adjust for phase, handle wraparound if necessary
				if(a_index - (i*(m_A/size)) < 0){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}

			for(k=0; k<n_B; k++){
				int b_index = 0;
				if(a_index % m_A < m_A/size){
					if(a_index >= m_A){
						b_index = k+(m_A * (a_index % m_A));
					}
					else{
						b_index = k+(m_A * a_index);
					}
				}
				else{
					if(a_index >= m_A){
						b_index = k+(m_A * ((a_index % (m_A/size)) % m_A));
					}
					else{
						b_index = k+(m_A * (a_index % (m_A/size)));
					}
				}
				double result = storage_A[a_index] * storage_B[b_index];
				C[(k+(m_A*(a_index/m_A)))] += result;
				flops += 1;
			}
		}
		MPI_Sendrecv(storage_B, n_B*(n_B/size), MPI_DOUBLE, ((rank + 1) % size), 1, storage_B, n_B*(n_B/size), MPI_DOUBLE,
		 (((rank - 1) + size) % size), 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
	}

	MPI_Barrier(MPI_COMM_WORLD);

	double io_start_2 = 0;
	GET_TIME(io_start_2);

	double *C_final = malloc(sizeof(double) * (m_C * n_C));

	MPI_Gather(C, n_C * (m_C/size), MPI_DOUBLE, C_final, n_C * (m_C/size), MPI_DOUBLE, 0, MPI_COMM_WORLD);

	if(rank == 0){
		write_matrix(c_out, m_C, n_C, C_final);
	}

	free(storage_A);
	free(storage_B);
	free(C);
	free(C_final);

	double io_finish_2 = 0;
	GET_TIME(io_finish_2);

	if(rank == 0){
		double io_time = (io_finish_2 - io_start_2) + (io_finish_1 - io_start_1);
		double mm_time = (io_start_2 - io_finish_1);
		double mflop_count = (flops * size)/1000000;
		printf("M: %i, N: %i\n", m_A, n_A);
		printf("Time Doing I/O: %f sec.\n", io_time);
		printf("Time Doing MM: %f sec.\n", mm_time);
		printf("Total Time: %f sec.\n", io_time + mm_time);
		printf("MFlops (w/o IO): %e, MFlops (w/ IO): %e\n", mflop_count/mm_time, mflop_count/(io_time + mm_time));
	}

	MPI_Finalize();

	return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xb8, %rsp
movq %rsi, %r14
movl %edi, %ebp
xorl %edi, %edi
xorl %esi, %esi
callq 0x10d0
leaq 0x68(%rsp), %rbx
movq %rbx, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%rbx), %xmm0
movsd %xmm0, (%rsp)
cvtsi2sdq 0x8(%rbx), %xmm0
movsd %xmm0, 0x30(%rsp)
movq 0x4c41(%rip), %rbx      # 0x5fd8
leaq 0xc(%rsp), %rsi
movq %rbx, %rdi
callq 0x1170
leaq 0x8(%rsp), %rsi
movq %rbx, %rdi
callq 0x1100
cmpl $0x7, %ebp
jne 0x18d3
movsd 0x30(%rsp), %xmm0
divsd 0x2c40(%rip), %xmm0     # 0x4008
addsd (%rsp), %xmm0
movsd %xmm0, 0x30(%rsp)
leaq 0x2c52(%rip), %r12      # 0x402c
pushq $0x7
popq %r13
leaq 0x2c33(%rip), %rbx      # 0x4018
movq 0x4bdc(%rip), %r15      # 0x5fc8
movl %r13d, %edi
movq %r14, %rsi
movq %r12, %rdx
callq 0x10a0
leal -0x3f(%rax), %ecx
cmpl $0x4, %ecx
ja 0x1410
movslq (%rbx,%rcx,4), %rax
addq %rbx, %rax
jmpq *%rax
movq (%r15), %rbp
jmp 0x13ec
cmpl $-0x1, %eax
jne 0x13ec
jmp 0x142d
movq (%r15), %rax
movq %rax, (%rsp)
jmp 0x13ec
movq (%r15), %rax
movq %rax, 0x80(%rsp)
jmp 0x13ec
subq $0x8, %rsp
movq 0x4b98(%rip), %r15      # 0x5fd0
leaq 0xb8(%rsp), %rsi
leaq 0x68(%rsp), %rdx
leaq 0x24(%rsp), %r8
leaq 0x5c(%rsp), %r14
movq %rbp, %rdi
movq %r15, %rcx
movq %r14, %r9
movq 0x4b79(%rip), %rbx      # 0x5fd8
pushq %rbx
callq 0x2418
popq %rax
popq %rcx
subq $0x8, %rsp
leaq 0xb0(%rsp), %rsi
leaq 0x60(%rsp), %rdx
leaq 0xac(%rsp), %r12
leaq 0x34(%rsp), %r9
movq 0x8(%rsp), %rdi
movq %r15, %rcx
movq %r12, %r8
pushq %rbx
callq 0x2418
popq %rax
popq %rcx
movl (%r14), %eax
cmpl (%r12), %eax
jne 0x18e1
movl 0x1c(%rsp), %ebp
movl 0x2c(%rsp), %r12d
movl %ebp, %eax
cltd
idivl 0x8(%rsp)
imull %r12d, %eax
movslq %eax, %rdi
pushq $0x8
popq %rsi
callq 0x11b0
movq %rax, %r15
leaq 0x68(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
movq (%r14), %rax
movq %rax, 0x78(%rsp)
cvtsi2sdq 0x8(%r14), %xmm0
divsd 0x2b1c(%rip), %xmm0     # 0x4008
movsd %xmm0, 0x20(%rsp)
xorpd %xmm0, %xmm0
xorl %edi, %edi
movl %ebp, 0x44(%rsp)
movl %r12d, 0x40(%rsp)
movl 0x8(%rsp), %esi
cmpl %esi, %edi
movsd %xmm0, (%rsp)
jge 0x16f5
movl 0x1c(%rsp), %ecx
movl %ecx, %eax
cltd
movq %rsi, 0x10(%rsp)
idivl %esi
movl %eax, %esi
movl %eax, %ebx
imull %eax, %ebx
movl %ecx, %eax
subl %esi, %eax
movl %eax, 0x50(%rsp)
movl 0xc(%rsp), %eax
movl %eax, %edx
imull %esi, %edx
movq %rdx, 0x90(%rsp)
movl %esi, %edx
imull %edi, %edx
movl %edx, 0x38(%rsp)
movq %rax, 0x88(%rsp)
movl %edi, 0x4c(%rsp)
subl %edi, %eax
imull %esi, %eax
movq %rax, 0x98(%rsp)
movl 0x2c(%rsp), %eax
movq 0x60(%rsp), %r8
movq 0x58(%rsp), %rdi
xorl %r9d, %r9d
testl %eax, %eax
movl $0x0, %r12d
movl %eax, 0x48(%rsp)
cmovgl %eax, %r12d
cmpl %ebx, %r9d
je 0x1686
cmpl %esi, %r9d
jge 0x15b8
movq 0x98(%rsp), %rax
addl %r9d, %eax
movl %eax, %r10d
sarl $0x1f, %r10d
andl %ecx, %r10d
addl %eax, %r10d
movsd (%rsp), %xmm1
movsd 0x2a5a(%rip), %xmm2     # 0x4010
jmp 0x1613
movl %r9d, %r10d
cmpl $0x1, 0x10(%rsp)
movsd (%rsp), %xmm1
movsd 0x2a43(%rip), %xmm2     # 0x4010
je 0x15ec
movl %r9d, %eax
cltd
idivl %esi
movl %eax, %r10d
imull 0x50(%rsp), %r10d
movq 0x90(%rsp), %rax
addl %r9d, %eax
addl %eax, %r10d
movl %r10d, %r11d
subl 0x38(%rsp), %r11d
movl %r11d, %eax
cltd
idivl %ecx
movl %eax, %ebp
movl %r10d, %eax
cltd
idivl %ecx
cmpl %eax, %ebp
movl %ecx, %r10d
movl $0x0, %eax
cmovel %eax, %r10d
addl %r11d, %r10d
movl %r10d, %r11d
movslq %r10d, %r14
xorl %r13d, %r13d
cmpl %r13d, %r12d
je 0x1679
movl %r11d, %eax
cltd
idivl %ecx
movl %edx, %ebp
cmpl %esi, %edx
jge 0x1638
cmpl %ecx, %r11d
movl %ebp, %edx
cmovll %r11d, %edx
jmp 0x1648
movl %r10d, %eax
cltd
idivl %esi
cmpl %ecx, %r11d
jl 0x1648
movl %edx, %eax
cltd
idivl %ecx
imull %ecx, %edx
movsd (%r8,%r14,8), %xmm0
movslq %edx, %rax
addq %r13, %rax
mulsd (%rdi,%rax,8), %xmm0
leal (%r11,%r13), %eax
subl %ebp, %eax
cltq
addsd (%r15,%rax,8), %xmm0
movsd %xmm0, (%r15,%rax,8)
addsd %xmm2, %xmm1
incq %r13
jmp 0x161c
movsd %xmm1, (%rsp)
incl %r9d
jmp 0x1583
movl 0x48(%rsp), %ecx
movl %ecx, %eax
cltd
movq 0x10(%rsp), %r8
idivl %r8d
movl %eax, %esi
imull %ecx, %esi
movq 0x88(%rsp), %r9
leal 0x1(%r9), %eax
cltd
idivl %r8d
movl %edx, %ecx
leal (%r8,%r9), %eax
decl %eax
cltd
idivl %r8d
movl %edx, %eax
movq 0x4911(%rip), %rdx      # 0x5fd0
pushq $0x1
popq %r8
movq %rdi, %r9
pushq $0x0
pushq 0x490a(%rip)            # 0x5fd8
pushq $0x1
pushq %rax
pushq %rdx
pushq %rsi
callq 0x1150
addq $0x30, %rsp
movl 0x4c(%rsp), %edi
incl %edi
movsd (%rsp), %xmm0
movl 0x44(%rsp), %ebp
movl 0x40(%rsp), %r12d
jmp 0x1501
movq 0x48dc(%rip), %rbx      # 0x5fd8
movq %rbx, %rdi
callq 0x1030
leaq 0x68(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1080
cvtsi2sdq (%r14), %xmm0
movsd %xmm0, 0x38(%rsp)
cvtsi2sdq 0x8(%r14), %xmm0
movsd %xmm0, 0x10(%rsp)
movl %r12d, %eax
imull %ebp, %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x10b0
movq %rax, %r14
movl %ebp, %eax
cltd
idivl 0x8(%rsp)
imull %r12d, %eax
movq 0x487f(%rip), %rdx      # 0x5fd0
movq %r15, %rdi
movl %eax, %esi
movq %r14, %rcx
movl %eax, %r8d
movq %rdx, %r9
pushq %rbx
pushq $0x0
callq 0x1130
popq %rcx
popq %rdx
cmpl $0x0, 0xc(%rsp)
jne 0x1785
movq 0x80(%rsp), %rdi
movl %ebp, %esi
movl %r12d, %edx
movq %r14, %rcx
callq 0x1a2b
movq 0x60(%rsp), %rdi
callq 0x10f0
movq 0x58(%rsp), %rdi
callq 0x10f0
movq %r15, %rdi
callq 0x10f0
movq %r14, %rdi
callq 0x10f0
leaq 0x68(%rsp), %rdi
xorl %esi, %esi
callq 0x1080
cmpl $0x0, 0xc(%rsp)
jne 0x18ba
cvtsi2sdq 0x78(%rsp), %xmm0
movsd 0x20(%rsp), %xmm3
addsd %xmm0, %xmm3
movsd 0x10(%rsp), %xmm4
divsd 0x2829(%rip), %xmm4     # 0x4008
cvtsi2sdq 0x68(%rsp), %xmm0
addsd 0x38(%rsp), %xmm4
cvtsi2sdq 0x70(%rsp), %xmm1
movsd 0x280d(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm0, %xmm1
subsd %xmm4, %xmm1
subsd %xmm3, %xmm4
movsd %xmm4, 0x10(%rsp)
subsd 0x30(%rsp), %xmm3
addsd %xmm1, %xmm3
movsd %xmm3, 0x20(%rsp)
cvtsi2sdl 0x8(%rsp), %xmm0
movsd (%rsp), %xmm1
mulsd %xmm0, %xmm1
divsd %xmm2, %xmm1
movsd %xmm1, (%rsp)
movl 0x1c(%rsp), %esi
movl 0x54(%rsp), %edx
leaq 0x27eb(%rip), %rdi      # 0x4033
xorl %eax, %eax
callq 0x1050
leaq 0x27eb(%rip), %rdi      # 0x4041
movsd 0x20(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
leaq 0x27f0(%rip), %rdi      # 0x405a
movsd 0x10(%rsp), %xmm0
movb $0x1, %al
callq 0x1050
movsd 0x20(%rsp), %xmm0
addsd 0x10(%rsp), %xmm0
movsd %xmm0, 0x20(%rsp)
leaq 0x27e2(%rip), %rdi      # 0x4072
movb $0x1, %al
callq 0x1050
movsd (%rsp), %xmm1
movapd %xmm1, %xmm0
divsd 0x10(%rsp), %xmm0
divsd 0x20(%rsp), %xmm1
leaq 0x27d4(%rip), %rdi      # 0x4087
movb $0x2, %al
callq 0x1050
callq 0x1190
xorl %eax, %eax
addq $0xb8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x27ff(%rip), %rdi      # 0x40d9
callq 0x1060
jmp 0x18f9
cmpl $0x0, 0xc(%rsp)
jne 0x18f4
leaq 0x27c1(%rip), %rdi      # 0x40b0
callq 0x1060
callq 0x1190
pushq $0x1
popq %rdi
callq 0x1070
nopl (%rax)