// File: /workspace/github_c_makefile_star2_O0/baileywjohnson[P]mpi-matrix-mult/mm-parallel.c
// Function: main
int main(int argc, char *argv[]){

	MPI_Init(NULL, NULL);

	double io_start_1 = 0;
	GET_TIME(io_start_1);

	int rank, size;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	MPI_Comm_size(MPI_COMM_WORLD, &size);

	char *a_in;
	char *b_in;
	char *c_out;

	int c;
	if(argc != 7){
		printf("USAGE: mm-serial -A <matrix 1 filename> -B <matrix 2 filename> -C <output filename>\n");
		exit(1);
	}
	while((c = getopt(argc, argv, "A:B:C:")) != -1){
		switch(c){
			case 'A':
				a_in = optarg;
				break;
			case 'B':
				b_in = optarg;
				break;
			case 'C':
				c_out = optarg;
				break;
			case '?':
				exit(1);
				break;
			default:
				break;
		}
	}

	int m_A, n_A;
	double **subs_A;
	double *storage_A;
	read_row_striped_matrix(a_in, (void***)&subs_A, (void**)&storage_A, MPI_DOUBLE, &m_A, &n_A, MPI_COMM_WORLD);

	int m_B, n_B;
	double **subs_B;
	double *storage_B;
	read_row_striped_matrix(b_in, (void***)&subs_B, (void**)&storage_B, MPI_DOUBLE, &m_B, &n_B, MPI_COMM_WORLD);

	if(n_A != m_B){
		if(rank == 0){
			printf("Matrices Incompatible for Multiplication\n");
		}
		MPI_Finalize();
		exit(1);
	}

	int m_C, n_C;
	double *C;
	m_C = m_A;
	n_C = n_B;
	C = calloc((n_C * (m_C/size)), sizeof(double));

	double io_finish_1 = 0;
	GET_TIME(io_finish_1);

	double flops = 0;

	int i, j, k;
	for(i=0; i<size; i++){

		for(j=0; j<((m_A/size)*(m_A/size)); j++){

			int a_index = 0;
			//not first row in block
			if(j >= m_A/size){
				//one processor
				if(size == 1){
					a_index = j;
				}
				//more than one processor
				else{
					a_index = j + ((j/(m_A/size)) * (m_A - (m_A/size))) + (rank * (m_A/size));
				}

				//adjust for phase, handle wraparound if necessary
				if(((a_index - (i*(m_A/size)))/m_A) != (a_index/m_A)){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}
			//first row
			else{
				a_index = j+(rank*(m_A/size));

				//adjust for phase, handle wraparound if necessary
				if(a_index - (i*(m_A/size)) < 0){
					a_index -= (i*(m_A/size));
					a_index += m_A;
				}
				else{
					a_index -= (i*(m_A/size));
				}
			}

			for(k=0; k<n_B; k++){
				int b_index = 0;
				if(a_index % m_A < m_A/size){
					if(a_index >= m_A){
						b_index = k+(m_A * (a_index % m_A));
					}
					else{
						b_index = k+(m_A * a_index);
					}
				}
				else{
					if(a_index >= m_A){
						b_index = k+(m_A * ((a_index % (m_A/size)) % m_A));
					}
					else{
						b_index = k+(m_A * (a_index % (m_A/size)));
					}
				}
				double result = storage_A[a_index] * storage_B[b_index];
				C[(k+(m_A*(a_index/m_A)))] += result;
				flops += 1;
			}
		}
		MPI_Sendrecv(storage_B, n_B*(n_B/size), MPI_DOUBLE, ((rank + 1) % size), 1, storage_B, n_B*(n_B/size), MPI_DOUBLE,
		 (((rank - 1) + size) % size), 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
	}

	MPI_Barrier(MPI_COMM_WORLD);

	double io_start_2 = 0;
	GET_TIME(io_start_2);

	double *C_final = malloc(sizeof(double) * (m_C * n_C));

	MPI_Gather(C, n_C * (m_C/size), MPI_DOUBLE, C_final, n_C * (m_C/size), MPI_DOUBLE, 0, MPI_COMM_WORLD);

	if(rank == 0){
		write_matrix(c_out, m_C, n_C, C_final);
	}

	free(storage_A);
	free(storage_B);
	free(C);
	free(C_final);

	double io_finish_2 = 0;
	GET_TIME(io_finish_2);

	if(rank == 0){
		double io_time = (io_finish_2 - io_start_2) + (io_finish_1 - io_start_1);
		double mm_time = (io_start_2 - io_finish_1);
		double mflop_count = (flops * size)/1000000;
		printf("M: %i, N: %i\n", m_A, n_A);
		printf("Time Doing I/O: %f sec.\n", io_time);
		printf("Time Doing MM: %f sec.\n", mm_time);
		printf("Total Time: %f sec.\n", io_time + mm_time);
		printf("MFlops (w/o IO): %e, MFlops (w/ IO): %e\n", mflop_count/mm_time, mflop_count/(io_time + mm_time));
	}

	MPI_Finalize();

	return 0;
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %rbx
subq $0x1b8, %rsp            # imm = 0x1B8
movl $0x0, -0xc(%rbp)
movl %edi, -0x10(%rbp)
movq %rsi, -0x18(%rbp)
xorl %eax, %eax
movl %eax, %esi
movq %rsi, %rdi
callq 0x10c0
xorps %xmm0, %xmm0
movsd %xmm0, -0x20(%rbp)
leaq -0x30(%rbp), %rdi
xorl %eax, %eax
movl %eax, %esi
callq 0x1070
cvtsi2sdq -0x30(%rbp), %xmm0
cvtsi2sdq -0x28(%rbp), %xmm1
movsd 0x2c79(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm1, %xmm0
movsd %xmm0, -0x20(%rbp)
movq 0x4c35(%rip), %rdi      # 0x5fd8
leaq -0x34(%rbp), %rsi
callq 0x1160
movq 0x4c25(%rip), %rdi      # 0x5fd8
leaq -0x38(%rbp), %rsi
callq 0x10f0
cmpl $0x7, -0x10(%rbp)
je 0x13da
leaq 0x2c63(%rip), %rdi      # 0x402c
movb $0x0, %al
callq 0x1050
movl $0x1, %edi
callq 0x1060
jmp 0x13dc
movl -0x10(%rbp), %edi
movq -0x18(%rbp), %rsi
leaq 0x2c97(%rip), %rdx      # 0x4081
callq 0x1090
movl %eax, -0x54(%rbp)
cmpl $-0x1, %eax
je 0x1463
movl -0x54(%rbp), %eax
addl $-0x3f, %eax
movl %eax, %ecx
movq %rcx, -0x130(%rbp)
subl $0x4, %eax
ja 0x145c
movq -0x130(%rbp), %rax
leaq 0x2bff(%rip), %rcx      # 0x4018
movslq (%rcx,%rax,4), %rax
addq %rcx, %rax
jmpq *%rax
movq 0x4b9f(%rip), %rax      # 0x5fc8
movq (%rax), %rax
movq %rax, -0x40(%rbp)
jmp 0x145e
movq 0x4b8f(%rip), %rax      # 0x5fc8
movq (%rax), %rax
movq %rax, -0x48(%rbp)
jmp 0x145e
movq 0x4b7f(%rip), %rax      # 0x5fc8
movq (%rax), %rax
movq %rax, -0x50(%rbp)
jmp 0x145e
movl $0x1, %edi
callq 0x1060
jmp 0x145e
jmp 0x13dc
movq -0x40(%rbp), %rdi
leaq -0x68(%rbp), %rsi
leaq -0x70(%rbp), %rdx
movq 0x4b5a(%rip), %rcx      # 0x5fd0
leaq -0x58(%rbp), %r8
leaq -0x5c(%rbp), %r9
movq 0x4b53(%rip), %rax      # 0x5fd8
movq %rax, (%rsp)
callq 0x2ad0
movq -0x48(%rbp), %rdi
leaq -0x80(%rbp), %rsi
leaq -0x88(%rbp), %rdx
movq 0x4b2c(%rip), %rcx      # 0x5fd0
leaq -0x74(%rbp), %r8
leaq -0x78(%rbp), %r9
movq 0x4b25(%rip), %rax      # 0x5fd8
movq %rax, (%rsp)
callq 0x2ad0
movl -0x5c(%rbp), %eax
cmpl -0x74(%rbp), %eax
je 0x14e7
cmpl $0x0, -0x34(%rbp)
jne 0x14d8
leaq 0x2bb7(%rip), %rdi      # 0x4088
movb $0x0, %al
callq 0x1050
callq 0x1180
movl $0x1, %edi
callq 0x1060
movl -0x58(%rbp), %eax
movl %eax, -0x8c(%rbp)
movl -0x78(%rbp), %eax
movl %eax, -0x90(%rbp)
movl -0x90(%rbp), %eax
movl %eax, -0x134(%rbp)
movl -0x8c(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %ecx
movl -0x134(%rbp), %eax
imull %ecx, %eax
movslq %eax, %rdi
movl $0x8, %esi
callq 0x11a0
movq %rax, -0x98(%rbp)
xorps %xmm0, %xmm0
movsd %xmm0, -0xa0(%rbp)
leaq -0xb0(%rbp), %rdi
xorl %eax, %eax
movl %eax, %esi
callq 0x1070
cvtsi2sdq -0xb0(%rbp), %xmm0
cvtsi2sdq -0xa8(%rbp), %xmm1
movsd 0x2aa5(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm1, %xmm0
movsd %xmm0, -0xa0(%rbp)
xorps %xmm0, %xmm0
movsd %xmm0, -0xb8(%rbp)
movl $0x0, -0xbc(%rbp)
movl -0xbc(%rbp), %eax
cmpl -0x38(%rbp), %eax
jge 0x1a3f
movl $0x0, -0xc0(%rbp)
movl -0xc0(%rbp), %eax
movl %eax, -0x138(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %edx
movl -0x138(%rbp), %eax
imull %edx, %ecx
cmpl %ecx, %eax
jge 0x19a6
movl $0x0, -0xc8(%rbp)
movl -0xc0(%rbp), %eax
movl %eax, -0x13c(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %ecx
movl -0x13c(%rbp), %eax
cmpl %ecx, %eax
jl 0x171d
cmpl $0x1, -0x38(%rbp)
jne 0x1611
movl -0xc0(%rbp), %eax
movl %eax, -0xc8(%rbp)
jmp 0x1685
movl -0xc0(%rbp), %eax
movl %eax, -0x144(%rbp)
movl -0xc0(%rbp), %eax
movl %eax, -0x14c(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %ecx
movl -0x14c(%rbp), %eax
cltd
idivl %ecx
movl %eax, %ecx
movl -0x58(%rbp), %eax
movl %eax, -0x148(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl -0x148(%rbp), %edx
movl %eax, %esi
movl -0x144(%rbp), %eax
subl %esi, %edx
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0x140(%rbp)
movl -0x34(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %edx
movl -0x140(%rbp), %eax
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0xc8(%rbp)
movl -0xc8(%rbp), %eax
movl %eax, -0x154(%rbp)
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %edx
movl -0x154(%rbp), %eax
imull %edx, %ecx
subl %ecx, %eax
cltd
idivl -0x58(%rbp)
movl %eax, -0x150(%rbp)
movl -0xc8(%rbp), %eax
cltd
idivl -0x58(%rbp)
movl %eax, %ecx
movl -0x150(%rbp), %eax
cmpl %ecx, %eax
je 0x16fa
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %ecx
movl -0xc8(%rbp), %eax
subl %ecx, %eax
movl %eax, -0xc8(%rbp)
movl -0x58(%rbp), %eax
addl -0xc8(%rbp), %eax
movl %eax, -0xc8(%rbp)
jmp 0x1718
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %ecx
movl -0xc8(%rbp), %eax
subl %ecx, %eax
movl %eax, -0xc8(%rbp)
jmp 0x17c0
movl -0xc0(%rbp), %eax
movl %eax, -0x15c(%rbp)
movl -0x34(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %edx
movl -0x15c(%rbp), %eax
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0xc8(%rbp)
movl -0xc8(%rbp), %eax
movl %eax, -0x158(%rbp)
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %edx
movl -0x158(%rbp), %eax
imull %edx, %ecx
subl %ecx, %eax
cmpl $0x0, %eax
jge 0x17a0
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %ecx
movl -0xc8(%rbp), %eax
subl %ecx, %eax
movl %eax, -0xc8(%rbp)
movl -0x58(%rbp), %eax
addl -0xc8(%rbp), %eax
movl %eax, -0xc8(%rbp)
jmp 0x17be
movl -0xbc(%rbp), %ecx
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %ecx
movl -0xc8(%rbp), %eax
subl %ecx, %eax
movl %eax, -0xc8(%rbp)
jmp 0x17c0
movl $0x0, -0xc4(%rbp)
movl -0xc4(%rbp), %eax
cmpl -0x78(%rbp), %eax
jge 0x1990
movl $0x0, -0xcc(%rbp)
movl -0xc8(%rbp), %eax
cltd
idivl -0x58(%rbp)
movl %edx, -0x160(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl -0x160(%rbp), %edx
cmpl %eax, %edx
jge 0x1858
movl -0xc8(%rbp), %eax
cmpl -0x58(%rbp), %eax
jl 0x183b
movl -0xc4(%rbp), %eax
movl %eax, -0x164(%rbp)
movl -0x58(%rbp), %ecx
movl -0xc8(%rbp), %eax
cltd
idivl -0x58(%rbp)
movl -0x164(%rbp), %eax
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0xcc(%rbp)
jmp 0x1853
movl -0xc4(%rbp), %eax
movl -0x58(%rbp), %ecx
imull -0xc8(%rbp), %ecx
addl %ecx, %eax
movl %eax, -0xcc(%rbp)
jmp 0x18e9
movl -0xc8(%rbp), %eax
cmpl -0x58(%rbp), %eax
jl 0x18a9
movl -0xc4(%rbp), %eax
movl %eax, -0x168(%rbp)
movl -0x58(%rbp), %ecx
movl -0xc8(%rbp), %eax
movl %eax, -0x16c(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %esi
movl -0x16c(%rbp), %eax
cltd
idivl %esi
movl %edx, %eax
cltd
idivl -0x58(%rbp)
movl -0x168(%rbp), %eax
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0xcc(%rbp)
jmp 0x18e7
movl -0xc4(%rbp), %eax
movl %eax, -0x170(%rbp)
movl -0x58(%rbp), %ecx
movl -0xc8(%rbp), %eax
movl %eax, -0x174(%rbp)
movl -0x58(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %eax, %esi
movl -0x174(%rbp), %eax
cltd
idivl %esi
movl -0x170(%rbp), %eax
imull %edx, %ecx
addl %ecx, %eax
movl %eax, -0xcc(%rbp)
jmp 0x18e9
movq -0x70(%rbp), %rax
movslq -0xc8(%rbp), %rcx
movsd (%rax,%rcx,8), %xmm0
movq -0x88(%rbp), %rax
movslq -0xcc(%rbp), %rcx
mulsd (%rax,%rcx,8), %xmm0
movsd %xmm0, -0xd8(%rbp)
movsd -0xd8(%rbp), %xmm0
movq -0x98(%rbp), %rax
movq %rax, -0x180(%rbp)
movl -0xc4(%rbp), %ecx
movl -0x58(%rbp), %eax
movl %eax, -0x184(%rbp)
movl -0xc8(%rbp), %eax
cltd
idivl -0x58(%rbp)
movl -0x184(%rbp), %edx
movl %eax, %esi
movq -0x180(%rbp), %rax
imull %esi, %edx
addl %edx, %ecx
movslq %ecx, %rcx
addsd (%rax,%rcx,8), %xmm0
movsd %xmm0, (%rax,%rcx,8)
movsd 0x26a4(%rip), %xmm0     # 0x4010
addsd -0xb8(%rbp), %xmm0
movsd %xmm0, -0xb8(%rbp)
movl -0xc4(%rbp), %eax
addl $0x1, %eax
movl %eax, -0xc4(%rbp)
jmp 0x17ca
jmp 0x1992
movl -0xc0(%rbp), %eax
addl $0x1, %eax
movl %eax, -0xc0(%rbp)
jmp 0x15a1
movq -0x88(%rbp), %rdi
movl -0x78(%rbp), %esi
movl -0x78(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %esi
movl -0x34(%rbp), %eax
addl $0x1, %eax
cltd
idivl -0x38(%rbp)
movl %edx, %ecx
movq -0x88(%rbp), %r9
movl -0x78(%rbp), %ebx
movl -0x78(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %ebx
movl -0x34(%rbp), %eax
subl $0x1, %eax
addl -0x38(%rbp), %eax
cltd
idivl -0x38(%rbp)
movl %edx, %r10d
movq 0x45df(%rip), %r11      # 0x5fd0
movl $0x1, %r8d
movq 0x45da(%rip), %rax      # 0x5fd8
xorl %edx, %edx
movq %r11, %rdx
movl %ebx, (%rsp)
movq %r11, 0x8(%rsp)
movl %r10d, 0x10(%rsp)
movl $0x1, 0x18(%rsp)
movq %rax, 0x20(%rsp)
movq $0x0, 0x28(%rsp)
callq 0x1140
movl -0xbc(%rbp), %eax
addl $0x1, %eax
movl %eax, -0xbc(%rbp)
jmp 0x1588
movq 0x4592(%rip), %rdi      # 0x5fd8
callq 0x1030
xorps %xmm0, %xmm0
movsd %xmm0, -0xe0(%rbp)
leaq -0xf0(%rbp), %rdi
xorl %eax, %eax
movl %eax, %esi
callq 0x1070
cvtsi2sdq -0xf0(%rbp), %xmm0
cvtsi2sdq -0xe8(%rbp), %xmm1
movsd 0x2588(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm1, %xmm0
movsd %xmm0, -0xe0(%rbp)
movl -0x8c(%rbp), %eax
imull -0x90(%rbp), %eax
movslq %eax, %rdi
shlq $0x3, %rdi
callq 0x10a0
movq %rax, -0xf8(%rbp)
movq -0x98(%rbp), %rdi
movl -0x90(%rbp), %esi
movl -0x8c(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %esi
movq -0xf8(%rbp), %rcx
movl -0x90(%rbp), %r8d
movl -0x8c(%rbp), %eax
cltd
idivl -0x38(%rbp)
imull %eax, %r8d
movq 0x44e3(%rip), %r9       # 0x5fd0
xorl %eax, %eax
movq 0x44e2(%rip), %rax      # 0x5fd8
movq %r9, %rdx
movl $0x0, (%rsp)
movq %rax, 0x8(%rsp)
callq 0x1120
cmpl $0x0, -0x34(%rbp)
jne 0x1b2c
movq -0x50(%rbp), %rdi
movl -0x8c(%rbp), %esi
movl -0x90(%rbp), %edx
movq -0xf8(%rbp), %rcx
callq 0x1e20
movq -0x70(%rbp), %rdi
callq 0x10e0
movq -0x88(%rbp), %rdi
callq 0x10e0
movq -0x98(%rbp), %rdi
callq 0x10e0
movq -0xf8(%rbp), %rdi
callq 0x10e0
xorps %xmm0, %xmm0
movsd %xmm0, -0x100(%rbp)
leaq -0x110(%rbp), %rdi
xorl %eax, %eax
movl %eax, %esi
callq 0x1070
cvtsi2sdq -0x110(%rbp), %xmm0
cvtsi2sdq -0x108(%rbp), %xmm1
movsd 0x247a(%rip), %xmm2     # 0x4008
divsd %xmm2, %xmm1
addsd %xmm1, %xmm0
movsd %xmm0, -0x100(%rbp)
cmpl $0x0, -0x34(%rbp)
jne 0x1ca6
movsd -0x100(%rbp), %xmm0
subsd -0xe0(%rbp), %xmm0
movsd -0xa0(%rbp), %xmm1
subsd -0x20(%rbp), %xmm1
addsd %xmm1, %xmm0
movsd %xmm0, -0x118(%rbp)
movsd -0xe0(%rbp), %xmm0
subsd -0xa0(%rbp), %xmm0
movsd %xmm0, -0x120(%rbp)
movsd -0xb8(%rbp), %xmm0
cvtsi2sdl -0x38(%rbp), %xmm1
mulsd %xmm1, %xmm0
movsd 0x2406(%rip), %xmm1     # 0x4008
divsd %xmm1, %xmm0
movsd %xmm0, -0x128(%rbp)
movl -0x58(%rbp), %esi
movl -0x5c(%rbp), %edx
leaq 0x2497(%rip), %rdi      # 0x40b2
movb $0x0, %al
callq 0x1050
movsd -0x118(%rbp), %xmm0
leaq 0x248f(%rip), %rdi      # 0x40c0
movb $0x1, %al
callq 0x1050
movsd -0x120(%rbp), %xmm0
leaq 0x2492(%rip), %rdi      # 0x40d9
movb $0x1, %al
callq 0x1050
movsd -0x118(%rbp), %xmm0
addsd -0x120(%rbp), %xmm0
leaq 0x248c(%rip), %rdi      # 0x40f1
movb $0x1, %al
callq 0x1050
movsd -0x128(%rbp), %xmm0
divsd -0x120(%rbp), %xmm0
movsd -0x128(%rbp), %xmm1
movsd -0x118(%rbp), %xmm2
addsd -0x120(%rbp), %xmm2
divsd %xmm2, %xmm1
leaq 0x2467(%rip), %rdi      # 0x4106
movb $0x2, %al
callq 0x1050
callq 0x1180
xorl %eax, %eax
addq $0x1b8, %rsp            # imm = 0x1B8
popq %rbx
popq %rbp
retq
nopw (%rax,%rax)