// File: /workspace/github_c_makefile_star2_O0/kkanellis[P]matvec-sparse/matvec_mpi_p2p.c
// Function: main
int main(int argc, char * argv[])
{
    char *in_file,
         *out_file = NULL;

    double t, comp_time, partition_time;
    int nprocs,     /* number of tasks/processes */
        rank;       /* id of task/process */
    
    /***** MPI MASTER (root) process only ******/
    proc_info_t *proc_info;

    int *buf_i_idx,     /* row index for all matrix elements */
        *buf_j_idx;     /* column index for all matrix elements */
    double *buf_values, /* value for all matrix elements */
           *buf_x,      /* value for all x vector elements */
           *res;        /* final result -> Ax */

    /*******************************************/

    /* Initialize MPI */
    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    create_mpi_datatypes(&proc_info_type);

    /* master thread reads matrix */
    if (rank == MASTER) {
        /* read arguments */
        if (argc < 2 || argc > 3) {
            printf("Usage: %s input_file [output_file]\n", argv[0]);
            return 0;
        }
        else {
            in_file = argv[1];
            if (argc == 3) 
                out_file = argv[2];
        }

        /* initialize proc_info array */
        proc_info = (proc_info_t *)malloc_or_exit( nprocs * sizeof(proc_info_t) );
        
        /* read matrix */
        if ( read_matrix(in_file, &buf_i_idx, &buf_j_idx, &buf_values,
                            &proc_info[MASTER].N, &proc_info[MASTER].NZ) != 0) {
            fprintf(stderr, "read_matrix: failed\n");
            exit(EXIT_FAILURE);
        }

        debug("[%d] Read matrix from '%s'!\n", rank, in_file);
        debug("[%d] Matrix properties: N = %d, NZ = %d\n\n",rank, 
                                    proc_info[MASTER].N, proc_info[MASTER].NZ);

        /* initialize process info */
        for (int p = 0; p < nprocs; p++) {
            if (p != MASTER) {
                proc_info[p] = proc_info[MASTER];
            }
        }

        /* allocate x, res vector */
        buf_x = (double *)malloc_or_exit( proc_info[MASTER].N * sizeof(double) );
        res = (double *)malloc_or_exit( proc_info[MASTER].N * sizeof(double) );

        /* generate random vector */
        //random_vec(buf_x, N, MAX_RANDOM_NUM);
        for (int i = 0; i < proc_info[MASTER].N; i++) {
            buf_x[i] = 1;
        }

        t = MPI_Wtime();
        
        /* divide work across processes */
        if (policy == EQUAL_ROWS) {
            debug("[%d] Policy: Equal number of ROWS\n", rank);
            partition_equal_rows(proc_info, nprocs, buf_i_idx);
        }
        else if (policy == EQUAL_NZ) {
            debug("[%d] Policy: Equal number of NZ ENTRIES\n", rank);
            partition_equal_nz_elements(proc_info, nprocs, buf_i_idx);
        }
        else {
            fprintf(stderr, "Wrong policy defined...");
            exit(EXIT_FAILURE);
        }
        
        partition_time = (MPI_Wtime() - t) * 1000.0;

        debug("[%d] Partition time: %10.3lf ms\n\n", rank, partition_time);
        debug("[%d] Starting algorithm...\n", rank);
    }

    /* Matrix-vector multiplication for each processes */
    res = mat_vec_mult_parallel(rank, nprocs, proc_info, buf_i_idx, 
                                buf_j_idx, buf_values, buf_x);
    
    double stdev = 0, mean = 0, runs[TOTAL_RUNS];
    for (int r = 0; r < TOTAL_RUNS; r++) {
        MPI_Barrier(MPI_COMM_WORLD);
        if (rank == MASTER) t = MPI_Wtime();
        res = mat_vec_mult_parallel(rank, nprocs, proc_info, buf_i_idx, 
                                buf_j_idx, buf_values, buf_x);
        //MPI_Barrier(MPI_COMM_WORLD);
        if (rank == MASTER){
            runs[r] = (MPI_Wtime() - t) * 1000.0; 
            mean += runs[r];
        }
    }

    /* print execution stats */
    if (rank == MASTER) {
        mean /= TOTAL_RUNS; 
        for (int r = 0; r < TOTAL_RUNS; r++) {
            stdev += (runs[r] - mean) * (runs[r] - mean);
        }
        stdev = sqrt(stdev);

        printf("[%d] Computation time: %10.3lf [%4.3lf] ms\n\n", rank, mean, stdev);
        printf("[%d] Total execution time: %10.3lf ms\n", rank, mean + partition_time);
        debug("Finished!\n");
    }

    /* write to output file */
    if (rank == MASTER) {
        
        if (out_file != NULL) {
            printf("Writing result to '%s'\n", out_file);

            /* open file */
            FILE *f;
            if ( !(f = fopen(out_file, "w")) ) {
                fprintf(stderr, "fopen: failed to open file '%s'", out_file);
                exit(EXIT_FAILURE);
            }

            /* write result */
            for (int i = 0; i < proc_info[MASTER].N; i++) {
                fprintf(f, "%.8lf\n", res[i]);
            }
            
            /* close file */
            if ( fclose(f) != 0) {
                fprintf(stderr, "fopen: failed to open file '%s'", out_file);
                exit(EXIT_FAILURE);
            }

            printf("Done!\n");
        }

        /* free the memory */
        free(buf_values);
        free(buf_i_idx);
        free(buf_j_idx);
        free(buf_x);
        free(res);
    }

    /* MPI: end */
    MPI_Finalize();

    return 0;
}

// Assembly:
subq $0x3c8, %rsp            # imm = 0x3C8
movl $0x0, 0x3c4(%rsp)
movl %edi, 0x3c0(%rsp)
movq %rsi, 0x3b8(%rsp)
movq $0x0, 0x3a8(%rsp)
leaq 0x3c0(%rsp), %rdi
leaq 0x3b8(%rsp), %rsi
callq 0x2100
movq 0x2f3f(%rip), %rdi      # 0x7fd8
leaq 0x38c(%rsp), %rsi
callq 0x2160
movq 0x2f2b(%rip), %rdi      # 0x7fd8
leaq 0x388(%rsp), %rsi
callq 0x21b0
leaq 0x30b7(%rip), %rdi      # 0x8178
callq 0x4ff0
cmpl $0x0, 0x388(%rsp)
jne 0x536e
cmpl $0x2, 0x3c0(%rsp)
jl 0x50e8
cmpl $0x3, 0x3c0(%rsp)
jle 0x5111
movq 0x3b8(%rsp), %rax
movq (%rax), %rsi
leaq 0x12a5(%rip), %rdi      # 0x639f
movb $0x0, %al
callq 0x2040
movl $0x0, 0x3c4(%rsp)
jmp 0x56fa
movq 0x3b8(%rsp), %rax
movq 0x8(%rax), %rax
movq %rax, 0x3b0(%rsp)
cmpl $0x3, 0x3c0(%rsp)
jne 0x5143
movq 0x3b8(%rsp), %rax
movq 0x10(%rax), %rax
movq %rax, 0x3a8(%rsp)
jmp 0x5145
movslq 0x38c(%rsp), %rax
imulq $0x18, %rax, %rdi
callq 0x3fd0
movq %rax, 0x380(%rsp)
movq 0x3b0(%rsp), %rdi
movq 0x380(%rsp), %r8
movq 0x380(%rsp), %r9
addq $0x4, %r9
leaq 0x378(%rsp), %rsi
leaq 0x370(%rsp), %rdx
leaq 0x368(%rsp), %rcx
callq 0x3750
cmpl $0x0, %eax
je 0x51be
movq 0x2e45(%rip), %rax      # 0x7fe8
movq (%rax), %rdi
leaq 0x1216(%rip), %rsi      # 0x63c3
movb $0x0, %al
callq 0x2290
movl $0x1, %edi
callq 0x2090
jmp 0x51c0
jmp 0x51c2
jmp 0x51c4
jmp 0x51c6
movl $0x0, 0x354(%rsp)
movl 0x354(%rsp), %eax
cmpl 0x38c(%rsp), %eax
jge 0x5235
cmpl $0x0, 0x354(%rsp)
je 0x5220
movq 0x380(%rsp), %rax
movslq 0x354(%rsp), %rcx
imulq $0x18, %rcx, %rcx
addq %rcx, %rax
movq 0x380(%rsp), %rcx
movq (%rcx), %rdx
movq %rdx, (%rax)
movq 0x8(%rcx), %rdx
movq %rdx, 0x8(%rax)
movq 0x10(%rcx), %rcx
movq %rcx, 0x10(%rax)
jmp 0x5222
movl 0x354(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x354(%rsp)
jmp 0x51d1
movq 0x380(%rsp), %rax
movslq (%rax), %rdi
shlq $0x3, %rdi
callq 0x3fd0
movq %rax, 0x360(%rsp)
movq 0x380(%rsp), %rax
movslq (%rax), %rdi
shlq $0x3, %rdi
callq 0x3fd0
movq %rax, 0x358(%rsp)
movl $0x0, 0x350(%rsp)
movl 0x350(%rsp), %eax
movq 0x380(%rsp), %rcx
cmpl (%rcx), %eax
jge 0x52bb
movq 0x360(%rsp), %rax
movslq 0x350(%rsp), %rcx
movsd 0x106d(%rip), %xmm0     # 0x6310
movsd %xmm0, (%rax,%rcx,8)
movl 0x350(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x350(%rsp)
jmp 0x5278
callq 0x2260
movsd %xmm0, 0x3a0(%rsp)
cmpl $0x0, 0x2e98(%rip)      # 0x8168
jne 0x52f4
jmp 0x52d4
jmp 0x52d6
movq 0x380(%rsp), %rdi
movl 0x38c(%rsp), %esi
movq 0x378(%rsp), %rdx
callq 0x3b10
jmp 0x5343
cmpl $0x1, 0x2e6d(%rip)      # 0x8168
jne 0x531f
jmp 0x52ff
jmp 0x5301
movq 0x380(%rsp), %rdi
movl 0x38c(%rsp), %esi
movq 0x378(%rsp), %rdx
callq 0x3c80
jmp 0x5341
movq 0x2cc2(%rip), %rax      # 0x7fe8
movq (%rax), %rdi
leaq 0x10a8(%rip), %rsi      # 0x63d8
movb $0x0, %al
callq 0x2290
movl $0x1, %edi
callq 0x2090
jmp 0x5343
callq 0x2260
subsd 0x3a0(%rsp), %xmm0
movsd 0xfa7(%rip), %xmm1      # 0x6300
mulsd %xmm1, %xmm0
movsd %xmm0, 0x390(%rsp)
jmp 0x5368
jmp 0x536a
jmp 0x536c
jmp 0x536e
movl 0x388(%rsp), %edi
movl 0x38c(%rsp), %esi
movq 0x380(%rsp), %rdx
movq 0x378(%rsp), %rcx
movq 0x370(%rsp), %r8
movq 0x368(%rsp), %r9
movq 0x360(%rsp), %rax
movq %rax, (%rsp)
callq 0x40c0
movq %rax, 0x358(%rsp)
xorps %xmm0, %xmm0
movsd %xmm0, 0x348(%rsp)
xorps %xmm0, %xmm0
movsd %xmm0, 0x340(%rsp)
movl $0x0, 0x1c(%rsp)
cmpl $0x64, 0x1c(%rsp)
jge 0x54a9
movq 0x2bf1(%rip), %rdi      # 0x7fd8
callq 0x2030
cmpl $0x0, 0x388(%rsp)
jne 0x5404
callq 0x2260
movsd %xmm0, 0x3a0(%rsp)
movl 0x388(%rsp), %edi
movl 0x38c(%rsp), %esi
movq 0x380(%rsp), %rdx
movq 0x378(%rsp), %rcx
movq 0x370(%rsp), %r8
movq 0x368(%rsp), %r9
movq 0x360(%rsp), %rax
movq %rax, (%rsp)
callq 0x40c0
movq %rax, 0x358(%rsp)
cmpl $0x0, 0x388(%rsp)
jne 0x5497
callq 0x2260
subsd 0x3a0(%rsp), %xmm0
movsd 0xe95(%rip), %xmm1      # 0x6300
mulsd %xmm1, %xmm0
movslq 0x1c(%rsp), %rax
movsd %xmm0, 0x20(%rsp,%rax,8)
movslq 0x1c(%rsp), %rax
movsd 0x20(%rsp,%rax,8), %xmm0
addsd 0x340(%rsp), %xmm0
movsd %xmm0, 0x340(%rsp)
jmp 0x5499
movl 0x1c(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x1c(%rsp)
jmp 0x53d5
cmpl $0x0, 0x388(%rsp)
jne 0x559c
movsd 0x340(%rsp), %xmm0
movsd 0xe40(%rip), %xmm1      # 0x6308
divsd %xmm1, %xmm0
movsd %xmm0, 0x340(%rsp)
movl $0x0, 0x18(%rsp)
cmpl $0x64, 0x18(%rsp)
jge 0x5533
movslq 0x18(%rsp), %rax
movsd 0x20(%rsp,%rax,8), %xmm0
subsd 0x340(%rsp), %xmm0
movslq 0x18(%rsp), %rax
movsd 0x20(%rsp,%rax,8), %xmm2
subsd 0x340(%rsp), %xmm2
movsd 0x348(%rsp), %xmm1
mulsd %xmm2, %xmm0
addsd %xmm1, %xmm0
movsd %xmm0, 0x348(%rsp)
movl 0x18(%rsp), %eax
addl $0x1, %eax
movl %eax, 0x18(%rsp)
jmp 0x54dd
movsd 0x348(%rsp), %xmm0
callq 0x22a0
movsd %xmm0, 0x348(%rsp)
movl 0x388(%rsp), %esi
movsd 0x340(%rsp), %xmm0
movsd 0x348(%rsp), %xmm1
leaq 0xe86(%rip), %rdi       # 0x63f0
movb $0x2, %al
callq 0x2040
movl 0x388(%rsp), %esi
movsd 0x340(%rsp), %xmm0
addsd 0x390(%rsp), %xmm0
leaq 0xe8c(%rip), %rdi       # 0x641d
movb $0x1, %al
callq 0x2040
jmp 0x559a
jmp 0x559c
cmpl $0x0, 0x388(%rsp)
jne 0x56ea
cmpq $0x0, 0x3a8(%rsp)
je 0x56a9
movq 0x3a8(%rsp), %rsi
leaq 0xe7c(%rip), %rdi       # 0x6444
movb $0x0, %al
callq 0x2040
movq 0x3a8(%rsp), %rdi
leaq 0xba2(%rip), %rsi       # 0x6180
callq 0x20f0
movq %rax, 0x10(%rsp)
cmpq $0x0, %rax
jne 0x5618
movq 0x29f3(%rip), %rax      # 0x7fe8
movq (%rax), %rdi
movq 0x3a8(%rsp), %rdx
leaq 0xe55(%rip), %rsi       # 0x645c
movb $0x0, %al
callq 0x2290
movl $0x1, %edi
callq 0x2090
movl $0x0, 0xc(%rsp)
movl 0xc(%rsp), %eax
movq 0x380(%rsp), %rcx
cmpl (%rcx), %eax
jge 0x5662
movq 0x10(%rsp), %rdi
movq 0x358(%rsp), %rax
movslq 0xc(%rsp), %rcx
movsd (%rax,%rcx,8), %xmm0
leaq 0xe2e(%rip), %rsi       # 0x647c
movb $0x1, %al
callq 0x2290
movl 0xc(%rsp), %eax
addl $0x1, %eax
movl %eax, 0xc(%rsp)
jmp 0x5620
movq 0x10(%rsp), %rdi
callq 0x2250
cmpl $0x0, %eax
je 0x569b
movq 0x2970(%rip), %rax      # 0x7fe8
movq (%rax), %rdi
movq 0x3a8(%rsp), %rdx
leaq 0xdd2(%rip), %rsi       # 0x645c
movb $0x0, %al
callq 0x2290
movl $0x1, %edi
callq 0x2090
leaq 0xde1(%rip), %rdi       # 0x6483
movb $0x0, %al
callq 0x2040
movq 0x368(%rsp), %rdi
callq 0x2130
movq 0x378(%rsp), %rdi
callq 0x2130
movq 0x370(%rsp), %rdi
callq 0x2130
movq 0x360(%rsp), %rdi
callq 0x2130
movq 0x358(%rsp), %rdi
callq 0x2130
callq 0x2200
movl $0x0, 0x3c4(%rsp)
movl 0x3c4(%rsp), %eax
addq $0x3c8, %rsp            # imm = 0x3C8
retq