// File: /workspace/github_c_makefile_star2_O3/kkanellis[P]matvec-sparse/matvec_mpi_bcast.c
// Function: main
int main(int argc, char * argv[])
{
    char *in_file,
         *out_file = NULL;

    double t, comp_time, partition_time;

    double *x; /* vector to be multiplied */

    int nprocs,     /* number of tasks/processes */
        rank;       /* id of task/process */
    
    /***** MPI MASTER (root) process only ******/
    proc_info_t *proc_info;

    int *buf_i_idx,     /* row index for all matrix elements */
        *buf_j_idx;     /* column index for all matrix elements */
    double *buf_values, /* value for all matrix elements */
           *buf_x,      /* value for all x vector elements */
           *res;        /* final result -> Ax */

    /*******************************************/

    /* Initialize MPI */
    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    create_mpi_datatypes(&proc_info_type);

    /* master thread reads matrix */
    if (rank == MASTER) {
        /* read arguments */
        if (argc < 2 || argc > 3) {
            printf("Usage: %s input_file [output_file]\n", argv[0]);
            return 0;
        }
        else {
            in_file = argv[1];
            if (argc == 3) 
                out_file = argv[2];
        }

        /* initialize proc_info array */
        proc_info = (proc_info_t *)malloc_or_exit( nprocs * sizeof(proc_info_t) );
        
        /* read matrix */
        if ( read_matrix(in_file, &buf_i_idx, &buf_j_idx, &buf_values,
                            &proc_info[MASTER].N, &proc_info[MASTER].NZ) != 0) {
            fprintf(stderr, "read_matrix: failed\n");
            exit(EXIT_FAILURE);
        }

        debug("[%d] Read matrix from '%s'!\n", rank, in_file);
        debug("[%d] Matrix properties: N = %d, NZ = %d\n\n",rank, 
                                    proc_info[MASTER].N, proc_info[MASTER].NZ);

        /* initialize process info */
        for (int p = 0; p < nprocs; p++) {
            if (p != MASTER) {
                proc_info[p] = proc_info[MASTER];
            }
        }

        /* allocate x, res vector */
        buf_x = (double *)malloc_or_exit( proc_info[MASTER].N * sizeof(double) );
        res = (double *)malloc_or_exit( proc_info[MASTER].N * sizeof(double) );

        /* generate random vector */
        //random_vec(buf_x, N, MAX_RANDOM_NUM);
        for (int i = 0; i < proc_info[MASTER].N; i++) {
            buf_x[i] = 1;
        }

        /* divide work across processes */
        t = MPI_Wtime();
        if (policy == EQUAL_ROWS) {
            debug("[%d] Policy: Equal number of ROWS\n", rank);
            partition_equal_rows(proc_info, nprocs, buf_i_idx);
        }
        else if (policy == EQUAL_NZ) {
            debug("[%d] Policy: Equal number of NZ ENTRIES\n", rank);
            partition_equal_nz_elements(proc_info, nprocs, buf_i_idx);
        }
        else {
            fprintf(stderr, "Wrong policy defined...");
            exit(EXIT_FAILURE);
        }
        partition_time = (MPI_Wtime() - t) * 1000.0;

        debug("[%d] Partition time: %10.3lf ms\n\n", rank, partition_time);
        debug("[%d] Starting algorithm...\n", rank);
        t = MPI_Wtime();
    }

    /* Matrix-vector multiplication for each processes */
    res = mat_vec_mult_parallel(rank, nprocs, proc_info, buf_i_idx, 
                                buf_j_idx, buf_values, buf_x);

    /* write to output file */
    if (rank == MASTER) {
        comp_time = (MPI_Wtime() - t) * 1000.0; 
        printf("[%d] Computation time: %10.3lf ms\n\n", rank, comp_time);

        printf("[%d] Total execution time: %10.3lf ms\n", rank, comp_time + partition_time);
        debug("Finished!\n");
        if (out_file != NULL) {
            printf("Writing result to '%s'\n", out_file);

            /* open file */
            FILE *f;
            if ( !(f = fopen(out_file, "w")) ) {
                fprintf(stderr, "fopen: failed to open file '%s'", out_file);
                exit(EXIT_FAILURE);
            }

            /* write result */
            for (int i = 0; i < proc_info[MASTER].N; i++) {
                fprintf(f, "%.8lf\n", res[i]);
            }
            
            /* close file */
            if ( fclose(f) != 0) {
                fprintf(stderr, "fopen: failed to open file '%s'", out_file);
                exit(EXIT_FAILURE);
            }

            printf("Done!\n");
        }

        /* free the memory */
        free(buf_values);
        free(buf_i_idx);
        free(buf_j_idx);
        free(buf_x);
        free(res);
    }

    /* MPI: end */
    MPI_Finalize();

        
    return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x48, %rsp
leaq 0x3c(%rsp), %rbx
movl %edi, (%rbx)
leaq 0x40(%rsp), %r14
movq %rsi, (%r14)
movq %rbx, %rdi
movq %r14, %rsi
callq 0x10a0
movq 0x2631(%rip), %r15      # 0x4fe0
leaq 0xc(%rsp), %rsi
movq %r15, %rdi
callq 0x1100
leaq 0x8(%rsp), %rsi
movq %r15, %rdi
callq 0x1130
leaq 0x2760(%rip), %rdi      # 0x5130
callq 0x2937
movl (%rbx), %ecx
leal -0x4(%rcx), %edx
movq (%r14), %rax
cmpl $-0x3, %edx
ja 0x2a04
movq (%rax), %rsi
leaq 0x96c(%rip), %rdi       # 0x3358
xorl %eax, %eax
callq 0x1030
xorl %eax, %eax
addq $0x48, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x8(%rax), %r15
cmpl $0x3, %ecx
jne 0x2a13
movq 0x10(%rax), %rbx
jmp 0x2a15
xorl %ebx, %ebx
movslq 0xc(%rsp), %rax
shlq $0x3, %rax
leaq (%rax,%rax,2), %rdi
callq 0x2632
movq %rax, %r14
leaq 0x4(%rax), %r9
leaq 0x18(%rsp), %rsi
leaq 0x30(%rsp), %rdx
leaq 0x28(%rsp), %rcx
movq %r15, %rdi
movq %rax, %r8
callq 0x20c0
movq 0x2599(%rip), %r13      # 0x4fe8
movq (%r13), %rdi
testl %eax, %eax
jne 0x2d85
movl 0x8(%rsp), %edx
leaq 0x92b(%rip), %rsi       # 0x3391
movq %r15, %rcx
xorl %eax, %eax
callq 0x1210
movq (%r13), %rdi
movl 0x8(%rsp), %edx
movl (%r14), %ecx
movl 0x4(%r14), %r8d
leaq 0x928(%rip), %rsi       # 0x33ae
xorl %eax, %eax
callq 0x1210
movslq 0xc(%rsp), %rax
testq %rax, %rax
jle 0x2ac3
shlq $0x3, %rax
leaq (%rax,%rax,2), %rax
xorl %ecx, %ecx
testq %rcx, %rcx
je 0x2aba
movq 0x10(%r14), %rdx
movq %rdx, 0x10(%r14,%rcx)
movdqu (%r14), %xmm0
movdqu %xmm0, (%r14,%rcx)
addq $0x18, %rcx
cmpq %rcx, %rax
jne 0x2aa1
movslq (%r14), %rdi
shlq $0x3, %rdi
callq 0x2632
movq %rax, %r15
movslq (%r14), %rdi
shlq $0x3, %rdi
callq 0x2632
movslq (%r14), %rcx
testq %rcx, %rcx
jle 0x2b83
leal 0x1(%rcx), %eax
decq %rcx
movq %rcx, %xmm0
pshufd $0x44, %xmm0, %xmm0     # xmm0 = xmm0[0,1,0,1]
shrl %eax
shlq $0x4, %rax
movdqa 0x818(%rip), %xmm1      # 0x3320
xorl %ecx, %ecx
movdqa 0x81e(%rip), %xmm2      # 0x3330
pxor %xmm2, %xmm0
movabsq $0x3ff0000000000000, %rdx # imm = 0x3FF0000000000000
pcmpeqd %xmm3, %xmm3
movdqa 0x814(%rip), %xmm4      # 0x3340
movdqa %xmm1, %xmm5
pxor %xmm2, %xmm5
movdqa %xmm5, %xmm6
pcmpgtd %xmm0, %xmm6
pcmpeqd %xmm0, %xmm5
pshufd $0xf5, %xmm5, %xmm7     # xmm7 = xmm5[1,1,3,3]
pand %xmm6, %xmm7
pshufd $0xf5, %xmm6, %xmm5     # xmm5 = xmm6[1,1,3,3]
por %xmm7, %xmm5
movd %xmm5, %esi
notl %esi
testb $0x1, %sil
je 0x2b62
movq %rdx, (%r15,%rcx)
pxor %xmm3, %xmm5
pextrw $0x4, %xmm5, %esi
testb $0x1, %sil
je 0x2b76
movq %rdx, 0x8(%r15,%rcx)
paddq %xmm4, %xmm1
addq $0x10, %rcx
cmpq %rcx, %rax
jne 0x2b2c
callq 0x11d0
movq %xmm0, 0x10(%rsp)
movl 0x258c(%rip), %eax      # 0x5120
movq (%r13), %rdi
cmpl $0x1, %eax
je 0x2bca
testl %eax, %eax
jne 0x2d8a
movl 0x8(%rsp), %edx
leaq 0x828(%rip), %rsi       # 0x33d8
xorl %eax, %eax
callq 0x1210
movl 0xc(%rsp), %esi
movq 0x18(%rsp), %rdx
movq %r14, %rdi
callq 0x23cc
jmp 0x2bed
movl 0x8(%rsp), %edx
leaq 0x826(%rip), %rsi       # 0x33fb
xorl %eax, %eax
callq 0x1210
movl 0xc(%rsp), %esi
movq 0x18(%rsp), %rdx
movq %r14, %rdi
callq 0x24a4
callq 0x11d0
subsd 0x10(%rsp), %xmm0
mulsd 0x750(%rip), %xmm0      # 0x3350
movq (%r13), %rdi
movl 0x8(%rsp), %edx
leaq 0x82d(%rip), %rsi       # 0x343c
movsd %xmm0, 0x10(%rsp)
movb $0x1, %al
callq 0x1210
movq (%r13), %rdi
movl 0x8(%rsp), %edx
leaq 0x833(%rip), %rsi       # 0x345e
xorl %eax, %eax
callq 0x1210
callq 0x11d0
movsd %xmm0, 0x20(%rsp)
movl 0x8(%rsp), %edi
movq 0x18(%rsp), %rcx
movq 0x30(%rsp), %r8
movq 0x28(%rsp), %r9
movl 0xc(%rsp), %esi
movq %r15, (%rsp)
movq %r14, %rdx
callq 0x2668
cmpl $0x0, 0x8(%rsp)
je 0x2c71
callq 0x1160
jmp 0x29f3
movq %rax, %r12
callq 0x11d0
subsd 0x20(%rsp), %xmm0
mulsd 0x6c9(%rip), %xmm0      # 0x3350
movsd %xmm0, 0x20(%rsp)
movl 0x8(%rsp), %esi
leaq 0x7e2(%rip), %rdi       # 0x347a
movb $0x1, %al
callq 0x1030
movl 0x8(%rsp), %esi
movsd 0x10(%rsp), %xmm0
addsd 0x20(%rsp), %xmm0
leaq 0x7e8(%rip), %rdi       # 0x349e
movb $0x1, %al
callq 0x1030
movq (%r13), %rcx
leaq 0x7fd(%rip), %rdi       # 0x34c5
movl $0xa, %esi
movl $0x1, %edx
callq 0x11e0
testq %rbx, %rbx
je 0x2d52
leaq 0x7ed(%rip), %rdi       # 0x34d0
movq %rbx, %rsi
xorl %eax, %eax
callq 0x1030
leaq 0x48c(%rip), %rsi       # 0x3180
movq %rbx, 0x10(%rsp)
movq %rbx, %rdi
callq 0x1090
testq %rax, %rax
je 0x2d8f
movq %rax, %r13
cmpl $0x0, (%r14)
jle 0x2d3a
xorl %ebx, %ebx
leaq 0x7ec(%rip), %rbp       # 0x3508
movsd (%r12,%rbx,8), %xmm0
movq %r13, %rdi
movq %rbp, %rsi
movb $0x1, %al
callq 0x1210
incq %rbx
movslq (%r14), %rax
cmpq %rax, %rbx
jl 0x2d1c
movq %r13, %rdi
callq 0x11c0
testl %eax, %eax
jne 0x2d99
leaq 0x7c2(%rip), %rdi       # 0x350f
callq 0x1050
movq 0x28(%rsp), %rdi
callq 0x10d0
movq 0x18(%rsp), %rdi
callq 0x10d0
movq 0x30(%rsp), %rdi
callq 0x10d0
movq %r15, %rdi
callq 0x10d0
movq %r12, %rdi
callq 0x10d0
jmp 0x2c67
callq 0x130b
callq 0x1371
movq 0x10(%rsp), %rdi
callq 0x134d
movq 0x10(%rsp), %rdi
callq 0x1329