// File: /workspace/github_cpp_makefile_star2_O2/alexandres[P]terashuf/terashuf.cc
// Function: main
int main()
{
    char const *sepStr = std::getenv("SEP");
    const char sep = (sepStr != NULL && strlen(sepStr)) ? sepStr[0] : '\n';

    char *seedStr = std::getenv("SEED");
    const int seed = (seedStr != NULL && strlen(seedStr)) ? strtol(seedStr, NULL, 10) : time(NULL);
    std::mt19937_64 rng(seed);

    char *skipStr = std::getenv("SKIP");
    const int skipLines = (skipStr != NULL && strlen(skipStr)) ? strtol(skipStr, NULL, 10) : 0;

    char const *tmpDir = std::getenv("TMPDIR");
    if (tmpDir == NULL)
        tmpDir = "/tmp";
    char *tmpNameTemplate = (char *)malloc((strlen(tmpDir) + sizeof(TMP_NAME_TEMPLATE)) * sizeof(char));
    strcpy(tmpNameTemplate, tmpDir);
    strcat(tmpNameTemplate, TMP_NAME_TEMPLATE);

    char *memoryStr = std::getenv("MEMORY");
    const float memory = (memoryStr != NULL && strlen(memoryStr)) ? strtof(memoryStr, NULL) : 4;

    std::vector<ll> shufIndexes;

    const ll bufBytes = sizeof(char) * (ll)(memory * 1024. * 1024. * 1024.);
    char *buf = (char *)malloc(bufBytes);
    fprintf(stderr, "trying to allocate %zu bytes\n", bufBytes);
    if (buf == NULL)
    {
        fprintf(stderr, "failed to allocate buf memory\n");
        return 1;
    }

    fprintf(stderr, "\nstarting read\n");

    for (int i = 0; i < skipLines;)
    {
        char c = fgetc(stdin);
        if (c == EOF)
            return 0;
        if (fputc(c, stdout) == EOF)
        {
            fprintf(stderr, "failed to write SKIP lines. is disk full?");
            return 1;
        }
        if (c == sep)
            i++;
    }
    fprintf(stderr, "skipped %d lines\n", skipLines);

    ll totalBytesRead = 0, totalLinesRead = 0;
    std::vector<TmpFile> files;
    bool reachedEof = false;
    while (!reachedEof)
    {
        reachedEof = fillBufAndMarkLines(stdin, buf, bufBytes, shufIndexes, sep, memory);
        TmpFile tmpFile;
        // default output is stdout. if input fit in buffer, no need to use
        // temp files, just write to stdout.
        tmpFile.f = stdout;
        if (!reachedEof || files.size() > 0)
        {
            // haven't reached eof so have to call fillBuf again.
            // or we did did reach eof but there is already a temp file,
            // so we are already in temp file mode, not direct to stdout.
            tmpFile.path = (char *)malloc((strlen(tmpNameTemplate) + 1) * sizeof(char));
            strcpy(tmpFile.path, tmpNameTemplate);
            int fd = mkstemp(tmpFile.path);
            if (fd == -1)
            {
                fprintf(stderr, "failed to create fd tmp file %s\n",
                        tmpNameTemplate);
                return 1;
            }
            tmpFile.f = fdopen(fd, "wb+");
            if (tmpFile.f == NULL)
            {
                fprintf(stderr, "failed to create tmp file %s\n",
                        tmpFile.path);
                return 1;
            }
        }
        totalBytesRead += shufFlushBuf(buf, shufIndexes, rng, sep, tmpFile.f);
        totalLinesRead += shufIndexes.size();
        tmpFile.lines = shufIndexes.size();
        fprintf(stderr, "\rlines read: %zu, gb read: %zu", totalLinesRead,
                totalBytesRead / (1024 * 1024 * 1024));
        files.push_back(tmpFile);
    }

    // inform totalLines and totalBytesRead
    fprintf(stderr, "\nRead %zu lines, %zu bytes, have %d tmp files\n",
            totalLinesRead, totalBytesRead, (int)files.size());

    if (files.size() == 1)
    {
        fprintf(
            stderr,
            "\nShuffled directly to stdout without using tmpfiles, done!\n");
        return 0;
    }

    std::vector<ll> linesRemainingPerFile;

    for (auto &file : files)
    {
        rewind(file.f);
        // for bufferedFgetc
        file.buf = (char *)malloc(IO_CHUNK);
        file.bufPos = 0;
        file.bufN = 0;
        file.eof = false;
        linesRemainingPerFile.push_back(file.lines);
    }

    auto fenwickTree = FenwickTree(linesRemainingPerFile);

    ll totalBytesWritten = 0, linesRemaining = totalLinesRead, totalBytesWrittenForProgress = 0;

    fprintf(stderr, "\nstarting write to output\n");

    while (linesRemaining)
    {
        ll randLine = std::uniform_int_distribution<ll>{0, linesRemaining - 1}(rng);
        auto fileIdx = fenwickTree.findIndexAndDraw(randLine + 1);
        auto linesRemainingInFile = fenwickTree.getCountAtIndex(fileIdx);
        linesRemaining--;
        TmpFile &file = files[fileIdx];
        ll bytesRead = readLine(buf, sep, file);
        if (!linesRemainingInFile)
        {
            fclose(file.f);
            unlink(file.path);
        }
        ll bytesWritten = fwrite(buf, sizeof(char), bytesRead, stdout);
        if (bytesRead != bytesWritten)
        {
            fprintf(stderr, "\nFATAL ERROR: failed to write line to disk. is there space left?\n");
            return 1;
        }
        totalBytesWritten += bytesWritten;
        totalBytesWrittenForProgress += bytesWritten;
        // progress report
        if (!linesRemaining || totalBytesWrittenForProgress >= bufBytes)
        {
            fprintf(stderr, "\rlines written: %zu, gb written: %zu",
                    totalLinesRead - linesRemaining, totalBytesWritten / (1024 * 1024 * 1024));
            totalBytesWrittenForProgress = 0;
        }
    }
    fprintf(stderr, "\ndone\n");
    return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xa88, %rsp            # imm = 0xA88
leaq 0x1a99(%rip), %rdi      # 0x3199
callq 0x1160
movb $0xa, %cl
testq %rax, %rax
je 0x1717
movzbl (%rax), %eax
testb %al, %al
pushq $0xa
popq %rcx
cmovnel %eax, %ecx
movl %ecx, (%rsp)
leaq 0x1a7c(%rip), %rdi      # 0x319d
callq 0x1160
testq %rax, %rax
je 0x1730
cmpb $0x0, (%rax)
jne 0x1739
xorl %edi, %edi
callq 0x10f0
jmp 0x1746
pushq $0xa
popq %rdx
movq %rax, %rdi
xorl %esi, %esi
callq 0x11a0
movslq %eax, %rsi
leaq 0xc0(%rsp), %rdi
callq 0x24de
leaq 0x1a45(%rip), %rdi      # 0x31a2
callq 0x1160
xorl %r12d, %r12d
testq %rax, %rax
je 0x177f
cmpb $0x0, (%rax)
je 0x177f
pushq $0xa
popq %rdx
movq %rax, %rdi
xorl %esi, %esi
callq 0x11a0
movq %rax, %r12
leaq 0x1a21(%rip), %rdi      # 0x31a7
callq 0x1160
testq %rax, %rax
leaq 0x1a19(%rip), %rbx      # 0x31ae
cmovneq %rax, %rbx
movq %rbx, %rdi
callq 0x1090
leaq 0x13(%rax), %rdi
callq 0x1180
movq %rax, %r13
movq %rax, %rdi
movq %rbx, %rsi
callq 0x1100
movq %r13, %rdi
callq 0x1090
movaps 0x1c29(%rip), %xmm0     # 0x33f0
movups %xmm0, (%r13,%rax)
movl $0x585858, 0xf(%r13,%rax) # imm = 0x585858
leaq 0x19d6(%rip), %rdi      # 0x31b3
callq 0x1160
movss 0x187e(%rip), %xmm0     # 0x3068
movss %xmm0, 0x8(%rsp)
testq %rax, %rax
je 0x180a
cmpb $0x0, (%rax)
je 0x180a
movq %rax, %rdi
xorl %esi, %esi
callq 0x1190
movss %xmm0, 0x8(%rsp)
xorps %xmm0, %xmm0
andq $0x0, 0x50(%rsp)
movaps %xmm0, 0x40(%rsp)
cvtss2sd 0x8(%rsp), %xmm0
movsd 0x1832(%rip), %xmm1     # 0x3058
mulsd %xmm1, %xmm0
mulsd %xmm1, %xmm0
mulsd %xmm1, %xmm0
cvttsd2si %xmm0, %rax
movq %rax, %rcx
sarq $0x3f, %rcx
subsd 0x181a(%rip), %xmm0     # 0x3060
cvttsd2si %xmm0, %r14
andq %rcx, %r14
orq %rax, %r14
movq %r14, %rdi
callq 0x1180
movq %rax, %rbp
movq 0x3775(%rip), %r15      # 0x4fd8
movq (%r15), %rdi
leaq 0x194d(%rip), %rsi      # 0x31ba
xorl %ebx, %ebx
movq %r14, 0x38(%rsp)
movq %r14, %rdx
xorl %eax, %eax
callq 0x11b0
movq (%r15), %rcx
movq %rbp, 0x10(%rsp)
testq %rbp, %rbp
jne 0x18a3
leaq 0x1946(%rip), %rdi      # 0x31d8
pushq $0x1e
popq %rsi
pushq $0x1
popq %rdx
callq 0x1220
pushq $0x1
popq %r14
jmp 0x1908
leaq 0x194d(%rip), %rdi      # 0x31f7
pushq $0xf
popq %rsi
pushq $0x1
popq %rdx
callq 0x1220
movsbl (%rsp), %ebp
movq 0x3710(%rip), %r15      # 0x4fd0
cmpl %r12d, %ebx
jge 0x1927
movq (%r15), %rdi
callq 0x11e0
movl %eax, %ecx
shll $0x18, %ecx
cmpl $0xff000000, %ecx       # imm = 0xFF000000
je 0x1905
movsbl %al, %r14d
movq 0x36d3(%rip), %rax      # 0x4fb8
movq (%rax), %rsi
movl %r14d, %edi
callq 0x1170
cmpl $-0x1, %eax
je 0x1a79
xorl %eax, %eax
cmpl %ebp, %r14d
sete %al
addl %eax, %ebx
jmp 0x18c0
xorl %r14d, %r14d
leaq 0x40(%rsp), %rdi
callq 0x20b4
movl %r14d, %eax
addq $0xa88, %rsp            # imm = 0xA88
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x36aa(%rip), %rbx      # 0x4fd8
movq (%rbx), %rdi
leaq 0x18f9(%rip), %rsi      # 0x3231
movq $0x0, (%rsp)
movl %r12d, %edx
xorl %eax, %eax
callq 0x11b0
xorpd %xmm0, %xmm0
leaq 0x20(%rsp), %rax
andq $0x0, 0x10(%rax)
movapd %xmm0, (%rax)
leaq 0x40(%rsp), %rbx
xorl %r12d, %r12d
xorl %r14d, %r14d
movq 0x10(%rsp), %rsi
testb $0x1, (%rsp)
jne 0x1a91
movq (%r15), %rdi
movq 0x38(%rsp), %rdx
movq %rbx, %rcx
movl %ebp, %r8d
movss 0x8(%rsp), %xmm0
callq 0x1492
movl %eax, %ecx
movq 0x3620(%rip), %rax      # 0x4fb8
movq (%rax), %rax
movq %rax, 0x58(%rsp)
movq %rcx, (%rsp)
testb %cl, %cl
je 0x19b4
movq 0x28(%rsp), %rcx
cmpq 0x20(%rsp), %rcx
je 0x1a10
movq %r14, %rbx
movq %r13, %rdi
callq 0x1090
leaq 0x1(%rax), %rdi
callq 0x1180
movq %rax, %r14
movq %rax, 0x60(%rsp)
movq %rax, %rdi
movq %r13, %rsi
callq 0x1100
movq %r14, %rdi
callq 0x1060
cmpl $-0x1, %eax
je 0x1afa
movl %eax, %edi
leaq 0x186f(%rip), %rsi      # 0x3264
callq 0x1140
movq %rax, 0x58(%rsp)
testq %rax, %rax
movq %rbx, %r14
leaq 0x40(%rsp), %rbx
je 0x1b10
movq 0x10(%rsp), %rdi
movq %rbx, %rsi
leaq 0xc0(%rsp), %rdx
movl %ebp, %ecx
movq %rax, %r8
callq 0x135c
addq %rax, %r14
movq 0x48(%rsp), %rax
subq 0x40(%rsp), %rax
sarq $0x3, %rax
addq %rax, %r12
movq %rax, 0x68(%rsp)
movq 0x358e(%rip), %rax      # 0x4fd8
movq (%rax), %rdi
movq %r14, %rcx
shrq $0x1e, %rcx
leaq 0x182b(%rip), %rsi      # 0x3286
movq %r12, %rdx
xorl %eax, %eax
callq 0x11b0
leaq 0x20(%rsp), %rdi
leaq 0x58(%rsp), %rsi
callq 0x1eb4
jmp 0x1967
movq 0x3558(%rip), %rax      # 0x4fd8
movq (%rax), %rcx
leaq 0x177d(%rip), %rdi      # 0x3207
pushq $0x29
jmp 0x1894
movq 0x3540(%rip), %rbx      # 0x4fd8
movq (%rbx), %rdi
movq 0x28(%rsp), %rax
subq 0x20(%rsp), %rax
pushq $0x38
popq %rcx
cqto
idivq %rcx
leaq 0x17f1(%rip), %rsi      # 0x32a5
movq %r14, %rcx
xorl %r14d, %r14d
movq %r12, %rdx
movl %eax, %r8d
xorl %eax, %eax
callq 0x11b0
movq 0x20(%rsp), %r13
movq %rbx, %rcx
movq 0x28(%rsp), %rbx
movq %rbx, %rax
subq %r13, %rax
cmpq $0x38, %rax
jne 0x1b36
movq (%rcx), %rcx
leaq 0x17ea(%rip), %rdi      # 0x32d4
pushq $0x3b
popq %rsi
pushq $0x1
popq %rdx
callq 0x1220
jmp 0x1d1a
movq 0x34d7(%rip), %rax      # 0x4fd8
movq (%rax), %rdi
leaq 0x1738(%rip), %rsi      # 0x3243
movq %r13, %rdx
jmp 0x1b26
movq 0x34c1(%rip), %rax      # 0x4fd8
movq (%rax), %rdi
movq 0x60(%rsp), %rdx
leaq 0x1742(%rip), %rsi      # 0x3268
xorl %eax, %eax
callq 0x11b0
pushq $0x1
popq %r14
jmp 0x1d1a
xorpd %xmm0, %xmm0
leaq 0xa0(%rsp), %r15
movapd %xmm0, (%r15)
andq $0x0, 0x10(%r15)
cmpq %rbx, %r13
je 0x1b8f
movq (%r13), %rdi
callq 0x1030
movl $0x10000, %edi          # imm = 0x10000
callq 0x1180
movq %rax, 0x18(%r13)
leaq 0x10(%r13), %r14
xorpd %xmm0, %xmm0
movupd %xmm0, 0x20(%r13)
movb $0x0, 0x30(%r13)
movq %r15, %rdi
movq %r14, %rsi
callq 0x1e94
addq $0x28, %r14
movq %r14, %r13
jmp 0x1b4c
leaq 0x58(%rsp), %rdi
leaq 0xa0(%rsp), %rsi
callq 0x1eea
movq 0x3430(%rip), %rbx      # 0x4fd8
movq (%rbx), %rcx
leaq 0x175e(%rip), %rdi      # 0x3310
pushq $0x1a
popq %rsi
pushq $0x1
popq %r14
movq %r14, %rdx
callq 0x1220
decq %r12
movq $0x0, (%rsp)
xorl %r15d, %r15d
movl %ebp, 0x1c(%rsp)
cmpq $-0x1, %r12
je 0x1cd0
movq %r15, 0x8(%rsp)
movq %r14, %rbp
andq $0x0, 0x90(%rsp)
movq %r12, %r13
movq %r12, 0x98(%rsp)
leaq 0x90(%rsp), %rdi
leaq 0xc0(%rsp), %rsi
callq 0x1f88
incq %rax
leaq 0x58(%rsp), %rdi
movq %rax, %rsi
callq 0x1f90
movslq 0x74(%rsp), %rcx
cltq
addq %rax, %rcx
movq 0x58(%rsp), %rdx
imulq $0x38, %rax, %r14
addq 0x20(%rsp), %r14
movq (%rdx,%rcx,8), %rbx
movq 0x10(%rsp), %r12
movq %r12, %rdi
movl 0x1c(%rsp), %esi
movq %r14, %rdx
callq 0x1455
movq %rax, %r15
testq %rbx, %rbx
jne 0x1c67
movq (%r14), %rdi
callq 0x1110
movq 0x8(%r14), %rdi
callq 0x10a0
movq 0x334a(%rip), %rax      # 0x4fb8
movq (%rax), %rcx
movq %r12, %rdi
pushq $0x1
popq %rsi
movq %r15, %rdx
callq 0x1220
cmpq %rax, %r15
movq 0x334f(%rip), %rbx      # 0x4fd8
jne 0x1cea
addq %r15, (%rsp)
movq %r13, %r12
addq $-0x1, %r12
movq %rbp, %r14
jae 0x1caf
addq 0x8(%rsp), %r15
cmpq 0x38(%rsp), %r15
jae 0x1caf
incq %r14
jmp 0x1bd3
movq (%rbx), %rdi
movq (%rsp), %rcx
shrq $0x1e, %rcx
leaq 0x16ac(%rip), %rsi      # 0x336d
movq %r14, %rdx
xorl %eax, %eax
callq 0x11b0
xorl %r15d, %r15d
jmp 0x1ca7
movq (%rbx), %rcx
leaq 0x16b8(%rip), %rdi      # 0x3392
pushq $0x6
popq %rsi
pushq $0x1
popq %rdx
callq 0x1220
xorl %r14d, %r14d
jmp 0x1d03
movq (%rbx), %rcx
leaq 0x1637(%rip), %rdi      # 0x332b
pushq $0x41
popq %rsi
pushq $0x1
popq %rdx
callq 0x1220
pushq $0x1
popq %r14
leaq 0x58(%rsp), %rdi
callq 0x20b4
leaq 0xa0(%rsp), %rdi
callq 0x20b4
leaq 0x20(%rsp), %rdi
callq 0x2520
jmp 0x1908
jmp 0x1d3a
movq %rax, %rbx
leaq 0x58(%rsp), %rdi
callq 0x20b4
jmp 0x1d3d
movq %rax, %rbx
leaq 0xa0(%rsp), %rdi
callq 0x20b4
jmp 0x1d53
jmp 0x1d50
jmp 0x1d50
movq %rax, %rbx
leaq 0x20(%rsp), %rdi
callq 0x2520
leaq 0x40(%rsp), %rdi
callq 0x20b4
movq %rbx, %rdi
callq 0x1200