Commit message: When decompressing with -N or -NT, strip any path from header name.

This uses the path of the compressed file combined with the name
from the header as the name of the decompressed output file.  Any
path information in the header name is stripped.  This avoids a
possible vulnerability where absolute or descending paths are put
in the gzip header.
CVE: CVE-2015-1191
CWE: CWE-22
Vulnerability description: Multiple directory traversal vulnerabilities in pigz 2.3.1 allow remote attackers to write to arbitrary files via a (1) full pathname or (2) .. (dot dot) in an archive.

Project: pigz
File: pigz.c

local unsigned long adler32_comb(unsigned long adler1, unsigned long adler2,
                                 size_t len2)
{
    unsigned long sum1;
    unsigned long sum2;
    unsigned rem;

    /* the derivation of this formula is left as an exercise for the reader */
    rem = (unsigned)(len2 % BASE);
    sum1 = adler1 & LOW16;
    sum2 = (rem * sum1) % BASE;
    sum1 += (adler2 & LOW16) + BASE - 1;
    sum2 += ((adler1 >> 16) & LOW16) + ((adler2 >> 16) & LOW16) + BASE - rem;
    if (sum1 >= BASE) sum1 -= BASE;
    if (sum1 >= BASE) sum1 -= BASE;
    if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1);
    if (sum2 >= BASE) sum2 -= BASE;
    return sum1 | (sum2 << 16);
}

Project: pigz
File: pigz.c

local void append_len(struct job *job, size_t len)
{
    struct space *lens;

    assert(len < 539000896UL);
    if (job->lens == NULL)
        job->lens = get_space(&lens_pool);
    lens = job->lens;
    if (lens->size < lens->len + 3)
        grow_space(lens);
    if (len < 64)
        lens->buf[lens->len++] = len + 128;
    else if (len < 32832U) {
        len -= 64;
        lens->buf[lens->len++] = len >> 8;
        lens->buf[lens->len++] = len;
    }
    else if (len < 2129984UL) {
        len -= 32832U;
        lens->buf[lens->len++] = (len >> 16) + 192;
        lens->buf[lens->len++] = len >> 8;
        lens->buf[lens->len++] = len;
    }
    else {
        len -= 2129984UL;
        lens->buf[lens->len++] = (len >> 24) + 224;
        lens->buf[lens->len++] = len >> 16;
        lens->buf[lens->len++] = len >> 8;
        lens->buf[lens->len++] = len;
    }
}

Project: pigz
File: pigz.c

local int bail(char *why, char *what)
{
    if (g.outd != -1 && g.outf != NULL)
        unlink(g.outf);
    complain("abort: %s%s", why, what);
    exit(1);
    return 0;
}

Project: pigz
File: pigz.c

local void cat(void)
{
    /* write first magic byte (if we're here, there's at least one byte) */
    writen(g.outd, &g.magic1, 1);
    g.out_tot = 1;

    /* copy the remainder of the input to the output (if there were any more
       bytes of input, then g.in_left is non-zero and g.in_next is pointing to
       the second magic byte) */
    while (g.in_left) {
        writen(g.outd, g.in_next, g.in_left);
        g.out_tot += g.in_left;
        g.in_left = 0;
        load();
    }
}

Project: pigz
File: pigz.c

local int complain(char *fmt, ...)
{
    va_list ap;

    if (g.verbosity > 0) {
        fprintf(stderr, "%s: ", g.prog);
        va_start(ap, fmt);
        vfprintf(stderr, fmt, ap);
        va_end(ap);
        putc('\n', stderr);
        fflush(stderr);
    }
    return 0;
}

Project: pigz
File: pigz.c

local void compress_thread(void *dummy)
{
    struct job *job;                /* job pulled and working on */
    struct job *here, **prior;      /* pointers for inserting in write list */
    unsigned long check;            /* check value of input */
    unsigned char *next;            /* pointer for blocks, check value data */
    size_t left;                    /* input left to process */
    size_t len;                     /* remaining bytes to compress/check */
#if ZLIB_VERNUM >= 0x1260
    int bits;                       /* deflate pending bits */
#endif
    struct space *temp = NULL;      /* temporary space for zopfli input */
    z_stream strm;                  /* deflate stream */

    (void)dummy;

    /* initialize the deflate stream for this thread */
    strm.zfree = ZFREE;
    strm.zalloc = ZALLOC;
    strm.opaque = OPAQUE;
    if (deflateInit2(&strm, 6, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK)
        bail("not enough memory", "");

    /* keep looking for work */
    for (;;) {
        /* get a job (like I tell my son) */
        possess(compress_have);
        wait_for(compress_have, NOT_TO_BE, 0);
        job = compress_head;
        assert(job != NULL);
        if (job->seq == -1)
            break;
        compress_head = job->next;
        if (job->next == NULL)
            compress_tail = &compress_head;
        twist(compress_have, BY, -1);

        /* got a job -- initialize and set the compression level (note that if
           deflateParams() is called immediately after deflateReset(), there is
           no need to initialize the input/output for the stream) */
        Trace(("-- compressing #%ld", job->seq));
        if (g.level <= 9) {
            (void)deflateReset(&strm);
            (void)deflateParams(&strm, g.level, Z_DEFAULT_STRATEGY);
        }
        else {
            temp = get_space(&out_pool);
            temp->len = 0;
        }

        /* set dictionary if provided, release that input or dictionary buffer
           (not NULL if g.setdict is true and if this is not the first work
           unit) */
        if (job->out != NULL) {
            len = job->out->len;
            left = len < DICT ? len : DICT;
            if (g.level <= 9)
                deflateSetDictionary(&strm, job->out->buf + (len - left),
                                     left);
            else {
                memcpy(temp->buf, job->out->buf + (len - left), left);
                temp->len = left;
            }
            drop_space(job->out);
        }

        /* set up input and output */
        job->out = get_space(&out_pool);
        if (g.level <= 9) {
            strm.next_in = job->in->buf;
            strm.next_out = job->out->buf;
        }
        else
            memcpy(temp->buf + temp->len, job->in->buf, job->in->len);

        /* compress each block, either flushing or finishing */
        next = job->lens == NULL ? NULL : job->lens->buf;
        left = job->in->len;
        job->out->len = 0;
        do {
            /* decode next block length from blocks list */
            len = next == NULL ? 128 : *next++;
            if (len < 128)                          /* 64..32831 */
                len = (len << 8) + (*next++) + 64;
            else if (len == 128)                    /* end of list */
                len = left;
            else if (len < 192)                     /* 1..63 */
                len &= 0x3f;
            else if (len < 224){                    /* 32832..2129983 */
                len = ((len & 0x1f) << 16) + (*next++ << 8);
                len += *next++ + 32832U;
            }
            else {                                  /* 2129984..539000895 */
                len = ((len & 0x1f) << 24) + (*next++ << 16);
                len += *next++ << 8;
                len += *next++ + 2129984UL;
            }
            left -= len;

            if (g.level <= 9) {
                /* run MAXP2-sized amounts of input through deflate -- this
                   loop is needed for those cases where the unsigned type is
                   smaller than the size_t type, or when len is close to the
                   limit of the size_t type */
                while (len > MAXP2) {
                    strm.avail_in = MAXP2;
                    deflate_engine(&strm, job->out, Z_NO_FLUSH);
                    len -= MAXP2;
                }

                /* run the last piece through deflate -- end on a byte
                   boundary, using a sync marker if necessary, or finish the
                   deflate stream if this is the last block */
                strm.avail_in = (unsigned)len;
                if (left || job->more) {
#if ZLIB_VERNUM >= 0x1260
                    deflate_engine(&strm, job->out, Z_BLOCK);

                    /* add enough empty blocks to get to a byte boundary */
                    (void)deflatePending(&strm, Z_NULL, &bits);
                    if (bits & 1)
                        deflate_engine(&strm, job->out, Z_SYNC_FLUSH);
                    else if (bits & 7) {
                        do {        /* add static empty blocks */
                            bits = deflatePrime(&strm, 10, 2);
                            assert(bits == Z_OK);
                            (void)deflatePending(&strm, Z_NULL, &bits);
                        } while (bits & 7);
                        deflate_engine(&strm, job->out, Z_BLOCK);
                    }
#else
                    deflate_engine(&strm, job->out, Z_SYNC_FLUSH);
#endif
                }
                else
                    deflate_engine(&strm, job->out, Z_FINISH);
            }
            else {
                /* compress len bytes using zopfli, bring to byte boundary */
                unsigned char bits, *out;
                size_t outsize;

                out = NULL;
                outsize = 0;
                bits = 0;
                ZopfliDeflatePart(&g.zopts, 2, !(left || job->more),
                                  temp->buf, temp->len, temp->len + len,
                                  &bits, &out, &outsize);
                assert(job->out->len + outsize + 5 <= job->out->size);
                memcpy(job->out->buf + job->out->len, out, outsize);
                free(out);
                job->out->len += outsize;
                if (left || job->more) {
                    bits &= 7;
                    if (bits & 1) {
                        if (bits == 7)
                            job->out->buf[job->out->len++] = 0;
                        job->out->buf[job->out->len++] = 0;
                        job->out->buf[job->out->len++] = 0;
                        job->out->buf[job->out->len++] = 0xff;
                        job->out->buf[job->out->len++] = 0xff;
                    }
                    else if (bits) {
                        do {
                            job->out->buf[job->out->len - 1] += 2 << bits;
                            job->out->buf[job->out->len++] = 0;
                            bits += 2;
                        } while (bits < 8);
                    }
                }
                temp->len += len;
            }
        } while (left);
        if (g.level > 9)
            drop_space(temp);
        if (job->lens != NULL) {
            drop_space(job->lens);
            job->lens = NULL;
        }
        Trace(("-- compressed #%ld%s", job->seq, job->more ? "" : " (last)"));

        /* reserve input buffer until check value has been calculated */
        use_space(job->in);

        /* insert write job in list in sorted order, alert write thread */
        possess(write_first);
        prior = &write_head;
        while ((here = *prior) != NULL) {
            if (here->seq > job->seq)
                break;
            prior = &(here->next);
        }
        job->next = here;
        *prior = job;
        twist(write_first, TO, write_head->seq);

        /* calculate the check value in parallel with writing, alert the write
           thread that the calculation is complete, and drop this usage of the
           input buffer */
        len = job->in->len;
        next = job->in->buf;
        check = CHECK(0L, Z_NULL, 0);
        while (len > MAXP2) {
            check = CHECK(check, next, MAXP2);
            len -= MAXP2;
            next += MAXP2;
        }
        check = CHECK(check, next, (unsigned)len);
        drop_space(job->in);
        job->check = check;
        Trace(("-- checked #%ld%s", job->seq, job->more ? "" : " (last)"));
        possess(job->calc);
        twist(job->calc, TO, 1);

        /* done with that one -- go find another job */
    }

    /* found job with seq == -1 -- free deflate memory and return to join */
    release(compress_have);
    (void)deflateEnd(&strm);
}

Project: pigz
File: pigz.c

local size_t compressed_suffix(char *nm)
{
    size_t len;

    len = strlen(nm);
    if (len > 4) {
        nm += len - 4;
        len = 4;
        if (strcmp(nm, ".zip") == 0 || strcmp(nm, ".ZIP") == 0 ||
            strcmp(nm, ".tgz") == 0)
            return 4;
    }
    if (len > 3) {
        nm += len - 3;
        len = 3;
        if (strcmp(nm, ".gz") == 0 || strcmp(nm, "-gz") == 0 ||
            strcmp(nm, ".zz") == 0 || strcmp(nm, "-zz") == 0)
            return 3;
    }
    if (len > 2) {
        nm += len - 2;
        if (strcmp(nm, ".z") == 0 || strcmp(nm, "-z") == 0 ||
            strcmp(nm, "_z") == 0 || strcmp(nm, ".Z") == 0)
            return 2;
    }
    return 0;
}

Project: pigz
File: pigz.c

local void copymeta(char *from, char *to)
{
    struct stat st;
    struct timeval times[2];

    /* get all of from's Unix meta data, return if not a regular file */
    if (stat(from, &st) != 0 || (st.st_mode & S_IFMT) != S_IFREG)
        return;

    /* set to's mode bits, ignore errors */
    (void)chmod(to, st.st_mode & 07777);

    /* copy owner's user and group, ignore errors */
    (void)chown(to, st.st_uid, st.st_gid);

    /* copy access and modify times, ignore errors */
    times[0].tv_sec = st.st_atime;
    times[0].tv_usec = 0;
    times[1].tv_sec = st.st_mtime;
    times[1].tv_usec = 0;
    (void)utimes(to, times);
}

Project: pigz
File: pigz.c

local unsigned long crc32_comb(unsigned long crc1, unsigned long crc2,
                               size_t len2)
{
    int n;
    unsigned long row;
    unsigned long even[32];     /* even-power-of-two zeros operator */
    unsigned long odd[32];      /* odd-power-of-two zeros operator */

    /* degenerate case */
    if (len2 == 0)
        return crc1;

    /* put operator for one zero bit in odd */
    odd[0] = 0xedb88320UL;          /* CRC-32 polynomial */
    row = 1;
    for (n = 1; n < 32; n++) {
        odd[n] = row;
        row <<= 1;
    }

    /* put operator for two zero bits in even */
    gf2_matrix_square(even, odd);

    /* put operator for four zero bits in odd */
    gf2_matrix_square(odd, even);

    /* apply len2 zeros to crc1 (first square will put the operator for one
       zero byte, eight zero bits, in even) */
    do {
        /* apply zeros operator for this bit of len2 */
        gf2_matrix_square(even, odd);
        if (len2 & 1)
            crc1 = gf2_matrix_times(even, crc1);
        len2 >>= 1;

        /* if no more bits set, then done */
        if (len2 == 0)
            break;

        /* another iteration of the loop with odd and even swapped */
        gf2_matrix_square(odd, even);
        if (len2 & 1)
            crc1 = gf2_matrix_times(odd, crc1);
        len2 >>= 1;

        /* if no more bits set, then done */
    } while (len2 != 0);

    /* return combined crc */
    crc1 ^= crc2;
    return crc1;
}

Project: pigz
File: pigz.c

local void cut_short(int sig)
{
    (void)sig;
    Trace(("termination by user"));
    if (g.outd != -1 && g.outf != NULL)
        unlink(g.outf);
    log_dump();
    _exit(1);
}

Project: pigz
File: pigz.c

local void deflate_engine(z_stream *strm, struct space *out, int flush)
{
    size_t room;

    do {
        room = out->size - out->len;
        if (room == 0) {
            grow_space(out);
            room = out->size - out->len;
        }
        strm->next_out = out->buf + out->len;
        strm->avail_out = room < UINT_MAX ? (unsigned)room : UINT_MAX;
        (void)deflate(strm, flush);
        out->len = strm->next_out - out->buf;
    } while (strm->avail_out == 0);
    assert(strm->avail_in == 0);
}

Project: pigz
File: pigz.c

local time_t dos2time(unsigned long dos)
{
    struct tm tm;

    if (dos == 0)
        return time(NULL);
    tm.tm_year = ((int)(dos >> 25) & 0x7f) + 80;
    tm.tm_mon  = ((int)(dos >> 21) & 0xf) - 1;
    tm.tm_mday = (int)(dos >> 16) & 0x1f;
    tm.tm_hour = (int)(dos >> 11) & 0x1f;
    tm.tm_min  = (int)(dos >> 5) & 0x3f;
    tm.tm_sec  = (int)(dos << 1) & 0x3e;
    tm.tm_isdst = -1;           /* figure out if DST or not */
    return mktime(&tm);
}

Project: pigz
File: pigz.c

local void drop_space(struct space *space)
{
    int use;
    struct pool *pool;

    possess(space->use);
    use = peek_lock(space->use);
    assert(use != 0);
    if (use == 1) {
        pool = space->pool;
        possess(pool->have);
        space->next = pool->head;
        pool->head = space;
        twist(pool->have, BY, +1);
    }
    twist(space->use, BY, -1);
}

Project: pigz
File: pigz.c

local void finish_jobs(void)
{
    struct job job;
    int caught;

    /* only do this once */
    if (compress_have == NULL)
        return;

    /* command all of the extant compress threads to return */
    possess(compress_have);
    job.seq = -1;
    job.next = NULL;
    compress_head = &job;
    compress_tail = &(job.next);
    twist(compress_have, BY, +1);       /* will wake them all up */

    /* join all of the compress threads, verify they all came back */
    caught = join_all();
    Trace(("-- joined %d compress threads", caught));
    assert(caught == cthreads);
    cthreads = 0;

    /* free the resources */
    caught = free_pool(&lens_pool);
    Trace(("-- freed %d block lengths buffers", caught));
    caught = free_pool(&dict_pool);
    Trace(("-- freed %d dictionary buffers", caught));
    caught = free_pool(&out_pool);
    Trace(("-- freed %d output buffers", caught));
    caught = free_pool(&in_pool);
    Trace(("-- freed %d input buffers", caught));
    free_lock(write_first);
    free_lock(compress_have);
    compress_have = NULL;
}

Project: pigz
File: pigz.c

local int free_pool(struct pool *pool)
{
    int count;
    struct space *space;

    possess(pool->have);
    count = 0;
    while ((space = pool->head) != NULL) {
        pool->head = space->next;
        FREE(space->buf);
        free_lock(space->use);
        FREE(space);
        count++;
    }
    assert(count == pool->made);
    release(pool->have);
    free_lock(pool->have);
    return count;
}

Project: pigz
File: pigz.c

local void free_track(struct mem_track_s *mem, void *ptr)
{
    size_t size;

    if (ptr != NULL) {
        size = MALLOC_SIZE(ptr);
        mem_track_grab(mem);
        mem->num--;
        mem->size -= size;
        mem_track_drop(mem);
        free(ptr);
    }
}

Project: pigz
File: pigz.c

local void gf2_matrix_square(unsigned long *square, unsigned long *mat)
{
    int n;

    for (n = 0; n < 32; n++)
        square[n] = gf2_matrix_times(mat, mat[n]);
}

Project: pigz
File: pigz.c

local unsigned long gf2_matrix_times(unsigned long *mat, unsigned long vec)
{
    unsigned long sum;

    sum = 0;
    while (vec) {
        if (vec & 1)
            sum ^= *mat;
        vec >>= 1;
        mat++;
    }
    return sum;
}

Project: pigz
File: pigz.c

local size_t grow(size_t size)
{
    size_t was, top;
    int shift;

    was = size;
    size += size >> 2;
    top = size;
    for (shift = 0; top > 7; shift++)
        top >>= 1;
    if (top == 7)
        size = (size_t)1 << (shift + 3);
    if (size < 16)
        size = 16;
    if (size <= was)
        size = (size_t)0 - 1;
    return size;
}

Project: pigz
File: pigz.c

local void grow_space(struct space *space)
{
    size_t more;

    /* compute next size up */
    more = grow(space->size);
    if (more == space->size)
        bail("not enough memory", "");

    /* reallocate the buffer */
    space->buf = REALLOC(space->buf, more);
    if (space->buf == NULL)
        bail("not enough memory", "");
    space->size = more;
}

Project: pigz
File: pigz.c

local void help(void)
{
    int n;

    if (g.verbosity == 0)
        return;
    for (n = 0; n < (int)(sizeof(helptext) / sizeof(char *)); n++)
        fprintf(stderr, "%s\n", helptext[n]);
    fflush(stderr);
    exit(0);
}

Project: pigz
File: pigz.c

local void in_init(void)
{
    g.in_left = 0;
    g.in_eof = 0;
    g.in_short = 0;
    g.in_tot = 0;
#ifndef NOTHREAD
    g.in_which = -1;
#endif
}

Project: pigz
File: pigz.c

local unsigned inb(void *desc, unsigned char **buf)
{
    (void)desc;
    load();
    *buf = g.in_next;
    return g.in_left;
}

Project: pigz
File: pigz.c

local void infchk(void)
{
    int ret, cont, was;
    unsigned long check, len;
    z_stream strm;
    unsigned tmp2;
    unsigned long tmp4;
    off_t clen;

    cont = 0;
    do {
        /* header already read -- set up for decompression */
        g.in_tot = g.in_left;       /* track compressed data length */
        g.out_tot = 0;
        g.out_check = CHECK(0L, Z_NULL, 0);
        strm.zalloc = ZALLOC;
        strm.zfree = ZFREE;
        strm.opaque = OPAQUE;
        ret = inflateBackInit(&strm, 15, out_buf);
        if (ret != Z_OK)
            bail("not enough memory", "");

        /* decompress, compute lengths and check value */
        strm.avail_in = g.in_left;
        strm.next_in = g.in_next;
        ret = inflateBack(&strm, inb, NULL, outb, NULL);
        if (ret != Z_STREAM_END)
            bail("corrupted input -- invalid deflate data: ", g.inf);
        g.in_left = strm.avail_in;
        g.in_next = strm.next_in;
        inflateBackEnd(&strm);
        outb(NULL, NULL, 0);        /* finish off final write and check */

        /* compute compressed data length */
        clen = g.in_tot - g.in_left;

        /* read and check trailer */
        if (g.form > 1) {           /* zip local trailer (if any) */
            if (g.form == 3) {      /* data descriptor follows */
                /* read original version of data descriptor */
                g.zip_crc = GET4();
                g.zip_clen = GET4();
                g.zip_ulen = GET4();
                if (g.in_eof)
                    bail("corrupted zip entry -- missing trailer: ", g.inf);

                /* if crc doesn't match, try info-zip variant with sig */
                if (g.zip_crc != g.out_check) {
                    if (g.zip_crc != 0x08074b50UL || g.zip_clen != g.out_check)
                        bail("corrupted zip entry -- crc32 mismatch: ", g.inf);
                    g.zip_crc = g.zip_clen;
                    g.zip_clen = g.zip_ulen;
                    g.zip_ulen = GET4();
                }

                /* handle incredibly rare cases where crc equals signature */
                else if (g.zip_crc == 0x08074b50UL &&
                         g.zip_clen == g.zip_crc &&
                         ((clen & LOW32) != g.zip_crc ||
                          g.zip_ulen == g.zip_crc)) {
                    g.zip_crc = g.zip_clen;
                    g.zip_clen = g.zip_ulen;
                    g.zip_ulen = GET4();
                }

                /* if second length doesn't match, try 64-bit lengths */
                if (g.zip_ulen != (g.out_tot & LOW32)) {
                    g.zip_ulen = GET4();
                    (void)GET4();
                }
                if (g.in_eof)
                    bail("corrupted zip entry -- missing trailer: ", g.inf);
            }
            if (g.zip_clen != (clen & LOW32) ||
                g.zip_ulen != (g.out_tot & LOW32))
                bail("corrupted zip entry -- length mismatch: ", g.inf);
            check = g.zip_crc;
        }
        else if (g.form == 1) {     /* zlib (big-endian) trailer */
            check = (unsigned long)(GET()) << 24;
            check += (unsigned long)(GET()) << 16;
            check += (unsigned)(GET()) << 8;
            check += GET();
            if (g.in_eof)
                bail("corrupted zlib stream -- missing trailer: ", g.inf);
            if (check != g.out_check)
                bail("corrupted zlib stream -- adler32 mismatch: ", g.inf);
        }
        else {                      /* gzip trailer */
            check = GET4();
            len = GET4();
            if (g.in_eof)
                bail("corrupted gzip stream -- missing trailer: ", g.inf);
            if (check != g.out_check)
                bail("corrupted gzip stream -- crc32 mismatch: ", g.inf);
            if (len != (g.out_tot & LOW32))
                bail("corrupted gzip stream -- length mismatch: ", g.inf);
        }

        /* show file information if requested */
        if (g.list) {
            g.in_tot = clen;
            show_info(8, check, g.out_tot, cont);
            cont = 1;
        }

        /* if a gzip entry follows a gzip entry, decompress it (don't replace
           saved header information from first entry) */
        was = g.form;
    } while (was == 0 && (ret = get_header(0)) == 8 && g.form == 0);

    /* gzip -cdf copies junk after gzip stream directly to output */
    if (was == 0 && ret == -2 && g.force && g.pipeout && g.decode != 2 &&
        !g.list)
        cat();
    else if (was > 1 && get_header(0) != -5)
        complain("entries after the first in %s were ignored", g.inf);
    else if ((was == 0 && ret != -1) || (was == 1 && (GET(), !g.in_eof)))
        complain("%s OK, has trailing junk which was ignored", g.inf);
}

Project: pigz
File: pigz.c

local void list_info(void)
{
    int method;             /* get_header() return value */
    size_t n;               /* available trailer bytes */
    off_t at;               /* used to calculate compressed length */
    unsigned char tail[8];  /* trailer containing check and length */
    unsigned long check, len;   /* check value and length from trailer */

    /* initialize input buffer */
    in_init();

    /* read header information and position input after header */
    method = get_header(1);
    if (method < 0) {
        RELEASE(g.hname);
        if (method != -1 && g.verbosity > 1)
            complain("%s not a compressed file -- skipping", g.inf);
        return;
    }

    /* list zip file */
    if (g.form > 1) {
        g.in_tot = g.zip_clen;
        show_info(method, g.zip_crc, g.zip_ulen, 0);
        return;
    }

    /* list zlib file */
    if (g.form == 1) {
        at = lseek(g.ind, 0, SEEK_END);
        if (at == -1) {
            check = 0;
            do {
                len = g.in_left < 4 ? g.in_left : 4;
                g.in_next += g.in_left - len;
                while (len--)
                    check = (check << 8) + *g.in_next++;
            } while (load() != 0);
            check &= LOW32;
        }
        else {
            g.in_tot = at;
            lseek(g.ind, -4, SEEK_END);
            readn(g.ind, tail, 4);
            check = PULL4M(tail);
        }
        g.in_tot -= 6;
        show_info(method, check, 0, 0);
        return;
    }

    /* list lzw file */
    if (method == 257) {
        at = lseek(g.ind, 0, SEEK_END);
        if (at == -1)
            while (load() != 0)
                ;
        else
            g.in_tot = at;
        g.in_tot -= 3;
        show_info(method, 0, 0, 0);
        return;
    }

    /* skip to end to get trailer (8 bytes), compute compressed length */
    if (g.in_short) {                   /* whole thing already read */
        if (g.in_left < 8) {
            complain("%s not a valid gzip file -- skipping", g.inf);
            return;
        }
        g.in_tot = g.in_left - 8;       /* compressed size */
        memcpy(tail, g.in_next + (g.in_left - 8), 8);
    }
    else if ((at = lseek(g.ind, -8, SEEK_END)) != -1) {
        g.in_tot = at - g.in_tot + g.in_left;   /* compressed size */
        readn(g.ind, tail, 8);          /* get trailer */
    }
    else {                              /* can't seek */
        at = g.in_tot - g.in_left;      /* save header size */
        do {
            n = g.in_left < 8 ? g.in_left : 8;
            memcpy(tail, g.in_next + (g.in_left - n), n);
            load();
        } while (g.in_left == BUF);     /* read until end */
        if (g.in_left < 8) {
            if (n + g.in_left < 8) {
                complain("%s not a valid gzip file -- skipping", g.inf);
                return;
            }
            if (g.in_left) {
                if (n + g.in_left > 8)
                    memcpy(tail, tail + n - (8 - g.in_left), 8 - g.in_left);
                memcpy(tail + 8 - g.in_left, g.in_next, g.in_left);
            }
        }
        else
            memcpy(tail, g.in_next + (g.in_left - 8), 8);
        g.in_tot -= at + 8;
    }
    if (g.in_tot < 2) {
        complain("%s not a valid gzip file -- skipping", g.inf);
        return;
    }

    /* convert trailer to check and uncompressed length (modulo 2^32) */
    check = PULL4L(tail);
    len = PULL4L(tail + 4);

    /* list information about contents */
    show_info(method, check, len, 0);
    RELEASE(g.hname);
}

Project: pigz
File: pigz.c

local size_t load(void)
{
    /* if already detected end of file, do nothing */
    if (g.in_short) {
        g.in_eof = 1;
        g.in_left = 0;
        return 0;
    }

#ifndef NOTHREAD
    /* if first time in or procs == 1, read a buffer to have something to
       return, otherwise wait for the previous read job to complete */
    if (g.procs > 1) {
        /* if first time, fire up the read thread, ask for a read */
        if (g.in_which == -1) {
            g.in_which = 1;
            g.load_state = new_lock(1);
            g.load_thread = launch(load_read, NULL);
        }

        /* wait for the previously requested read to complete */
        possess(g.load_state);
        wait_for(g.load_state, TO_BE, 0);
        release(g.load_state);

        /* set up input buffer with the data just read */
        g.in_next = g.in_which ? g.in_buf : g.in_buf2;
        g.in_left = g.in_len;

        /* if not at end of file, alert read thread to load next buffer,
           alternate between g.in_buf and g.in_buf2 */
        if (g.in_len == BUF) {
            g.in_which = 1 - g.in_which;
            possess(g.load_state);
            twist(g.load_state, TO, 1);
        }

        /* at end of file -- join read thread (already exited), clean up */
        else {
            join(g.load_thread);
            free_lock(g.load_state);
            g.in_which = -1;
        }
    }
    else
#endif
    {
        /* don't use threads -- simply read a buffer into g.in_buf */
        g.in_left = readn(g.ind, g.in_next = g.in_buf, BUF);
    }

    /* note end of file */
    if (g.in_left < BUF) {
        g.in_short = 1;

        /* if we got bupkis, now is the time to mark eof */
        if (g.in_left == 0)
            g.in_eof = 1;
    }

    /* update the total and return the available bytes */
    g.in_tot += g.in_left;
    return g.in_left;
}

Project: pigz
File: pigz.c

local void load_read(void *dummy)
{
    size_t len;

    (void)dummy;

    Trace(("-- launched decompress read thread"));
    do {
        possess(g.load_state);
        wait_for(g.load_state, TO_BE, 1);
        g.in_len = len = readn(g.ind, g.in_which ? g.in_buf : g.in_buf2, BUF);
        Trace(("-- decompress read thread read %lu bytes", len));
        twist(g.load_state, TO, 0);
    } while (len == BUF);
    Trace(("-- exited decompress read thread"));
}

Project: pigz
File: pigz.c

local void log_add(char *fmt, ...)
{
    struct timeval now;
    struct log *me;
    va_list ap;
    char msg[MAXMSG];

    gettimeofday(&now, NULL);
    me = MALLOC(sizeof(struct log));
    if (me == NULL)
        bail("not enough memory", "");
    me->when = now;
    va_start(ap, fmt);
    vsnprintf(msg, MAXMSG, fmt, ap);
    va_end(ap);
    me->msg = MALLOC(strlen(msg) + 1);
    if (me->msg == NULL) {
        FREE(me);
        bail("not enough memory", "");
    }
    strcpy(me->msg, msg);
    me->next = NULL;
#ifndef NOTHREAD
    assert(log_lock != NULL);
    possess(log_lock);
#endif
    *log_tail = me;
    log_tail = &(me->next);
#ifndef NOTHREAD
    twist(log_lock, BY, +1);
#endif
}

Project: pigz
File: pigz.c

local void log_dump(void)
{
    if (log_tail == NULL)
        return;
    while (log_show())
        ;
    log_free();
    if (mem_track.num || mem_track.size)
        complain("memory leak: %lu allocs of %lu bytes total",
                 mem_track.num, mem_track.size);
    if (mem_track.max)
        fprintf(stderr, "%lu bytes of memory used\n", mem_track.max);
}

Project: pigz
File: pigz.c

local void log_free(void)
{
    struct log *me;

    if (log_tail != NULL) {
#ifndef NOTHREAD
        possess(log_lock);
#endif
        while ((me = log_head) != NULL) {
            log_head = me->next;
            FREE(me->msg);
            FREE(me);
        }
#ifndef NOTHREAD
        twist(log_lock, TO, 0);
        free_lock(log_lock);
        log_lock = NULL;
        yarn_mem(malloc, free);
        free_lock(mem_track.lock);
#endif
        log_tail = NULL;
    }
}

Project: pigz
File: pigz.c

local void log_init(void)
{
    if (log_tail == NULL) {
        mem_track.num = 0;
        mem_track.size = 0;
        mem_track.max = 0;
#ifndef NOTHREAD
        mem_track.lock = new_lock(0);
        yarn_mem(yarn_malloc, yarn_free);
        log_lock = new_lock(0);
#endif
        log_head = NULL;
        log_tail = &log_head;
    }
}

Project: pigz
File: pigz.c

local void *malloc_track(struct mem_track_s *mem, size_t size)
{
    void *ptr;

    ptr = malloc(size);
    if (ptr != NULL) {
        size = MALLOC_SIZE(ptr);
        mem_track_grab(mem);
        mem->num++;
        mem->size += size;
        if (mem->size > mem->max)
            mem->max = mem->size;
        mem_track_drop(mem);
    }
    return ptr;
}

Project: pigz
File: pigz.c

local void new_pool(struct pool *pool, size_t size, int limit)
{
    pool->have = new_lock(0);
    pool->head = NULL;
    pool->size = size;
    pool->limit = limit;
    pool->made = 0;
}

Project: pigz
File: pigz.c

local int nprocs(int n)
{
#  ifdef _SC_NPROCESSORS_ONLN
    n = (int)sysconf(_SC_NPROCESSORS_ONLN);
#  else
#    ifdef _SC_NPROC_ONLN
    n = (int)sysconf(_SC_NPROC_ONLN);
#    else
#      ifdef __hpux
    struct pst_dynamic psd;

    if (pstat_getdynamic(&psd, sizeof(psd), (size_t)1, 0) != -1)
        n = psd.psd_proc_cnt;
#      endif
#    endif
#  endif
    return n;
}

Project: pigz
File: pigz.c

local size_t num(char *arg)
{
    char *str = arg;
    size_t val = 0;

    if (*str == 0)
        bail("internal error: empty parameter", "");
    do {
        if (*str < '0' || *str > '9' ||
            (val && ((~(size_t)0) - (*str - '0')) / val < 10))
            bail("invalid numeric parameter: ", arg);
        val = val * 10 + (*str - '0');
    } while (*++str);
    return val;
}

Project: pigz
File: pigz.c

local int option(char *arg)
{
    static int get = 0;     /* if not zero, look for option parameter */
    char bad[3] = "-X";     /* for error messages (X is replaced) */

    /* if no argument or dash option, check status of get */
    if (get && (arg == NULL || *arg == '-')) {
        bad[1] = "bpSIM"[get - 1];
        bail("missing parameter after ", bad);
    }
    if (arg == NULL)
        return 0;

    /* process long option or short options */
    if (*arg == '-') {
        /* a single dash will be interpreted as stdin */
        if (*++arg == 0)
            return 1;

        /* process long option (fall through with equivalent short option) */
        if (*arg == '-') {
            int j;

            arg++;
            for (j = NLOPTS - 1; j >= 0; j--)
                if (strcmp(arg, longopts[j][0]) == 0) {
                    arg = longopts[j][1];
                    break;
                }
            if (j < 0)
                bail("invalid option: ", arg - 2);
        }

        /* process short options (more than one allowed after dash) */
        do {
            /* if looking for a parameter, don't process more single character
               options until we have the parameter */
            if (get) {
                if (get == 3)
                    bail("invalid usage: -s must be followed by space", "");
                break;      /* allow -pnnn and -bnnn, fall to parameter code */
            }

            /* process next single character option or compression level */
            bad[1] = *arg;
            switch (*arg) {
            case '0': case '1': case '2': case '3': case '4':
            case '5': case '6': case '7': case '8': case '9':
                g.level = *arg - '0';
                while (arg[1] >= '0' && arg[1] <= '9') {
                    if (g.level && (INT_MAX - (arg[1] - '0')) / g.level < 10)
                        bail("only levels 0..9 and 11 are allowed", "");
                    g.level = g.level * 10 + *++arg - '0';
                }
                if (g.level == 10 || g.level > 11)
                    bail("only levels 0..9 and 11 are allowed", "");
                new_opts();
                break;
            case 'F':  g.zopts.blocksplittinglast = 1;  break;
            case 'I':  get = 4;  break;
            case 'K':  g.form = 2;  g.sufx = ".zip";  break;
            case 'L':
                fputs(VERSION, stderr);
                fputs("Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013"
                      " Mark Adler\n",
                      stderr);
                fputs("Subject to the terms of the zlib license.\n",
                      stderr);
                fputs("No warranty is provided or implied.\n", stderr);
                exit(0);
            case 'M':  get = 5;  break;
            case 'N':  g.headis |= 0xf;  break;
            case 'O':  g.zopts.blocksplitting = 0;  break;
            case 'R':  g.rsync = 1;  break;
            case 'S':  get = 3;  break;
            case 'T':  g.headis &= ~0xa;  break;
            case 'V':  fputs(VERSION, stderr);  exit(0);
            case 'Z':
                bail("invalid option: LZW output not supported: ", bad);
            case 'a':
                bail("invalid option: ascii conversion not supported: ", bad);
            case 'b':  get = 1;  break;
            case 'c':  g.pipeout = 1;  break;
            case 'd':  if (!g.decode) g.headis >>= 2;  g.decode = 1;  break;
            case 'f':  g.force = 1;  break;
            case 'h':  help();  break;
            case 'i':  g.setdict = 0;  break;
            case 'k':  g.keep = 1;  break;
            case 'l':  g.list = 1;  break;
            case 'n':  g.headis &= ~5;  break;
            case 'p':  get = 2;  break;
            case 'q':  g.verbosity = 0;  break;
            case 'r':  g.recurse = 1;  break;
            case 't':  g.decode = 2;  break;
            case 'v':  g.verbosity++;  break;
            case 'z':  g.form = 1;  g.sufx = ".zz";  break;
            default:
                bail("invalid option: ", bad);
            }
        } while (*++arg);
        if (*arg == 0)
            return 0;
    }

    /* process option parameter for -b, -p, -S, -I, or -M */
    if (get) {
        size_t n;

        if (get == 1) {
            n = num(arg);
            g.block = n << 10;                  /* chunk size */
            if (g.block < DICT)
                bail("block size too small (must be >= 32K)", "");
            if (n != g.block >> 10 ||
                OUTPOOL(g.block) < g.block ||
                (ssize_t)OUTPOOL(g.block) < 0 ||
                g.block > (1UL << 29))          /* limited by append_len() */
                bail("block size too large: ", arg);
            new_opts();
        }
        else if (get == 2) {
            n = num(arg);
            g.procs = (int)n;                   /* # processes */
            if (g.procs < 1)
                bail("invalid number of processes: ", arg);
            if ((size_t)g.procs != n || INBUFS(g.procs) < 1)
                bail("too many processes: ", arg);
#ifdef NOTHREAD
            if (g.procs > 1)
                bail("compiled without threads", "");
#endif
            new_opts();
        }
        else if (get == 3)
            g.sufx = arg;                       /* gz suffix */
        else if (get == 4)
            g.zopts.numiterations = num(arg);   /* optimization iterations */
        else if (get == 5)
            g.zopts.blocksplittingmax = num(arg);   /* max block splits */
        get = 0;
        return 0;
    }

    /* neither an option nor parameter */
    return 1;
}

Project: pigz
File: pigz.c

local int outb(void *desc, unsigned char *buf, unsigned len)
{
#ifndef NOTHREAD
    static thread *wr, *ch;

    if (g.procs > 1) {
        /* if first time, initialize state and launch threads */
        if (outb_write_more == NULL) {
            outb_write_more = new_lock(0);
            outb_check_more = new_lock(0);
            wr = launch(outb_write, NULL);
            ch = launch(outb_check, NULL);
        }

        /* wait for previous write and check threads to complete */
        possess(outb_check_more);
        wait_for(outb_check_more, TO_BE, 0);
        possess(outb_write_more);
        wait_for(outb_write_more, TO_BE, 0);

        /* copy the output and alert the worker bees */
        out_len = len;
        g.out_tot += len;
        memcpy(out_copy, buf, len);
        twist(outb_write_more, TO, 1);
        twist(outb_check_more, TO, 1);

        /* if requested with len == 0, clean up -- terminate and join write and
           check threads, free lock */
        if (len == 0) {
            join(ch);
            join(wr);
            free_lock(outb_check_more);
            free_lock(outb_write_more);
            outb_write_more = NULL;
        }

        /* return for more decompression while last buffer is being written
           and having its check value calculated -- we wait for those to finish
           the next time this function is called */
        return 0;
    }
#endif

    (void)desc;

    /* if just one process or no threads, then do it without threads */
    if (len) {
        if (g.decode == 1)
            writen(g.outd, buf, len);
        g.out_check = CHECK(g.out_check, buf, len);
        g.out_tot += len;
    }
    return 0;
}

Project: pigz
File: pigz.c

local void outb_write(void *dummy)
{
    size_t len;

    (void)dummy;

    Trace(("-- launched decompress write thread"));
    do {
        possess(outb_write_more);
        wait_for(outb_write_more, TO_BE, 1);
        len = out_len;
        if (len && g.decode == 1)
            writen(g.outd, out_copy, len);
        Trace(("-- decompress wrote %lu bytes", len));
        twist(outb_write_more, TO, 0);
    } while (len);
    Trace(("-- exited decompress write thread"));
}

Project: pigz
File: pigz.c

local void parallel_compress(void)
{
    long seq;                       /* sequence number */
    struct space *curr;             /* input data to compress */
    struct space *next;             /* input data that follows curr */
    struct space *hold;             /* input data that follows next */
    struct space *dict;             /* dictionary for next compression */
    struct job *job;                /* job for compress, then write */
    int more;                       /* true if more input to read */
    unsigned hash;                  /* hash for rsyncable */
    unsigned char *scan;            /* next byte to compute hash on */
    unsigned char *end;             /* after end of data to compute hash on */
    unsigned char *last;            /* position after last hit */
    size_t left;                    /* last hit in curr to end of curr */
    size_t len;                     /* for various length computations */

    /* if first time or after an option change, setup the job lists */
    setup_jobs();

    /* start write thread */
    writeth = launch(write_thread, NULL);

    /* read from input and start compress threads (write thread will pick up
       the output of the compress threads) */
    seq = 0;
    next = get_space(&in_pool);
    next->len = readn(g.ind, next->buf, next->size);
    hold = NULL;
    dict = NULL;
    scan = next->buf;
    hash = RSYNCHIT;
    left = 0;
    do {
        /* create a new job */
        job = MALLOC(sizeof(struct job));
        if (job == NULL)
            bail("not enough memory", "");
        job->calc = new_lock(0);

        /* update input spaces */
        curr = next;
        next = hold;
        hold = NULL;

        /* get more input if we don't already have some */
        if (next == NULL) {
            next = get_space(&in_pool);
            next->len = readn(g.ind, next->buf, next->size);
        }

        /* if rsyncable, generate block lengths and prepare curr for job to
           likely have less than size bytes (up to the last hash hit) */
        job->lens = NULL;
        if (g.rsync && curr->len) {
            /* compute the hash function starting where we last left off to
               cover either size bytes or to EOF, whichever is less, through
               the data in curr (and in the next loop, through next) -- save
               the block lengths resulting from the hash hits in the job->lens
               list */
            if (left == 0) {
                /* scan is in curr */
                last = curr->buf;
                end = curr->buf + curr->len;
                while (scan < end) {
                    hash = ((hash << 1) ^ *scan++) & RSYNCMASK;
                    if (hash == RSYNCHIT) {
                        len = scan - last;
                        append_len(job, len);
                        last = scan;
                    }
                }

                /* continue scan in next */
                left = scan - last;
                scan = next->buf;
            }

            /* scan in next for enough bytes to fill curr, or what is available
               in next, whichever is less (if next isn't full, then we're at
               the end of the file) -- the bytes in curr since the last hit,
               stored in left, counts towards the size of the first block */
            last = next->buf;
            len = curr->size - curr->len;
            if (len > next->len)
                len = next->len;
            end = next->buf + len;
            while (scan < end) {
                hash = ((hash << 1) ^ *scan++) & RSYNCMASK;
                if (hash == RSYNCHIT) {
                    len = (scan - last) + left;
                    left = 0;
                    append_len(job, len);
                    last = scan;
                }
            }
            append_len(job, 0);

            /* create input in curr for job up to last hit or entire buffer if
               no hits at all -- save remainder in next and possibly hold */
            len = (job->lens->len == 1 ? scan : last) - next->buf;
            if (len) {
                /* got hits in next, or no hits in either -- copy to curr */
                memcpy(curr->buf + curr->len, next->buf, len);
                curr->len += len;
                memmove(next->buf, next->buf + len, next->len - len);
                next->len -= len;
                scan -= len;
                left = 0;
            }
            else if (job->lens->len != 1 && left && next->len) {
                /* had hits in curr, but none in next, and last hit in curr
                   wasn't right at the end, so we have input there to save --
                   use curr up to the last hit, save the rest, moving next to
                   hold */
                hold = next;
                next = get_space(&in_pool);
                memcpy(next->buf, curr->buf + (curr->len - left), left);
                next->len = left;
                curr->len -= left;
            }
            else {
                /* else, last match happened to be right at the end of curr,
                   or we're at the end of the input compressing the rest */
                left = 0;
            }
        }

        /* compress curr->buf to curr->len -- compress thread will drop curr */
        job->in = curr;

        /* set job->more if there is more to compress after curr */
        more = next->len != 0;
        job->more = more;

        /* provide dictionary for this job, prepare dictionary for next job */
        job->out = dict;
        if (more && g.setdict) {
            if (curr->len >= DICT || job->out == NULL) {
                dict = curr;
                use_space(dict);
            }
            else {
                dict = get_space(&dict_pool);
                len = DICT - curr->len;
                memcpy(dict->buf, job->out->buf + (job->out->len - len), len);
                memcpy(dict->buf + len, curr->buf, curr->len);
                dict->len = DICT;
            }
        }

        /* preparation of job is complete */
        job->seq = seq;
        Trace(("-- read #%ld%s", seq, more ? "" : " (last)"));
        if (++seq < 1)
            bail("input too long: ", g.inf);

        /* start another compress thread if needed */
        if (cthreads < seq && cthreads < g.procs) {
            (void)launch(compress_thread, NULL);
            cthreads++;
        }

        /* put job at end of compress list, let all the compressors know */
        possess(compress_have);
        job->next = NULL;
        *compress_tail = job;
        compress_tail = &(job->next);
        twist(compress_have, BY, +1);
    } while (more);
    drop_space(next);

    /* wait for the write thread to complete (we leave the compress threads out
       there and waiting in case there is another stream to compress) */
    join(writeth);
    writeth = NULL;
    Trace(("-- write thread joined"));
}

Project: pigz
File: pigz.c

local unsigned long put_header(void)
{
    unsigned long len;
    unsigned char head[30];

    if (g.form > 1) {               /* zip */
        /* write local header */
        PUT4L(head, 0x04034b50UL);  /* local header signature */
        PUT2L(head + 4, 20);        /* version needed to extract (2.0) */
        PUT2L(head + 6, 8);         /* flags: data descriptor follows data */
        PUT2L(head + 8, 8);         /* deflate */
        PUT4L(head + 10, time2dos(g.mtime));
        PUT4L(head + 14, 0);        /* crc (not here) */
        PUT4L(head + 18, 0);        /* compressed length (not here) */
        PUT4L(head + 22, 0);        /* uncompressed length (not here) */
        PUT2L(head + 26, g.name == NULL ? 1 :   /* length of name */
                                          strlen(g.name));
        PUT2L(head + 28, 9);        /* length of extra field (see below) */
        writen(g.outd, head, 30);   /* write local header */
        len = 30;

        /* write file name (use "-" for stdin) */
        if (g.name == NULL)
            writen(g.outd, (unsigned char *)"-", 1);
        else
            writen(g.outd, (unsigned char *)g.name, strlen(g.name));
        len += g.name == NULL ? 1 : strlen(g.name);

        /* write extended timestamp extra field block (9 bytes) */
        PUT2L(head, 0x5455);        /* extended timestamp signature */
        PUT2L(head + 2, 5);         /* number of data bytes in this block */
        head[4] = 1;                /* flag presence of mod time */
        PUT4L(head + 5, g.mtime);   /* mod time */
        writen(g.outd, head, 9);    /* write extra field block */
        len += 9;
    }
    else if (g.form) {              /* zlib */
        head[0] = 0x78;             /* deflate, 32K window */
        head[1] = (g.level >= 9 ? 3 :
                   (g.level == 1 ? 0 :
                    (g.level >= 6 || g.level == Z_DEFAULT_COMPRESSION ?
                        1 : 2))) << 6;
        head[1] += 31 - (((head[0] << 8) + head[1]) % 31);
        writen(g.outd, head, 2);
        len = 2;
    }
    else {                          /* gzip */
        head[0] = 31;
        head[1] = 139;
        head[2] = 8;                /* deflate */
        head[3] = g.name != NULL ? 8 : 0;
        PUT4L(head + 4, g.mtime);
        head[8] = g.level >= 9 ? 2 : (g.level == 1 ? 4 : 0);
        head[9] = 3;                /* unix */
        writen(g.outd, head, 10);
        len = 10;
        if (g.name != NULL)
            writen(g.outd, (unsigned char *)g.name, strlen(g.name) + 1);
        if (g.name != NULL)
            len += strlen(g.name) + 1;
    }
    return len;
}

Project: pigz
File: pigz.c

local void put_trailer(unsigned long ulen, unsigned long clen,
                       unsigned long check, unsigned long head)
{
    unsigned char tail[46];

    if (g.form > 1) {               /* zip */
        unsigned long cent;

        /* write data descriptor (as promised in local header) */
        PUT4L(tail, 0x08074b50UL);
        PUT4L(tail + 4, check);
        PUT4L(tail + 8, clen);
        PUT4L(tail + 12, ulen);
        writen(g.outd, tail, 16);

        /* write central file header */
        PUT4L(tail, 0x02014b50UL);  /* central header signature */
        tail[4] = 63;               /* obeyed version 6.3 of the zip spec */
        tail[5] = 255;              /* ignore external attributes */
        PUT2L(tail + 6, 20);        /* version needed to extract (2.0) */
        PUT2L(tail + 8, 8);         /* data descriptor is present */
        PUT2L(tail + 10, 8);        /* deflate */
        PUT4L(tail + 12, time2dos(g.mtime));
        PUT4L(tail + 16, check);    /* crc */
        PUT4L(tail + 20, clen);     /* compressed length */
        PUT4L(tail + 24, ulen);     /* uncompressed length */
        PUT2L(tail + 28, g.name == NULL ? 1 :   /* length of name */
                                          strlen(g.name));
        PUT2L(tail + 30, 9);        /* length of extra field (see below) */
        PUT2L(tail + 32, 0);        /* no file comment */
        PUT2L(tail + 34, 0);        /* disk number 0 */
        PUT2L(tail + 36, 0);        /* internal file attributes */
        PUT4L(tail + 38, 0);        /* external file attributes (ignored) */
        PUT4L(tail + 42, 0);        /* offset of local header */
        writen(g.outd, tail, 46);   /* write central file header */
        cent = 46;

        /* write file name (use "-" for stdin) */
        if (g.name == NULL)
            writen(g.outd, (unsigned char *)"-", 1);
        else
            writen(g.outd, (unsigned char *)g.name, strlen(g.name));
        cent += g.name == NULL ? 1 : strlen(g.name);

        /* write extended timestamp extra field block (9 bytes) */
        PUT2L(tail, 0x5455);        /* extended timestamp signature */
        PUT2L(tail + 2, 5);         /* number of data bytes in this block */
        tail[4] = 1;                /* flag presence of mod time */
        PUT4L(tail + 5, g.mtime);   /* mod time */
        writen(g.outd, tail, 9);    /* write extra field block */
        cent += 9;

        /* write end of central directory record */
        PUT4L(tail, 0x06054b50UL);  /* end of central directory signature */
        PUT2L(tail + 4, 0);         /* number of this disk */
        PUT2L(tail + 6, 0);         /* disk with start of central directory */
        PUT2L(tail + 8, 1);         /* number of entries on this disk */
        PUT2L(tail + 10, 1);        /* total number of entries */
        PUT4L(tail + 12, cent);     /* size of central directory */
        PUT4L(tail + 16, head + clen + 16); /* offset of central directory */
        PUT2L(tail + 20, 0);        /* no zip file comment */
        writen(g.outd, tail, 22);   /* write end of central directory record */
    }
    else if (g.form) {              /* zlib */
        PUT4M(tail, check);
        writen(g.outd, tail, 4);
    }
    else {                          /* gzip */
        PUT4L(tail, check);
        PUT4L(tail + 4, ulen);
        writen(g.outd, tail, 8);
    }
}

Project: pigz
File: pigz.c

local int read_extra(unsigned len, int save)
{
    unsigned id, size, tmp2;
    unsigned long tmp4;

    /* process extra blocks */
    while (len >= 4) {
        id = GET2();
        size = GET2();
        if (g.in_eof)
            return -1;
        len -= 4;
        if (size > len)
            break;
        len -= size;
        if (id == 0x0001) {
            /* Zip64 Extended Information Extra Field */
            if (g.zip_ulen == LOW32 && size >= 8) {
                g.zip_ulen = GET4();
                SKIP(4);
                size -= 8;
            }
            if (g.zip_clen == LOW32 && size >= 8) {
                g.zip_clen = GET4();
                SKIP(4);
                size -= 8;
            }
        }
        if (save) {
            if ((id == 0x000d || id == 0x5855) && size >= 8) {
                /* PKWare Unix or Info-ZIP Type 1 Unix block */
                SKIP(4);
                g.stamp = tolong(GET4());
                size -= 8;
            }
            if (id == 0x5455 && size >= 5) {
                /* Extended Timestamp block */
                size--;
                if (GET() & 1) {
                    g.stamp = tolong(GET4());
                    size -= 4;
                }
            }
        }
        SKIP(size);
    }
    SKIP(len);
    return 0;
}

Project: pigz
File: pigz.c

local size_t readn(int desc, unsigned char *buf, size_t len)
{
    ssize_t ret;
    size_t got;

    got = 0;
    while (len) {
        ret = read(desc, buf, len);
        if (ret < 0)
            bail("read error on ", g.inf);
        if (ret == 0)
            break;
        buf += ret;
        len -= ret;
        got += ret;
    }
    return got;
}

Project: pigz
File: pigz.c

local void *realloc_track(struct mem_track_s *mem, void *ptr, size_t size)
{
    size_t was;

    if (ptr == NULL)
        return malloc_track(mem, size);
    was = MALLOC_SIZE(ptr);
    ptr = realloc(ptr, size);
    if (ptr != NULL) {
        size = MALLOC_SIZE(ptr);
        mem_track_grab(mem);
        mem->size -= was;
        mem->size += size;
        if (mem->size > mem->max)
            mem->max = mem->size;
        mem_track_drop(mem);
    }
    return ptr;
}

Project: pigz
File: pigz.c

local void setup_jobs(void)
{
    /* set up only if not already set up*/
    if (compress_have != NULL)
        return;

    /* allocate locks and initialize lists */
    compress_have = new_lock(0);
    compress_head = NULL;
    compress_tail = &compress_head;
    write_first = new_lock(-1);
    write_head = NULL;

    /* initialize buffer pools (initial size for out_pool not critical, since
       buffers will be grown in size if needed -- initial size chosen to make
       this unlikely -- same for lens_pool) */
    new_pool(&in_pool, g.block, INBUFS(g.procs));
    new_pool(&out_pool, OUTPOOL(g.block), -1);
    new_pool(&dict_pool, DICT, -1);
    new_pool(&lens_pool, g.block >> (RSYNCBITS - 1), -1);
}

Project: pigz
File: pigz.c

local void show_info(int method, unsigned long check, off_t len, int cont)
{
    size_t max;             /* maximum name length for current verbosity */
    size_t n;               /* name length without suffix */
    time_t now;             /* for getting current year */
    char mod[26];           /* modification time in text */
    char tag[NAMEMAX1+1];   /* header or file name, possibly truncated */

    /* create abbreviated name from header file name or actual file name */
    max = g.verbosity > 1 ? NAMEMAX2 : NAMEMAX1;
    memset(tag, 0, max + 1);
    if (cont)
        strncpy(tag, "<...>", max + 1);
    else if (g.hname == NULL) {
        n = strlen(g.inf) - compressed_suffix(g.inf);
        strncpy(tag, g.inf, n > max + 1 ? max + 1 : n);
        if (strcmp(g.inf + n, ".tgz") == 0 && n < max + 1)
            strncpy(tag + n, ".tar", max + 1 - n);
    }
    else
        strncpy(tag, g.hname, max + 1);
    if (tag[max])
        strcpy(tag + max - 3, "...");

    /* convert time stamp to text */
    if (g.stamp) {
        strcpy(mod, ctime(&g.stamp));
        now = time(NULL);
        if (strcmp(mod + 20, ctime(&now) + 20) != 0)
            strcpy(mod + 11, mod + 19);
    }
    else
        strcpy(mod + 4, "------ -----");
    mod[16] = 0;

    /* if first time, print header */
    if (g.first) {
        if (g.verbosity > 1)
            fputs("method    check    timestamp    ", stdout);
        if (g.verbosity > 0)
            puts("compressed   original reduced  name");
        g.first = 0;
    }

    /* print information */
    if (g.verbosity > 1) {
        if (g.form == 3 && !g.decode)
            printf("zip%3d  --------  %s  ", method, mod + 4);
        else if (g.form > 1)
            printf("zip%3d  %08lx  %s  ", method, check, mod + 4);
        else if (g.form == 1)
            printf("zlib%2d  %08lx  %s  ", method, check, mod + 4);
        else if (method == 257)
            printf("lzw     --------  %s  ", mod + 4);
        else
            printf("gzip%2d  %08lx  %s  ", method, check, mod + 4);
    }
    if (g.verbosity > 0) {
        if ((g.form == 3 && !g.decode) ||
            (method == 8 && g.in_tot > (len + (len >> 10) + 12)) ||
            (method == 257 && g.in_tot > len + (len >> 1) + 3))
#if __STDC_VERSION__-0 >= 199901L || __GNUC__-0 >= 3
            printf("%10jd %10jd?  unk    %s\n",
                   (intmax_t)g.in_tot, (intmax_t)len, tag);
        else
            printf("%10jd %10jd %6.1f%%  %s\n",
                   (intmax_t)g.in_tot, (intmax_t)len,
                   len == 0 ? 0 : 100 * (len - g.in_tot)/(double)len,
                   tag);
#else
            printf(sizeof(off_t) == sizeof(long) ?
                   "%10ld %10ld?  unk    %s\n" : "%10lld %10lld?  unk    %s\n",
                   g.in_tot, len, tag);
        else
            printf(sizeof(off_t) == sizeof(long) ?
                   "%10ld %10ld %6.1f%%  %s\n" : "%10lld %10lld %6.1f%%  %s\n",
                   g.in_tot, len,
                   len == 0 ? 0 : 100 * (len - g.in_tot)/(double)len,
                   tag);
#endif
    }
}

Project: pigz
File: pigz.c

local void single_compress(int reset)
{
    size_t got;                     /* amount of data in in[] */
    size_t more;                    /* amount of data in next[] (0 if eof) */
    size_t start;                   /* start of data in next[] */
    size_t have;                    /* bytes in current block for -i */
    size_t hist;                    /* offset of permitted history */
    int fresh;                      /* if true, reset compression history */
    unsigned hash;                  /* hash for rsyncable */
    unsigned char *scan;            /* pointer for hash computation */
    size_t left;                    /* bytes left to compress after hash hit */
    unsigned long head;             /* header length */
    unsigned long ulen;             /* total uncompressed size (overflow ok) */
    unsigned long clen;             /* total compressed size (overflow ok) */
    unsigned long check;            /* check value of uncompressed data */
    static unsigned out_size;       /* size of output buffer */
    static unsigned char *in, *next, *out;  /* reused i/o buffers */
    static z_stream *strm = NULL;   /* reused deflate structure */

    /* if requested, just release the allocations and return */
    if (reset) {
        if (strm != NULL) {
            (void)deflateEnd(strm);
            FREE(strm);
            FREE(out);
            FREE(next);
            FREE(in);
            strm = NULL;
        }
        return;
    }

    /* initialize the deflate structure if this is the first time */
    if (strm == NULL) {
        out_size = g.block > MAXP2 ? MAXP2 : (unsigned)g.block;
        if ((in = MALLOC(g.block + DICT)) == NULL ||
            (next = MALLOC(g.block + DICT)) == NULL ||
            (out = MALLOC(out_size)) == NULL ||
            (strm = MALLOC(sizeof(z_stream))) == NULL)
            bail("not enough memory", "");
        strm->zfree = ZFREE;
        strm->zalloc = ZALLOC;
        strm->opaque = OPAQUE;
        if (deflateInit2(strm, 6, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) !=
                         Z_OK)
            bail("not enough memory", "");
    }

    /* write header */
    head = put_header();

    /* set compression level in case it changed */
    if (g.level <= 9) {
        (void)deflateReset(strm);
        (void)deflateParams(strm, g.level, Z_DEFAULT_STRATEGY);
    }

    /* do raw deflate and calculate check value */
    got = 0;
    more = readn(g.ind, next, g.block);
    ulen = (unsigned long)more;
    start = 0;
    hist = 0;
    clen = 0;
    have = 0;
    check = CHECK(0L, Z_NULL, 0);
    hash = RSYNCHIT;
    do {
        /* get data to compress, see if there is any more input */
        if (got == 0) {
            scan = in;  in = next;  next = scan;
            strm->next_in = in + start;
            got = more;
            if (g.level > 9) {
                left = start + more - hist;
                if (left > DICT)
                    left = DICT;
                memcpy(next, in + ((start + more) - left), left);
                start = left;
                hist = 0;
            }
            else
                start = 0;
            more = readn(g.ind, next + start, g.block);
            ulen += (unsigned long)more;
        }

        /* if rsyncable, compute hash until a hit or the end of the block */
        left = 0;
        if (g.rsync && got) {
            scan = strm->next_in;
            left = got;
            do {
                if (left == 0) {
                    /* went to the end -- if no more or no hit in size bytes,
                       then proceed to do a flush or finish with got bytes */
                    if (more == 0 || got == g.block)
                        break;

                    /* fill in[] with what's left there and as much as possible
                       from next[] -- set up to continue hash hit search */
                    if (g.level > 9) {
                        left = (strm->next_in - in) - hist;
                        if (left > DICT)
                            left = DICT;
                    }
                    memmove(in, strm->next_in - left, left + got);
                    hist = 0;
                    strm->next_in = in + left;
                    scan = in + left + got;
                    left = more > g.block - got ? g.block - got : more;
                    memcpy(scan, next + start, left);
                    got += left;
                    more -= left;
                    start += left;

                    /* if that emptied the next buffer, try to refill it */
                    if (more == 0) {
                        more = readn(g.ind, next, g.block);
                        ulen += (unsigned long)more;
                        start = 0;
                    }
                }
                left--;
                hash = ((hash << 1) ^ *scan++) & RSYNCMASK;
            } while (hash != RSYNCHIT);
            got -= left;
        }

        /* clear history for --independent option */
        fresh = 0;
        if (!g.setdict) {
            have += got;
            if (have > g.block) {
                fresh = 1;
                have = got;
            }
        }

        if (g.level <= 9) {
            /* clear history if requested */
            if (fresh)
                (void)deflateReset(strm);

            /* compress MAXP2-size chunks in case unsigned type is small */
            while (got > MAXP2) {
                strm->avail_in = MAXP2;
                check = CHECK(check, strm->next_in, strm->avail_in);
                DEFLATE_WRITE(Z_NO_FLUSH);
                got -= MAXP2;
            }

            /* compress the remainder, emit a block, finish if end of input */
            strm->avail_in = (unsigned)got;
            got = left;
            check = CHECK(check, strm->next_in, strm->avail_in);
            if (more || got) {
#if ZLIB_VERNUM >= 0x1260
                int bits;

                DEFLATE_WRITE(Z_BLOCK);
                (void)deflatePending(strm, Z_NULL, &bits);
                if (bits & 1)
                    DEFLATE_WRITE(Z_SYNC_FLUSH);
                else if (bits & 7) {
                    do {
                        bits = deflatePrime(strm, 10, 2);
                        assert(bits == Z_OK);
                        (void)deflatePending(strm, Z_NULL, &bits);
                    } while (bits & 7);
                    DEFLATE_WRITE(Z_NO_FLUSH);
                }
#else
                DEFLATE_WRITE(Z_SYNC_FLUSH);
#endif
            }
            else
                DEFLATE_WRITE(Z_FINISH);
        }
        else {
            /* compress got bytes using zopfli, bring to byte boundary */
            unsigned char bits, *out;
            size_t outsize, off;

            /* discard history if requested */
            off = strm->next_in - in;
            if (fresh)
                hist = off;

            out = NULL;
            outsize = 0;
            bits = 0;
            ZopfliDeflatePart(&g.zopts, 2, !(more || left),
                              in + hist, off - hist, (off - hist) + got,
                              &bits, &out, &outsize);
            bits &= 7;
            if ((more || left) && bits) {
                if (bits & 1) {
                    writen(g.outd, out, outsize);
                    if (bits == 7)
                        writen(g.outd, (unsigned char *)"\0", 1);
                    writen(g.outd, (unsigned char *)"\0\0\xff\xff", 4);
                }
                else {
                    assert(outsize > 0);
                    writen(g.outd, out, outsize - 1);
                    do {
                        out[outsize - 1] += 2 << bits;
                        writen(g.outd, out + outsize - 1, 1);
                        out[outsize - 1] = 0;
                        bits += 2;
                    } while (bits < 8);
                    writen(g.outd, out + outsize - 1, 1);
                }
            }
            else
                writen(g.outd, out, outsize);
            free(out);
            while (got > MAXP2) {
                check = CHECK(check, strm->next_in, MAXP2);
                strm->next_in += MAXP2;
                got -= MAXP2;
            }
            check = CHECK(check, strm->next_in, (unsigned)got);
            strm->next_in += got;
            got = left;
        }

        /* do until no more input */
    } while (more || got);

    /* write trailer */
    put_trailer(ulen, clen, check, head);
}

Project: pigz
File: pigz.c

local unsigned long time2dos(time_t t)
{
    struct tm *tm;
    unsigned long dos;

    if (t == 0)
        t = time(NULL);
    tm = localtime(&t);
    if (tm->tm_year < 80 || tm->tm_year > 207)
        return 0;
    dos = (tm->tm_year - 80) << 25;
    dos += (tm->tm_mon + 1) << 21;
    dos += tm->tm_mday << 16;
    dos += tm->tm_hour << 11;
    dos += tm->tm_min << 5;
    dos += (tm->tm_sec + 1) >> 1;   /* round to double-seconds */
    return dos;
}

Project: pigz
File: pigz.c

local long tolong(unsigned long val)
{
    return (long)(val & 0x7fffffffUL) - (long)(val & 0x80000000UL);
}

Project: pigz
File: pigz.c

local void unlzw(void)
{
    unsigned bits;              /* current bits per code (9..16) */
    unsigned mask;              /* mask for current bits codes = (1<<bits)-1 */
    bits_t buf;                 /* bit buffer (need 23 bits) */
    unsigned left;              /* bits left in buf (0..7 after code pulled) */
    off_t mark;                 /* offset where last change in bits began */
    unsigned code;              /* code, table traversal index */
    unsigned max;               /* maximum bits per code for this stream */
    unsigned flags;             /* compress flags, then block compress flag */
    unsigned end;               /* last valid entry in prefix/suffix tables */
    unsigned prev;              /* previous code */
    unsigned final;             /* last character written for previous code */
    unsigned stack;             /* next position for reversed string */
    unsigned outcnt;            /* bytes in output buffer */
    /* memory for unlzw() -- the first 256 entries of prefix[] and suffix[] are
       never used, so could have offset the index but it's faster to waste a
       little memory */
    uint_least16_t prefix[65536];       /* index to LZW prefix string */
    unsigned char suffix[65536];        /* one-character LZW suffix */
    unsigned char match[65280 + 2];     /* buffer for reversed match */

    /* process remainder of compress header -- a flags byte */
    g.out_tot = 0;
    if (NOMORE())
        bail("lzw premature end: ", g.inf);
    flags = NEXT();
    if (flags & 0x60)
        bail("unknown lzw flags set: ", g.inf);
    max = flags & 0x1f;
    if (max < 9 || max > 16)
        bail("lzw bits out of range: ", g.inf);
    if (max == 9)                           /* 9 doesn't really mean 9 */
        max = 10;
    flags &= 0x80;                          /* true if block compress */

    /* mark the start of the compressed data for computing the first flush */
    mark = g.in_tot - g.in_left;

    /* clear table, start at nine bits per symbol */
    bits = 9;
    mask = 0x1ff;
    end = flags ? 256 : 255;

    /* set up: get first 9-bit code, which is the first decompressed byte, but
       don't create a table entry until the next code */
    if (NOMORE())                           /* no compressed data is ok */
        return;
    buf = NEXT();
    if (NOMORE())
        bail("lzw premature end: ", g.inf); /* need at least nine bits */
    buf += NEXT() << 8;
    final = prev = buf & mask;              /* code */
    buf >>= bits;
    left = 16 - bits;
    if (prev > 255)
        bail("invalid lzw code: ", g.inf);
    out_buf[0] = final;                     /* write first decompressed byte */
    outcnt = 1;

    /* decode codes */
    stack = 0;
    for (;;) {
        /* if the table will be full after this, increment the code size */
        if (end >= mask && bits < max) {
            /* flush unused input bits and bytes to next 8*bits bit boundary
               (this is a vestigial aspect of the compressed data format
               derived from an implementation that made use of a special VAX
               machine instruction!) */
            {
                unsigned rem = ((g.in_tot - g.in_left) - mark) % bits;
                if (rem)
                    rem = bits - rem;
                while (rem > g.in_left) {
                    rem -= g.in_left;
                    if (load() == 0)
                        break;
                }
                g.in_left -= rem;
                g.in_next += rem;
            }
            buf = 0;
            left = 0;

            /* mark this new location for computing the next flush */
            mark = g.in_tot - g.in_left;

            /* go to the next number of bits per symbol */
            bits++;
            mask <<= 1;
            mask++;
        }

        /* get a code of bits bits */
        if (NOMORE())
            break;                          /* end of compressed data */
        buf += (bits_t)(NEXT()) << left;
        left += 8;
        if (left < bits) {
            if (NOMORE())
                bail("lzw premature end: ", g.inf);
            buf += (bits_t)(NEXT()) << left;
            left += 8;
        }
        code = buf & mask;
        buf >>= bits;
        left -= bits;

        /* process clear code (256) */
        if (code == 256 && flags) {
            /* flush unused input bits and bytes to next 8*bits bit boundary */
            {
                unsigned rem = ((g.in_tot - g.in_left) - mark) % bits;
                if (rem)
                    rem = bits - rem;
                while (rem > g.in_left) {
                    rem -= g.in_left;
                    if (load() == 0)
                        break;
                }
                g.in_left -= rem;
                g.in_next += rem;
            }
            buf = 0;
            left = 0;

            /* mark this new location for computing the next flush */
            mark = g.in_tot - g.in_left;

            /* go back to nine bits per symbol */
            bits = 9;                       /* initialize bits and mask */
            mask = 0x1ff;
            end = 255;                      /* empty table */
            continue;                       /* get next code */
        }

        /* special code to reuse last match */
        {
            unsigned temp = code;           /* save the current code */
            if (code > end) {
                /* Be picky on the allowed code here, and make sure that the
                   code we drop through (prev) will be a valid index so that
                   random input does not cause an exception. */
                if (code != end + 1 || prev > end)
                    bail("invalid lzw code: ", g.inf);
                match[stack++] = final;
                code = prev;
            }

            /* walk through linked list to generate output in reverse order */
            while (code >= 256) {
                match[stack++] = suffix[code];
                code = prefix[code];
            }
            match[stack++] = code;
            final = code;

            /* link new table entry */
            if (end < mask) {
                end++;
                prefix[end] = prev;
                suffix[end] = final;
            }

            /* set previous code for next iteration */
            prev = temp;
        }

        /* write output in forward order */
        while (stack > OUTSIZE - outcnt) {
            while (outcnt < OUTSIZE)
                out_buf[outcnt++] = match[--stack];
            g.out_tot += outcnt;
            if (g.decode == 1)
                writen(g.outd, out_buf, outcnt);
            outcnt = 0;
        }
        do {
            out_buf[outcnt++] = match[--stack];
        } while (stack);
    }

    /* write any remaining buffered output */
    g.out_tot += outcnt;
    if (outcnt && g.decode == 1)
        writen(g.outd, out_buf, outcnt);
}

Project: pigz
File: pigz.c

local void use_space(struct space *space)
{
    possess(space->use);
    twist(space->use, BY, +1);
}

Project: pigz
File: pigz.c

local void writen(int desc, unsigned char *buf, size_t len)
{
    ssize_t ret;

    while (len) {
        ret = write(desc, buf, len);
        if (ret < 1) {
            complain("write error code %d", errno);
            bail("write error on ", g.outf);
        }
        buf += ret;
        len -= ret;
    }
}

Project: pigz
File: pigz.c

local void yarn_free(void *ptr)
{
    return free_track(&mem_track, ptr);
}

Project: pigz
File: pigz.c

local void *yarn_malloc(size_t size)
{
    return malloc_track(&mem_track, size);
}

Project: pigz
File: pigz.c

local voidpf zlib_alloc(voidpf opaque, uInt items, uInt size)
{
    return malloc_track(opaque, items * (size_t)size);
}

Project: pigz
File: pigz.c

local void zlib_free(voidpf opaque, voidpf address)
{
    free_track(opaque, address);
}