Commit message: jpc_math: split jpc_firstone() in int/jpc_fix_t overloads

Fixes CVE-2018-9055 (denial of service via a reachable assertion due
to integer overflow).

Based on a patch from Fridrich Strba <fstrba@suse.com>.  Instead of
switching to `int_fast32_t`, this patch splits jpc_firstone() into two
overloads, one for `int` and one for `jpc_fix_t`.  This is safer
against future changes on `jpc_fix_t`.  To avoid the overhead of 64
bit integer math on 32 bit CPUs, this leaves the `int` overload
around.

Closes https://github.com/jasper-maint/jasper/issues/9
CVE: CVE-2018-9055
CWE: CWE-617
Vulnerability description: JasPer 2.0.14 allows denial of service via a reachable assertion in the function jpc_firstone in libjasper/jpc/jpc_math.c.

Project: jasper
File: jpc_t1enc.c

static int jpc_encrawsigpass(jpc_bitstream_t *out, int bitpos, int vcausalflag, jas_matrix_t *flags,
  jas_matrix_t *data, int term, long *nmsedec)
{
	int i;
	int j;
	int k;
	int one;
	int vscanlen;
	int width;
	int height;
	int frowstep;
	int drowstep;
	int fstripestep;
	int dstripestep;
	jpc_fix_t *fstripestart;
	jpc_fix_t *dstripestart;
	jpc_fix_t *fp;
	jpc_fix_t *dp;
	jpc_fix_t *fvscanstart;
	jpc_fix_t *dvscanstart;

	*nmsedec = 0;
	width = jas_matrix_numcols(data);
	height = jas_matrix_numrows(data);
	frowstep = jas_matrix_rowstep(flags);
	drowstep = jas_matrix_rowstep(data);
	fstripestep = frowstep << 2;
	dstripestep = drowstep << 2;

	one = 1 << (bitpos + JPC_NUMEXTRABITS);

	fstripestart = jas_matrix_getref(flags, 1, 1);
	dstripestart = jas_matrix_getref(data, 0, 0);
	for (i = height; i > 0; i -= 4, fstripestart += fstripestep,
	  dstripestart += dstripestep) {
		fvscanstart = fstripestart;
		dvscanstart = dstripestart;
		vscanlen = JAS_MIN(i, 4);
		for (j = width; j > 0; --j, ++fvscanstart, ++dvscanstart) {
			fp = fvscanstart;
			dp = dvscanstart;
			k = vscanlen;

			rawsigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, out, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;

			rawsigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, out, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;

			rawsigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, out, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;

			rawsigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, out, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;

		}
	}

	if (term) {
		jpc_bitstream_outalign(out, 0x2a);
	}

	return 0;
}

Project: jasper
File: jpc_t2enc.c

void jpc_restore_t2state(jpc_enc_t *enc)
{

	jpc_enc_tcmpt_t *comp;
	jpc_enc_tcmpt_t *endcomps;
	jpc_enc_rlvl_t *lvl;
	jpc_enc_rlvl_t *endlvls;
	jpc_enc_band_t *band;
	jpc_enc_band_t *endbands;
	jpc_enc_cblk_t *cblk;
	jpc_enc_cblk_t *endcblks;
	jpc_enc_tile_t *tile;
	int prcno;
	jpc_enc_prc_t *prc;

	tile = enc->curtile;

	endcomps = &tile->tcmpts[tile->numtcmpts];
	for (comp = tile->tcmpts; comp != endcomps; ++comp) {
		endlvls = &comp->rlvls[comp->numrlvls];
		for (lvl = comp->rlvls; lvl != endlvls; ++lvl) {
			if (!lvl->bands) {
				continue;
			}
			endbands = &lvl->bands[lvl->numbands];
			for (band = lvl->bands; band != endbands; ++band) {
				if (!band->data) {
					continue;
				}
				for (prcno = 0, prc = band->prcs; prcno < lvl->numprcs; ++prcno, ++prc) {
					if (!prc->cblks) {
						continue;
					}
					jpc_tagtree_copy(prc->incltree, prc->savincltree);
					jpc_tagtree_copy(prc->nlibtree, prc->savnlibtree);
					endcblks = &prc->cblks[prc->numcblks];
					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						cblk->curpass = cblk->savedcurpass;
						cblk->numencpasses = cblk->savednumencpasses;
						cblk->numlenbits = cblk->savednumlenbits;
					}
				}
			}
		}
	}
}

Project: jasper
File: jpc_t2enc.c

static int jpc_putcommacode(jpc_bitstream_t *out, int n)
{
	assert(n >= 0);

	while (--n >= 0) {
		if (jpc_bitstream_putbit(out, 1) == EOF) {
			return -1;
		}
	}
	if (jpc_bitstream_putbit(out, 0) == EOF) {
		return -1;
	}
	return 0;
}

Project: jasper
File: jpc_t1enc.c

static int getthebyte(jas_stream_t *in, long off)
{
	int c;
	long oldpos;
	oldpos = jas_stream_tell(in);
	assert(oldpos >= 0);
	jas_stream_seek(in, off, SEEK_SET);
	c = jas_stream_peekc(in);
	jas_stream_seek(in, oldpos, SEEK_SET);
	return c;
}

Project: jasper
File: jpc_t2enc.c

void jpc_init_t2state(jpc_enc_t *enc, int raflag)
{
/* It is assumed that band->numbps and cblk->numbps precomputed */

	jpc_enc_tcmpt_t *comp;
	jpc_enc_tcmpt_t *endcomps;
	jpc_enc_rlvl_t *lvl;
	jpc_enc_rlvl_t *endlvls;
	jpc_enc_band_t *band;
	jpc_enc_band_t *endbands;
	jpc_enc_cblk_t *cblk;
	jpc_enc_cblk_t *endcblks;
	jpc_enc_pass_t *pass;
	jpc_enc_pass_t *endpasses;
	jpc_tagtreenode_t *leaf;
	jpc_enc_tile_t *tile;
	int prcno;
	jpc_enc_prc_t *prc;

	tile = enc->curtile;

	endcomps = &tile->tcmpts[tile->numtcmpts];
	for (comp = tile->tcmpts; comp != endcomps; ++comp) {
		endlvls = &comp->rlvls[comp->numrlvls];
		for (lvl = comp->rlvls; lvl != endlvls; ++lvl) {
			if (!lvl->bands) {
				continue;
			}
			endbands = &lvl->bands[lvl->numbands];
			for (band = lvl->bands; band != endbands; ++band) {
				if (!band->data) {
					continue;
				}
				for (prcno = 0, prc = band->prcs; prcno < lvl->numprcs; ++prcno, ++prc) {
					if (!prc->cblks) {
						continue;
					}
					jpc_tagtree_reset(prc->incltree);
					jpc_tagtree_reset(prc->nlibtree);
					endcblks = &prc->cblks[prc->numcblks];
					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						if (jas_stream_rewind(cblk->stream)) {
							assert(0);
						}
						cblk->curpass = (cblk->numpasses > 0) ? cblk->passes : 0;
						cblk->numencpasses = 0;
						cblk->numlenbits = 3;
						cblk->numimsbs = band->numbps - cblk->numbps;
						assert(cblk->numimsbs >= 0);
						leaf = jpc_tagtree_getleaf(prc->nlibtree, cblk - prc->cblks);
						jpc_tagtree_setvalue(prc->nlibtree, leaf, cblk->numimsbs);

						if (raflag) {
							endpasses = &cblk->passes[cblk->numpasses];
							for (pass = cblk->passes; pass != endpasses; ++pass) {
								pass->lyrno = -1;
								pass->lyrno = 0;
							}
						}
					}
				}
			}
		}
	}

}

Project: jasper
File: jpc_t2enc.c

int jpc_enc_encpkt(jpc_enc_t *enc, jas_stream_t *out, int compno, int lvlno, int prcno, int lyrno)
{
	jpc_enc_tcmpt_t *comp;
	jpc_enc_rlvl_t *lvl;
	jpc_enc_band_t *band;
	jpc_enc_band_t *endbands;
	jpc_enc_cblk_t *cblk;
	jpc_enc_cblk_t *endcblks;
	jpc_bitstream_t *outb;
	jpc_enc_pass_t *pass;
	jpc_enc_pass_t *startpass;
	jpc_enc_pass_t *lastpass;
	jpc_enc_pass_t *endpass;
	jpc_enc_pass_t *endpasses;
	int i;
	int included;
	int ret;
	jpc_tagtreenode_t *leaf;
	int n;
	int t1;
	int t2;
	int adjust;
	int maxadjust;
	int datalen;
	int numnewpasses;
	int passcount;
	jpc_enc_tile_t *tile;
	jpc_enc_prc_t *prc;
	jpc_enc_cp_t *cp;
	jpc_ms_t *ms;

	JAS_DBGLOG(10, ("encoding packet begin %d %d %d %d\n", compno, lvlno,
	  prcno, lyrno));

	tile = enc->curtile;
	cp = enc->cp;

	if (cp->tcp.csty & JPC_COD_SOP) {
		if (!(ms = jpc_ms_create(JPC_MS_SOP))) {
			return -1;
		}
		ms->parms.sop.seqno = jpc_pi_getind(tile->pi);
		if (jpc_putms(out, enc->cstate, ms)) {
			return -1;
		}
		jpc_ms_destroy(ms);
	}

	if (!(outb = jpc_bitstream_sopen(out, "w+"))) {
		abort();
	}

	if (jpc_bitstream_putbit(outb, 1) == EOF) {
		return -1;
	}
	//JAS_DBGLOG(10, ("\n"));
	JAS_DBGLOG(10, ("present.\n"));

	comp = &tile->tcmpts[compno];
	lvl = &comp->rlvls[lvlno];
	endbands = &lvl->bands[lvl->numbands];
	for (band = lvl->bands; band != endbands; ++band) {
		if (!band->data) {
			continue;
		}
		prc = &band->prcs[prcno];
		if (!prc->cblks) {
			continue;
		}

		endcblks = &prc->cblks[prc->numcblks];
		for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
			if (!lyrno) {
				leaf = jpc_tagtree_getleaf(prc->nlibtree, cblk - prc->cblks);
				jpc_tagtree_setvalue(prc->nlibtree, leaf, cblk->numimsbs);
			}
			pass = cblk->curpass;
			included = (pass && pass->lyrno == lyrno);
			if (included && (!cblk->numencpasses)) {
				assert(pass->lyrno == lyrno);
				leaf = jpc_tagtree_getleaf(prc->incltree,
				  cblk - prc->cblks);
				jpc_tagtree_setvalue(prc->incltree, leaf, pass->lyrno);
			}
		}

		endcblks = &prc->cblks[prc->numcblks];
		for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
			pass = cblk->curpass;
			included = (pass && pass->lyrno == lyrno);
			if (!cblk->numencpasses) {
				leaf = jpc_tagtree_getleaf(prc->incltree,
				  cblk - prc->cblks);
				if (jpc_tagtree_encode(prc->incltree, leaf, lyrno + 1, outb) <
				  0) {
					return -1;
				}
			} else {
				if (jpc_bitstream_putbit(outb, included) == EOF) {
					return -1;
				}
			}
			JAS_DBGLOG(10, ("included=%d ", included));
			if (!included) {
				continue;
			}
			if (!cblk->numencpasses) {
				i = 1;
				leaf = jpc_tagtree_getleaf(prc->nlibtree, cblk - prc->cblks);
				for (;;) {
					if ((ret = jpc_tagtree_encode(prc->nlibtree, leaf, i,
					  outb)) < 0) {
						return -1;
					}
					if (ret) {
						break;
					}
					++i;
				}
				assert(leaf->known_ && i == leaf->value_ + 1);
			}

			endpasses = &cblk->passes[cblk->numpasses];
			startpass = pass;
			endpass = startpass;
			while (endpass != endpasses && endpass->lyrno == lyrno){
				++endpass;
			}
			numnewpasses = endpass - startpass;
			if (jpc_putnumnewpasses(outb, numnewpasses)) {
				return -1;
			}
			JAS_DBGLOG(10, ("numnewpasses=%d ", numnewpasses));

			lastpass = endpass - 1;
			n = startpass->start;
			passcount = 1;
			maxadjust = 0;
			for (pass = startpass; pass != endpass; ++pass) {
				if (pass->term || pass == lastpass) {
					datalen = pass->end - n;
					t1 = jpc_int_firstone(datalen) + 1;
					t2 = cblk->numlenbits + jpc_floorlog2(passcount);
					adjust = JAS_MAX(t1 - t2, 0);
					maxadjust = JAS_MAX(adjust, maxadjust);
					n += datalen;
					passcount = 1;
				} else {
					++passcount;
				}
			}
			if (jpc_putcommacode(outb, maxadjust)) {
				return -1;
			}
			cblk->numlenbits += maxadjust;

			lastpass = endpass - 1;
			n = startpass->start;
			passcount = 1;
			for (pass = startpass; pass != endpass; ++pass) {
				if (pass->term || pass == lastpass) {
					datalen = pass->end - n;
					assert(jpc_int_firstone(datalen) < cblk->numlenbits +
					  jpc_floorlog2(passcount));
					if (jpc_bitstream_putbits(outb, cblk->numlenbits +
					  jpc_floorlog2(passcount), datalen) == EOF) {
						return -1;
					}
					n += datalen;
					passcount = 1;
				} else {
					++passcount;
				}
			}
		}
	}

	jpc_bitstream_outalign(outb, 0);
	jpc_bitstream_close(outb);

	if (cp->tcp.csty & JPC_COD_EPH) {
		if (!(ms = jpc_ms_create(JPC_MS_EPH))) {
			return -1;
		}
		if (jpc_putms(out, enc->cstate, ms)) {
			return -1;
		}
		jpc_ms_destroy(ms);
	}

	comp = &tile->tcmpts[compno];
	lvl = &comp->rlvls[lvlno];
	endbands = &lvl->bands[lvl->numbands];
	for (band = lvl->bands; band != endbands; ++band) {
		if (!band->data) {
			continue;
		}
		prc = &band->prcs[prcno];
		if (!prc->cblks) {
			continue;
		}
		endcblks = &prc->cblks[prc->numcblks];
		for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
			pass = cblk->curpass;

			if (!pass) {
				continue;
			}
			if (pass->lyrno != lyrno) {
				assert(pass->lyrno < 0 || pass->lyrno > lyrno);
				continue;
			}

			endpasses = &cblk->passes[cblk->numpasses];
			startpass = pass;
			endpass = startpass;
			while (endpass != endpasses && endpass->lyrno == lyrno){
				++endpass;
			}
			lastpass = endpass - 1;
			numnewpasses = endpass - startpass;

			jas_stream_seek(cblk->stream, startpass->start, SEEK_SET);
			assert(jas_stream_tell(cblk->stream) == startpass->start);
			if (jas_stream_copy(out, cblk->stream, lastpass->end -
			  startpass->start)) {
				return -1;
			}
			cblk->curpass = (endpass != endpasses) ? endpass : 0;
			cblk->numencpasses += numnewpasses;

		}
	}

	JAS_DBGLOG(10, ("encoding packet end\n"));

	return 0;
}

Project: jasper
File: jpc_t1enc.c

int jpc_enc_enccblk(jpc_enc_t *enc, jas_stream_t *out, jpc_enc_tcmpt_t *tcmpt, jpc_enc_band_t *band, jpc_enc_cblk_t *cblk)
{
	jpc_enc_pass_t *pass;
	jpc_enc_pass_t *endpasses;
	int bitpos;
	int n;
	int adjust;
	int ret;
	int passtype;
	int t;
	jpc_bitstream_t *bout;
	jpc_enc_pass_t *termpass;
	jpc_enc_rlvl_t *rlvl;
	int vcausal;
	int segsym;
	int termmode;
	int c;

	bout = 0;
	rlvl = band->rlvl;

	cblk->stream = jas_stream_memopen(0, 0);
	assert(cblk->stream);
	cblk->mqenc = jpc_mqenc_create(JPC_NUMCTXS, cblk->stream);
	assert(cblk->mqenc);
	jpc_mqenc_setctxs(cblk->mqenc, JPC_NUMCTXS, jpc_mqctxs);

	cblk->numpasses = (cblk->numbps > 0) ? (3 * cblk->numbps - 2) : 0;
	if (cblk->numpasses > 0) {
		cblk->passes = jas_alloc2(cblk->numpasses, sizeof(jpc_enc_pass_t));
		assert(cblk->passes);
	} else {
		cblk->passes = 0;
	}
	endpasses = (cblk->passes) ? &cblk->passes[cblk->numpasses] : 0;
	for (pass = cblk->passes; pass != endpasses; ++pass) {
		pass->start = 0;
		pass->end = 0;
		pass->term = JPC_ISTERMINATED(pass - cblk->passes, 0, cblk->numpasses, (tcmpt->cblksty & JPC_COX_TERMALL) != 0, (tcmpt->cblksty & JPC_COX_LAZY) != 0);
		pass->type = JPC_SEGTYPE(pass - cblk->passes, 0, (tcmpt->cblksty & JPC_COX_LAZY) != 0);
		pass->lyrno = -1;
if (pass == endpasses - 1) {
assert(pass->term == 1);
	pass->term = 1;
}
	}

	cblk->flags = jas_matrix_create(jas_matrix_numrows(cblk->data) + 2,
	  jas_matrix_numcols(cblk->data) + 2);
	assert(cblk->flags);


	bitpos = cblk->numbps - 1;
	pass = cblk->passes;
	n = cblk->numpasses;
	while (--n >= 0) {

		if (pass->type == JPC_SEG_MQ) {
			/* NOP */
		} else {
			assert(pass->type == JPC_SEG_RAW);
			if (!bout) {
				bout = jpc_bitstream_sopen(cblk->stream, "w");
				assert(bout);
			}
		}

#if 1
		passtype = (pass - cblk->passes + 2) % 3;
#else
		passtype = JPC_PASSTYPE(pass - cblk->passes + 2);
#endif
		pass->start = jas_stream_tell(cblk->stream);
#if 0
assert(jas_stream_tell(cblk->stream) == jas_stream_getrwcount(cblk->stream));
#endif
		assert(bitpos >= 0);
		vcausal = (tcmpt->cblksty & JPC_COX_VSC) != 0;
		segsym = (tcmpt->cblksty & JPC_COX_SEGSYM) != 0;
		if (pass->term) {
			termmode = ((tcmpt->cblksty & JPC_COX_PTERM) ?
			  JPC_MQENC_PTERM : JPC_MQENC_DEFTERM) + 1;
		} else {
			termmode = 0;
		}
		switch (passtype) {
		case JPC_SIGPASS:
			ret = (pass->type == JPC_SEG_MQ) ? jpc_encsigpass(cblk->mqenc,
			  bitpos, band->orient, vcausal, cblk->flags,
			  cblk->data, termmode, &pass->nmsedec) :
			  jpc_encrawsigpass(bout, bitpos, vcausal, cblk->flags,
			  cblk->data, termmode, &pass->nmsedec);
			break;
		case JPC_REFPASS:
			ret = (pass->type == JPC_SEG_MQ) ? jpc_encrefpass(cblk->mqenc,
			  bitpos, vcausal, cblk->flags, cblk->data, termmode,
			  &pass->nmsedec) : jpc_encrawrefpass(bout, bitpos,
			  vcausal, cblk->flags, cblk->data, termmode,
			  &pass->nmsedec);
			break;
		case JPC_CLNPASS:
			assert(pass->type == JPC_SEG_MQ);
			ret = jpc_encclnpass(cblk->mqenc, bitpos, band->orient,
			  vcausal, segsym, cblk->flags, cblk->data, termmode,
			  &pass->nmsedec);
			break;
		default:
			assert(0);
			break;
		}

		if (pass->type == JPC_SEG_MQ) {
			if (pass->term) {
				jpc_mqenc_init(cblk->mqenc);
			}
			jpc_mqenc_getstate(cblk->mqenc, &pass->mqencstate);
			pass->end = jas_stream_tell(cblk->stream);
			if (tcmpt->cblksty & JPC_COX_RESET) {
				jpc_mqenc_setctxs(cblk->mqenc, JPC_NUMCTXS, jpc_mqctxs);
			}
		} else {
			if (pass->term) {
				if (jpc_bitstream_pending(bout)) {
					jpc_bitstream_outalign(bout, 0x2a);
				}
				jpc_bitstream_close(bout);
				bout = 0;
				pass->end = jas_stream_tell(cblk->stream);
			} else {
				pass->end = jas_stream_tell(cblk->stream) +
				  jpc_bitstream_pending(bout);
/* NOTE - This will not work.  need to adjust by # of pending output bytes */
			}
		}
#if 0
/* XXX - This assertion fails sometimes when various coding modes are used.
This seems to be harmless, but why does it happen at all? */
assert(jas_stream_tell(cblk->stream) == jas_stream_getrwcount(cblk->stream));
#endif

		pass->wmsedec = jpc_fixtodbl(band->rlvl->tcmpt->synweight) *
		  jpc_fixtodbl(band->rlvl->tcmpt->synweight) *
		  jpc_fixtodbl(band->synweight) *
		  jpc_fixtodbl(band->synweight) *
		  jpc_fixtodbl(band->absstepsize) * jpc_fixtodbl(band->absstepsize) *
		  ((double) (1 << bitpos)) * ((double)(1 << bitpos)) *
		  jpc_fixtodbl(pass->nmsedec);
		pass->cumwmsedec = pass->wmsedec;
		if (pass != cblk->passes) {
			pass->cumwmsedec += pass[-1].cumwmsedec;
		}
		if (passtype == JPC_CLNPASS) {
			--bitpos;
		}
		++pass;
	}

#if 0
dump_passes(cblk->passes, cblk->numpasses, cblk);
#endif

	n = 0;
	endpasses = (cblk->passes) ? &cblk->passes[cblk->numpasses] : 0;
	for (pass = cblk->passes; pass != endpasses; ++pass) {
		if (pass->start < n) {
			pass->start = n;
		}
		if (pass->end < n) {
			pass->end = n;
		}
		if (!pass->term) {
			termpass = pass;
			while (termpass - pass < cblk->numpasses &&
			  !termpass->term) {
				++termpass;
			}
			if (pass->type == JPC_SEG_MQ) {
				t = (pass->mqencstate.lastbyte == 0xff) ? 1 : 0;
				if (pass->mqencstate.ctreg >= 5) {
					adjust = 4 + t;
				} else {
					adjust = 5 + t;
				}
				pass->end += adjust;
			}
			if (pass->end > termpass->end) {
				pass->end = termpass->end;
			}
			if ((c = getthebyte(cblk->stream, pass->end - 1)) == EOF) {
				abort();
			}
			if (c == 0xff) {
				++pass->end;
			}
			n = JAS_MAX(n, pass->end);
		} else {
			n = JAS_MAX(n, pass->end);
		}
	}

#if 0
dump_passes(cblk->passes, cblk->numpasses, cblk);
#endif

	if (bout) {
		jpc_bitstream_close(bout);
	}

	return 0;
}

Project: jasper
File: jpc_t1enc.c

static int jpc_encrawrefpass(jpc_bitstream_t *out, int bitpos, int vcausalflag, jas_matrix_t *flags,
  jas_matrix_t *data, int term, long *nmsedec)
{
	int i;
	int j;
	int k;
	int one;
	int vscanlen;
	int width;
	int height;
	int frowstep;
	int drowstep;
	int fstripestep;
	int dstripestep;
	jpc_fix_t *fstripestart;
	jpc_fix_t *dstripestart;
	jpc_fix_t *fvscanstart;
	jpc_fix_t *dvscanstart;
	jpc_fix_t *dp;
	jpc_fix_t *fp;

	*nmsedec = 0;
	width = jas_matrix_numcols(data);
	height = jas_matrix_numrows(data);
	frowstep = jas_matrix_rowstep(flags);
	drowstep = jas_matrix_rowstep(data);
	fstripestep = frowstep << 2;
	dstripestep = drowstep << 2;

	one = 1 << (bitpos + JPC_NUMEXTRABITS);

	fstripestart = jas_matrix_getref(flags, 1, 1);
	dstripestart = jas_matrix_getref(data, 0, 0);
	for (i = height; i > 0; i -= 4, fstripestart += fstripestep,
	  dstripestart += dstripestep) {
		fvscanstart = fstripestart;
		dvscanstart = dstripestart;
		vscanlen = JAS_MIN(i, 4);
		for (j = width; j > 0; --j, ++fvscanstart, ++dvscanstart) {
			fp = fvscanstart;
			dp = dvscanstart;
			k = vscanlen;

			rawrefpass_step(fp, dp, bitpos, one, nmsedec,
			  out, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			rawrefpass_step(fp, dp, bitpos, one, nmsedec,
			  out, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			rawrefpass_step(fp, dp, bitpos, one, nmsedec,
			  out, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			rawrefpass_step(fp, dp, bitpos, one, nmsedec,
			  out, vcausalflag);

		}
	}

	if (term) {
		jpc_bitstream_outalign(out, 0x2a);
	}

	return 0;
}

Project: jasper
File: jpc_t2enc.c

void jpc_save_t2state(jpc_enc_t *enc)
{
/* stream pos in embedded T1 stream may be wrong since not saved/restored! */

	jpc_enc_tcmpt_t *comp;
	jpc_enc_tcmpt_t *endcomps;
	jpc_enc_rlvl_t *lvl;
	jpc_enc_rlvl_t *endlvls;
	jpc_enc_band_t *band;
	jpc_enc_band_t *endbands;
	jpc_enc_cblk_t *cblk;
	jpc_enc_cblk_t *endcblks;
	jpc_enc_tile_t *tile;
	int prcno;
	jpc_enc_prc_t *prc;

	tile = enc->curtile;

	endcomps = &tile->tcmpts[tile->numtcmpts];
	for (comp = tile->tcmpts; comp != endcomps; ++comp) {
		endlvls = &comp->rlvls[comp->numrlvls];
		for (lvl = comp->rlvls; lvl != endlvls; ++lvl) {
			if (!lvl->bands) {
				continue;
			}
			endbands = &lvl->bands[lvl->numbands];
			for (band = lvl->bands; band != endbands; ++band) {
				if (!band->data) {
					continue;
				}
				for (prcno = 0, prc = band->prcs; prcno < lvl->numprcs; ++prcno, ++prc) {
					if (!prc->cblks) {
						continue;
					}
					jpc_tagtree_copy(prc->savincltree, prc->incltree);
					jpc_tagtree_copy(prc->savnlibtree, prc->nlibtree);
					endcblks = &prc->cblks[prc->numcblks];
					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						cblk->savedcurpass = cblk->curpass;
						cblk->savednumencpasses = cblk->numencpasses;
						cblk->savednumlenbits = cblk->numlenbits;
					}
				}
			}
		}
	}

}

Project: jasper
File: jpc_t1enc.c

static int jpc_encclnpass(jpc_mqenc_t *mqenc, int bitpos, int orient, int vcausalflag, int segsymflag, jas_matrix_t *flags,
  jas_matrix_t *data, int term, long *nmsedec)
{
	int i;
	int j;
	int k;
	int vscanlen;
	int v;
	int runlen;
	jpc_fix_t *fp;
	int width;
	int height;
	jpc_fix_t *dp;
	int one;
	int frowstep;
	int drowstep;
	int fstripestep;
	int dstripestep;
	jpc_fix_t *fstripestart;
	jpc_fix_t *dstripestart;
	jpc_fix_t *fvscanstart;
	jpc_fix_t *dvscanstart;

	*nmsedec = 0;
	width = jas_matrix_numcols(data);
	height = jas_matrix_numrows(data);
	frowstep = jas_matrix_rowstep(flags);
	drowstep = jas_matrix_rowstep(data);
	fstripestep = frowstep << 2;
	dstripestep = drowstep << 2;

	one = 1 << (bitpos + JPC_NUMEXTRABITS);

	fstripestart = jas_matrix_getref(flags, 1, 1);
	dstripestart = jas_matrix_getref(data, 0, 0);
	for (i = height; i > 0; i -= 4, fstripestart += fstripestep,
	  dstripestart += dstripestep) {
		fvscanstart = fstripestart;
		dvscanstart = dstripestart;
		vscanlen = JAS_MIN(i, 4);
		for (j = width; j > 0; --j, ++fvscanstart, ++dvscanstart) {

			fp = fvscanstart;
			if (vscanlen >= 4 && !((*fp) & (JPC_SIG | JPC_VISIT |
			  JPC_OTHSIGMSK)) && (fp += frowstep, !((*fp) & (JPC_SIG |
			  JPC_VISIT | JPC_OTHSIGMSK))) && (fp += frowstep, !((*fp) &
			  (JPC_SIG | JPC_VISIT | JPC_OTHSIGMSK))) && (fp += frowstep,
			  !((*fp) & (JPC_SIG | JPC_VISIT | JPC_OTHSIGMSK)))) {
				dp = dvscanstart;
				for (k = 0; k < vscanlen; ++k) {
					v = (JAS_ABS(*dp) & one) ? 1 : 0;
					if (v) {
						break;
					}
					dp += drowstep;
				}
				runlen = k;
				if (runlen >= 4) {
					jpc_mqenc_setcurctx(mqenc, JPC_AGGCTXNO);
					jpc_mqenc_putbit(mqenc, 0);
					continue;
				}
				jpc_mqenc_setcurctx(mqenc, JPC_AGGCTXNO);
				jpc_mqenc_putbit(mqenc, 1);
				jpc_mqenc_setcurctx(mqenc, JPC_UCTXNO);
				jpc_mqenc_putbit(mqenc, runlen >> 1);
				jpc_mqenc_putbit(mqenc, runlen & 1);
				fp = fvscanstart + frowstep * runlen;
				dp = dvscanstart + drowstep * runlen;
				k = vscanlen - runlen;
				switch (runlen) {
				case 0:
					goto clnpass_partial0;
					break;
				case 1:
					goto clnpass_partial1;
					break;
				case 2:
					goto clnpass_partial2;
					break;
				case 3:
					goto clnpass_partial3;
					break;
				}
			} else {
				runlen = 0;
				fp = fvscanstart;
				dp = dvscanstart;
				k = vscanlen;
				goto clnpass_full0;
			}
			clnpass_step(fp, frowstep, dp, bitpos, one,
			  orient, nmsedec, mqenc, clnpass_full0:, clnpass_partial0:, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			clnpass_step(fp, frowstep, dp, bitpos, one,
				orient, nmsedec, mqenc, ;, clnpass_partial1:, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			clnpass_step(fp, frowstep, dp, bitpos, one,
				orient, nmsedec, mqenc, ;, clnpass_partial2:, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			clnpass_step(fp, frowstep, dp, bitpos, one,
				orient, nmsedec, mqenc, ;, clnpass_partial3:, 0);
		}
	}

	if (segsymflag) {
		jpc_mqenc_setcurctx(mqenc, JPC_UCTXNO);
		jpc_mqenc_putbit(mqenc, 1);
		jpc_mqenc_putbit(mqenc, 0);
		jpc_mqenc_putbit(mqenc, 1);
		jpc_mqenc_putbit(mqenc, 0);
	}

	if (term) {
		jpc_mqenc_flush(mqenc, term - 1);
	}

	return jpc_mqenc_error(mqenc) ? (-1) : 0;
}

Project: jasper
File: jpc_t2enc.c

static int jpc_putnumnewpasses(jpc_bitstream_t *out, int n)
{
	int ret;

	if (n <= 0) {
		return -1;
	} else if (n == 1) {
		ret = jpc_bitstream_putbit(out, 0);
	} else if (n == 2) {
		ret = jpc_bitstream_putbits(out, 2, 2);
	} else if (n <= 5) {
		ret = jpc_bitstream_putbits(out, 4, 0xc | (n - 3));
	} else if (n <= 36) {
		ret = jpc_bitstream_putbits(out, 9, 0x1e0 | (n - 6));
	} else if (n <= 164) {
		ret = jpc_bitstream_putbits(out, 16, 0xff80 | (n - 37));
	} else {
		/* The standard has no provision for encoding a larger value.
		In practice, however, it is highly unlikely that this
		limitation will ever be encountered. */
		return -1;
	}

	return (ret != EOF) ? 0 : (-1);
}

Project: jasper
File: jpc_t1enc.c

int jpc_enc_enccblks(jpc_enc_t *enc)
{
	jpc_enc_tcmpt_t *tcmpt;
	jpc_enc_tcmpt_t *endcomps;
	jpc_enc_rlvl_t *lvl;
	jpc_enc_rlvl_t *endlvls;
	jpc_enc_band_t *band;
	jpc_enc_band_t *endbands;
	jpc_enc_cblk_t *cblk;
	jpc_enc_cblk_t *endcblks;
	int i;
	int j;
	jpc_fix_t mx;
	jpc_fix_t bmx;
	jpc_fix_t v;
	jpc_enc_tile_t *tile;
	uint_fast32_t prcno;
	jpc_enc_prc_t *prc;

	tile = enc->curtile;

	endcomps = &tile->tcmpts[tile->numtcmpts];
	for (tcmpt = tile->tcmpts; tcmpt != endcomps; ++tcmpt) {
		endlvls = &tcmpt->rlvls[tcmpt->numrlvls];
		for (lvl = tcmpt->rlvls; lvl != endlvls; ++lvl) {
			if (!lvl->bands) {
				continue;
			}
			endbands = &lvl->bands[lvl->numbands];
			for (band = lvl->bands; band != endbands; ++band) {
				if (!band->data) {
					continue;
				}
				for (prcno = 0, prc = band->prcs; prcno < lvl->numprcs; ++prcno, ++prc) {
					if (!prc->cblks) {
						continue;
					}
					bmx = 0;
					endcblks = &prc->cblks[prc->numcblks];
					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						mx = 0;
						for (i = 0; i < jas_matrix_numrows(cblk->data); ++i) {
							for (j = 0; j < jas_matrix_numcols(cblk->data); ++j) {
								v = JAS_ABS(jas_matrix_get(cblk->data, i, j));
								if (v > mx) {
									mx = v;
								}
							}
						}
						if (mx > bmx) {
							bmx = mx;
						}
						cblk->numbps = JAS_MAX(jpc_fix_firstone(mx) + 1 - JPC_NUMEXTRABITS, 0);
					}

					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						cblk->numimsbs = band->numbps - cblk->numbps;
						assert(cblk->numimsbs >= 0);
					}

					for (cblk = prc->cblks; cblk != endcblks; ++cblk) {
						if (jpc_enc_enccblk(enc, cblk->stream, tcmpt, band, cblk)) {
							return -1;
						}
					}
				}
			}
		}
	}
	return 0;
}

Project: jasper
File: jpc_enc.c

static int jpc_calcssexp(jpc_fix_t stepsize)
{
	return jpc_fix_firstone(stepsize) - JPC_FIX_FRACBITS;
}

Project: jasper
File: jpc_t2enc.c

jpc_pi_t *jpc_enc_pi_create(jpc_enc_cp_t *cp, jpc_enc_tile_t *tile)
{
	jpc_pi_t *pi;
	int compno;
	jpc_picomp_t *picomp;
	jpc_pirlvl_t *pirlvl;
	jpc_enc_tcmpt_t *tcomp;
	int rlvlno;
	jpc_enc_rlvl_t *rlvl;
	int prcno;
	int *prclyrno;

	if (!(pi = jpc_pi_create0())) {
		return 0;
	}
	pi->pktno = -1;
	pi->numcomps = cp->numcmpts;
	if (!(pi->picomps = jas_alloc2(pi->numcomps, sizeof(jpc_picomp_t)))) {
		jpc_pi_destroy(pi);
		return 0;
	}
	for (compno = 0, picomp = pi->picomps; compno < pi->numcomps; ++compno,
	  ++picomp) {
		picomp->pirlvls = 0;
	}

	for (compno = 0, tcomp = tile->tcmpts, picomp = pi->picomps;
	  compno < pi->numcomps; ++compno, ++tcomp, ++picomp) {
		picomp->numrlvls = tcomp->numrlvls;
		if (!(picomp->pirlvls = jas_alloc2(picomp->numrlvls,
		  sizeof(jpc_pirlvl_t)))) {
			jpc_pi_destroy(pi);
			return 0;
		}
		for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
		  picomp->numrlvls; ++rlvlno, ++pirlvl) {
			pirlvl->prclyrnos = 0;
		}
		for (rlvlno = 0, pirlvl = picomp->pirlvls, rlvl = tcomp->rlvls;
		  rlvlno < picomp->numrlvls; ++rlvlno, ++pirlvl, ++rlvl) {
/* XXX sizeof(long) should be sizeof different type */
			pirlvl->numprcs = rlvl->numprcs;
			if (rlvl->numprcs) {
				if (!(pirlvl->prclyrnos = jas_alloc2(pirlvl->numprcs,
				  sizeof(long)))) {
					jpc_pi_destroy(pi);
					return 0;
				}
			} else {
				pirlvl->prclyrnos = 0;
			}
		}
	}

	pi->maxrlvls = 0;
	for (compno = 0, tcomp = tile->tcmpts, picomp = pi->picomps;
	  compno < pi->numcomps; ++compno, ++tcomp, ++picomp) {
		picomp->hsamp = cp->ccps[compno].sampgrdstepx;
		picomp->vsamp = cp->ccps[compno].sampgrdstepy;
		for (rlvlno = 0, pirlvl = picomp->pirlvls, rlvl = tcomp->rlvls;
		  rlvlno < picomp->numrlvls; ++rlvlno, ++pirlvl, ++rlvl) {
			pirlvl->prcwidthexpn = rlvl->prcwidthexpn;
			pirlvl->prcheightexpn = rlvl->prcheightexpn;
			for (prcno = 0, prclyrno = pirlvl->prclyrnos;
			  prcno < pirlvl->numprcs; ++prcno, ++prclyrno) {
				*prclyrno = 0;
			}
			pirlvl->numhprcs = rlvl->numhprcs;
		}
		if (pi->maxrlvls < tcomp->numrlvls) {
			pi->maxrlvls = tcomp->numrlvls;
		}
	}

	pi->numlyrs = tile->numlyrs;
	pi->xstart = tile->tlx;
	pi->ystart = tile->tly;
	pi->xend = tile->brx;
	pi->yend = tile->bry;

	pi->picomp = 0;
	pi->pirlvl = 0;
	pi->x = 0;
	pi->y = 0;
	pi->compno = 0;
	pi->rlvlno = 0;
	pi->prcno = 0;
	pi->lyrno = 0;
	pi->xstep = 0;
	pi->ystep = 0;

	pi->pchgno = -1;

	pi->defaultpchg.prgord = tile->prg;
	pi->defaultpchg.compnostart = 0;
	pi->defaultpchg.compnoend = pi->numcomps;
	pi->defaultpchg.rlvlnostart = 0;
	pi->defaultpchg.rlvlnoend = pi->maxrlvls;
	pi->defaultpchg.lyrnoend = pi->numlyrs;
	pi->pchg = 0;

	pi->valid = 0;

	return pi;
}

Project: jasper
File: jpc_math.c

int jpc_int_firstone(int x)
{
	int n;

	/* The argument must be nonnegative. */
	assert(x >= 0);

	n = -1;
	while (x > 0) {
		x >>= 1;
		++n;
	}
	return n;
}

Project: jasper
File: jpc_t1enc.c

static int jpc_encsigpass(jpc_mqenc_t *mqenc, int bitpos, int orient, int vcausalflag,
  jas_matrix_t *flags, jas_matrix_t *data, int term, long *nmsedec)
{
	int i;
	int j;
	int one;
	int vscanlen;
	int width;
	int height;
	int frowstep;
	int drowstep;
	int fstripestep;
	int dstripestep;
	jpc_fix_t *fstripestart;
	jpc_fix_t *dstripestart;
	jpc_fix_t *fp;
	jpc_fix_t *dp;
	jpc_fix_t *fvscanstart;
	jpc_fix_t *dvscanstart;
	int k;

	*nmsedec = 0;
	width = jas_matrix_numcols(data);
	height = jas_matrix_numrows(data);
	frowstep = jas_matrix_rowstep(flags);
	drowstep = jas_matrix_rowstep(data);
	fstripestep = frowstep << 2;
	dstripestep = drowstep << 2;

	one = 1 << (bitpos + JPC_NUMEXTRABITS);

	fstripestart = jas_matrix_getref(flags, 1, 1);
	dstripestart = jas_matrix_getref(data, 0, 0);
	for (i = height; i > 0; i -= 4, fstripestart += fstripestep,
	  dstripestart += dstripestep) {
		fvscanstart = fstripestart;
		dvscanstart = dstripestart;
		vscanlen = JAS_MIN(i, 4);
		for (j = width; j > 0; --j, ++fvscanstart, ++dvscanstart) {
			fp = fvscanstart;
			dp = dvscanstart;
			k = vscanlen;

			sigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, orient, mqenc, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			sigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, orient, mqenc, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			sigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, orient, mqenc, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			sigpass_step(fp, frowstep, dp, bitpos, one,
			  nmsedec, orient, mqenc, 0);

		}
	}

	if (term) {
		jpc_mqenc_flush(mqenc, term - 1);
	}

	return jpc_mqenc_error(mqenc) ? (-1) : 0;
}

Project: jasper
File: jpc_math.c

int jpc_fix_firstone(jpc_fix_t x)
{
	int n;

	/* The argument must be nonnegative. */
	assert(x >= 0);

	n = -1;
	while (x > 0) {
		x >>= 1;
		++n;
	}
	return n;
}

Project: jasper
File: jpc_t2enc.c

int jpc_enc_encpkts(jpc_enc_t *enc, jas_stream_t *out)
{
	jpc_enc_tile_t *tile;
	jpc_pi_t *pi;

	tile = enc->curtile;

	jpc_init_t2state(enc, 0);
	pi = tile->pi;
	jpc_pi_init(pi);

	if (!jpc_pi_next(pi)) {
		for (;;) {
			if (jpc_enc_encpkt(enc, out, jpc_pi_cmptno(pi), jpc_pi_rlvlno(pi),
			  jpc_pi_prcno(pi), jpc_pi_lyrno(pi))) {
				return -1;
			}
			if (jpc_pi_next(pi)) {
				break;
			}
		}
	}
	
	return 0;
}

Project: jasper
File: jpc_math.c

int jpc_floorlog2(int x)
{
	int y;

	/* The argument must be positive. */
	assert(x > 0);

	y = 0;
	while (x > 1) {
		x >>= 1;
		++y;
	}
	return y;
}

Project: jasper
File: jpc_enc.c

static int jpc_calcssmant(jpc_fix_t stepsize)
{
	int n;
	int e;
	int m;

	n = jpc_fix_firstone(stepsize);
	e = n - JPC_FIX_FRACBITS;
	if (n >= 11) {
		m = (stepsize >> (n - 11)) & 0x7ff;
	} else {
		m = (stepsize & ((1 << n) - 1)) << (11 - n);
	}
	return m;
}

Project: jasper
File: jpc_t1enc.c

static int jpc_encrefpass(jpc_mqenc_t *mqenc, int bitpos, int vcausalflag, jas_matrix_t *flags, jas_matrix_t *data,
  int term, long *nmsedec)
{
	int i;
	int j;
	int one;
	int vscanlen;
	int d;
	int width;
	int height;
	int frowstep;
	int drowstep;
	int fstripestep;
	int dstripestep;
	jpc_fix_t *fstripestart;
	jpc_fix_t *dstripestart;
	jpc_fix_t *fvscanstart;
	jpc_fix_t *dvscanstart;
	jpc_fix_t *dp;
	jpc_fix_t *fp;
int k;

	*nmsedec = 0;
	width = jas_matrix_numcols(data);
	height = jas_matrix_numrows(data);
	frowstep = jas_matrix_rowstep(flags);
	drowstep = jas_matrix_rowstep(data);
	fstripestep = frowstep << 2;
	dstripestep = drowstep << 2;

	one = 1 << (bitpos + JPC_NUMEXTRABITS);

	fstripestart = jas_matrix_getref(flags, 1, 1);
	dstripestart = jas_matrix_getref(data, 0, 0);
	for (i = height; i > 0; i -= 4, fstripestart += fstripestep,
	  dstripestart += dstripestep) {
		fvscanstart = fstripestart;
		dvscanstart = dstripestart;
		vscanlen = JAS_MIN(i, 4);
		for (j = width; j > 0; --j, ++fvscanstart, ++dvscanstart) {
			fp = fvscanstart;
			dp = dvscanstart;
			k = vscanlen;

			refpass_step(fp, dp, bitpos, one, nmsedec,
			  mqenc, vcausalflag);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			refpass_step(fp, dp, bitpos, one, nmsedec,
			  mqenc, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			refpass_step(fp, dp, bitpos, one, nmsedec,
			  mqenc, 0);
			if (--k <= 0) {
				continue;
			}
			fp += frowstep;
			dp += drowstep;
			refpass_step(fp, dp, bitpos, one, nmsedec,
			  mqenc, 0);

		}
	}

	if (term) {
		jpc_mqenc_flush(mqenc, term - 1);
	}

	return jpc_mqenc_error(mqenc) ? (-1) : 0;
}