Vulnerability: Division-by-zero vulnerabilities in the functions pi_next_pcrl, pi_next_cprl, and pi_next_rpcl in openmj2/pi.c in OpenJPEG through 2.3.0 allow remote attackers to cause a denial of service (application crash).
Vulnerability ID: CVE-2018-20845
CWE: CWE-369
Severity score: 2.9
Project: https://github.com/uclouvain/openjpeg
Published: 2019-06-26
File: src/lib/openmj2/pi.c
Commit message: [MJ2] To avoid divisions by zero / undefined behaviour on shift

Signed-off-by: Young_X <YangX92@hotmail.com>
Commit author: Young_X

Vulnerable version (before fix):
 static opj_bool pi_next_rpcl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue; static opj_bool pi_next_pcrl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue; static opj_bool pi_next_cprl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue;

Fixed version (after fix):
 static opj_bool pi_next_rpcl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                      /* To avoid divisions by zero / undefined behaviour on shift */                     if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||                             rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {                         continue;                     }                      if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue; static opj_bool pi_next_pcrl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                      /* To avoid divisions by zero / undefined behaviour on shift */                     if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||                             rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {                         continue;                     }                      if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue; static opj_bool pi_next_cprl(opj_pi_iterator_t * pi)                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);                     rpx = res->pdx + levelno;                     rpy = res->pdy + levelno;                      /* To avoid divisions by zero / undefined behaviour on shift */                     if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||                             rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {                         continue;                     }                      if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&                             ((try0 << levelno) % (1 << rpy))))) {                         continue;

Patch:
patch: @@ -224,6 +224,13 @@ static opj_bool pi_next_rpcl(opj_pi_iterator_t * pi)
                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
+
+                    /* To avoid divisions by zero / undefined behaviour on shift */
+                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
+                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                        continue;
+                    }
+
                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
                             ((try0 << levelno) % (1 << rpy))))) {
                         continue;
@@ -317,6 +324,13 @@ static opj_bool pi_next_pcrl(opj_pi_iterator_t * pi)
                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
+
+                    /* To avoid divisions by zero / undefined behaviour on shift */
+                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
+                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                        continue;
+                    }
+
                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
                             ((try0 << levelno) % (1 << rpy))))) {
                         continue;
@@ -408,6 +422,13 @@ static opj_bool pi_next_cprl(opj_pi_iterator_t * pi)
                     try1 = int_ceildiv(pi->ty1, comp->dy << levelno);
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
+
+                    /* To avoid divisions by zero / undefined behaviour on shift */
+                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
+                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                        continue;
+                    }
+
                     if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
                             ((try0 << levelno) % (1 << rpy))))) {
                         continue;
path: src/lib/openmj2/pi.c