static int SetOidValue(byte* out, word32 outSz, const byte *oid, word32 oidSz,
                       byte *in, word32 inSz)
{
    int idx = 0;

    if (out == NULL || oid == NULL || in == NULL)
        return BAD_FUNC_ARG;

    if (outSz < 3)
        return BUFFER_E;

    /* sequence,  + 1 => byte to put value size */
    idx = SetSequence(inSz + oidSz + 1, out);

    if ((idx + inSz + oidSz + 1) > outSz)
        return BUFFER_E;

    XMEMCPY(out+idx, oid, oidSz);
    idx += oidSz;
    out[idx++] = (byte)inSz;
    XMEMCPY(out+idx, in, inSz);

    return (idx+inSz);
}


int wc_MIME_parse_headers(char* in, int inLen, MimeHdr** headers)
{
    MimeHdr* nextHdr = NULL;
    MimeHdr* curHdr = NULL;
    MimeParam* nextParam = NULL;
    size_t start = 0;
    size_t end = 0;
    char* nameAttr = NULL;
    char* bodyVal = NULL;
    MimeTypes mimeType = MIME_HDR;
    MimeStatus mimeStatus = MIME_NAMEATTR;
    int ret = -1;
    size_t pos = 0;
    size_t lineLen = 0;
    char* curLine = NULL;
    char* ptr = NULL;

    if (in == NULL || inLen <= 0 || in[inLen] != '\0' || headers == NULL) {
        ret = BAD_FUNC_ARG;
        goto error;
    }
    nextHdr = (MimeHdr*)XMALLOC(sizeof(MimeHdr), NULL, DYNAMIC_TYPE_PKCS7);
    nextParam = (MimeParam*)XMALLOC(sizeof(MimeParam), NULL,
                                    DYNAMIC_TYPE_PKCS7);
    if (nextHdr == NULL || nextParam == NULL) {
        ret = MEMORY_E;
        goto error;
    }
    XMEMSET(nextHdr, 0, (word32)sizeof(MimeHdr));
    XMEMSET(nextParam, 0, (word32)sizeof(MimeParam));

    curLine = XSTRTOK(in, "\r\n", &ptr);
    if (curLine == NULL) {
        ret = ASN_PARSE_E;
        goto error;
    }

    while (curLine != NULL) {
        /* Leftover from previous line, add params to previous header. */
        if (curLine[0] == ' ' && curHdr) {
            mimeType = MIME_PARAM;
        }
        else {
            mimeType = MIME_HDR;
        }
        start = 0;
        lineLen = XSTRLEN(curLine);
        if (lineLen == 0) {
            ret = BAD_FUNC_ARG;
            goto error;
        }

        for (pos = 0; pos < lineLen; pos++) {
            char cur = curLine[pos];

            if (mimeStatus == MIME_NAMEATTR && ((cur == ':' &&
                mimeType == MIME_HDR) || (cur == '=' &&
                mimeType == MIME_PARAM)) && pos >= 1) {
                mimeStatus = MIME_BODYVAL;
                end = pos-1;
                ret = wc_MIME_header_strip(curLine, &nameAttr, start, end);
                if (ret) {
                    goto error;
                }
                start = pos+1;
            }
            else if (mimeStatus == MIME_BODYVAL && cur == ';' && pos >= 1) {
                end = pos-1;
                ret = wc_MIME_header_strip(curLine, &bodyVal, start, end);
                if (ret) {
                    goto error;
                }
                if (mimeType == MIME_HDR) {
                    nextHdr->name = nameAttr;
                    nameAttr = NULL;
                    nextHdr->body = bodyVal;
                    bodyVal = NULL;
                    nextHdr->next = curHdr;
                    curHdr = nextHdr;
                    nextHdr = (MimeHdr*)XMALLOC(sizeof(MimeHdr), NULL,
                                                DYNAMIC_TYPE_PKCS7);
                    if (nextHdr == NULL) {
                        ret = MEMORY_E;
                        goto error;
                    }
                    XMEMSET(nextHdr, 0, (word32)sizeof(MimeHdr));
                }
                else {
                    nextParam->attribute = nameAttr;
                    nameAttr = NULL;
                    nextParam->value = bodyVal;
                    bodyVal = NULL;
                    nextParam->next = curHdr->params;
                    curHdr->params = nextParam;
                    nextParam = (MimeParam*)XMALLOC(sizeof(MimeParam), NULL,
                                                    DYNAMIC_TYPE_PKCS7);
                    if (nextParam == NULL) {
                        ret = MEMORY_E;
                        goto error;
                    }
                    XMEMSET(nextParam, 0, (word32)sizeof(MimeParam));
                }
                mimeType = MIME_PARAM;
                mimeStatus = MIME_NAMEATTR;
                start = pos+1;
            }
        }

        end = lineLen-1;
        /* Omit newline characters. */
        while ((curLine[end] == '\r' || curLine[end] == '\n') && end > 0) {
            end--;
        }
        if (end >= start && mimeStatus == MIME_BODYVAL) {
            ret = wc_MIME_header_strip(curLine, &bodyVal, start, end);
            if (ret) {
                goto error;
            }
            if (mimeType == MIME_HDR) {
                nextHdr->name = nameAttr;
                nameAttr = NULL;
                nextHdr->body = bodyVal;
                bodyVal = NULL;
                nextHdr->next = curHdr;
                curHdr = nextHdr;
                nextHdr = (MimeHdr*)XMALLOC(sizeof(MimeHdr), NULL,
                                            DYNAMIC_TYPE_PKCS7);
                if (nextHdr == NULL) {
                    ret = MEMORY_E;
                    goto error;
                }
                XMEMSET(nextHdr, 0, (word32)sizeof(MimeHdr));
            } else {
                nextParam->attribute = nameAttr;
                nameAttr = NULL;
                nextParam->value = bodyVal;
                bodyVal = NULL;
                nextParam->next = curHdr->params;
                curHdr->params = nextParam;
                nextParam = (MimeParam*)XMALLOC(sizeof(MimeParam), NULL,
                                                DYNAMIC_TYPE_PKCS7);
                if (nextParam == NULL) {
                    ret = MEMORY_E;
                    goto error;
                }
                XMEMSET(nextParam, 0, (word32)sizeof(MimeParam));
            }
        }

        curLine = XSTRTOK(NULL, "\r\n", &ptr);
        mimeStatus = MIME_NAMEATTR;
    }

    *headers = curHdr;
    ret = 0; /* success if at this point */

error:
    if (ret != 0)
        wc_MIME_free_hdrs(curHdr);
    wc_MIME_free_hdrs(nextHdr);
    if (nameAttr != NULL)
        XFREE(nameAttr, NULL, DYNAMIC_TYPE_PKCS7);
    if (bodyVal != NULL)
        XFREE(bodyVal, NULL, DYNAMIC_TYPE_PKCS7);
    XFREE(nextParam, NULL, DYNAMIC_TYPE_PKCS7);

    return ret;
}


static word32 SetOctetString8Bit(word32 len, byte* output)
{
    output[0] = ASN_OCTET_STRING;
    output[1] = (byte)len;
    return 2;
}


int ToTraditionalEnc(byte* input, word32 sz, const char* password,
                     int passwordSz, word32* algId)
{
    int ret;

    ret = wc_DecryptPKCS8Key(input, sz, password, passwordSz);
    if (ret > 0) {
        ret = ToTraditional_ex(input, ret, algId);
    }

    return ret;
}


static int ParseCRL_Extensions(DecodedCRL* dcrl, const byte* buf,
        word32* inOutIdx, word32 sz)
{
    int length;
    word32 idx;
    word32 ext_bound; /* boundary index for the sequence of extensions */
    word32 oid;
    byte tag;

    WOLFSSL_ENTER("ParseCRL_Extensions");
    (void)dcrl;

    if (inOutIdx == NULL)
        return BAD_FUNC_ARG;

    idx = *inOutIdx;

    /* CRL Extensions are optional */
    if ((idx + 1) > sz)
        return 0;

    /* CRL Extensions are optional */
    if (GetASNTag(buf, &idx, &tag, sz) < 0)
        return 0;

    /* CRL Extensions are optional */
    if (tag != (ASN_CONSTRUCTED | ASN_CONTEXT_SPECIFIC | 0))
        return 0;

    if (GetLength(buf, &idx, &length, sz) < 0)
        return ASN_PARSE_E;

    if (GetSequence(buf, &idx, &length, sz) < 0)
        return ASN_PARSE_E;

    ext_bound = idx + length;

    while (idx < (word32)ext_bound) {
        word32 localIdx;
        int ret;

        if (GetSequence(buf, &idx, &length, sz) < 0) {
            WOLFSSL_MSG("\tfail: should be a SEQUENCE");
            return ASN_PARSE_E;
        }

        oid = 0;
        if (GetObjectId(buf, &idx, &oid, oidCrlExtType, sz) < 0) {
            WOLFSSL_MSG("\tfail: OBJECT ID");
            return ASN_PARSE_E;
        }

        /* check for critical flag */
        if ((idx + 1) > (word32)sz) {
            WOLFSSL_MSG("\tfail: malformed buffer");
            return BUFFER_E;
        }

        localIdx = idx;
        if (GetASNTag(buf, &localIdx, &tag, sz) == 0 && tag == ASN_BOOLEAN) {
            WOLFSSL_MSG("\tfound optional critical flag, moving past");
            ret = GetBoolean(buf, &idx, sz);
            if (ret < 0)
                return ret;
        }

        ret = GetOctetString(buf, &idx, &length, sz);
        if (ret < 0)
            return ret;

        if (oid == AUTH_KEY_OID) {
        #ifndef NO_SKID
            ret = ParseCRL_AuthKeyIdExt(buf + idx, length, dcrl);
            if (ret < 0) {
                WOLFSSL_MSG("\tcouldn't parse AuthKeyId extension");
                return ret;
            }
        #endif
        }

        idx += length;
    }

    *inOutIdx = idx;

    return 0;
}


void FreeSigner(Signer* signer, void* heap)
{
    XFREE(signer->name, heap, DYNAMIC_TYPE_SUBJECT_CN);
    XFREE((void*)signer->publicKey, heap, DYNAMIC_TYPE_PUBLIC_KEY);
#ifndef IGNORE_NAME_CONSTRAINTS
    if (signer->permittedNames)
        FreeNameSubtrees(signer->permittedNames, heap);
    if (signer->excludedNames)
        FreeNameSubtrees(signer->excludedNames, heap);
#endif
#ifdef WOLFSSL_SIGNER_DER_CERT
    FreeDer(&signer->derCert);
#endif
    XFREE(signer, heap, DYNAMIC_TYPE_SIGNER);

    (void)heap;
}


static WC_INLINE const char* SkipEndOfLineChars(const char* line,
                                                const char* endOfLine)
{
    /* eat end of line characters */
    while (line < endOfLine &&
              (line[0] == '\r' || line[0] == '\n')) {
        line++;
    }
    return line;
}


int wc_DsaKeyToParamsDer(DsaKey* key, byte* output, word32 inLen)
{
    if (!key || !output)
        return BAD_FUNC_ARG;

    return DsaKeyIntsToDer(key, output, inLen, DSA_PARAM_INTS, 0);
}


static WC_INLINE void FreeTmpRsas(byte** tmps, void* heap)
{
    int i;

    (void)heap;

    for (i = 0; i < RSA_INTS; i++)
        XFREE(tmps[i], heap, DYNAMIC_TYPE_RSA);
}


static int IndefItems_AddItem(IndefItems* items, word32 start)
{
    int ret = 0;
    int i;

    if (items->cnt == INDEF_ITEMS_MAX) {
        ret = MEMORY_E;
    }
    else {
        i = items->cnt++;
        items->len[i].start = start;
        items->len[i].depth = items->depth++;
        items->len[i].headerLen = 1;
        items->len[i].len = 0;
        items->idx = i;
    }

    return ret;
}


int wc_PemPubKeyToDer(const char* fileName,
                           unsigned char* derBuf, int derSz)
{
#ifdef WOLFSSL_SMALL_STACK
    byte   staticBuffer[1]; /* force XMALLOC */
#else
    byte   staticBuffer[FILE_BUFFER_SIZE];
#endif
    byte*  fileBuf = staticBuffer;
    int    dynamic = 0;
    int    ret     = 0;
    long   sz      = 0;
    XFILE  file;
    DerBuffer* converted = NULL;

    WOLFSSL_ENTER("wc_PemPubKeyToDer");

    if (fileName == NULL) {
        ret = BAD_FUNC_ARG;
    }
    else {
        file = XFOPEN(fileName, "rb");
        if (file == XBADFILE) {
            ret = BUFFER_E;
        }
    }

    if (ret == 0) {
        if(XFSEEK(file, 0, XSEEK_END) != 0)
            ret = BUFFER_E;
        sz = XFTELL(file);
        XREWIND(file);

        if (sz <= 0) {
            ret = BUFFER_E;
        }
        else if (sz > (long)sizeof(staticBuffer)) {
        #ifdef WOLFSSL_STATIC_MEMORY
            WOLFSSL_MSG("File was larger then static buffer");
            return MEMORY_E;
        #endif
            fileBuf = (byte*)XMALLOC(sz, NULL, DYNAMIC_TYPE_FILE);
            if (fileBuf == NULL)
                ret = MEMORY_E;
            else
                dynamic = 1;
        }
        if (ret == 0) {
            if ((size_t)XFREAD(fileBuf, 1, sz, file) != (size_t)sz) {
                ret = BUFFER_E;
            }
        #ifdef WOLFSSL_PEM_TO_DER
            else {
                ret = PemToDer(fileBuf, sz, PUBLICKEY_TYPE, &converted,
                               0, NULL, NULL);
            }
        #endif

            if (ret == 0) {
                if (converted->length < (word32)derSz) {
                    XMEMCPY(derBuf, converted->buffer, converted->length);
                    ret = converted->length;
                }
                else
                    ret = BUFFER_E;
            }

            FreeDer(&converted);
        }

        XFCLOSE(file);
        if (dynamic)
            XFREE(fileBuf, NULL, DYNAMIC_TYPE_FILE);
    }

    return ret;
}


static int ConfirmNameConstraints(Signer* signer, DecodedCert* cert)
{
    if (signer == NULL || cert == NULL)
        return 0;

    /* Check against the excluded list */
    if (signer->excludedNames) {
        Base_entry* base = signer->excludedNames;

        while (base != NULL) {
            switch (base->type) {
                case ASN_DNS_TYPE:
                {
                    DNS_entry* name = cert->altNames;
                    while (name != NULL) {
                        if (MatchBaseName(ASN_DNS_TYPE,
                                          name->name, name->len,
                                          base->name, base->nameSz)) {
                            return 0;
                        }
                        name = name->next;
                    }
                    break;
                }
                case ASN_RFC822_TYPE:
                {
                    DNS_entry* name = cert->altEmailNames;
                    while (name != NULL) {
                        if (MatchBaseName(ASN_RFC822_TYPE,
                                          name->name, name->len,
                                          base->name, base->nameSz)) {
                            return 0;
                        }
                        name = name->next;
                    }
                    break;
                }
                case ASN_DIR_TYPE:
                {
                    /* allow permitted dirName smaller than actual subject */
                    if (cert->subjectRawLen >= base->nameSz &&
                        XMEMCMP(cert->subjectRaw, base->name,
                                                        base->nameSz) == 0) {
                        return 0;
                    }
                    #ifndef WOLFSSL_NO_ASN_STRICT
                    /* RFC 5280 section 4.2.1.10
                       "Restrictions of the form directoryName MUST be
                        applied to the subject field .... and to any names
                        of type directoryName in the subjectAltName
                        extension"
                    */
                    if (cert->altDirNames != NULL) {
                        DNS_entry* cur = cert->altDirNames;
                        while (cur != NULL) {
                            if (XMEMCMP(cur->name, base->name, base->nameSz)
                                    == 0) {
                                WOLFSSL_MSG("DIR alt name constraint err");
                                return 0;
                            }
                            cur = cur->next;
                        }
                    }
                    #endif /* !WOLFSSL_NO_ASN_STRICT */
                    break;
                }
                default:
                    break;
            }; /* switch */
            base = base->next;
        }
    }

    /* Check against the permitted list */
    if (signer->permittedNames != NULL) {
        int needDns = 0;
        int matchDns = 0;
        int needEmail = 0;
        int matchEmail = 0;
        int needDir = 0;
        int matchDir = 0;
        Base_entry* base = signer->permittedNames;

        while (base != NULL) {
            switch (base->type) {
                case ASN_DNS_TYPE:
                {
                    DNS_entry* name = cert->altNames;

                    if (name != NULL)
                        needDns = 1;

                    while (name != NULL) {
                        matchDns = MatchBaseName(ASN_DNS_TYPE,
                                          name->name, name->len,
                                          base->name, base->nameSz);
                        name = name->next;
                    }
                    break;
                }
                case ASN_RFC822_TYPE:
                {
                    DNS_entry* name = cert->altEmailNames;

                    if (name != NULL)
                        needEmail = 1;

                    while (name != NULL) {
                        matchEmail = MatchBaseName(ASN_DNS_TYPE,
                                          name->name, name->len,
                                          base->name, base->nameSz);
                        name = name->next;
                    }
                    break;
                }
                case ASN_DIR_TYPE:
                {
                    /* allow permitted dirName smaller than actual subject */
                    needDir = 1;
                    if (cert->subjectRaw != NULL &&
                        cert->subjectRawLen >= base->nameSz &&
                        XMEMCMP(cert->subjectRaw, base->name,
                                                        base->nameSz) == 0) {
                        matchDir = 1;

                        #ifndef WOLFSSL_NO_ASN_STRICT
                        /* RFC 5280 section 4.2.1.10
                           "Restrictions of the form directoryName MUST be
                            applied to the subject field .... and to any names
                            of type directoryName in the subjectAltName
                            extension"
                        */
                        if (cert->altDirNames != NULL) {
                            DNS_entry* cur = cert->altDirNames;
                            while (cur != NULL) {
                                if (XMEMCMP(cur->name, base->name, base->nameSz)
                                        != 0) {
                                    WOLFSSL_MSG("DIR alt name constraint err");
                                    matchDir = 0; /* did not match */
                                }
                                cur = cur->next;
                            }
                        }
                        #endif /* !WOLFSSL_NO_ASN_STRICT */
                    }
                    break;
                }
                default:
                    break;
            } /* switch */
            base = base->next;
        }

        if ((needDns   && !matchDns) ||
            (needEmail && !matchEmail) ||
            (needDir   && !matchDir)) {
            return 0;
        }
    }

    return 1;
}


static int SetKeyIdFromPublicKey(Cert *cert, RsaKey *rsakey, ecc_key *eckey,
                                 byte *ntruKey, word16 ntruKeySz,
                                 ed25519_key* ed25519Key, ed448_key* ed448Key,
                                 int kid_type)
{
    byte *buf;
    int   bufferSz, ret;

    if (cert == NULL ||
        (rsakey == NULL && eckey == NULL && ntruKey == NULL &&
                                      ed25519Key == NULL && ed448Key == NULL) ||
        (kid_type != SKID_TYPE && kid_type != AKID_TYPE))
        return BAD_FUNC_ARG;

    buf = (byte *)XMALLOC(MAX_PUBLIC_KEY_SZ, cert->heap,
                                                       DYNAMIC_TYPE_TMP_BUFFER);
    if (buf == NULL)
        return MEMORY_E;

    /* Public Key */
    bufferSz = -1;
#ifndef NO_RSA
    /* RSA public key */
    if (rsakey != NULL)
        bufferSz = SetRsaPublicKey(buf, rsakey, MAX_PUBLIC_KEY_SZ, 0);
#endif
#ifdef HAVE_ECC
    /* ECC public key */
    if (eckey != NULL)
        bufferSz = SetEccPublicKey(buf, eckey, 0);
#endif
#ifdef HAVE_NTRU
    /* NTRU public key */
    if (ntruKey != NULL) {
        bufferSz = MAX_PUBLIC_KEY_SZ;
        ret = ntru_crypto_ntru_encrypt_publicKey2SubjectPublicKeyInfo(
                        ntruKeySz, ntruKey, (word16 *)(&bufferSz), buf);
        if (ret != NTRU_OK)
            bufferSz = -1;
    }
#else
    (void)ntruKeySz;
#endif
#ifdef HAVE_ED25519
    /* ED25519 public key */
    if (ed25519Key != NULL)
        bufferSz = SetEd25519PublicKey(buf, ed25519Key, 0);
#endif
#ifdef HAVE_ED448
    /* ED448 public key */
    if (ed448Key != NULL)
        bufferSz = SetEd448PublicKey(buf, ed448Key, 0);
#endif

    if (bufferSz <= 0) {
        XFREE(buf, cert->heap, DYNAMIC_TYPE_TMP_BUFFER);
        return PUBLIC_KEY_E;
    }

    /* Compute SKID by hashing public key */
    if (kid_type == SKID_TYPE) {
        ret = CalcHashId(buf, bufferSz, cert->skid);
        cert->skidSz = KEYID_SIZE;
    }
    else if (kid_type == AKID_TYPE) {
        ret = CalcHashId(buf, bufferSz, cert->akid);
        cert->akidSz = KEYID_SIZE;
    }
    else
        ret = BAD_FUNC_ARG;

    XFREE(buf, cert->heap, DYNAMIC_TYPE_TMP_BUFFER);
    return ret;
}


int wc_Ed25519PrivateKeyToDer(ed25519_key* key, byte* output, word32 inLen)
{
    return wc_BuildEd25519KeyDer(key, output, inLen, 0);
}


int wc_DhKeyDecode(const byte* input, word32* inOutIdx, DhKey* key, word32 inSz)
{
    int ret = 0;
    int length;
#ifdef WOLFSSL_DH_EXTRA
    #if !defined(HAVE_FIPS) || \
        (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION > 2))
    word32 oid = 0, temp = 0;
    #endif
#endif

    WOLFSSL_ENTER("wc_DhKeyDecode");

    if (inOutIdx == NULL)
        return BAD_FUNC_ARG;

    if (GetSequence(input, inOutIdx, &length, inSz) < 0)
        return ASN_PARSE_E;

#ifdef WOLFSSL_DH_EXTRA
    #if !defined(HAVE_FIPS) || \
        (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION > 2))
    temp = *inOutIdx;
    #endif
#endif
    /* Assume input started after 1.2.840.113549.1.3.1 dhKeyAgreement */
    if (GetInt(&key->p, input, inOutIdx, inSz) < 0) {
        ret = ASN_DH_KEY_E;
    }
    if (ret == 0 && GetInt(&key->g, input, inOutIdx, inSz) < 0) {
        mp_clear(&key->p);
        ret = ASN_DH_KEY_E;
    }

#ifdef WOLFSSL_DH_EXTRA
    #if !defined(HAVE_FIPS) || \
        (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION > 2))
    /* If ASN_DH_KEY_E: Check if input started at beginning of key */
    if (ret == ASN_DH_KEY_E) {
        *inOutIdx = temp;

        /* the version (0) - private only (for public skip) */
        if (GetASNInt(input, inOutIdx, &length, inSz) == 0) {
            *inOutIdx += length;
        }

        /* Size of dhKeyAgreement section */
        if (GetSequence(input, inOutIdx, &length, inSz) < 0)
            return ASN_PARSE_E;

        /* Check for dhKeyAgreement */
        ret = GetObjectId(input, inOutIdx, &oid, oidKeyType, inSz);
        if (oid != DHk || ret < 0)
            return ASN_DH_KEY_E;

        if (GetSequence(input, inOutIdx, &length, inSz) < 0)
            return ASN_PARSE_E;

        if (GetInt(&key->p, input, inOutIdx, inSz) < 0) {
            return ASN_DH_KEY_E;
        }
        if (ret == 0 && GetInt(&key->g, input, inOutIdx, inSz) < 0) {
            mp_clear(&key->p);
            return ASN_DH_KEY_E;
        }
    }

    temp = *inOutIdx;
    ret = (CheckBitString(input, inOutIdx, &length, inSz, 0, NULL) == 0);
    if (ret > 0) {
        /* Found Bit String */
        if (GetInt(&key->pub, input, inOutIdx, inSz) == 0) {
            WOLFSSL_MSG("Found Public Key");
            ret = 0;
        }
    } else {
        *inOutIdx = temp;
        ret = (GetOctetString(input, inOutIdx, &length, inSz) >= 0);
        if (ret > 0) {
            /* Found Octet String */
            if (GetInt(&key->priv, input, inOutIdx, inSz) == 0) {
                WOLFSSL_MSG("Found Private Key");

                /* Compute public */
                ret = mp_exptmod(&key->g, &key->priv, &key->p, &key->pub);
            }
        } else {
            /* Don't use length from failed CheckBitString/GetOctetString */
            *inOutIdx = temp;
            ret = 0;
        }
    }
    #endif /* !HAVE_FIPS || HAVE_FIPS_VERSION > 2 */
#endif /* WOLFSSL_DH_EXTRA */

    WOLFSSL_LEAVE("wc_DhKeyDecode", ret);

    return ret;
}


void FreeDer(DerBuffer** pDer)
{
    if (pDer && *pDer)
    {
        DerBuffer* der = (DerBuffer*)*pDer;

        /* ForceZero private keys */
        if (der->type == PRIVATEKEY_TYPE && der->buffer != NULL) {
            ForceZero(der->buffer, der->length);
        }
        der->buffer = NULL;
        der->length = 0;
        XFREE(der, der->heap, der->dynType);

        *pDer = NULL;
    }
}


int AddSignature(byte* buf, int bodySz, const byte* sig, int sigSz,
                        int sigAlgoType)
{
    byte seq[MAX_SEQ_SZ];
    int  idx = bodySz, seqSz;

    /* algo */
    idx += SetAlgoID(sigAlgoType, buf ? buf + idx : NULL, oidSigType, 0);
    /* bit string */
    idx += SetBitString(sigSz, 0, buf ? buf + idx : NULL);
    /* signature */
    if (buf)
        XMEMCPY(buf + idx, sig, sigSz);
    idx += sigSz;

    /* make room for overall header */
    seqSz = SetSequence(idx, seq);
    if (buf) {
        XMEMMOVE(buf + seqSz, buf, idx);
        XMEMCPY(buf, seq, seqSz);
    }

    return idx + seqSz;
}


int wc_EccPrivateKeyDecode(const byte* input, word32* inOutIdx, ecc_key* key,
                        word32 inSz)
{
    word32 oidSum;
    int    version, length;
    int    privSz, pubSz = 0;
    byte   b;
    int    ret = 0;
    int    curve_id = ECC_CURVE_DEF;
#ifdef WOLFSSL_SMALL_STACK
    byte* priv;
    byte* pub = NULL;
#else
    byte priv[ECC_MAXSIZE+1];
    byte pub[2*(ECC_MAXSIZE+1)]; /* public key has two parts plus header */
#endif
    byte* pubData = NULL;

    if (input == NULL || inOutIdx == NULL || key == NULL || inSz == 0)
        return BAD_FUNC_ARG;

    if (GetSequence(input, inOutIdx, &length, inSz) < 0)
        return ASN_PARSE_E;

    if (GetMyVersion(input, inOutIdx, &version, inSz) < 0)
        return ASN_PARSE_E;

    if (*inOutIdx >= inSz)
        return ASN_PARSE_E;

    b = input[*inOutIdx];
    *inOutIdx += 1;

    /* priv type */
    if (b != 4 && b != 6 && b != 7)
        return ASN_PARSE_E;

    if (GetLength(input, inOutIdx, &length, inSz) < 0)
        return ASN_PARSE_E;
    privSz = length;

    if (privSz > ECC_MAXSIZE)
        return BUFFER_E;

#ifdef WOLFSSL_SMALL_STACK
    priv = (byte*)XMALLOC(privSz, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
    if (priv == NULL)
        return MEMORY_E;
#endif

    /* priv key */
    XMEMCPY(priv, &input[*inOutIdx], privSz);
    *inOutIdx += length;

    if ((*inOutIdx + 1) < inSz) {
        /* prefix 0, may have */
        b = input[*inOutIdx];
        if (b == ECC_PREFIX_0) {
            *inOutIdx += 1;

            if (GetLength(input, inOutIdx, &length, inSz) <= 0)
                ret = ASN_PARSE_E;
            else {
                ret = GetObjectId(input, inOutIdx, &oidSum, oidIgnoreType,
                                  inSz);
                if (ret == 0) {
                    if ((ret = CheckCurve(oidSum)) < 0)
                        ret = ECC_CURVE_OID_E;
                    else {
                        curve_id = ret;
                        ret = 0;
                    }
                }
            }
        }
    }

    if (ret == 0 && (*inOutIdx + 1) < inSz) {
        /* prefix 1 */
        b = input[*inOutIdx];
        *inOutIdx += 1;

        if (b != ECC_PREFIX_1) {
            ret = ASN_ECC_KEY_E;
        }
        else if (GetLength(input, inOutIdx, &length, inSz) <= 0) {
            ret = ASN_PARSE_E;
        }
        else {
            /* key header */
            ret = CheckBitString(input, inOutIdx, &length, inSz, 0, NULL);
            if (ret == 0) {
                /* pub key */
                pubSz = length;
                if (pubSz > 2*(ECC_MAXSIZE+1))
                    ret = BUFFER_E;
                else {
            #ifdef WOLFSSL_SMALL_STACK
                    pub = (byte*)XMALLOC(pubSz, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
                    if (pub == NULL)
                        ret = MEMORY_E;
                    else
            #endif
                    {
                        XMEMCPY(pub, &input[*inOutIdx], pubSz);
                        *inOutIdx += length;
                        pubData = pub;
                    }
                }
            }
        }
    }

    if (ret == 0) {
        ret = wc_ecc_import_private_key_ex(priv, privSz, pubData, pubSz, key,
                                                                      curve_id);
    }

#ifdef WOLFSSL_SMALL_STACK
    XFREE(priv, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
    XFREE(pub,  key->heap, DYNAMIC_TYPE_TMP_BUFFER);
#endif

    return ret;
}


int GetASNTag(const byte* input, word32* inOutIdx, byte* tag, word32 maxIdx)
{
    word32 idx;

    if (tag == NULL || inOutIdx == NULL || input == NULL) {
        return BAD_FUNC_ARG;
    }

    idx = *inOutIdx;
    if (idx + ASN_TAG_SZ > maxIdx) {
        WOLFSSL_MSG("Buffer too small for ASN tag");
        return BUFFER_E;
    }

    *tag = input[idx];
    *inOutIdx = idx + ASN_TAG_SZ;
    return 0;
}


int wc_RsaPublicKeyDecodeRaw(const byte* n, word32 nSz, const byte* e,
                             word32 eSz, RsaKey* key)
{
    if (n == NULL || e == NULL || key == NULL)
        return BAD_FUNC_ARG;

    key->type = RSA_PUBLIC;

    if (mp_init(&key->n) != MP_OKAY)
        return MP_INIT_E;

    if (mp_read_unsigned_bin(&key->n, n, nSz) != 0) {
        mp_clear(&key->n);
        return ASN_GETINT_E;
    }
#ifdef HAVE_WOLF_BIGINT
    if ((int)nSz > 0 && wc_bigint_from_unsigned_bin(&key->n.raw, n, nSz) != 0) {
        mp_clear(&key->n);
        return ASN_GETINT_E;
    }
#endif /* HAVE_WOLF_BIGINT */

    if (mp_init(&key->e) != MP_OKAY) {
        mp_clear(&key->n);
        return MP_INIT_E;
    }

    if (mp_read_unsigned_bin(&key->e, e, eSz) != 0) {
        mp_clear(&key->n);
        mp_clear(&key->e);
        return ASN_GETINT_E;
    }
#ifdef HAVE_WOLF_BIGINT
    if ((int)eSz > 0 && wc_bigint_from_unsigned_bin(&key->e.raw, e, eSz) != 0) {
        mp_clear(&key->n);
        mp_clear(&key->e);
        return ASN_GETINT_E;
    }
#endif /* HAVE_WOLF_BIGINT */

#ifdef WOLFSSL_XILINX_CRYPT
    if (wc_InitRsaHw(key) != 0) {
        return BAD_STATE_E;
    }
#endif

    return 0;
}


int wc_SetExtKeyUsage(Cert *cert, const char *value)
{
    int ret = 0;
    char *token, *str, *ptr;
    word32 len;

    if (cert == NULL || value == NULL)
        return BAD_FUNC_ARG;

    cert->extKeyUsage = 0;

    /* duplicate string (including terminator) */
    len = (word32)XSTRLEN(value);
    str = (char*)XMALLOC(len+1, cert->heap, DYNAMIC_TYPE_TMP_BUFFER);
    if (str == NULL)
        return MEMORY_E;
    XMEMCPY(str, value, len+1);

    /* parse value, and set corresponding Key Usage value */
    if ((token = XSTRTOK(str, ",", &ptr)) == NULL) {
        XFREE(str, cert->heap, DYNAMIC_TYPE_TMP_BUFFER);
        return EXTKEYUSAGE_E;
    }

    while (token != NULL)
    {
        len = (word32)XSTRLEN(token);

        if (!XSTRNCASECMP(token, "any", len))
            cert->extKeyUsage |= EXTKEYUSE_ANY;
        else if (!XSTRNCASECMP(token, "serverAuth", len))
            cert->extKeyUsage |= EXTKEYUSE_SERVER_AUTH;
        else if (!XSTRNCASECMP(token, "clientAuth", len))
            cert->extKeyUsage |= EXTKEYUSE_CLIENT_AUTH;
        else if (!XSTRNCASECMP(token, "codeSigning", len))
            cert->extKeyUsage |= EXTKEYUSE_CODESIGN;
        else if (!XSTRNCASECMP(token, "emailProtection", len))
            cert->extKeyUsage |= EXTKEYUSE_EMAILPROT;
        else if (!XSTRNCASECMP(token, "timeStamping", len))
            cert->extKeyUsage |= EXTKEYUSE_TIMESTAMP;
        else if (!XSTRNCASECMP(token, "OCSPSigning", len))
            cert->extKeyUsage |= EXTKEYUSE_OCSP_SIGN;
        else {
            ret = EXTKEYUSAGE_E;
            break;
        }

        token = XSTRTOK(NULL, ",", &ptr);
    }

    XFREE(str, cert->heap, DYNAMIC_TYPE_TMP_BUFFER);
    return ret;
}


struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
				       enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;

	if (mem_cgroup_disabled())
		return &zone->lruvec;

	pc = lookup_page_cgroup(page);
	memcg = pc->mem_cgroup;
	mz = page_cgroup_zoneinfo(memcg, page);
	/* compound_order() is stabilized through lru_lock */
	MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
	return &mz->lruvec;
}


smb_sendv(struct TCP_Server_Info *server, struct kvec *iov, int n_vec)
{
	struct smb_rqst rqst = { .rq_iov = iov,
				 .rq_nvec = n_vec };

	return smb_send_rqst(server, &rqst);
}


send_lock_cancel(const unsigned int xid, struct cifs_tcon *tcon,
			struct smb_hdr *in_buf,
			struct smb_hdr *out_buf)
{
	int bytes_returned;
	struct cifs_ses *ses = tcon->ses;
	LOCK_REQ *pSMB = (LOCK_REQ *)in_buf;

	/* We just modify the current in_buf to change
	   the type of lock from LOCKING_ANDX_SHARED_LOCK
	   or LOCKING_ANDX_EXCLUSIVE_LOCK to
	   LOCKING_ANDX_CANCEL_LOCK. */

	pSMB->LockType = LOCKING_ANDX_CANCEL_LOCK|LOCKING_ANDX_LARGE_FILES;
	pSMB->Timeout = 0;
	pSMB->hdr.Mid = get_next_mid(ses->server);

	return SendReceive(xid, ses, in_buf, out_buf,
			&bytes_returned, 0);
}


cifs_delete_mid(struct mid_q_entry *mid)
{
	spin_lock(&GlobalMid_Lock);
	list_del(&mid->qhead);
	spin_unlock(&GlobalMid_Lock);

	DeleteMidQEntry(mid);
}


is_max_address (dwarf_vma addr, unsigned int pointer_size)
{
  dwarf_vma mask = ~(~(dwarf_vma) 1 << (pointer_size * 8 - 1));
  return ((addr & mask) == mask);
}


init_dwarf_regnames_s390 (void)
{
  dwarf_regnames = dwarf_regnames_s390;
  dwarf_regnames_count = ARRAY_SIZE (dwarf_regnames_s390);
  dwarf_regnames_lookup_func = regname_internal_by_table_only;
}


read_cie (unsigned char *start, unsigned char *end,
	  Frame_Chunk **p_cie, int *p_version,
	  bfd_size_type *p_aug_len, unsigned char **p_aug)
{
  int version;
  Frame_Chunk *fc;
  unsigned char *augmentation_data = NULL;
  bfd_size_type augmentation_data_len = 0;

  * p_cie = NULL;
  /* PR 17512: file: 001-228113-0.004.  */
  if (start >= end)
    return end;

  fc = (Frame_Chunk *) xmalloc (sizeof (Frame_Chunk));
  memset (fc, 0, sizeof (Frame_Chunk));

  fc->col_type = (short int *) xmalloc (sizeof (short int));
  fc->col_offset = (int *) xmalloc (sizeof (int));

  version = *start++;

  fc->augmentation = (char *) start;
  /* PR 17512: file: 001-228113-0.004.
     Skip past augmentation name, but avoid running off the end of the data.  */
  while (start < end)
    if (* start ++ == '\0')
      break;
  if (start == end)
    {
      warn (_("No terminator for augmentation name\n"));
      goto fail;
    }

  if (strcmp (fc->augmentation, "eh") == 0)
    {
      if (eh_addr_size > (size_t) (end - start))
	goto fail;
      start += eh_addr_size;
    }

  if (version >= 4)
    {
      if (2 > (size_t) (end - start))
	goto fail;
      GET (fc->ptr_size, 1);
      if (fc->ptr_size < 1 || fc->ptr_size > 8)
	{
	  warn (_("Invalid pointer size (%d) in CIE data\n"), fc->ptr_size);
	  goto fail;
	}

      GET (fc->segment_size, 1);
      /* PR 17512: file: e99d2804.  */
      if (fc->segment_size > 8 || fc->segment_size + fc->ptr_size > 8)
	{
	  warn (_("Invalid segment size (%d) in CIE data\n"), fc->segment_size);
	  goto fail;
	}

      eh_addr_size = fc->ptr_size;
    }
  else
    {
      fc->ptr_size = eh_addr_size;
      fc->segment_size = 0;
    }

  READ_ULEB (fc->code_factor, start, end);
  READ_SLEB (fc->data_factor, start, end);

  if (start >= end)
    goto fail;

  if (version == 1)
    {
      GET (fc->ra, 1);
    }
  else
    {
      READ_ULEB (fc->ra, start, end);
    }

  if (fc->augmentation[0] == 'z')
    {
      if (start >= end)
	goto fail;
      READ_ULEB (augmentation_data_len, start, end);
      augmentation_data = start;
      /* PR 17512: file: 11042-2589-0.004.  */
      if (augmentation_data_len > (bfd_size_type) (end - start))
	{
	  warn (_("Augmentation data too long: 0x%s, expected at most %#lx\n"),
		dwarf_vmatoa ("x", augmentation_data_len),
		(unsigned long) (end - start));
	  goto fail;
	}
      start += augmentation_data_len;
    }

  if (augmentation_data_len)
    {
      unsigned char *p;
      unsigned char *q;
      unsigned char *qend;

      p = (unsigned char *) fc->augmentation + 1;
      q = augmentation_data;
      qend = q + augmentation_data_len;

      while (p < end && q < qend)
	{
	  if (*p == 'L')
	    q++;
	  else if (*p == 'P')
	    q += 1 + size_of_encoded_value (*q);
	  else if (*p == 'R')
	    fc->fde_encoding = *q++;
	  else if (*p == 'S')
	    ;
	  else if (*p == 'B')
	    ;
	  else
	    break;
	  p++;
	}
      /* Note - it is OK if this loop terminates with q < qend.
	 Padding may have been inserted to align the end of the CIE.  */
    }

  *p_cie = fc;
  if (p_version)
    *p_version = version;
  if (p_aug_len)
    {
      *p_aug_len = augmentation_data_len;
      *p_aug = augmentation_data;
    }
  return start;

 fail:
  free (fc->col_offset);
  free (fc->col_type);
  free (fc);
  return end;
}


init_dwarf_regnames_x86_64 (void)
{
  dwarf_regnames = dwarf_regnames_x86_64;
  dwarf_regnames_count = ARRAY_SIZE (dwarf_regnames_x86_64);
  dwarf_regnames_lookup_func = regname_internal_by_table_only;
}


display_debug_aranges (struct dwarf_section *section,
		       void *file ATTRIBUTE_UNUSED)
{
  unsigned char *start = section->start;
  unsigned char *end = start + section->size;

  introduce (section, false);

  /* It does not matter if this load fails,
     we test for that later on.  */
  load_debug_info (file);

  while (start < end)
    {
      unsigned char *hdrptr;
      DWARF2_Internal_ARange arange;
      unsigned char *addr_ranges;
      dwarf_vma length;
      dwarf_vma address;
      dwarf_vma     sec_off;
      unsigned char address_size;
      unsigned int  offset_size;
      unsigned char *end_ranges;

      hdrptr = start;
      sec_off = hdrptr - section->start;

      SAFE_BYTE_GET_AND_INC (arange.ar_length, hdrptr, 4, end);
      if (arange.ar_length == 0xffffffff)
	{
	  SAFE_BYTE_GET_AND_INC (arange.ar_length, hdrptr, 8, end);
	  offset_size = 8;
	}
      else
	offset_size = 4;

      if (arange.ar_length > (size_t) (end - hdrptr))
	{
	  warn (_("Debug info is corrupted, %s header at %#lx has length %s\n"),
		section->name,
		(unsigned long) sec_off,
		dwarf_vmatoa ("x", arange.ar_length));
	  break;
	}
      end_ranges = hdrptr + arange.ar_length;

      SAFE_BYTE_GET_AND_INC (arange.ar_version, hdrptr, 2, end_ranges);
      SAFE_BYTE_GET_AND_INC (arange.ar_info_offset, hdrptr, offset_size,
			     end_ranges);

      if (num_debug_info_entries != DEBUG_INFO_UNAVAILABLE
	  && num_debug_info_entries > 0
	  && find_debug_info_for_offset (arange.ar_info_offset) == NULL)
	warn (_(".debug_info offset of 0x%lx in %s section does not point to a CU header.\n"),
	      (unsigned long) arange.ar_info_offset, section->name);

      SAFE_BYTE_GET_AND_INC (arange.ar_pointer_size, hdrptr, 1, end_ranges);
      SAFE_BYTE_GET_AND_INC (arange.ar_segment_size, hdrptr, 1, end_ranges);

      if (arange.ar_version != 2 && arange.ar_version != 3)
	{
	  /* PR 19872: A version number of 0 probably means that there is
	     padding at the end of the .debug_aranges section.  Gold puts
	     it there when performing an incremental link, for example.
	     So do not generate a warning in this case.  */
	  if (arange.ar_version)
	    warn (_("Only DWARF 2 and 3 aranges are currently supported.\n"));
	  break;
	}

      printf (_("  Length:                   %ld\n"),
	      (long) arange.ar_length);
      printf (_("  Version:                  %d\n"), arange.ar_version);
      printf (_("  Offset into .debug_info:  0x%lx\n"),
	      (unsigned long) arange.ar_info_offset);
      printf (_("  Pointer Size:             %d\n"), arange.ar_pointer_size);
      printf (_("  Segment Size:             %d\n"), arange.ar_segment_size);

      address_size = arange.ar_pointer_size + arange.ar_segment_size;

      /* PR 17512: file: 001-108546-0.001:0.1.  */
      if (address_size == 0 || address_size > 8)
	{
	  error (_("Invalid address size in %s section!\n"),
		 section->name);
	  break;
	}

      /* The DWARF spec does not require that the address size be a power
	 of two, but we do.  This will have to change if we ever encounter
	 an uneven architecture.  */
      if ((address_size & (address_size - 1)) != 0)
	{
	  warn (_("Pointer size + Segment size is not a power of two.\n"));
	  break;
	}

      if (address_size > 4)
	printf (_("\n    Address            Length\n"));
      else
	printf (_("\n    Address    Length\n"));

      addr_ranges = hdrptr;

      /* Must pad to an alignment boundary that is twice the address size.  */
      addr_ranges += (2 * address_size - 1
		      - (hdrptr - start - 1) % (2 * address_size));

      while (2 * address_size <= end_ranges - addr_ranges)
	{
	  SAFE_BYTE_GET_AND_INC (address, addr_ranges, address_size,
				 end_ranges);
	  SAFE_BYTE_GET_AND_INC (length, addr_ranges, address_size,
				 end_ranges);
	  printf ("    ");
	  print_dwarf_vma (address, address_size);
	  print_dwarf_vma (length, address_size);
	  putchar ('\n');
	}

      start = end_ranges;
    }

  printf ("\n");

  return 1;
}


cmalloc (size_t nmemb, size_t size)
{
  /* Check for overflow.  */
  if (nmemb >= ~(size_t) 0 / size)
    return NULL;

  return xmalloc (nmemb * size);
}


display_debug_not_supported (struct dwarf_section *section,
			     void *file ATTRIBUTE_UNUSED)
{
  printf (_("Displaying the debug contents of section %s is not yet supported.\n"),
	    section->name);

  return 1;
}


display_data (bfd_size_type          printed,
	      const unsigned char *  data,
	      const bfd_size_type    len)
{
  if (do_wide || len < ((80 - printed) / 3))
    for (printed = 0; printed < len; ++printed)
      printf (" %02x", data[printed]);
  else
    {
      for (printed = 0; printed < len; ++printed)
	{
	  if (printed % (80 / 3) == 0)
	    putchar ('\n');
	  printf (" %02x", data[printed]);
	}
    }
}


get_encoded_value (unsigned char **pdata,
		   int encoding,
		   struct dwarf_section *section,
		   unsigned char * end)
{
  unsigned char * data = * pdata;
  unsigned int size = size_of_encoded_value (encoding);
  dwarf_vma val;

  if (data >= end || size > (size_t) (end - data))
    {
      warn (_("Encoded value extends past end of section\n"));
      * pdata = end;
      return 0;
    }

  /* PR 17512: file: 002-829853-0.004.  */
  if (size > 8)
    {
      warn (_("Encoded size of %d is too large to read\n"), size);
      * pdata = end;
      return 0;
    }

  /* PR 17512: file: 1085-5603-0.004.  */
  if (size == 0)
    {
      warn (_("Encoded size of 0 is too small to read\n"));
      * pdata = end;
      return 0;
    }

  if (encoding & DW_EH_PE_signed)
    val = byte_get_signed (data, size);
  else
    val = byte_get (data, size);

  if ((encoding & 0x70) == DW_EH_PE_pcrel)
    val += section->address + (data - section->start);

  * pdata = data + size;
  return val;
}


free_all_abbrevs (void)
{
  while (abbrev_lists)
    abbrev_lists = free_abbrev_list (abbrev_lists);

  free (cu_abbrev_map);
  cu_abbrev_map = NULL;
  next_free_abbrev_map_entry = 0;
}


load_separate_debug_files (void * file, const char * filename)
{
  /* Skip this operation if we are not interested in debug links.  */
  if (! do_follow_links && ! do_debug_links)
    return false;

  /* See if there are any dwo links.  */
  if (load_debug_section (str, file)
      && load_debug_section (abbrev, file)
      && load_debug_section (info, file))
    {
      /* Load the .debug_addr section, if it exists.  */
      load_debug_section (debug_addr, file);
      /* Load the .debug_str_offsets section, if it exists.  */
      load_debug_section (str_index, file);
      /* Load the .debug_loclists section, if it exists.  */
      load_debug_section (loclists, file);
      /* Load the .debug_rnglists section, if it exists.  */
      load_debug_section (rnglists, file);

      free_dwo_info ();

      if (process_debug_info (& debug_displays[info].section, file, abbrev,
			      true, false))
	{
	  bool introduced = false;
	  dwo_info *dwinfo;
	  const char *dir = NULL;
	  const char *id = NULL;
	  const char *name = NULL;

	  for (dwinfo = first_dwo_info; dwinfo != NULL; dwinfo = dwinfo->next)
	    {
	      /* Accumulate NAME, DIR and ID fields.  */
	      switch (dwinfo->type)
		{
		case DWO_NAME:
		  if (name != NULL)
		    warn (_("Multiple DWO_NAMEs encountered for the same CU\n"));
		  name = dwinfo->value;
		  break;

		case DWO_DIR:
		  /* There can be multiple DW_AT_comp_dir entries in a CU,
		     so do not complain.  */
		  dir = dwinfo->value;
		  break;

		case DWO_ID:
		  if (id != NULL)
		    warn (_("multiple DWO_IDs encountered for the same CU\n"));
		  id = dwinfo->value;
		  break;

		default:
		  error (_("Unexpected DWO INFO type"));
		  break;
		}

	      /* If we have reached the end of our list, or we are changing
		 CUs, then display the information that we have accumulated
		 so far.  */
	      if (name != NULL
		  && (dwinfo->next == NULL
		      || dwinfo->next->cu_offset != dwinfo->cu_offset))
		{
		  if (do_debug_links)
		    {
		      if (! introduced)
			{
			  printf (_("The %s section contains link(s) to dwo file(s):\n\n"),
				  debug_displays [info].section.uncompressed_name);
			  introduced = true;
			}

		      printf (_("  Name:      %s\n"), name);
		      printf (_("  Directory: %s\n"), dir ? dir : _("<not-found>"));
		      if (id != NULL)
			display_data (printf (_("  ID:       ")), (unsigned char *) id, 8);
		      else if (debug_information[0].dwarf_version != 5)
			printf (_("  ID:        <not specified>\n"));
		      printf ("\n\n");
		    }

		  if (do_follow_links)
		    load_dwo_file (filename, name, dir, id);

		  name = dir = id = NULL;
		}
	    }
	}
    }

  if (! do_follow_links)
    /* The other debug links will be displayed by display_debug_links()
       so we do not need to do any further processing here.  */
    return false;

  /* FIXME: We do not check for the presence of both link sections in the same file.  */
  /* FIXME: We do not check for the presence of multiple, same-name debuglink sections.  */
  /* FIXME: We do not check for the presence of a dwo link as well as a debuglink.  */

  check_for_and_load_links (file, filename);
  if (first_separate_info != NULL)
    return true;

  do_follow_links = 0;
  return false;
}


print_dwarf_view (dwarf_vma value, unsigned num_bytes, int force)
{
  int len;
  if (!num_bytes)
    len = 4;
  else
    len = num_bytes * 2;

  assert (value == (unsigned long) value);
  if (value || force)
    printf ("v%0*lx ", len - 1, (unsigned long) value);
  else
    printf ("%*s", len + 1, "");
}


free_dwo_info (void)
{
  dwo_info * dwinfo;
  dwo_info * next;

  for (dwinfo = first_dwo_info; dwinfo != NULL; dwinfo = next)
    {
      next = dwinfo->next;
      free (dwinfo);
    }
  first_dwo_info = NULL;
}


cifs_read_from_socket(struct TCP_Server_Info *server, char *buf,
		      unsigned int to_read)
{
	struct kvec iov;

	iov.iov_base = buf;
	iov.iov_len = to_read;

	return cifs_readv_from_socket(server, &iov, 1, to_read);
}


cifs_readv_from_socket(struct TCP_Server_Info *server, struct kvec *iov_orig,
		       unsigned int nr_segs, unsigned int to_read)
{
	int length = 0;
	int total_read;
	unsigned int segs;
	struct msghdr smb_msg;
	struct kvec *iov;

	iov = get_server_iovec(server, nr_segs);
	if (!iov)
		return -ENOMEM;

	smb_msg.msg_control = NULL;
	smb_msg.msg_controllen = 0;

	for (total_read = 0; to_read; total_read += length, to_read -= length) {
		try_to_freeze();

		if (server_unresponsive(server)) {
			total_read = -EAGAIN;
			break;
		}

		segs = kvec_array_init(iov, iov_orig, nr_segs, total_read);

		length = kernel_recvmsg(server->ssocket, &smb_msg,
					iov, segs, to_read, 0);

		if (server->tcpStatus == CifsExiting) {
			total_read = -ESHUTDOWN;
			break;
		} else if (server->tcpStatus == CifsNeedReconnect) {
			cifs_reconnect(server);
			total_read = -EAGAIN;
			break;
		} else if (length == -ERESTARTSYS ||
			   length == -EAGAIN ||
			   length == -EINTR) {
			/*
			 * Minimum sleep to prevent looping, allowing socket
			 * to clear and app threads to set tcpStatus
			 * CifsNeedReconnect if server hung.
			 */
			usleep_range(1000, 2000);
			length = 0;
			continue;
		} else if (length <= 0) {
			cifs_dbg(FYI, "Received no data or error: expecting %d\n"
				 "got %d", to_read, length);
			cifs_reconnect(server);
			total_read = -EAGAIN;
			break;
		}
	}
	return total_read;
}


get_server_iovec(struct TCP_Server_Info *server, unsigned int nr_segs)
{
	struct kvec *new_iov;

	if (server->iov && nr_segs <= server->nr_iov)
		return server->iov;

	/* not big enough -- allocate a new one and release the old */
	new_iov = kmalloc(sizeof(*new_iov) * nr_segs, GFP_NOFS);
	if (new_iov) {
		kfree(server->iov);
		server->iov = new_iov;
		server->nr_iov = nr_segs;
	}
	return new_iov;
}


cifs_echo_request(struct work_struct *work)
{
	int rc;
	struct TCP_Server_Info *server = container_of(work,
					struct TCP_Server_Info, echo.work);

	/*
	 * We cannot send an echo if it is disabled or until the
	 * NEGOTIATE_PROTOCOL request is done, which is indicated by
	 * server->ops->need_neg() == true. Also, no need to ping if
	 * we got a response recently.
	 */
	if (!server->ops->need_neg || server->ops->need_neg(server) ||
	    (server->ops->can_echo && !server->ops->can_echo(server)) ||
	    time_before(jiffies, server->lstrp + SMB_ECHO_INTERVAL - HZ))
		goto requeue_echo;

	rc = server->ops->echo ? server->ops->echo(server) : -ENOSYS;
	if (rc)
		cifs_dbg(FYI, "Unable to send echo request to server: %s\n",
			 server->hostname);

requeue_echo:
	queue_delayed_work(cifsiod_wq, &server->echo, SMB_ECHO_INTERVAL);
}


cifs_negotiate_protocol(const unsigned int xid, struct cifs_ses *ses)
{
	int rc = 0;
	struct TCP_Server_Info *server = ses->server;

	if (!server->ops->need_neg || !server->ops->negotiate)
		return -ENOSYS;

	/* only send once per connect */
	if (!server->ops->need_neg(server))
		return 0;

	set_credits(server, 1);

	rc = server->ops->negotiate(xid, ses);
	if (rc == 0) {
		spin_lock(&GlobalMid_Lock);
		if (server->tcpStatus == CifsNeedNegotiate)
			server->tcpStatus = CifsGood;
		else
			rc = -EHOSTDOWN;
		spin_unlock(&GlobalMid_Lock);
	}

	return rc;
}


ip_rfc1001_connect(struct TCP_Server_Info *server)
{
	int rc = 0;
	/*
	 * some servers require RFC1001 sessinit before sending
	 * negprot - BB check reconnection in case where second
	 * sessinit is sent but no second negprot
	 */
	struct rfc1002_session_packet *ses_init_buf;
	struct smb_hdr *smb_buf;
	ses_init_buf = kzalloc(sizeof(struct rfc1002_session_packet),
			       GFP_KERNEL);
	if (ses_init_buf) {
		ses_init_buf->trailer.session_req.called_len = 32;

		if (server->server_RFC1001_name &&
		    server->server_RFC1001_name[0] != 0)
			rfc1002mangle(ses_init_buf->trailer.
				      session_req.called_name,
				      server->server_RFC1001_name,
				      RFC1001_NAME_LEN_WITH_NULL);
		else
			rfc1002mangle(ses_init_buf->trailer.
				      session_req.called_name,
				      DEFAULT_CIFS_CALLED_NAME,
				      RFC1001_NAME_LEN_WITH_NULL);

		ses_init_buf->trailer.session_req.calling_len = 32;

		/*
		 * calling name ends in null (byte 16) from old smb
		 * convention.
		 */
		if (server->workstation_RFC1001_name &&
		    server->workstation_RFC1001_name[0] != 0)
			rfc1002mangle(ses_init_buf->trailer.
				      session_req.calling_name,
				      server->workstation_RFC1001_name,
				      RFC1001_NAME_LEN_WITH_NULL);
		else
			rfc1002mangle(ses_init_buf->trailer.
				      session_req.calling_name,
				      "LINUX_CIFS_CLNT",
				      RFC1001_NAME_LEN_WITH_NULL);

		ses_init_buf->trailer.session_req.scope1 = 0;
		ses_init_buf->trailer.session_req.scope2 = 0;
		smb_buf = (struct smb_hdr *)ses_init_buf;

		/* sizeof RFC1002_SESSION_REQUEST with no scope */
		smb_buf->smb_buf_length = cpu_to_be32(0x81000044);
		rc = smb_send(server, smb_buf, 0x44);
		kfree(ses_init_buf);
		/*
		 * RFC1001 layer in at least one server
		 * requires very short break before negprot
		 * presumably because not expecting negprot
		 * to follow so fast.  This is a simple
		 * solution that works without
		 * complicating the code and causes no
		 * significant slowing down on mount
		 * for everyone else
		 */
		usleep_range(1000, 2000);
	}
	/*
	 * else the negprot may still work without this
	 * even though malloc failed
	 */

	return rc;
}


static int cifs_parse_security_flavors(char *value,
				       struct smb_vol *vol)
{

	substring_t args[MAX_OPT_ARGS];

	switch (match_token(value, cifs_secflavor_tokens, args)) {
	case Opt_sec_krb5:
		vol->secFlg |= CIFSSEC_MAY_KRB5 | CIFSSEC_MAY_SIGN;
		break;
	case Opt_sec_krb5i:
		vol->secFlg |= CIFSSEC_MAY_KRB5 | CIFSSEC_MUST_SIGN;
		break;
	case Opt_sec_krb5p:
		/* vol->secFlg |= CIFSSEC_MUST_SEAL | CIFSSEC_MAY_KRB5; */
		cifs_dbg(VFS, "Krb5 cifs privacy not supported\n");
		break;
	case Opt_sec_ntlmssp:
		vol->secFlg |= CIFSSEC_MAY_NTLMSSP;
		break;
	case Opt_sec_ntlmsspi:
		vol->secFlg |= CIFSSEC_MAY_NTLMSSP | CIFSSEC_MUST_SIGN;
		break;
	case Opt_ntlm:
		/* ntlm is default so can be turned off too */
		vol->secFlg |= CIFSSEC_MAY_NTLM;
		break;
	case Opt_sec_ntlmi:
		vol->secFlg |= CIFSSEC_MAY_NTLM | CIFSSEC_MUST_SIGN;
		break;
	case Opt_sec_ntlmv2:
		vol->secFlg |= CIFSSEC_MAY_NTLMV2;
		break;
	case Opt_sec_ntlmv2i:
		vol->secFlg |= CIFSSEC_MAY_NTLMV2 | CIFSSEC_MUST_SIGN;
		break;
#ifdef CONFIG_CIFS_WEAK_PW_HASH
	case Opt_sec_lanman:
		vol->secFlg |= CIFSSEC_MAY_LANMAN;
		break;
#endif
	case Opt_sec_none:
		vol->nullauth = 1;
		vol->secFlg |= CIFSSEC_MAY_NTLM;
		break;
	default:
		cifs_dbg(VFS, "bad security option: %s\n", value);
		return 1;
	}

	return 0;
}


cifs_parse_devname(const char *devname, struct smb_vol *vol)
{
	char *pos;
	const char *delims = "/\\";
	size_t len;

	/* make sure we have a valid UNC double delimiter prefix */
	len = strspn(devname, delims);
	if (len != 2)
		return -EINVAL;

	/* find delimiter between host and sharename */
	pos = strpbrk(devname + 2, delims);
	if (!pos)
		return -EINVAL;

	/* skip past delimiter */
	++pos;

	/* now go until next delimiter or end of string */
	len = strcspn(pos, delims);

	/* move "pos" up to delimiter or NULL */
	pos += len;
	vol->UNC = kstrndup(devname, pos - devname, GFP_KERNEL);
	if (!vol->UNC)
		return -ENOMEM;

	convert_delimiter(vol->UNC, '\\');

	/* If pos is NULL, or is a bogus trailing delimiter then no prepath */
	if (!*pos++ || !*pos)
		return 0;

	vol->prepath = kstrdup(pos, GFP_KERNEL);
	if (!vol->prepath)
		return -ENOMEM;

	return 0;
}


cifs_prune_tlinks(struct work_struct *work)
{
	struct cifs_sb_info *cifs_sb = container_of(work, struct cifs_sb_info,
						    prune_tlinks.work);
	struct rb_root *root = &cifs_sb->tlink_tree;
	struct rb_node *node = rb_first(root);
	struct rb_node *tmp;
	struct tcon_link *tlink;

	/*
	 * Because we drop the spinlock in the loop in order to put the tlink
	 * it's not guarded against removal of links from the tree. The only
	 * places that remove entries from the tree are this function and
	 * umounts. Because this function is non-reentrant and is canceled
	 * before umount can proceed, this is safe.
	 */
	spin_lock(&cifs_sb->tlink_tree_lock);
	node = rb_first(root);
	while (node != NULL) {
		tmp = node;
		node = rb_next(tmp);
		tlink = rb_entry(tmp, struct tcon_link, tl_rbnode);

		if (test_bit(TCON_LINK_MASTER, &tlink->tl_flags) ||
		    atomic_read(&tlink->tl_count) != 0 ||
		    time_after(tlink->tl_time + TLINK_IDLE_EXPIRE, jiffies))
			continue;

		cifs_get_tlink(tlink);
		clear_bit(TCON_LINK_IN_TREE, &tlink->tl_flags);
		rb_erase(tmp, root);

		spin_unlock(&cifs_sb->tlink_tree_lock);
		cifs_put_tlink(tlink);
		spin_lock(&cifs_sb->tlink_tree_lock);
	}
	spin_unlock(&cifs_sb->tlink_tree_lock);

	queue_delayed_work(cifsiod_wq, &cifs_sb->prune_tlinks,
				TLINK_IDLE_EXPIRE);
}


void LstmCell(const uint8* input_data_uint8, const Dims<4>& input_dims,
              const uint8* prev_activ_data_uint8,
              const Dims<4>& prev_activ_dims, const uint8* weights_data_uint8,
              const Dims<4>& weights_dims, const int32* bias_data_int32,
              const Dims<4>& bias_dims, const int16* prev_state_data_int16,
              const Dims<4>& prev_state_dims, int16* output_state_data_int16,
              const Dims<4>& output_state_dims, uint8* output_activ_data_uint8,
              const Dims<4>& output_activ_dims, uint8* concat_temp_data_uint8,
              const Dims<4>& concat_temp_dims, int16* activ_temp_data_int16,
              const Dims<4>& activ_temp_dims, int32 weights_zero_point,
              int32 accum_multiplier, int accum_shift,
              gemmlowp::GemmContext* gemmlowp_context) {
  tflite::LstmCellParams op_params;
  op_params.weights_zero_point = weights_zero_point;
  op_params.accum_multiplier = accum_multiplier;
  op_params.accum_shift = accum_shift;

  LstmCell<StateIntegerBits>(
      op_params, DimsToShape(input_dims), input_data_uint8,
      DimsToShape(prev_activ_dims), prev_activ_data_uint8,
      DimsToShape(weights_dims), weights_data_uint8, DimsToShape(bias_dims),
      bias_data_int32, DimsToShape(prev_state_dims), prev_state_data_int16,
      DimsToShape(output_state_dims), output_state_data_int16,
      DimsToShape(output_activ_dims), output_activ_data_uint8,
      DimsToShape(concat_temp_dims), concat_temp_data_uint8,
      DimsToShape(activ_temp_dims), activ_temp_data_int16, gemmlowp_context);
}


inline void ShapeFromDims(const tflite::Dims<4>& dims, RuntimeShape* shape) {
  shape->BuildFrom(
      {dims.sizes[3], dims.sizes[2], dims.sizes[1], dims.sizes[0]});
}


void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
                  const T* input2_data, const Dims<4>& input2_dims,
                  T* output_data, const Dims<4>& output_dims) {
  T output_activation_min, output_activation_max;
  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);

  BroadcastAdd(input1_data, input1_dims, input2_data, input2_dims,
               output_activation_min, output_activation_max, output_data,
               output_dims);
}


TfLiteRegistration* Register_L2_POOL_2D() {
  return Register_L2_POOL_GENERIC_OPT();
}


inline void Conv(const ConvParams& params, const RuntimeShape& input_shape,
                 const uint8* input_data, const RuntimeShape& filter_shape,
                 const uint8* filter_data, const RuntimeShape& bias_shape,
                 const int32* bias_data, const RuntimeShape& output_shape,
                 uint8* output_data, const RuntimeShape& im2col_shape,
                 uint8* im2col_data, gemmlowp::GemmContext* gemmlowp_context) {
  ruy::profiler::ScopeLabel label("Conv/8bit");
  const int stride_width = params.stride_width;
  const int stride_height = params.stride_height;
  const int dilation_width_factor = params.dilation_width_factor;
  const int dilation_height_factor = params.dilation_height_factor;
  const int32 input_offset = params.input_offset;
  const int32 filter_offset = params.weights_offset;
  const int32 output_offset = params.output_offset;
  const int32 output_multiplier = params.output_multiplier;
  const int output_shift = params.output_shift;
  const int32 output_activation_min = params.quantized_activation_min;
  const int32 output_activation_max = params.quantized_activation_max;
  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
  TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);

  const uint8* gemm_input_data = nullptr;
  const RuntimeShape* gemm_input_shape = nullptr;
  const int filter_width = filter_shape.Dims(2);
  const int filter_height = filter_shape.Dims(1);
  const bool need_dilated_im2col =
      dilation_width_factor != 1 || dilation_height_factor != 1;
  const bool need_im2col = stride_width != 1 || stride_height != 1 ||
                           filter_width != 1 || filter_height != 1;
  if (need_dilated_im2col) {
    TFLITE_DCHECK(im2col_data);
    const int input_zero_point = -input_offset;
    TFLITE_DCHECK_GE(input_zero_point, 0);
    TFLITE_DCHECK_LE(input_zero_point, 255);
    DilatedIm2col(params, input_zero_point, input_shape, input_data,
                  filter_shape, output_shape, im2col_data);
    gemm_input_data = im2col_data;
    gemm_input_shape = &im2col_shape;
  } else if (need_im2col) {
    TFLITE_DCHECK(im2col_data);
    const int input_zero_point = -input_offset;
    TFLITE_DCHECK_GE(input_zero_point, 0);
    TFLITE_DCHECK_LE(input_zero_point, 255);
    Im2col(params, filter_height, filter_width, input_zero_point, input_shape,
           input_data, im2col_shape, im2col_data);
    gemm_input_data = im2col_data;
    gemm_input_shape = &im2col_shape;
  } else {
    TFLITE_DCHECK(!im2col_data);
    gemm_input_data = input_data;
    gemm_input_shape = &input_shape;
  }

  const int gemm_input_rows = gemm_input_shape->Dims(3);
  // Using FlatSizeSkipDim causes segfault in some contexts (see b/79927784).
  // The root cause has not yet been identified though. Same applies below for
  // the other calls commented out. This is a partial rollback of cl/196819423.
  // const int gemm_input_cols = FlatSizeSkipDim(*gemm_input_shape, 3);
  const int gemm_input_cols = gemm_input_shape->Dims(0) *
                              gemm_input_shape->Dims(1) *
                              gemm_input_shape->Dims(2);
  const int filter_rows = filter_shape.Dims(0);
  // See b/79927784.
  // const int filter_cols = FlatSizeSkipDim(filter_shape, 0);
  const int filter_cols =
      filter_shape.Dims(1) * filter_shape.Dims(2) * filter_shape.Dims(3);
  const int output_rows = output_shape.Dims(3);
  // See b/79927784.
  // const int output_cols = FlatSizeSkipDim(output_shape, 3);
  const int output_cols =
      output_shape.Dims(0) * output_shape.Dims(1) * output_shape.Dims(2);
  TFLITE_DCHECK_EQ(output_rows, filter_rows);
  TFLITE_DCHECK_EQ(output_cols, gemm_input_cols);
  TFLITE_DCHECK_EQ(filter_cols, gemm_input_rows);
  TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_rows);

#ifdef USE_NEON
  if (gemm_input_cols == 1 && output_rows >= 4) {
    RuntimeShape fc_filter_shape{
        filter_shape.Dims(0),
        filter_shape.Dims(filter_shape.DimensionsCount() - 1)};

    return FullyConnectedAsGEMV(
        *gemm_input_shape, gemm_input_data, input_offset, fc_filter_shape,
        filter_data, filter_offset, bias_shape, bias_data, output_offset,
        output_multiplier, output_shift, output_activation_min,
        output_activation_max, output_shape, output_data, gemmlowp_context);
  }
#endif

  gemmlowp::MatrixMap<const uint8, gemmlowp::MapOrder::RowMajor> filter_matrix(
      filter_data, filter_rows, filter_cols);
  gemmlowp::MatrixMap<const uint8, gemmlowp::MapOrder::ColMajor> input_matrix(
      gemm_input_data, gemm_input_rows, gemm_input_cols);
  gemmlowp::MatrixMap<uint8, gemmlowp::MapOrder::ColMajor> output_matrix(
      output_data, output_rows, output_cols);
  const auto& output_pipeline = GemmlowpOutputPipeline::MakeExp(
      bias_data, output_rows, output_offset, output_multiplier, output_shift,
      output_activation_min, output_activation_max);
  gemmlowp::GemmWithOutputPipeline<uint8, uint8,
                                   gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
      gemmlowp_context, filter_matrix, input_matrix, &output_matrix,
      filter_offset, input_offset, output_pipeline);
}


inline void BinaryFunction(const T1* input1_data, const Dims<4>& input1_dims,
                           const T2* input2_data, const Dims<4>& input2_dims,
                           R* output_data, const Dims<4>& output_dims,
                           R (*func)(T1, T2)) {
  BinaryFunction(DimsToShape(input1_dims), input1_data,
                 DimsToShape(input2_dims), input2_data,
                 DimsToShape(output_dims), output_data, func);
}


inline void PadV2(const T* input_data, const Dims<4>& input_dims,
                  const std::vector<int>& left_paddings,
                  const std::vector<int>& right_paddings, T* output_data,
                  const Dims<4>& output_dims, const T pad_value) {
  TFLITE_DCHECK_EQ(left_paddings.size(), 4);
  TFLITE_DCHECK_EQ(right_paddings.size(), 4);
  tflite::PadParams op_params;
  op_params.left_padding_count = 4;
  op_params.right_padding_count = 4;
  for (int i = 0; i < 4; ++i) {
    op_params.left_padding[i] = left_paddings[3 - i];
    op_params.right_padding[i] = right_paddings[3 - i];
  }
  // SetFloatOrInt(pad_value, &op_params.pad_value);
  const T pad_value_copy = pad_value;

  Pad(op_params, DimsToShape(input_dims), input_data, &pad_value_copy,
      DimsToShape(output_dims), output_data);
}


void Conv(const uint8* input_data, const Dims<4>& input_dims,
          int32 input_offset, const uint8* filter_data,
          const Dims<4>& filter_dims, int32 filter_offset,
          const int32* bias_data, const Dims<4>& bias_dims, int stride,
          int pad_width, int pad_height, int32 output_offset,
          int32 output_multiplier, int output_shift,
          int32 output_activation_min, int32 output_activation_max,
          uint8* output_data, const Dims<4>& output_dims, uint8* im2col_data,
          const Dims<4>& im2col_dims, gemmlowp::GemmContext* gemmlowp_context) {
  static_assert(Ac == FusedActivationFunctionType::kNone ||
                    Ac == FusedActivationFunctionType::kRelu ||
                    Ac == FusedActivationFunctionType::kRelu6 ||
                    Ac == FusedActivationFunctionType::kRelu1,
                "");
  Conv(input_data, input_dims, input_offset, filter_data, filter_dims,
       filter_offset, bias_data, bias_dims, stride, stride, pad_width,
       pad_height, output_offset, output_multiplier, output_shift,
       output_activation_min, output_activation_max, output_data, output_dims,
       im2col_data, im2col_dims, gemmlowp_context);
}


TfLiteStatus MaxEval(TfLiteContext* context, TfLiteNode* node) {
  auto* params = reinterpret_cast<TfLitePoolParams*>(node->builtin_data);
  OpData* data = reinterpret_cast<OpData*>(node->user_data);

  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
  const TfLiteTensor* input;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input));
  switch (input->type) {  // Already know in/out types are same.
    case kTfLiteFloat32:
      MaxEvalFloat<kernel_type>(context, node, params, data, input, output);
      break;
    case kTfLiteUInt8:
      MaxEvalQuantizedUInt8<kernel_type>(context, node, params, data, input,
                                         output);
      break;
    case kTfLiteInt8:
      MaxEvalQuantizedInt8<kernel_type>(context, node, params, data, input,
                                        output);
      break;
    case kTfLiteInt16:
      MaxEvalQuantizedInt16<kernel_type>(context, node, params, data, input,
                                         output);
      break;
    default:
      TF_LITE_KERNEL_LOG(context, "Type %s not currently supported.",
                         TfLiteTypeGetName(input->type));
      return kTfLiteError;
  }
  return kTfLiteOk;
}


inline void Concatenation(int concat_dim, const Scalar* const* input_data,
                          const Dims<4>* const* input_dims, int inputs_count,
                          Scalar* output_data, const Dims<4>& output_dims) {
  // For now we don't have a model with a Concatenation with fused activation.
  TFLITE_DCHECK_EQ(Ac, FusedActivationFunctionType::kNone);

  std::vector<RuntimeShape> input_shapes(inputs_count);
  std::vector<const RuntimeShape*> input_shapes_indirect(inputs_count);
  for (int i = 0; i < inputs_count; ++i) {
    ShapeFromDims(*input_dims[i], &input_shapes[i]);
    input_shapes_indirect[i] = &input_shapes[i];
  }
  tflite::ConcatenationParams op_params;
  op_params.axis = 3 - concat_dim;
  op_params.inputs_count = inputs_count;

  Concatenation(op_params, input_shapes_indirect.data(), input_data,
                DimsToShape(output_dims), output_data);
}


void Add(const int32* input1_data, const Dims<4>& input1_dims,
         const int32* input2_data, const Dims<4>& input2_dims,
         int32* output_data, const Dims<4>& output_dims) {
  ruy::profiler::ScopeLabel label("Add/int32");
  TFLITE_DCHECK(Ac == FusedActivationFunctionType::kNone);

  tflite::ArithmeticParams op_params;
  op_params.quantized_activation_min = std::numeric_limits<int32>::min();
  op_params.quantized_activation_max = std::numeric_limits<int32>::max();
  Add(op_params, DimsToShape(input1_dims), input1_data,
      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
      output_data);
}


inline void RankOneSelect(const D* input_condition_data,
                          const Dims<4>& input_condition_dims,
                          const T* input_x_data, const Dims<4>& input_x_dims,
                          const T* input_y_data, const Dims<4>& input_y_dims,
                          T* output_data, const Dims<4>& output_dims) {
  RankOneSelect(DimsToShape(input_condition_dims), input_condition_data,
                DimsToShape(input_x_dims), input_x_data,
                DimsToShape(input_y_dims), input_y_data,
                DimsToShape(output_dims), output_data);
}


inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
                int input1_shift, const int16* input2_data,
                const Dims<4>& input2_dims, int input2_shift,
                int16 output_activation_min, int16 output_activation_max,
                int16* output_data, const Dims<4>& output_dims) {
  constexpr int kReverseShift = -1;
  static_assert(Ac == FusedActivationFunctionType::kNone ||
                    Ac == FusedActivationFunctionType::kRelu ||
                    Ac == FusedActivationFunctionType::kRelu6 ||
                    Ac == FusedActivationFunctionType::kRelu1,
                "");
  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
  if (Ac == FusedActivationFunctionType::kNone) {
    TFLITE_DCHECK_EQ(output_activation_min, -32768);
    TFLITE_DCHECK_EQ(output_activation_max, 32767);
  }

  tflite::ArithmeticParams op_params;
  op_params.input1_shift = kReverseShift * input1_shift;
  op_params.input2_shift = kReverseShift * input2_shift;
  op_params.quantized_activation_min = output_activation_min;
  op_params.quantized_activation_max = output_activation_max;
  Add(op_params, DimsToShape(input1_dims), input1_data,
      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
      output_data);
}


inline void GEMVForLstmCellWithSymmetricRange(
    const RuntimeShape& input_shape, const uint8* input_data,
    const RuntimeShape& weights_shape, const uint8* weights_data,
    const RuntimeShape& bias_shape, const int32* bias_data,
    int32 accum_multiplier, int accum_shift, const RuntimeShape& output_shape,
    int16* output_data) {
  ruy::profiler::ScopeLabel label("GEMVForLstmCellWithSymmetricRange");
  TFLITE_DCHECK_GE(input_shape.DimensionsCount(), 1);
  TFLITE_DCHECK_GE(weights_shape.DimensionsCount(), 2);
  TFLITE_DCHECK_GE(output_shape.DimensionsCount(), 1);
  const int output_dim_count = output_shape.DimensionsCount();
  const int weights_dim_count = weights_shape.DimensionsCount();
  TFLITE_DCHECK_EQ(FlatSizeSkipDim(output_shape, output_dim_count - 1), 1);
  const int input_size = FlatSizeSkipDim(input_shape, 0);
  const int output_size = MatchingDim(weights_shape, weights_dim_count - 2,
                                      output_shape, output_dim_count - 1);
  // This special fast path for quantized LSTM cells does not try to support
  // odd sizes that we haven't encountered in any LSTM cell, that would
  // require special code (that would go untested until any LSTM cell
  // exercises it). We just guard our assumptions about size evenness with
  // the following assertions.
  TFLITE_DCHECK(!(output_size % 4));
  TFLITE_DCHECK(!(input_size % 64));
  const int32* bias_ptr = bias_data;
  int16* output_ptr = output_data;
  const uint8x16_t signbit = vdupq_n_u8(0x80);
  for (int in = 0; in < input_size; in += 32) {
    optimized_ops_preload_l1_keep(input_data + in);
  }
  const int left_shift = accum_shift > 0 ? accum_shift : 0;
  const int right_shift = accum_shift > 0 ? 0 : -accum_shift;
  for (int out = 0; out < output_size; out += 4) {
    // Load the bias values
    int32x4_t bias_vec = vld1q_s32(bias_ptr);
    bias_ptr += 4;

    // Clear accumulators. We use 2 accumulator registers per row,
    // for 4 rows. row_accumRN is the N-th accumulator for row R.
    int32x4_t row_accum00 = vdupq_n_s32(0);
    int32x4_t row_accum01 = vdupq_n_s32(0);
    int32x4_t row_accum10 = vdupq_n_s32(0);
    int32x4_t row_accum11 = vdupq_n_s32(0);
    int32x4_t row_accum20 = vdupq_n_s32(0);
    int32x4_t row_accum21 = vdupq_n_s32(0);
    int32x4_t row_accum30 = vdupq_n_s32(0);
    int32x4_t row_accum31 = vdupq_n_s32(0);

    // kReadAhead parametrizes how far ahead we prefetch weights into L1 cache.
    const int kReadAhead = 512;
    // Prefetch the first weights values.
    for (int k = 0; k < kReadAhead; k += 64) {
      optimized_ops_preload_l1_stream(weights_data + (out + 0) * input_size +
                                      k);
      optimized_ops_preload_l1_stream(weights_data + (out + 1) * input_size +
                                      k);
      optimized_ops_preload_l1_stream(weights_data + (out + 2) * input_size +
                                      k);
      optimized_ops_preload_l1_stream(weights_data + (out + 3) * input_size +
                                      k);
    }
    // Loop along the rows, handling 64 bytes per iteration because that's
    // cache line size on most current ARM-architecture CPUs.
    for (int in = 0; in < input_size; in += 64) {
      // Prefetch some future weights values.
      optimized_ops_preload_l1_stream(weights_data + (out + 0) * input_size +
                                      in + kReadAhead);
      optimized_ops_preload_l1_stream(weights_data + (out + 1) * input_size +
                                      in + kReadAhead);
      optimized_ops_preload_l1_stream(weights_data + (out + 2) * input_size +
                                      in + kReadAhead);
      optimized_ops_preload_l1_stream(weights_data + (out + 3) * input_size +
                                      in + kReadAhead);

      // We will use 2 local 16-bit accumulators per row, for 2 rows.
      // See below (*) for the rationale of processing only 2 rows at a time.
      // local_accumRN is the N-th local accumulator for row R.
      int16x8_t local_accum00;
      int16x8_t local_accum01;
      int16x8_t local_accum10;
      int16x8_t local_accum11;

      // Load 64 bytes of input activations values. Convert to signed int8
      // by flipping the sign bit (i.e. subtracting 128, the required
      // zero_point value).
      int8x16_t input0 = vreinterpretq_s8_u8(
          veorq_u8(signbit, vld1q_u8(input_data + in + 16 * 0)));
      int8x16_t input1 = vreinterpretq_s8_u8(
          veorq_u8(signbit, vld1q_u8(input_data + in + 16 * 1)));
      int8x16_t input2 = vreinterpretq_s8_u8(
          veorq_u8(signbit, vld1q_u8(input_data + in + 16 * 2)));
      int8x16_t input3 = vreinterpretq_s8_u8(
          veorq_u8(signbit, vld1q_u8(input_data + in + 16 * 3)));

      // Beginning of the core accumulation. Notice how while we have 4
      // rows to process, this code is taking care of only 2 rows at a time,
      // thus being divided into two parts looking similar ("Rows 0 and 1" and
      // "Rows 2 and 3").
      //
      // (*) The rationale for handling only 2 rows at a time is to avoid
      // cache aliasing issues on 4-way set-associative L1-cache CPUs, such
      // as Cortex-A53. With sufficiently large, power-of-two matrix dimensions,
      // we may find ourselves in a situation where rows alias each other in
      // the L1 cache, and moreover may also mutually alias with the input
      // activations. If we try to load 4 rows at a time, together with the
      // input activations, that may be 5 mutually-aliasing vectors, resulting
      // in constant mutual eviction from L1 cache. Handling 2 rows at a time
      // here largely mitigates these issues, and seems at least to be very
      // effective on Cortex-A53:
      //                          Before       After
      // big (Cortex-A73)         2.85 ms      2.85 ms
      // little (Cortex-A53)      11.0 ms      5.16 ms

      // Rows 0 and 1:
      // Load 64 bytes of weights values from each row. Convert to signed int8
      // by flipping the sign bit (i.e. subtracting 128, the required
      // zero_point value).
      int8x16_t weights00 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 0) * input_size + in + 16 * 0)));
      int8x16_t weights01 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 0) * input_size + in + 16 * 1)));
      int8x16_t weights02 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 0) * input_size + in + 16 * 2)));
      int8x16_t weights03 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 0) * input_size + in + 16 * 3)));
      int8x16_t weights10 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 1) * input_size + in + 16 * 0)));
      int8x16_t weights11 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 1) * input_size + in + 16 * 1)));
      int8x16_t weights12 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 1) * input_size + in + 16 * 2)));
      int8x16_t weights13 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 1) * input_size + in + 16 * 3)));
      // Multiply-accumulate into local 16-bit accumulators.
      // We can accumulate two products without overflow because weights are
      // required to never be -128, so each product is at most 127^2 in absolute
      // value.
      local_accum00 = vmull_s8(vget_low_s8(weights00), vget_low_s8(input0));
      local_accum01 = vmull_s8(vget_low_s8(weights01), vget_low_s8(input1));
      local_accum10 = vmull_s8(vget_low_s8(weights10), vget_low_s8(input0));
      local_accum11 = vmull_s8(vget_low_s8(weights11), vget_low_s8(input1));
      local_accum00 = vmlal_s8(local_accum00, vget_high_s8(weights00),
                               vget_high_s8(input0));
      local_accum01 = vmlal_s8(local_accum01, vget_high_s8(weights01),
                               vget_high_s8(input1));
      local_accum10 = vmlal_s8(local_accum10, vget_high_s8(weights10),
                               vget_high_s8(input0));
      local_accum11 = vmlal_s8(local_accum11, vget_high_s8(weights11),
                               vget_high_s8(input1));
      // Pairwise add and accumulate into 32-bit accumulators
      row_accum00 = vpadalq_s16(row_accum00, local_accum00);
      row_accum01 = vpadalq_s16(row_accum01, local_accum01);
      row_accum10 = vpadalq_s16(row_accum10, local_accum10);
      row_accum11 = vpadalq_s16(row_accum11, local_accum11);
      // Multiply-accumulate into local 16-bit accumulators.
      // We can accumulate two products without overflow because weights are
      // required to never be -128, so each product is at most 127^2 in absolute
      // value.
      local_accum00 = vmull_s8(vget_low_s8(weights02), vget_low_s8(input2));
      local_accum01 = vmull_s8(vget_low_s8(weights03), vget_low_s8(input3));
      local_accum10 = vmull_s8(vget_low_s8(weights12), vget_low_s8(input2));
      local_accum11 = vmull_s8(vget_low_s8(weights13), vget_low_s8(input3));
      local_accum00 = vmlal_s8(local_accum00, vget_high_s8(weights02),
                               vget_high_s8(input2));
      local_accum01 = vmlal_s8(local_accum01, vget_high_s8(weights03),
                               vget_high_s8(input3));
      local_accum10 = vmlal_s8(local_accum10, vget_high_s8(weights12),
                               vget_high_s8(input2));
      local_accum11 = vmlal_s8(local_accum11, vget_high_s8(weights13),
                               vget_high_s8(input3));
      // Pairwise add and accumulate into 32-bit accumulators
      row_accum00 = vpadalq_s16(row_accum00, local_accum00);
      row_accum01 = vpadalq_s16(row_accum01, local_accum01);
      row_accum10 = vpadalq_s16(row_accum10, local_accum10);
      row_accum11 = vpadalq_s16(row_accum11, local_accum11);

      // Rows 2 and 3:
      // Load 64 bytes of weights values from each row. Convert to signed int8
      // by flipping the sign bit (i.e. subtracting 128, the required
      // zero_point value).
      weights00 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 2) * input_size + in + 16 * 0)));
      weights01 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 2) * input_size + in + 16 * 1)));
      weights02 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 2) * input_size + in + 16 * 2)));
      weights03 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 2) * input_size + in + 16 * 3)));
      weights10 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 3) * input_size + in + 16 * 0)));
      weights11 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 3) * input_size + in + 16 * 1)));
      weights12 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 3) * input_size + in + 16 * 2)));
      weights13 = vreinterpretq_s8_u8(veorq_u8(
          signbit,
          vld1q_u8(weights_data + (out + 3) * input_size + in + 16 * 3)));
      // Multiply-accumulate into local 16-bit accumulators.
      // We can accumulate two products without overflow because weights are
      // required to never be -128, so each product is at most 127^2 in absolute
      // value.
      local_accum00 = vmull_s8(vget_low_s8(weights00), vget_low_s8(input0));
      local_accum01 = vmull_s8(vget_low_s8(weights01), vget_low_s8(input1));
      local_accum10 = vmull_s8(vget_low_s8(weights10), vget_low_s8(input0));
      local_accum11 = vmull_s8(vget_low_s8(weights11), vget_low_s8(input1));
      local_accum00 = vmlal_s8(local_accum00, vget_high_s8(weights00),
                               vget_high_s8(input0));
      local_accum01 = vmlal_s8(local_accum01, vget_high_s8(weights01),
                               vget_high_s8(input1));
      local_accum10 = vmlal_s8(local_accum10, vget_high_s8(weights10),
                               vget_high_s8(input0));
      local_accum11 = vmlal_s8(local_accum11, vget_high_s8(weights11),
                               vget_high_s8(input1));
      // Pairwise add and accumulate into 32-bit accumulators
      row_accum20 = vpadalq_s16(row_accum20, local_accum00);
      row_accum21 = vpadalq_s16(row_accum21, local_accum01);
      row_accum30 = vpadalq_s16(row_accum30, local_accum10);
      row_accum31 = vpadalq_s16(row_accum31, local_accum11);
      // Multiply-accumulate into local 16-bit accumulators.
      // We can accumulate two products without overflow because weights are
      // required to never be -128, so each product is at most 127^2 in absolute
      // value.
      local_accum00 = vmull_s8(vget_low_s8(weights02), vget_low_s8(input2));
      local_accum01 = vmull_s8(vget_low_s8(weights03), vget_low_s8(input3));
      local_accum10 = vmull_s8(vget_low_s8(weights12), vget_low_s8(input2));
      local_accum11 = vmull_s8(vget_low_s8(weights13), vget_low_s8(input3));
      local_accum00 = vmlal_s8(local_accum00, vget_high_s8(weights02),
                               vget_high_s8(input2));
      local_accum01 = vmlal_s8(local_accum01, vget_high_s8(weights03),
                               vget_high_s8(input3));
      local_accum10 = vmlal_s8(local_accum10, vget_high_s8(weights12),
                               vget_high_s8(input2));
      local_accum11 = vmlal_s8(local_accum11, vget_high_s8(weights13),
                               vget_high_s8(input3));
      // Pairwise add and accumulate into 32-bit accumulators
      row_accum20 = vpadalq_s16(row_accum20, local_accum00);
      row_accum21 = vpadalq_s16(row_accum21, local_accum01);
      row_accum30 = vpadalq_s16(row_accum30, local_accum10);
      row_accum31 = vpadalq_s16(row_accum31, local_accum11);
    }

    row_accum00 = vaddq_s32(row_accum00, row_accum01);
    row_accum10 = vaddq_s32(row_accum10, row_accum11);
    row_accum20 = vaddq_s32(row_accum20, row_accum21);
    row_accum30 = vaddq_s32(row_accum30, row_accum31);
    // Horizontally reduce accumulators
    int32x2_t pairwise_reduced_acc_0, pairwise_reduced_acc_1,
        pairwise_reduced_acc_2, pairwise_reduced_acc_3;
    pairwise_reduced_acc_0 =
        vpadd_s32(vget_low_s32(row_accum00), vget_high_s32(row_accum00));
    pairwise_reduced_acc_1 =
        vpadd_s32(vget_low_s32(row_accum10), vget_high_s32(row_accum10));
    pairwise_reduced_acc_2 =
        vpadd_s32(vget_low_s32(row_accum20), vget_high_s32(row_accum20));
    pairwise_reduced_acc_3 =
        vpadd_s32(vget_low_s32(row_accum30), vget_high_s32(row_accum30));
    const int32x2_t reduced_lo =
        vpadd_s32(pairwise_reduced_acc_0, pairwise_reduced_acc_1);
    const int32x2_t reduced_hi =
        vpadd_s32(pairwise_reduced_acc_2, pairwise_reduced_acc_3);
    int32x4_t reduced = vcombine_s32(reduced_lo, reduced_hi);
    // Add bias values.
    reduced = vaddq_s32(reduced, bias_vec);
    reduced = vshlq_s32(reduced, vdupq_n_s32(left_shift));
    // Multiply by the fixed-point multiplier.
    reduced = vqrdmulhq_n_s32(reduced, accum_multiplier);
    // Rounding-shift-right.
    using gemmlowp::RoundingDivideByPOT;
    reduced = RoundingDivideByPOT(reduced, right_shift);
    // Narrow values down to 16 bit signed.
    const int16x4_t res16 = vqmovn_s32(reduced);
    vst1_s16(output_ptr, res16);
    output_ptr += 4;
  }
}


  LegacyPerChannelDepthwiseConvWorkerTask(
      const DepthwiseParams& params, const int32* output_multiplier,
      const int32* output_shift, const RuntimeShape& input_shape,
      const T* input_data, const RuntimeShape& filter_shape,
      const T* filter_data, const RuntimeShape& bias_shape, const TS* bias_data,
      const RuntimeShape& output_shape, T* output_data, int thread_start,
      int thread_end, int thread_dim)
      : params_(params),
        output_multiplier_(output_multiplier),
        output_shift_(output_shift),
        input_shape_(input_shape),
        input_data_(input_data),
        filter_shape_(filter_shape),
        filter_data_(filter_data),
        bias_shape_(bias_shape),
        bias_data_(bias_data),
        output_shape_(output_shape),
        output_data_(output_data),
        thread_start_(thread_start),
        thread_end_(thread_end),
        thread_dim_(thread_dim) {}


inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                           const int32* output_size_data,
                           const Dims<4>& output_size_dims, T* output_data,
                           const Dims<4>& output_dims, bool align_corners) {
  tflite::ResizeBilinearParams op_params;
  op_params.align_corners = align_corners;
  op_params.half_pixel_centers = false;
  ResizeBilinear(op_params, DimsToShape(input_dims), input_data,
                 DimsToShape(output_size_dims), output_size_data,
                 DimsToShape(output_dims), output_data);
}


void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
                       const T* input2_data, T* output_data,
                       const Dims<4>& output_dims) {
  Maximum(DimsToShape(input1_dims), input1_data, input2_data,
          DimsToShape(output_dims), output_data);
}


TfLiteStatus AverageEvalQuantizedInt16(TfLiteContext* context, TfLiteNode* node,
                                       TfLitePoolParams* params, OpData* data,
                                       const TfLiteTensor* input,
                                       TfLiteTensor* output) {
  int32_t activation_min;
  int32_t activation_max;
  CalculateActivationRangeQuantized(context, params->activation, output,
                                    &activation_min, &activation_max);
#define TF_LITE_AVERAGE_POOL(type)                                            \
  tflite::PoolParams op_params;                                               \
  op_params.stride_height = params->stride_height;                            \
  op_params.stride_width = params->stride_width;                              \
  op_params.filter_height = params->filter_height;                            \
  op_params.filter_width = params->filter_width;                              \
  op_params.padding_values.height = data->padding.height;                     \
  op_params.padding_values.width = data->padding.width;                       \
  op_params.quantized_activation_min = activation_min;                        \
  op_params.quantized_activation_max = activation_max;                        \
  TF_LITE_ENSURE(context, type::AveragePool(op_params, GetTensorShape(input), \
                                            GetTensorData<int16_t>(input),    \
                                            GetTensorShape(output),           \
                                            GetTensorData<int16_t>(output)))
  TF_LITE_AVERAGE_POOL(reference_integer_ops);
#undef TF_LITE_AVERAGE_POOL
  return kTfLiteOk;
}


inline void DepthwiseConv(const float* input_data, const Dims<4>& input_dims,
                          const float* filter_data, const Dims<4>& filter_dims,
                          const float* bias_data, const Dims<4>& bias_dims,
                          int stride_width, int stride_height, int pad_width,
                          int pad_height, int depth_multiplier,
                          float output_activation_min,
                          float output_activation_max, float* output_data,
                          const Dims<4>& output_dims) {
  DepthwiseConv(input_data, input_dims, filter_data, filter_dims, bias_data,
                bias_dims, stride_width, stride_height, 1, 1, pad_width,
                pad_height, depth_multiplier, output_activation_min,
                output_activation_max, output_data, output_dims);
}


void ArgMinMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims,
               T2* output_data, const Dims<4>& output_dims, const Cmp& cmp) {
  ArgMinMax(axis, DimsToShape(input_dims), input_data, DimsToShape(output_dims),
            output_data, cmp);
}


inline void BroadcastComparison(const T* input1_data,
                                const Dims<4>& input1_dims,
                                const T* input2_data,
                                const Dims<4>& input2_dims, bool* output_data,
                                const Dims<4>& output_dims) {
  ComparisonParams op_params;
  // No parameters needed.
  BroadcastComparison4DSlowImpl<T, F>(op_params, DimsToShape(input1_dims),
                                      input1_data, DimsToShape(input2_dims),
                                      input2_data, DimsToShape(output_dims),
                                      output_data);
}


inline void BatchToSpaceND(const T* input_data, const Dims<4>& input_dims,
                           const int32* block_shape_data,
                           const Dims<4>& block_shape_dims,
                           const int32* crops_data, const Dims<4>& crops_dims,
                           T* output_data, const Dims<4>& output_dims) {
  BatchToSpaceND(DimsToShape(input_dims), input_data,
                 DimsToShape(block_shape_dims), block_shape_data,
                 DimsToShape(crops_dims), crops_data, DimsToShape(output_dims),
                 output_data);
}


void MaxEvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node,
                          TfLitePoolParams* params, OpData* data,
                          const TfLiteTensor* input, TfLiteTensor* output) {
  int32_t activation_min;
  int32_t activation_max;
  (void)CalculateActivationRangeQuantized(context, params->activation, output,
                                          &activation_min, &activation_max);
#define TF_LITE_MAX_POOL(type)                                        \
  tflite::PoolParams op_params;                                       \
  op_params.stride_height = params->stride_height;                    \
  op_params.stride_width = params->stride_width;                      \
  op_params.filter_height = params->filter_height;                    \
  op_params.filter_width = params->filter_width;                      \
  op_params.padding_values.height = data->padding.height;             \
  op_params.padding_values.width = data->padding.width;               \
  op_params.quantized_activation_min = activation_min;                \
  op_params.quantized_activation_max = activation_max;                \
  type::MaxPool(op_params, GetTensorShape(input),                     \
                GetTensorData<int8_t>(input), GetTensorShape(output), \
                GetTensorData<int8_t>(output))
  if (kernel_type == kReference) {
    TF_LITE_MAX_POOL(reference_integer_ops);
  } else {
    TF_LITE_MAX_POOL(optimized_integer_ops);
  }
#undef TF_LITE_MAX_POOL
}


inline void ShuffledFullyConnected(
    const FullyConnectedParams& params, const RuntimeShape& input_shape,
    const uint8* input_data, const RuntimeShape& weights_shape,
    const uint8* shuffled_weights_data, const RuntimeShape& bias_shape,
    const int32* bias_data, const RuntimeShape& output_shape,
    int16* output_data, uint8* shuffled_input_workspace_data,
    gemmlowp::GemmContext* gemmlowp_context) {
  ruy::profiler::ScopeLabel label("ShuffledFullyConnected/8bit");
  const int32 output_multiplier = params.output_multiplier;
  const int output_shift = params.output_shift;
  const int32 output_activation_min = params.quantized_activation_min;
  const int32 output_activation_max = params.quantized_activation_max;
  (void)gemmlowp_context;  // only used in optimized code.
  TFLITE_DCHECK_EQ(output_activation_min, -32768);
  TFLITE_DCHECK_EQ(output_activation_max, 32767);
  TFLITE_DCHECK_GE(input_shape.DimensionsCount(), 1);
  TFLITE_DCHECK_GE(weights_shape.DimensionsCount(), 2);
  TFLITE_DCHECK_GE(output_shape.DimensionsCount(), 1);
  // TODO(b/62193649): This really should be:
  //     const int batches = ArraySize(output_dims, 1);
  // but the current --variable_batch hack consists in overwriting the 3rd
  // dimension with the runtime batch size, as we don't keep track for each
  // array of which dimension is the batch dimension in it.
  const int output_dim_count = output_shape.DimensionsCount();
  const int weights_dim_count = weights_shape.DimensionsCount();
  const int batches = FlatSizeSkipDim(output_shape, output_dim_count - 1);
  const int output_depth = MatchingDim(weights_shape, weights_dim_count - 2,
                                       output_shape, output_dim_count - 1);
  const int accum_depth = weights_shape.Dims(weights_dim_count - 1);
  TFLITE_DCHECK((accum_depth % 16) == 0);
  TFLITE_DCHECK((output_depth % 4) == 0);
  // Shuffled weights have had their sign bit (0x80) pre-flipped (xor'd)
  // so that just reinterpreting them as int8 values is equivalent to
  // subtracting 128 from them, thus implementing for free the subtraction of
  // the zero_point value 128.
  const int8* int8_shuffled_weights_data =
      reinterpret_cast<const int8*>(shuffled_weights_data);

  // Shuffling and xoring of input activations into the workspace buffer
  if (batches == 1) {
#ifdef USE_NEON
    const uint8x16_t signbit = vdupq_n_u8(0x80);
    for (int i = 0; i < accum_depth; i += 16) {
      uint8x16_t val = vld1q_u8(input_data + i);
      val = veorq_u8(val, signbit);
      vst1q_u8(shuffled_input_workspace_data + i, val);
    }
#else
    for (int i = 0; i < accum_depth; i++) {
      shuffled_input_workspace_data[i] = input_data[i] ^ 0x80;
    }
#endif
  } else if (batches == 4) {
    uint8* shuffled_input_workspace_ptr = shuffled_input_workspace_data;
    int c = 0;
#ifdef USE_NEON
    const uint8x16_t signbit = vdupq_n_u8(0x80);
    for (c = 0; c < accum_depth; c += 16) {
      const uint8* src_data_ptr = input_data + c;
      uint8x16_t val0 = vld1q_u8(src_data_ptr + 0 * accum_depth);
      uint8x16_t val1 = vld1q_u8(src_data_ptr + 1 * accum_depth);
      uint8x16_t val2 = vld1q_u8(src_data_ptr + 2 * accum_depth);
      uint8x16_t val3 = vld1q_u8(src_data_ptr + 3 * accum_depth);
      val0 = veorq_u8(val0, signbit);
      val1 = veorq_u8(val1, signbit);
      val2 = veorq_u8(val2, signbit);
      val3 = veorq_u8(val3, signbit);
      vst1q_u8(shuffled_input_workspace_ptr + 0, val0);
      vst1q_u8(shuffled_input_workspace_ptr + 16, val1);
      vst1q_u8(shuffled_input_workspace_ptr + 32, val2);
      vst1q_u8(shuffled_input_workspace_ptr + 48, val3);
      shuffled_input_workspace_ptr += 64;
    }
#else
    for (c = 0; c < accum_depth; c += 16) {
      for (int b = 0; b < 4; b++) {
        const uint8* src_data_ptr = input_data + b * accum_depth + c;
        for (int j = 0; j < 16; j++) {
          uint8 src_val = *src_data_ptr++;
          // Flip the sign bit, so that the kernel will only need to
          // reinterpret these uint8 values as int8, getting for free the
          // subtraction of the zero_point value 128.
          uint8 dst_val = src_val ^ 0x80;
          *shuffled_input_workspace_ptr++ = dst_val;
        }
      }
    }
#endif
  } else {
    TFLITE_DCHECK(false);
    return;
  }

  static constexpr int kKernelRows = 4;
  const int thread_count = gemmlowp::HowManyThreads<kKernelRows>(
      gemmlowp_context->max_num_threads(), output_depth, batches, accum_depth);
  if (thread_count == 1) {
    // Single-thread case: do the computation on the current thread, don't
    // use a threadpool
    ShuffledFullyConnectedWorkerImpl(
        shuffled_input_workspace_data, int8_shuffled_weights_data, batches,
        output_depth, output_depth, accum_depth, bias_data, output_multiplier,
        output_shift, output_data);
    return;
  }

  // Multi-threaded case: use the gemmlowp context's threadpool.
  TFLITE_DCHECK_GT(thread_count, 1);
  std::vector<gemmlowp::Task*> tasks(thread_count);
  const int kRowsPerWorker = gemmlowp::RoundUp<kKernelRows>(
      gemmlowp::CeilQuotient(output_depth, thread_count));
  int row_start = 0;
  for (int i = 0; i < thread_count; i++) {
    int row_end = std::min(output_depth, row_start + kRowsPerWorker);
    tasks[i] = new LegacyShuffledFullyConnectedWorkerTask(
        shuffled_input_workspace_data,
        int8_shuffled_weights_data + row_start * accum_depth, batches,
        row_end - row_start, output_depth, accum_depth, bias_data + row_start,
        output_multiplier, output_shift, output_data + row_start);
    row_start = row_end;
  }
  TFLITE_DCHECK_EQ(row_start, output_depth);
  gemmlowp_context->workers_pool()->LegacyExecuteAndDestroyTasks(tasks);
}


void TensorFlowMinimum(const T* input1_data, const Dims<4>& input1_dims,
                       const T* input2_data, T* output_data,
                       const Dims<4>& output_dims) {
  Minimum(DimsToShape(input1_dims), input1_data, input2_data,
          DimsToShape(output_dims), output_data);
}