Commit message: SERVER-17264: improve bson validation for utf-8 strings

(cherry picked from commit 394a8569ff14a215c0691aa34440227b2e62a4de)
CVE: CVE-2015-1609
Vulnerability description: MongoDB before 2.4.13 and 2.6.x before 2.6.8 allows remote attackers to cause a denial of service via a crafted UTF-8 string in a BSON request.

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, ErrorWithId) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        ob.append("_id", 1);
        appendInvalidStringElement("not_id", &bb);
        const BSONObj x = ob.done();
        const Status status = validateBSON(x.objdata(), x.objsize());
        ASSERT_NOT_OK(status);
        ASSERT_EQUALS(status.reason(), "not null terminated string in object with _id: 1");
    }

Project: mongo
File: bson_validate.cpp

            bool skip( uint64_t sz ) {
                _position += sz;
                return _position < _maxLength;
            }

Project: mongo
File: bson_validate.cpp

            void setIdElem(BSONElement idElem) {
                _idElem = idElem;
            }

Project: mongo
File: bson_validate_test.cpp

    TEST( BSONValidateFast, RegEx ) {
        BSONObjBuilder b;
        b.appendRegex( "foo", "i" );
        BSONObj x = b.obj();
        ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );
    }

Project: mongo
File: bson_validate.cpp

    Status validateBSON( const char* originalBuffer, uint64_t maxLength ) {
        if ( maxLength < 5 ) {
            return Status( ErrorCodes::InvalidBSON, "bson data has to be at least 5 bytes" );
        }

        Buffer buf( originalBuffer, maxLength );
        return validateBSONIterative( &buf );
    }

Project: mongo
File: bson_validate.cpp

            int startPosition() const { return _startPosition & ~(1 << 31); }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, ErrorBeforeId) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        appendInvalidStringElement("not_id", &bb);
        ob.append("_id", 1);
        const BSONObj x = ob.done();
        const Status status = validateBSON(x.objdata(), x.objsize());
        ASSERT_NOT_OK(status);
        ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id");
    }

Project: mongo
File: bson_validate.cpp

            Status readCString( StringData* out ) {
                const void* x = memchr( _buffer + _position, 0, _maxLength - _position );
                if ( !x )
                    return makeError("no end of c-string", _idElem);
                uint64_t len = static_cast<uint64_t>( static_cast<const char*>(x) - ( _buffer + _position ) );

                StringData data( _buffer + _position, len );
                _position += len + 1;

                if ( out ) {
                    *out = data;
                }
                return Status::OK();
            }

Project: mongo
File: bson_validate_test.cpp

    TEST( BSONValidate, Fuzz ) {
        int64_t seed = time( 0 );
        log() << "BSONValidate Fuzz random seed: " << seed << endl;
        PseudoRandom randomSource( seed );

        BSONObj original = BSON( "one" << 3 <<
                                 "two" << 5 <<
                                 "three" << BSONObj() <<
                                 "four" << BSON( "five" << BSON( "six" << 11 ) ) <<
                                 "seven" << BSON_ARRAY( "a" << "bb" << "ccc" << 5 ) <<
                                 "eight" << BSONDBRef( "rrr", OID( "01234567890123456789aaaa" ) ) <<
                                 "_id" << OID( "deadbeefdeadbeefdeadbeef" ) <<
                                 "nine" << BSONBinData( "\x69\xb7", 2, BinDataGeneral ) <<
                                 "ten" << Date_t( 44 ) <<
                                 "eleven" << BSONRegEx( "foooooo", "i" ) );
        
        int32_t fuzzFrequencies[] = { 2, 10, 20, 100, 1000 };
        for( size_t i = 0; i < sizeof( fuzzFrequencies ) / sizeof( int32_t ); ++i ) {
            int32_t fuzzFrequency = fuzzFrequencies[ i ];

            // Copy the 'original' BSONObj to 'buffer'.
            scoped_array<char> buffer( new char[ original.objsize() ] );
            memcpy( buffer.get(), original.objdata(), original.objsize() );

            // Randomly flip bits in 'buffer', with probability determined by 'fuzzFrequency'. The
            // first four bytes, representing the size of the object, are excluded from bit
            // flipping.
            for( int32_t byteIdx = 4; byteIdx < original.objsize(); ++byteIdx ) {
                for( int32_t bitIdx = 0; bitIdx < 8; ++bitIdx ) {
                    if ( randomSource.nextInt32( fuzzFrequency ) == 0 ) {
                        reinterpret_cast<unsigned char&>( buffer[ byteIdx ] ) ^= ( 1U << bitIdx );
                    }
                }
            }
            BSONObj fuzzed( buffer.get() );

            // Check that the two validation implementations agree (and neither crashes).
            ASSERT_EQUALS( fuzzed.valid(),
                           validateBSON( fuzzed.objdata(), fuzzed.objsize() ).isOK() );
        }
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, Simple3 ) {
        BSONObjBuilder b;
        char buf[64];
        for ( int i=1; i<=JSTypeMax; i++ ) {
            sprintf( buf, "foo%d", i );
            b.appendMinForType( buf, i );
            sprintf( buf, "bar%d", i );
            b.appendMaxForType( buf, i );
        }
        BSONObj x = b.obj();
        ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, NestedObject) {
        BSONObj x = BSON( "a" << 1 << "b" << BSON("c" << 2 << "d" << BSONArrayBuilder().obj() << "e" << BSON_ARRAY("1" << 2 << 3)));
        ASSERT_OK(validateBSON(x.objdata(), x.objsize()));
        ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize() / 2));
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidate, RandomData) {
        PseudoRandom r(17);

        int numValid = 0;
        int numToRun = 1000;
        long long jsonSize = 0;

        for ( int i=0; i<numToRun; i++ ) {
            int size = 1234;

            char* x = new char[size];
            DataView(x).writeLE(size);

            for ( int i=4; i<size; i++ ) {
                x[i] = r.nextInt32( 255 );
            }

            x[size-1] = 0;

            BSONObj o( x );

            ASSERT_EQUALS( size, o.objsize() );

            if ( o.valid() ) {
                numValid++;
                jsonSize += o.jsonString().size();
                ASSERT_OK( validateBSON( o.objdata(), o.objsize() ) );
            }
            else {
                ASSERT_NOT_OK( validateBSON( o.objdata(), o.objsize() ) );
            }

            delete[] x;
        }

        log() << "RandomData: didn't crash valid/total: " << numValid << "/" << numToRun
              << " (want few valid ones)"
              << " jsonSize: " << jsonSize << endl;
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, Simple2 ) {
        char buf[64];
        for ( int i=1; i<=JSTypeMax; i++ ) {
            BSONObjBuilder b;
            sprintf( buf, "foo%d", i );
            b.appendMinForType( buf, i );
            sprintf( buf, "bar%d", i );
            b.appendMaxForType( buf, i );
            BSONObj x = b.obj();
            ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );
        }
    }

Project: mongo
File: bson_validate.cpp

            Buffer( const char* buffer, uint64_t maxLength )
                : _buffer( buffer ), _position( 0 ), _maxLength( maxLength ) {
            }

Project: mongo
File: bson_validate.cpp

            bool readNumber( N* out ) {
                if ( ( _position + sizeof(N) ) > _maxLength )
                    return false;
                if ( out ) {
                    *out = ConstDataView(_buffer).readLE<N>(_position);
                }
                _position += sizeof(N);
                return true;
            }

Project: mongo
File: bson_validate.cpp

            const char* getBasePtr() const {
                return _buffer;
            }

Project: mongo
File: bson_validate.cpp

        Status validateElementInfo(Buffer* buffer,
                                   ValidationState::State* nextState,
                                   BSONElement idElem) {
            Status status = Status::OK();

            signed char type;
            if ( !buffer->readNumber<signed char>(&type) )
                return makeError("invalid bson", idElem);

            if ( type == EOO ) {
                *nextState = ValidationState::EndObj;
                return Status::OK();
            }

            StringData name;
            status = buffer->readCString( &name );
            if ( !status.isOK() )
                return status;

            switch ( type ) {
            case MinKey:
            case MaxKey:
            case jstNULL:
            case Undefined:
                return Status::OK();

            case jstOID:
                if ( !buffer->skip( OID::kOIDSize ) )
                    return makeError("invalid bson", idElem);
                return Status::OK();

            case NumberInt:
                if ( !buffer->skip( sizeof(int32_t) ) )
                    return makeError("invalid bson", idElem);
                return Status::OK();

            case Bool:
                if ( !buffer->skip( sizeof(int8_t) ) )
                    return makeError("invalid bson", idElem);
                return Status::OK();


            case NumberDouble:
            case NumberLong:
            case Timestamp:
            case Date:
                if ( !buffer->skip( sizeof(int64_t) ) )
                    return makeError("invalid bson", idElem);
                return Status::OK();

            case DBRef:
                status = buffer->readUTF8String( NULL );
                if ( !status.isOK() )
                    return status;
                buffer->skip( OID::kOIDSize );
                return Status::OK();

            case RegEx:
                status = buffer->readCString( NULL );
                if ( !status.isOK() )
                    return status;
                status = buffer->readCString( NULL );
                if ( !status.isOK() )
                    return status;

                return Status::OK();

            case Code:
            case Symbol:
            case String:
                status = buffer->readUTF8String( NULL );
                if ( !status.isOK() )
                    return status;
                return Status::OK();

            case BinData: {
                int sz;
                if ( !buffer->readNumber<int>( &sz ) )
                    return makeError("invalid bson", idElem);
                if ( !buffer->skip( 1 + sz ) )
                    return makeError("invalid bson", idElem);
                return Status::OK();
            }
            case CodeWScope:
                *nextState = ValidationState::BeginCodeWScope;
                return Status::OK();
            case Object:
            case Array:
                *nextState = ValidationState::BeginObj;
                return Status::OK();

            default:
                return makeError("invalid bson type", idElem);
            }
        }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, ErrorNoId) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        appendInvalidStringElement("not_id", &bb);
        const BSONObj x = ob.done();
        const Status status = validateBSON(x.objdata(), x.objsize());
        ASSERT_NOT_OK(status);
        ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id");
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidate, MuckingData1) {

        BSONObj theObject;

        {
            BSONObjBuilder b;
            b.append( "name" , "eliot was here" );
            b.append( "yippee" , "asd" );
            BSONArrayBuilder a( b.subarrayStart( "arr" ) );
            for ( int i=0; i<100; i++ ) {
                a.append( BSON( "x" << i << "who" << "me" << "asd" << "asd" ) );
            }
            a.done();
            b.done();

            theObject = b.obj();
        }

        int numValid = 0;
        int numToRun = 1000;
        long long jsonSize = 0;

        for ( int i=4; i<theObject.objsize()-1; i++ ) {
            BSONObj mine = theObject.copy();

            char* data = const_cast<char*>(mine.objdata());

            data[ i ] = 200;

            numToRun++;
            if ( mine.valid() ) {
                numValid++;
                jsonSize += mine.jsonString().size();
                ASSERT_OK( validateBSON( mine.objdata(), mine.objsize() ) );
            }
            else {
                ASSERT_NOT_OK( validateBSON( mine.objdata(), mine.objsize() ) );
            }

        }

        log() << "MuckingData1: didn't crash valid/total: " << numValid << "/" << numToRun
              << " (want few valid ones) "
              << " jsonSize: " << jsonSize << endl;
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, NonTopLevelId) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        ob.append("not_id1", BSON("_id" << "not the real _id"));
        appendInvalidStringElement("not_id2", &bb);
        const BSONObj x = ob.done();
        const Status status = validateBSON(x.objdata(), x.objsize());
        ASSERT_NOT_OK(status);
        ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id");
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, Simple0 ) {
        BSONObj x;
        ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );

        x = BSON( "foo" << 17 << "bar" << "eliot" );
        ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );

    }

Project: mongo
File: bson_validate.cpp

            bool isCodeWithScope() const { return _startPosition & (1 << 31); }

Project: mongo
File: bson_validate_test.cpp

    void appendInvalidStringElement(const char* fieldName, BufBuilder* bb) {
        // like a BSONObj string, but without a NUL terminator.
        bb->appendChar(String);
        bb->appendStr(fieldName, /*withNUL*/true);
        bb->appendNum(4);
        bb->appendStr("asdf", /*withNUL*/false);
    }

Project: mongo
File: bson_validate.cpp

            uint64_t position() const {
                return _position;
            }

Project: mongo
File: bson_validate.cpp

            void setIsCodeWithScope(bool isCodeWithScope) {
                if (isCodeWithScope) {
                    _startPosition |= 1 << 31;
                }
                else {
                    _startPosition &= ~(1 << 31);
                }
            }

Project: mongo
File: bson_validate.cpp

        Status validateBSONIterative(Buffer* buffer) {
            std::deque<ValidationObjectFrame> frames;
            ValidationObjectFrame* curr = NULL;
            ValidationState::State state = ValidationState::BeginObj;

            uint64_t idElemStartPos = 0; // will become idElem once validated
            BSONElement idElem;

            while (state != ValidationState::Done) {
                switch (state) {
                case ValidationState::BeginObj:
                    frames.push_back(ValidationObjectFrame());
                    curr = &frames.back();
                    curr->setStartPosition(buffer->position());
                    curr->setIsCodeWithScope(false);
                    if (!buffer->readNumber<int>(&curr->expectedSize)) {
                        return makeError("bson size is larger than buffer size", idElem);
                    }
                    state = ValidationState::WithinObj;
                    // fall through
                case ValidationState::WithinObj: {
                    const bool atTopLevel = frames.size() == 1;
                    // check if we've finished validating idElem and are at start of next element.
                    if (atTopLevel && idElemStartPos) {
                        idElem = BSONElement(buffer->getBasePtr() + idElemStartPos);
                        buffer->setIdElem(idElem);
                        idElemStartPos = 0;
                    }

                    const uint64_t elemStartPos = buffer->position();
                    ValidationState::State nextState = state;
                    Status status = validateElementInfo(buffer, &nextState, idElem);
                    if (!status.isOK())
                        return status;

                    // we've already validated that fieldname is safe to access as long as we aren't
                    // at the end of the object, since EOO doesn't have a fieldname.
                    if (nextState != ValidationState::EndObj && idElem.eoo() && atTopLevel) {
                        if (strcmp(buffer->getBasePtr() + elemStartPos + 1/*type*/, "_id") == 0) {
                            idElemStartPos = elemStartPos;
                        }
                    }

                    state = nextState;
                    break;
                }
                case ValidationState::EndObj: {
                    int actualLength = buffer->position() - curr->startPosition();
                    if ( actualLength != curr->expectedSize ) {
                        return makeError("bson length doesn't match what we found", idElem);
                    }
                    frames.pop_back();
                    if (frames.empty()) {
                        state = ValidationState::Done;
                    }
                    else {
                        curr = &frames.back();
                        if (curr->isCodeWithScope())
                            state = ValidationState::EndCodeWScope;
                        else
                            state = ValidationState::WithinObj;
                    }
                    break;
                }
                case ValidationState::BeginCodeWScope: {
                    frames.push_back(ValidationObjectFrame());
                    curr = &frames.back();
                    curr->setStartPosition(buffer->position());
                    curr->setIsCodeWithScope(true);
                    if ( !buffer->readNumber<int>( &curr->expectedSize ) )
                        return makeError("invalid bson CodeWScope size", idElem);
                    Status status = buffer->readUTF8String( NULL );
                    if ( !status.isOK() )
                        return status;
                    state = ValidationState::BeginObj;
                    break;
                }
                case ValidationState::EndCodeWScope: {
                    int actualLength = buffer->position() - curr->startPosition();
                    if ( actualLength != curr->expectedSize ) {
                        return makeError("bson length for CodeWScope doesn't match what we found",
                                         idElem);
                    }
                    frames.pop_back();
                    if (frames.empty())
                        return makeError("unnested CodeWScope", idElem);
                    curr = &frames.back();
                    state = ValidationState::WithinObj;
                    break;
                }
                case ValidationState::Done:
                    break;
                }
            }

            return Status::OK();
        }

Project: mongo
File: bson_validate_test.cpp

    TEST( BSONValidateFast, Empty ) {
        BSONObj x;
        ASSERT_OK( validateBSON( x.objdata(), x.objsize() ) );
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, StringHasSomething) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        bb.appendChar(String);
        bb.appendStr("x", /*withNUL*/true);
        bb.appendNum(0);
        const BSONObj x = ob.done();
        ASSERT_EQUALS(5 // overhead
                      + 1 // type
                      + 2 // name
                      + 4 // size
                      , x.objsize());
        ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize()));
    }

Project: mongo
File: bson_validate.cpp

        Status makeError(std::string baseMsg, BSONElement idElem) {
            if (idElem.eoo()) {
                baseMsg += " in object with unknown _id";
            }
            else {
                baseMsg += " in object with " + idElem.toString(/*field name=*/true, /*full=*/true);
            }
            return Status(ErrorCodes::InvalidBSON, baseMsg);
        }

Project: mongo
File: bson_validate.cpp

            Status readUTF8String( StringData* out ) {
                int sz;
                if ( !readNumber<int>( &sz ) )
                    return makeError("invalid bson", _idElem);

                if ( sz <= 0 ) {
                    // must have NULL at the very least
                    return makeError("invalid bson", _idElem);
                }

                if ( out ) {
                    *out = StringData( _buffer + _position, sz );
                }

                if ( !skip( sz - 1 ) )
                    return makeError("invalid bson", _idElem);

                char c;
                if ( !readNumber<char>( &c ) )
                    return makeError("invalid bson", _idElem);

                if ( c != 0 )
                    return makeError("not null terminated string", _idElem);

                return Status::OK();
            }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidateFast, ErrorIsInId) {
        BufBuilder bb;
        BSONObjBuilder ob(bb);
        appendInvalidStringElement("_id", &bb);
        const BSONObj x = ob.done();
        const Status status = validateBSON(x.objdata(), x.objsize());
        ASSERT_NOT_OK(status);
        ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id");
    }

Project: mongo
File: bson_validate_test.cpp

    TEST(BSONValidate, Basic) {
        BSONObj x;
        ASSERT_TRUE( x.valid() );

        x = BSON( "x" << 1 );
        ASSERT_TRUE( x.valid() );
    }

Project: mongo
File: bson_validate.cpp

            void setStartPosition(int pos) {
                _startPosition = (_startPosition & (1 << 31)) | (pos & ~(1 << 31));
            }