int Lex_input_stream::lex_one_token(YYSTYPE *yylval, THD *thd)
{
  uchar UNINIT_VAR(c);
  bool comment_closed;
  int tokval;
  uint length;
  enum my_lex_states state;
  LEX *lex= thd->lex;
  CHARSET_INFO *const cs= thd->charset();
  const uchar *const state_map= cs->state_map;
  const uchar *const ident_map= cs->ident_map;

  start_token();
  state= next_state;
  next_state= MY_LEX_OPERATOR_OR_IDENT;
  for (;;)
  {
    switch (state) {
    case MY_LEX_OPERATOR_OR_IDENT:        // Next is operator or keyword
    case MY_LEX_START:                    // Start of token
      // Skip starting whitespace
      while(state_map[c= yyPeek()] == MY_LEX_SKIP)
      {
        if (c == '\n')
          yylineno++;

        yySkip();
      }

      /* Start of real token */
      restart_token();
      c= yyGet();
      state= (enum my_lex_states) state_map[c];
      break;
    case MY_LEX_ESCAPE:
      if (!eof() && yyGet() == 'N')
      {                                        // Allow \N as shortcut for NULL
        yylval->lex_str.str= (char*) "\\N";
        yylval->lex_str.length= 2;
        return NULL_SYM;
      }
      /* Fall through */
    case MY_LEX_CHAR:                          // Unknown or single char token
      if (c == '%' && (m_thd->variables.sql_mode & MODE_ORACLE))
      {
        next_state= MY_LEX_START;
        return PERCENT_ORACLE_SYM;
      }
      if (c == '[' && (m_thd->variables.sql_mode & MODE_MSSQL))
        return scan_ident_delimited(thd, &yylval->ident_cli, ']');
      /* Fall through */
    case MY_LEX_SKIP:                          // This should not happen
      if (c != ')')
        next_state= MY_LEX_START;         // Allow signed numbers
      yylval->kwd.set_keyword(m_tok_start, 1);
      return((int) c);

    case MY_LEX_MINUS_OR_COMMENT:
      if (yyPeek() == '-' &&
          (my_isspace(cs,yyPeekn(1)) ||
           my_iscntrl(cs,yyPeekn(1))))
      {
        state=MY_LEX_COMMENT;
        break;
      }
      next_state= MY_LEX_START;        // Allow signed numbers
      return((int) c);

    case MY_LEX_PLACEHOLDER:
      /*
        Check for a placeholder: it should not precede a possible identifier
        because of binlogging: when a placeholder is replaced with
        its value in a query for the binlog, the query must stay
        grammatically correct.
      */
      next_state= MY_LEX_START;        // Allow signed numbers
      if (stmt_prepare_mode && !ident_map[(uchar) yyPeek()])
        return(PARAM_MARKER);
      return((int) c);

    case MY_LEX_COMMA:
      next_state= MY_LEX_START;        // Allow signed numbers
      /*
        Warning:
        This is a work around, to make the "remember_name" rule in
        sql/sql_yacc.yy work properly.
        The problem is that, when parsing "select expr1, expr2",
        the code generated by bison executes the *pre* action
        remember_name (see select_item) *before* actually parsing the
        first token of expr2.
      */
      restart_token();
      return((int) c);

    case MY_LEX_IDENT_OR_NCHAR:
    {
      uint sep;
      if (yyPeek() != '\'')
      {
        state= MY_LEX_IDENT;
        break;
      }
      /* Found N'string' */
      yySkip();                         // Skip '
      if (get_text(&yylval->lex_string_with_metadata, (sep= yyGetLast()), 2, 1))
      {
        state= MY_LEX_CHAR;                    // Read char by char
        break;
      }

      body_utf8_append(m_cpp_text_start);
      body_utf8_append_escape(thd, &yylval->lex_string_with_metadata,
                                   national_charset_info,
                                   m_cpp_text_end, sep);
      return(NCHAR_STRING);
    }
    case MY_LEX_IDENT_OR_HEX:
      if (yyPeek() == '\'')
      {                                      // Found x'hex-number'
        state= MY_LEX_HEX_NUMBER;
        break;
      }
      /* fall through */
    case MY_LEX_IDENT_OR_BIN:
      if (yyPeek() == '\'')
      {                                 // Found b'bin-number'
        state= MY_LEX_BIN_NUMBER;
        break;
      }
      /* fall through */
    case MY_LEX_IDENT:
    {
      tokval= scan_ident_middle(thd, &yylval->ident_cli,
                                &yylval->charset, &state);
      if (!tokval)
        continue;
      if (tokval == UNDERSCORE_CHARSET)
        m_underscore_cs= yylval->charset;
      return tokval;
    }

    case MY_LEX_IDENT_SEP:                  // Found ident and now '.'
      yylval->lex_str.str= (char*) get_ptr();
      yylval->lex_str.length= 1;
      c= yyGet();                          // should be '.'
      if (lex->parsing_options.lookup_keywords_after_qualifier)
        next_state= MY_LEX_IDENT_OR_KEYWORD;
      else
        next_state= MY_LEX_IDENT_START;    // Next is ident (not keyword)
      if (!ident_map[(uchar) yyPeek()])    // Probably ` or "
        next_state= MY_LEX_START;
      return((int) c);

    case MY_LEX_NUMBER_IDENT:                   // number or ident which num-start
      if (yyGetLast() == '0')
      {
        c= yyGet();
        if (c == 'x')
        {
          while (my_isxdigit(cs, (c = yyGet()))) ;
          if ((yyLength() >= 3) && !ident_map[c])
          {
            /* skip '0x' */
            yylval->lex_str= get_token(2, yyLength() - 2);
            return (HEX_NUM);
          }
          yyUnget();
          state= MY_LEX_IDENT_START;
          break;
        }
        else if (c == 'b')
        {
          while ((c= yyGet()) == '0' || c == '1')
            ;
          if ((yyLength() >= 3) && !ident_map[c])
          {
            /* Skip '0b' */
            yylval->lex_str= get_token(2, yyLength() - 2);
            return (BIN_NUM);
          }
          yyUnget();
          state= MY_LEX_IDENT_START;
          break;
        }
        yyUnget();
      }

      while (my_isdigit(cs, (c= yyGet()))) ;
      if (!ident_map[c])
      {                                        // Can't be identifier
        state=MY_LEX_INT_OR_REAL;
        break;
      }
      if (c == 'e' || c == 'E')
      {
        // The following test is written this way to allow numbers of type 1e1
        if (my_isdigit(cs, yyPeek()) ||
            (c=(yyGet())) == '+' || c == '-')
        {                                       // Allow 1E+10
          if (my_isdigit(cs, yyPeek()))         // Number must have digit after sign
          {
            yySkip();
            while (my_isdigit(cs, yyGet())) ;
            yylval->lex_str= get_token(0, yyLength());
            return(FLOAT_NUM);
          }
        }
        /*
          We've found:
          - A sequence of digits
          - Followed by 'e' or 'E'
          - Followed by some byte XX which is not a known mantissa start,
            and it's known to be a valid identifier part.
            XX can be either a 8bit identifier character, or a multi-byte head.
        */
        yyUnget();
        return scan_ident_start(thd, &yylval->ident_cli);
      }
      /*
        We've found:
        - A sequence of digits
        - Followed by some character XX, which is neither 'e' nor 'E',
          and it's known to be a valid identifier part.
          XX can be a 8bit identifier character, or a multi-byte head.
      */
      yyUnget();
      return scan_ident_start(thd, &yylval->ident_cli);

    case MY_LEX_IDENT_START:                    // We come here after '.'
      return scan_ident_start(thd, &yylval->ident_cli);

    case MY_LEX_USER_VARIABLE_DELIMITER:        // Found quote char
      return scan_ident_delimited(thd, &yylval->ident_cli, m_tok_start[0]);

    case MY_LEX_INT_OR_REAL:                    // Complete int or incomplete real
      if (c != '.' || yyPeek() == '.')
      {
        /*
          Found a complete integer number:
          - the number is either not followed by a dot at all, or
          - the number is followed by a double dot as in: FOR i IN 1..10
        */
        yylval->lex_str= get_token(0, yyLength());
        return int_token(yylval->lex_str.str, (uint) yylval->lex_str.length);
      }
      // fall through
    case MY_LEX_REAL:                           // Incomplete real number
      while (my_isdigit(cs, c= yyGet())) ;

      if (c == 'e' || c == 'E')
      {
        c= yyGet();
        if (c == '-' || c == '+')
          c= yyGet();                           // Skip sign
        if (!my_isdigit(cs, c))
	  return ABORT_SYM; // No digit after sign
        while (my_isdigit(cs, yyGet())) ;
        yylval->lex_str= get_token(0, yyLength());
        return(FLOAT_NUM);
      }
      yylval->lex_str= get_token(0, yyLength());
      return(DECIMAL_NUM);

    case MY_LEX_HEX_NUMBER:             // Found x'hexstring'
      yySkip();                    // Accept opening '
      while (my_isxdigit(cs, (c= yyGet()))) ;
      if (c != '\'')
        return(ABORT_SYM);              // Illegal hex constant
      yySkip();                    // Accept closing '
      length= yyLength();          // Length of hexnum+3
      if ((length % 2) == 0)
        return(ABORT_SYM);              // odd number of hex digits
      yylval->lex_str= get_token(2,            // skip x'
                                 length - 3);  // don't count x' and last '
      return HEX_STRING;

    case MY_LEX_BIN_NUMBER:           // Found b'bin-string'
      yySkip();                  // Accept opening '
      while ((c= yyGet()) == '0' || c == '1')
        ;
      if (c != '\'')
        return(ABORT_SYM);            // Illegal hex constant
      yySkip();                  // Accept closing '
      length= yyLength();        // Length of bin-num + 3
      yylval->lex_str= get_token(2,           // skip b'
                                 length - 3); // don't count b' and last '
      return (BIN_NUM);

    case MY_LEX_CMP_OP:                     // Incomplete comparison operator
      next_state= MY_LEX_START;        // Allow signed numbers
      if (state_map[(uchar) yyPeek()] == MY_LEX_CMP_OP ||
          state_map[(uchar) yyPeek()] == MY_LEX_LONG_CMP_OP)
      {
        yySkip();
        if ((tokval= find_keyword(&yylval->kwd, 2, 0)))
          return(tokval);
        yyUnget();
      }
      return(c);

    case MY_LEX_LONG_CMP_OP:                // Incomplete comparison operator
      next_state= MY_LEX_START;
      if (state_map[(uchar) yyPeek()] == MY_LEX_CMP_OP ||
          state_map[(uchar) yyPeek()] == MY_LEX_LONG_CMP_OP)
      {
        yySkip();
        if (state_map[(uchar) yyPeek()] == MY_LEX_CMP_OP)
        {
          yySkip();
          if ((tokval= find_keyword(&yylval->kwd, 3, 0)))
            return(tokval);
          yyUnget();
        }
        if ((tokval= find_keyword(&yylval->kwd, 2, 0)))
          return(tokval);
        yyUnget();
      }
      return(c);

    case MY_LEX_BOOL:
      if (c != yyPeek())
      {
        state= MY_LEX_CHAR;
        break;
      }
      yySkip();
      tokval= find_keyword(&yylval->kwd, 2, 0);  // Is a bool operator
      next_state= MY_LEX_START;                  // Allow signed numbers
      return(tokval);

    case MY_LEX_STRING_OR_DELIMITER:
      if (thd->variables.sql_mode & MODE_ANSI_QUOTES)
      {
        state= MY_LEX_USER_VARIABLE_DELIMITER;
        break;
      }
      /* " used for strings */
      /* fall through */
    case MY_LEX_STRING:                        // Incomplete text string
    {
      uint sep;
      if (get_text(&yylval->lex_string_with_metadata, (sep= yyGetLast()), 1, 1))
      {
        state= MY_LEX_CHAR;                     // Read char by char
        break;
      }
      CHARSET_INFO *strcs= m_underscore_cs ? m_underscore_cs : cs;
      body_utf8_append(m_cpp_text_start);

      body_utf8_append_escape(thd, &yylval->lex_string_with_metadata,
                                   strcs, m_cpp_text_end, sep);
      m_underscore_cs= NULL;
      return(TEXT_STRING);
    }
    case MY_LEX_COMMENT:                       //  Comment
      lex->lex_options|= OPTION_LEX_FOUND_COMMENT;
      while ((c= yyGet()) != '\n' && c) ;
      yyUnget();                          // Safety against eof
      state= MY_LEX_START;                     // Try again
      break;
    case MY_LEX_LONG_COMMENT:                  // Long C comment?
      if (yyPeek() != '*')
      {
        state= MY_LEX_CHAR;                     // Probable division
        break;
      }
      lex->lex_options|= OPTION_LEX_FOUND_COMMENT;
      /* Reject '/' '*', since we might need to turn off the echo */
      yyUnget();

      save_in_comment_state();

      if (yyPeekn(2) == '!' ||
          (yyPeekn(2) == 'M' && yyPeekn(3) == '!'))
      {
        bool maria_comment_syntax= yyPeekn(2) == 'M';
        in_comment= DISCARD_COMMENT;
        /* Accept '/' '*' '!', but do not keep this marker. */
        set_echo(FALSE);
        yySkipn(maria_comment_syntax ? 4 : 3);

        /*
          The special comment format is very strict:
          '/' '*' '!', followed by an optional 'M' and exactly
          1-2 digits (major), 2 digits (minor), then 2 digits (dot).
          32302  -> 3.23.02
          50032  -> 5.0.32
          50114  -> 5.1.14
          100000 -> 10.0.0
        */
        if (  my_isdigit(cs, yyPeekn(0))
           && my_isdigit(cs, yyPeekn(1))
           && my_isdigit(cs, yyPeekn(2))
           && my_isdigit(cs, yyPeekn(3))
           && my_isdigit(cs, yyPeekn(4))
           )
        {
          ulong version;
          uint length= 5;
          char *end_ptr= (char*) get_ptr() + length;
          int error;
          if (my_isdigit(cs, yyPeekn(5)))
          {
            end_ptr++;                          // 6 digit number
            length++;
          }

          version= (ulong) my_strtoll10(get_ptr(), &end_ptr, &error);

          /*
            MySQL-5.7 has new features and might have new SQL syntax that
            MariaDB-10.0 does not understand. Ignore all versioned comments
            with MySQL versions in the range 50700-999999, but
            do not ignore MariaDB specific comments for the same versions.
          */ 
          if (version <= MYSQL_VERSION_ID &&
              (version < 50700 || version > 99999 || maria_comment_syntax))
          {
            /* Accept 'M' 'm' 'm' 'd' 'd' */
            yySkipn(length);
            /* Expand the content of the special comment as real code */
            set_echo(TRUE);
            state=MY_LEX_START;
            break;  /* Do not treat contents as a comment.  */
          }
          else
          {
#ifdef WITH_WSREP
            if (WSREP(thd) && version == 99997 && wsrep_thd_is_local(thd))
            {
              WSREP_DEBUG("consistency check: %s", thd->query());
              thd->wsrep_consistency_check= CONSISTENCY_CHECK_DECLARED;
              yySkipn(5);
              set_echo(TRUE);
              state= MY_LEX_START;
              break;  /* Do not treat contents as a comment.  */
            }
#endif /* WITH_WSREP */
            /*
              Patch and skip the conditional comment to avoid it
              being propagated infinitely (eg. to a slave).
            */
            char *pcom= yyUnput(' ');
            comment_closed= ! consume_comment(1);
            if (! comment_closed)
            {
              *pcom= '!';
            }
            /* version allowed to have one level of comment inside. */
          }
        }
        else
        {
          /* Not a version comment. */
          state=MY_LEX_START;
          set_echo(TRUE);
          break;
        }
      }
      else
      {
        in_comment= PRESERVE_COMMENT;
        yySkip();                  // Accept /
        yySkip();                  // Accept *
        comment_closed= ! consume_comment(0);
        /* regular comments can have zero comments inside. */
      }
      /*
        Discard:
        - regular '/' '*' comments,
        - special comments '/' '*' '!' for a future version,
        by scanning until we find a closing '*' '/' marker.

        Nesting regular comments isn't allowed.  The first 
        '*' '/' returns the parser to the previous state.

        /#!VERSI oned containing /# regular #/ is allowed #/

                Inside one versioned comment, another versioned comment
                is treated as a regular discardable comment.  It gets
                no special parsing.
      */

      /* Unbalanced comments with a missing '*' '/' are a syntax error */
      if (! comment_closed)
        return (ABORT_SYM);
      state = MY_LEX_START;             // Try again
      restore_in_comment_state();
      break;
    case MY_LEX_END_LONG_COMMENT:
      if ((in_comment != NO_COMMENT) && yyPeek() == '/')
      {
        /* Reject '*' '/' */
        yyUnget();
        /* Accept '*' '/', with the proper echo */
        set_echo(in_comment == PRESERVE_COMMENT);
        yySkipn(2);
        /* And start recording the tokens again */
        set_echo(TRUE);
        in_comment= NO_COMMENT;
        state=MY_LEX_START;
      }
      else
        state= MY_LEX_CHAR;              // Return '*'
      break;
    case MY_LEX_SET_VAR:                // Check if ':='
      if (yyPeek() != '=')
      {
        next_state= MY_LEX_START;
        if (m_thd->variables.sql_mode & MODE_ORACLE)
        {
          yylval->kwd.set_keyword(m_tok_start, 1);
          return COLON_ORACLE_SYM;
        }
        return (int) ':';
      }
      yySkip();
      return (SET_VAR);
    case MY_LEX_SEMICOLON:              // optional line terminator
      state= MY_LEX_CHAR;               // Return ';'
      break;
    case MY_LEX_EOL:
      if (eof())
      {
        yyUnget();                 // Reject the last '\0'
        set_echo(FALSE);
        yySkip();
        set_echo(TRUE);
        /* Unbalanced comments with a missing '*' '/' are a syntax error */
        if (in_comment != NO_COMMENT)
          return (ABORT_SYM);
        next_state= MY_LEX_END;     // Mark for next loop
        return(END_OF_INPUT);
      }
      state=MY_LEX_CHAR;
      break;
    case MY_LEX_END:
      next_state= MY_LEX_END;
      return(0);                        // We found end of input last time

      /* Actually real shouldn't start with . but allow them anyhow */
    case MY_LEX_REAL_OR_POINT:
      if (my_isdigit(cs, (c= yyPeek())))
        state = MY_LEX_REAL;            // Real
      else if (c == '.')
      {
        yySkip();
        return DOT_DOT_SYM;
      }
      else
      {
        state= MY_LEX_IDENT_SEP;        // return '.'
        yyUnget();                 // Put back '.'
      }
      break;
    case MY_LEX_USER_END:               // end '@' of user@hostname
      switch (state_map[(uchar) yyPeek()]) {
      case MY_LEX_STRING:
      case MY_LEX_USER_VARIABLE_DELIMITER:
      case MY_LEX_STRING_OR_DELIMITER:
        break;
      case MY_LEX_USER_END:
        next_state= MY_LEX_SYSTEM_VAR;
        break;
      default:
        next_state= MY_LEX_HOSTNAME;
        break;
      }
      yylval->lex_str.str= (char*) get_ptr();
      yylval->lex_str.length= 1;
      return((int) '@');
    case MY_LEX_HOSTNAME:               // end '@' of user@hostname
      for (c= yyGet() ;
           my_isalnum(cs, c) || c == '.' || c == '_' ||  c == '$';
           c= yyGet()) ;
      yylval->lex_str= get_token(0, yyLength());
      return(LEX_HOSTNAME);
    case MY_LEX_SYSTEM_VAR:
      yylval->lex_str.str= (char*) get_ptr();
      yylval->lex_str.length= 1;
      yySkip();                                    // Skip '@'
      next_state= (state_map[(uchar) yyPeek()] ==
                        MY_LEX_USER_VARIABLE_DELIMITER ?
                        MY_LEX_OPERATOR_OR_IDENT :
                        MY_LEX_IDENT_OR_KEYWORD);
      return((int) '@');
    case MY_LEX_IDENT_OR_KEYWORD:
      /*
        We come here when we have found two '@' in a row.
        We should now be able to handle:
        [(global | local | session) .]variable_name
      */
      return scan_ident_sysvar(thd, &yylval->ident_cli);
    }
  }
}