int Lex_input_stream::lex_token(YYSTYPE *yylval, THD *thd)
{
  int token;
  const int left_paren= (int) '(';

  if (lookahead_token >= 0)
  {
    /*
      The next token was already parsed in advance,
      return it.
    */
    token= lookahead_token;
    lookahead_token= -1;
    *yylval= *(lookahead_yylval);
    lookahead_yylval= NULL;
    return token;
  }

  token= lex_one_token(yylval, thd);
  add_digest_token(token, yylval);

  SELECT_LEX *curr_sel= thd->lex->current_select;

  switch(token) {
  case WITH:
    /*
      Parsing 'WITH' 'ROLLUP' or 'WITH' 'CUBE' requires 2 look ups,
      which makes the grammar LALR(2).
      Replace by a single 'WITH_ROLLUP' or 'WITH_CUBE' token,
      to transform the grammar into a LALR(1) grammar,
      which sql_yacc.yy can process.
    */
    token= lex_one_token(yylval, thd);
    add_digest_token(token, yylval);
    switch(token) {
    case CUBE_SYM:
      return WITH_CUBE_SYM;
    case ROLLUP_SYM:
      return WITH_ROLLUP_SYM;
    case SYSTEM:
      return WITH_SYSTEM_SYM;
    default:
      /*
        Save the token following 'WITH'
      */
      lookahead_yylval= yylval;
      lookahead_token= token;
      return WITH;
    }
    break;
  case FOR_SYM:
    /*
     * Additional look-ahead to resolve doubtful cases like:
     * SELECT ... FOR UPDATE
     * SELECT ... FOR SYSTEM_TIME ... .
     */
    token= lex_one_token(yylval, thd);
    add_digest_token(token, yylval);
    switch(token) {
    case SYSTEM_TIME_SYM:
      return FOR_SYSTEM_TIME_SYM;
    default:
      /*
        Save the token following 'FOR_SYM'
      */
      lookahead_yylval= yylval;
      lookahead_token= token;
      return FOR_SYM;
    }
    break;
  case VALUES:
    if (curr_sel &&
        (curr_sel->parsing_place == BEFORE_OPT_LIST ||
         curr_sel->parsing_place == AFTER_LIST))
    {
      curr_sel->parsing_place= NO_MATTER;
      break;
    }
    if (curr_sel &&
        (curr_sel->parsing_place == IN_UPDATE_ON_DUP_KEY ||
         curr_sel->parsing_place == IN_PART_FUNC))
      return VALUE_SYM;
    token= lex_one_token(yylval, thd);
    add_digest_token(token, yylval);
    switch(token) {
    case LESS_SYM:
      return VALUES_LESS_SYM;
    case IN_SYM:
      return VALUES_IN_SYM;
    default:
      lookahead_yylval= yylval;
      lookahead_token= token;
      return VALUES;
    }
  case VALUE_SYM:
    if (curr_sel &&
        (curr_sel->parsing_place == BEFORE_OPT_LIST ||
         curr_sel->parsing_place == AFTER_LIST))
    {
      curr_sel->parsing_place= NO_MATTER;
      return VALUES;
    }
    break;
  case PARTITION_SYM:
  case SELECT_SYM:
  case UNION_SYM:
    if (curr_sel &&
        (curr_sel->parsing_place == BEFORE_OPT_LIST ||
         curr_sel->parsing_place == AFTER_LIST))
    {
      curr_sel->parsing_place= NO_MATTER;
    }
    break;
  case left_paren:
    if (!curr_sel ||
        curr_sel->parsing_place != BEFORE_OPT_LIST)
      return token;
    token= lex_one_token(yylval, thd);
    add_digest_token(token, yylval);
    lookahead_yylval= yylval;
    yylval= NULL;
    lookahead_token= token;
    curr_sel->parsing_place= NO_MATTER;
    if (token == LIKE)
      return LEFT_PAREN_LIKE;
    if (token == WITH)
      return LEFT_PAREN_WITH;
    if (token != left_paren && token != SELECT_SYM && token != VALUES)
      return LEFT_PAREN_ALT;
    else
      return left_paren;
    break;
  default:
    break;
  }
  return token;
}