revert second lookahead in the lexer

it's not enough to look for NOT NULL IS, this also fails queries like
SELECT NOT NULL <=> NULL;
and adds no value anymore, as the grammar now requires parentheses

mysql-test/r/func_test.result

@@ -320,7 +320,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (not((`test`.`t1`.`a` + 0)))
 select not 1, not null, not not null, 1 is not null;
-not 1	NULL	not not null	1 is not null
+not 1	not null	not not null	1 is not null
 0	NULL	NULL	1
 drop table t1;
 #

mysql-test/r/null.result

@@ -50,6 +50,9 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
 Note	1003	select inet_ntoa(NULL) AS `inet_ntoa(null)`,inet_aton(NULL) AS `inet_aton(null)`,inet_aton('122.256') AS `inet_aton("122.256")`,inet_aton('122.226.') AS `inet_aton("122.226.")`,inet_aton('') AS `inet_aton("")`
+select not null is true, not null or true, not null and false, not null <=> null;
+not null is true	not null or true	not null and false	not null <=> null
+1	1	0	0
 create table t1 (x int);
 insert into t1 values (null);
 select * from t1 where x != 0;

mysql-test/r/subselect.result

@@ -1932,7 +1932,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/r/subselect_no_exists_to_in.result

@@ -1936,7 +1936,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/r/subselect_no_mat.result

@@ -1939,7 +1939,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/r/subselect_no_opts.result

@@ -1935,7 +1935,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/r/subselect_no_scache.result

@@ -1938,7 +1938,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/r/subselect_no_semijoin.result

@@ -1935,7 +1935,7 @@ create table t1(id int);
 create table t2(id int);
 create table t3(flag int);
 select (select * from t3 where id not null) from t1, t2;
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'not null) from t1, t2' at line 1
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'null) from t1, t2' at line 1
 drop table t1,t2,t3;
 CREATE TABLE t1 (id INT);
 CREATE TABLE t2 (id INT);

mysql-test/t/null.test

@@ -22,6 +22,7 @@ SELECT (NULL OR NULL) IS NULL;
 select NULL AND 0, 0 and NULL;
 select inet_ntoa(null),inet_aton(null),inet_aton("122.256"),inet_aton("122.226."),inet_aton("");
 explain extended select inet_ntoa(null),inet_aton(null),inet_aton("122.256"),inet_aton("122.226."),inet_aton("");
+select not null is true, not null or true, not null and false, not null <=> null;
 
 create table t1 (x int);
 insert into t1 values (null);

sql/sql_lex.cc

@@ -279,8 +279,8 @@ Lex_input_stream::reset(char *buffer, unsigned int length)
 {
   yylineno= 1;
   yylval= NULL;
-  lookahead_token= lookahead_token2= -1;
-  lookahead_yylval= lookahead_yylval2= NULL;
+  lookahead_token= -1;
+  lookahead_yylval= NULL;
   m_ptr= buffer;
   m_tok_start= NULL;
   m_tok_end= NULL;
@@ -1241,13 +1241,9 @@ int MYSQLlex(YYSTYPE *yylval, THD *thd)
       return it.
     */
     token= lip->lookahead_token;
+    lip->lookahead_token= -1;
     *yylval= *(lip->lookahead_yylval);
-    lip->lookahead_token=  lip->lookahead_token2;
-    lip->lookahead_yylval= lip->lookahead_yylval2;
-    lip->m_cpp_tok_start=  lip->lookahead_cpp_start;
-    lip->m_cpp_tok_end=    lip->lookahead_cpp_end;
-    lip->lookahead_token2= -1;
-    lip->lookahead_yylval2= NULL;
+    lip->lookahead_yylval= NULL;
     return token;
   }
 
@@ -1277,70 +1273,9 @@ int MYSQLlex(YYSTYPE *yylval, THD *thd)
       lip->lookahead_yylval= lip->yylval;
       lip->yylval= NULL;
       lip->lookahead_token= token;
-      lip->lookahead_cpp_start= lip->get_cpp_tok_start();
-      lip->lookahead_cpp_end=   lip->get_cpp_tok_end();
       return WITH;
     }
     break;
-  case NOT_SYM:
-  {
-    const char *cpp_tok_start, *m_tok_end, *tok_start;
-    /*
-      To be able to handle "DEFAULT 1 NOT NULL" which clashes with
-      "DEFAULT 1 NOT IN (..)" we must combine NOT NULL to one lex token
-      NOT_NULL_SYM. We also have to ensure that NOT NULL IS are still
-      separate tokens to ensure that NOT NULL IS TRUE is evaluated as
-      NOT (NULL IS TRUE)
-    */
-    cpp_tok_start= lip->get_cpp_tok_start();      // Save position of NOT
-    tok_start=     lip->get_tok_start();          // For errors
-    m_tok_end=     lip->get_cpp_tok_end();
-
-    token= lex_one_token(yylval, thd);
-    lip->add_digest_token(token, yylval);
-
-    if (token == NULL_SYM)
-    {
-      /* Check that next is not 'IS' */
-      token= lex_one_token(yylval, thd);
-      lip->add_digest_token(token, yylval);
-      if (token != IS)
-      {
-        /* Save the token following 'NOT NULL' */
-        lip->lookahead_token=     token;
-        lip->lookahead_yylval=    lip->yylval;
-        lip->lookahead_cpp_start= lip->get_cpp_tok_start();
-        lip->lookahead_cpp_end=   lip->get_cpp_tok_end();
-        token= NOT_NULL_SYM;
-      }
-      else
-      {
-        /* Save NULL and IS and return NOT */
-        lip->lookahead_token2=  IS;
-        lip->lookahead_token=   NULL_SYM;
-        lip->lookahead_yylval2= lip->yylval;
-        lip->lookahead_yylval=  lip->yylval;
-        lip->lookahead_cpp_start= cpp_tok_start;
-        lip->lookahead_cpp_end= lip->get_cpp_tok_end();
-        token= NOT_SYM;
-      }
-    }
-    else
-    {
-      /* Save the token following 'NOT' */
-      lip->lookahead_token= token;
-      lip->lookahead_yylval= lip->yylval;
-      lip->lookahead_cpp_start= lip->get_cpp_tok_start();
-      lip->lookahead_cpp_end=   lip->get_cpp_tok_end();
-      token= NOT_SYM;
-    }
-    lip->yylval= NULL;
-    /* Restore parser position for the current token */
-    lip->m_cpp_tok_start= cpp_tok_start;
-    lip->m_cpp_tok_end= m_tok_end;
-    lip->m_tok_start=   tok_start;
-    break;
-  }
   default:
     break;
   }

sql/sql_lex.h

@@ -2179,12 +2179,10 @@ class Lex_input_stream
     or -1, if no token was parsed in advance.
     Note: 0 is a legal token, and represents YYEOF.
   */
-  int lookahead_token, lookahead_token2;
+  int lookahead_token;
 
-  /** LALR(3) resolution, value of the look ahead token.*/
+  /** LALR(2) resolution, value of the look ahead token.*/
   LEX_YYSTYPE lookahead_yylval;
-  LEX_YYSTYPE lookahead_yylval2;
-  const char *lookahead_cpp_start, *lookahead_cpp_end;
 
   bool get_text(LEX_STRING *to, uint sep, int pre_skip, int post_skip);
 
@@ -2312,7 +2310,6 @@ class Lex_input_stream
     Current statement digest instrumentation. 
   */
   sql_digest_state* m_digest;
-  friend int MYSQLlex(union YYSTYPE *yylval, THD *thd);
 };
 
 /**

sql/sql_yacc.yy

@@ -1452,7 +1452,6 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  NO_SYM                        /* SQL-2003-R */
 %token  NO_WAIT_SYM
 %token  NO_WRITE_TO_BINLOG
-%token  NOT_NULL_SYM
 %token  NTILE_SYM
 %token  NULL_SYM                      /* SQL-2003-R */
 %token  NUM
@@ -6743,7 +6742,7 @@ opt_attribute_list:
 
 attribute:
           NULL_SYM { Lex->last_field->flags&= ~ NOT_NULL_FLAG; }
-        | NOT_NULL_SYM { Lex->last_field->flags|= NOT_NULL_FLAG; }
+        | not NULL_SYM { Lex->last_field->flags|= NOT_NULL_FLAG; }
         | DEFAULT column_default_expr { Lex->last_field->default_value= $2; }
         | ON UPDATE_SYM NOW_SYM opt_default_time_precision
           {
@@ -9185,12 +9184,6 @@ bool_pri:
             if ($$ == NULL)
               MYSQL_YYABORT;
           }
-        | bool_pri IS NOT_NULL_SYM %prec IS
-          {
-            $$= new (thd->mem_root) Item_func_isnotnull(thd, $1);
-            if ($$ == NULL)
-              MYSQL_YYABORT;
-          }
         | bool_pri EQUAL_SYM predicate %prec EQUAL_SYM
           {
             $$= new (thd->mem_root) Item_func_equal(thd, $1, $3);
@@ -9560,13 +9553,6 @@ column_default_non_parenthesized_expr:
         | variable
         | sum_expr
         | window_func_expr
-        | NOT_NULL_SYM
-          {
-            /* Replace NOT NULL with NULL */
-            $$= new (thd->mem_root) Item_null(thd);
-            if ($$ == NULL)
-              MYSQL_YYABORT;
-           }
         | ROW_SYM '(' expr ',' expr_list ')'
           {
             $5->push_front($3, thd->mem_root);