Merge paul@bk-internal.mysql.com:/home/bk/mysql-4.1

into ice.snake.net:/Volumes/ice2/MySQL/bk/mysql-4.1

BitKeeper/etc/logging_ok

@@ -26,6 +26,7 @@ bell@laptop.sanja.is.com.ua
 bell@sanja.is.com.ua
 bk@admin.bk
 bk@mysql.r18.ru
+brian@avenger.(none)
 brian@brian-akers-computer.local
 carsten@tsort.bitbybit.dk
 davida@isil.mysql.com

acconfig.h

@@ -115,6 +115,9 @@
 /* Builds Example DB */
 #undef HAVE_EXAMPLE_DB
 
+/* Builds Archive Storage Engine */
+#undef HAVE_ARCHIVE_DB
+
 /* fp_except from ieeefp.h */
 #undef HAVE_FP_EXCEPT
 

acinclude.m4

@@ -1332,6 +1332,36 @@ dnl ---------------------------------------------------------------------------
 dnl END OF MYSQL_CHECK_EXAMPLE SECTION
 dnl ---------------------------------------------------------------------------
 
+dnl ---------------------------------------------------------------------------
+dnl Macro: MYSQL_CHECK_ARCHIVEDB
+dnl Sets HAVE_ARCHIVE_DB if --with-archive-storage-engine is used
+dnl ---------------------------------------------------------------------------
+AC_DEFUN([MYSQL_CHECK_ARCHIVEDB], [
+  AC_ARG_WITH([archive-storage-engine],
+              [
+  --with-archive-storage-engine
+                          Enable the Archive Storge Engine],
+              [archivedb="$withval"],
+              [archivedb=no])
+  AC_MSG_CHECKING([for archive storage engine])
+
+  case "$archivedb" in
+    yes )
+      AC_DEFINE(HAVE_ARCHIVE_DB)
+      AC_MSG_RESULT([yes])
+      [archivedb=yes]
+      ;;
+    * )
+      AC_MSG_RESULT([no])
+      [archivedb=no]
+      ;;
+  esac
+
+])
+dnl ---------------------------------------------------------------------------
+dnl END OF MYSQL_CHECK_ARCHIVE SECTION
+dnl ---------------------------------------------------------------------------
+
 dnl ---------------------------------------------------------------------------
 dnl Macro: MYSQL_CHECK_NDBCLUSTER
 dnl Sets HAVE_NDBCLUSTER_DB if --with-ndbcluster is used

client/mysqldump.c

@@ -394,7 +394,11 @@ static void write_header(FILE *sql_file, char *db_name)
 	      mysql_get_server_info(&mysql_connection));
     }
     if (opt_set_charset)
-      fprintf(sql_file,"\n/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT, CHARACTER_SET_CLIENT=%s */;\n",default_charset);
+      fprintf(sql_file,
+"\n/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;"
+"\n/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;"
+"\n/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;"
+"\n/*!40101 SET NAMES %s */;\n",default_charset);
     if (!path)
     {
       fprintf(md_result_file,"\
@@ -425,7 +429,9 @@ static void write_footer(FILE *sql_file)
     }
     if (opt_set_charset)
       fprintf(sql_file,
-	      "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n");
+"/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n"
+"/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;\n"
+"/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n");
     fputs("\n", sql_file);
   }
 } /* write_footer */

configure.in

@@ -2652,6 +2652,7 @@ MYSQL_CHECK_ISAM
 MYSQL_CHECK_BDB
 MYSQL_CHECK_INNODB
 MYSQL_CHECK_EXAMPLEDB
+MYSQL_CHECK_ARCHIVEDB
 MYSQL_CHECK_NDBCLUSTER
 
 # If we have threads generate some library functions and test programs

mysql-test/include/have_archive.inc

+-- require r/have_archive.require
+disable_query_log;
+show variables like "have_archive";
+enable_query_log;

mysql-test/r/ctype_utf8.result

@@ -218,3 +218,25 @@ b
 select * from t1 where a = 'b' and a != 'b';
 a
 drop table t1;
+set names utf8;
+select  'вася'  rlike '[[:<:]]вася[[:>:]]';
+'вася'  rlike '[[:<:]]вася[[:>:]]'
+1
+select  'вася ' rlike '[[:<:]]вася[[:>:]]';
+'вася ' rlike '[[:<:]]вася[[:>:]]'
+1
+select ' вася'  rlike '[[:<:]]вася[[:>:]]';
+' вася'  rlike '[[:<:]]вася[[:>:]]'
+1
+select ' вася ' rlike '[[:<:]]вася[[:>:]]';
+' вася ' rlike '[[:<:]]вася[[:>:]]'
+1
+select  'васяz' rlike '[[:<:]]вася[[:>:]]';
+'васяz' rlike '[[:<:]]вася[[:>:]]'
+0
+select 'zвася'  rlike '[[:<:]]вася[[:>:]]';
+'zвася'  rlike '[[:<:]]вася[[:>:]]'
+0
+select 'zвасяz' rlike '[[:<:]]вася[[:>:]]';
+'zвасяz' rlike '[[:<:]]вася[[:>:]]'
+0

mysql-test/r/have_archive.require

+Variable_name	Value
+have_archive	YES

mysql-test/t/ctype_utf8.test

@@ -141,3 +141,19 @@ select * from t1 where a = 'b';
 select * from t1 where a = 'b' and a = 'b';
 select * from t1 where a = 'b' and a != 'b';
 drop table t1;
+
+#
+# Bug #3928 regexp [[:>:]] and UTF-8
+#
+set names utf8;
+
+# This should return TRUE
+select  'вася'  rlike '[[:<:]]вася[[:>:]]';
+select  'вася ' rlike '[[:<:]]вася[[:>:]]';
+select ' вася'  rlike '[[:<:]]вася[[:>:]]';
+select ' вася ' rlike '[[:<:]]вася[[:>:]]';
+
+# This should return FALSE
+select  'васяz' rlike '[[:<:]]вася[[:>:]]';
+select 'zвася'  rlike '[[:<:]]вася[[:>:]]';
+select 'zвасяz' rlike '[[:<:]]вася[[:>:]]';

sql-common/client.c

@@ -1835,40 +1835,39 @@ CLI_MYSQL_REAL_CONNECT(MYSQL *mysql,const char *host, const char *user,
   }
 
   /* Set character set */
-  if (mysql->options.charset_name)
+  if (!mysql->options.charset_name &&
+      !(mysql->options.charset_name= 
+       my_strdup(MYSQL_DEFAULT_CHARSET_NAME,MYF(MY_WME))))
+    goto error;
+  
   {
     const char *save= charsets_dir;
     if (mysql->options.charset_dir)
       charsets_dir=mysql->options.charset_dir;
     mysql->charset=get_charset_by_csname(mysql->options.charset_name,
-					 MY_CS_PRIMARY,
-					 MYF(MY_WME));
+                                         MY_CS_PRIMARY, MYF(MY_WME));
     charsets_dir= save;
-
-    if (!mysql->charset)
-    {
-      net->last_errno=CR_CANT_READ_CHARSET;
-      strmov(net->sqlstate, unknown_sqlstate);
-      if (mysql->options.charset_dir)
-        my_snprintf(net->last_error, sizeof(net->last_error)-1,
-		    ER(net->last_errno),
-		    mysql->options.charset_name,
-		    mysql->options.charset_dir);
-      else
-      {
-        char cs_dir_name[FN_REFLEN];
-        get_charsets_dir(cs_dir_name);
-        my_snprintf(net->last_error, sizeof(net->last_error)-1,
-		  ER(net->last_errno),
-		  mysql->options.charset_name,
-		  cs_dir_name);
-      }
-      goto error;
-    }
   }
-  else
+  
+  if (!mysql->charset)
   {
-    mysql->charset= default_charset_info;
+    net->last_errno=CR_CANT_READ_CHARSET;
+    strmov(net->sqlstate, unknown_sqlstate);
+    if (mysql->options.charset_dir)
+      my_snprintf(net->last_error, sizeof(net->last_error)-1,
+                  ER(net->last_errno),
+                  mysql->options.charset_name,
+                  mysql->options.charset_dir);
+    else
+    {
+      char cs_dir_name[FN_REFLEN];
+      get_charsets_dir(cs_dir_name);
+      my_snprintf(net->last_error, sizeof(net->last_error)-1,
+                  ER(net->last_errno),
+                  mysql->options.charset_name,
+                  cs_dir_name);
+    }
+    goto error;
   }
 
 

sql/Makefile.am

@@ -58,7 +58,7 @@ noinst_HEADERS =	item.h item_func.h item_sum.h item_cmpfunc.h \
 			log_event.h sql_repl.h slave.h \
 			stacktrace.h sql_sort.h sql_cache.h set_var.h \
 			spatial.h gstream.h client_settings.h \
-			examples/ha_example.h
+			examples/ha_example.h examples/ha_archive.h
 mysqld_SOURCES =	sql_lex.cc sql_handler.cc \
 			item.cc item_sum.cc item_buff.cc item_func.cc \
 			item_cmpfunc.cc item_strfunc.cc item_timefunc.cc \
@@ -88,7 +88,7 @@ mysqld_SOURCES =	sql_lex.cc sql_handler.cc \
 			client.c sql_client.cc mini_client_errors.c pack.c\
 			stacktrace.c repl_failsafe.h repl_failsafe.cc \
 			gstream.cc spatial.cc sql_help.cc protocol_cursor.cc \
-			examples/ha_example.cc
+			examples/ha_example.cc examples/ha_archive.cc
 gen_lex_hash_SOURCES =	gen_lex_hash.cc
 gen_lex_hash_LDADD =	$(LDADD) $(CXXLDFLAGS)
 

sql/examples/ha_archive.h

+/* Copyright (C) 2003 MySQL AB
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifdef __GNUC__
+#pragma interface			/* gcc class implementation */
+#endif
+
+#include <zlib.h>
+
+/*
+  Please read ha_archive.cc first. If you are looking for more general
+  answers on how storage engines work, look at ha_example.cc and 
+  ha_example.h.
+*/
+
+typedef struct st_archive_share {
+  char *table_name;
+  char data_file_name[FN_REFLEN];
+  uint table_name_length,use_count;
+  pthread_mutex_t mutex;
+  THR_LOCK lock;
+  gzFile archive_write;             /* Archive file we are working with */
+  bool dirty;                       /* Flag for if a flush should occur */
+} ARCHIVE_SHARE;
+
+/* 
+  Version for file format.
+  1 - Initial Version
+*/
+#define ARCHIVE_VERSION 1
+
+class ha_archive: public handler
+{
+  THR_LOCK_DATA lock;        /* MySQL lock */
+  ARCHIVE_SHARE *share;      /* Shared lock info */
+  gzFile archive;            /* Archive file we are working with */
+  z_off_t current_position;  /* The position of the row we just read */
+  byte byte_buffer[IO_SIZE]; /* Initial buffer for our string */
+  String buffer;             /* Buffer used for blob storage */
+  unsigned int version;      /* Used for recording version */
+
+public:
+  ha_archive(TABLE *table): handler(table)
+  {
+    /* Set our original buffer from pre-allocated memory */
+    buffer.set(byte_buffer, IO_SIZE, system_charset_info);
+
+    /* The size of the offset value we will use for position() */
+    ref_length = sizeof(z_off_t);
+  }
+  ~ha_archive() 
+  {
+  }
+  const char *table_type() const { return "ARCHIVE"; }
+  const char *index_type(uint inx) { return "NONE"; }
+  const char **bas_ext() const;
+  ulong table_flags() const
+  {
+    return (HA_REC_NOT_IN_SEQ | HA_NOT_EXACT_COUNT | HA_NO_WRITE_DELAYED |
+            HA_NO_AUTO_INCREMENT);
+  }
+  ulong index_flags(uint inx) const
+  {
+    return 0;
+  }
+  /* 
+    This is just a default, there is no real limit as far as
+    archive is concerned.
+  */
+  uint max_record_length() const { return HA_MAX_REC_LENGTH; }
+  uint max_keys()          const { return 0; }
+  uint max_key_parts()     const { return 0; }
+  uint max_key_length()    const { return 0; }
+  /*
+    Called in test_quick_select to determine if indexes should be used.
+  */
+  virtual double scan_time() { return (double) (records) / 20.0+10; }
+  /* The next method will never be called */
+  virtual double read_time(ha_rows rows) { return (double) rows /  20.0+1; }
+  int open(const char *name, int mode, uint test_if_locked);
+  int close(void);
+  int write_row(byte * buf);
+  int update_row(const byte * old_data, byte * new_data);
+  int delete_row(const byte * buf);
+  int index_read(byte * buf, const byte * key,
+                 uint key_len, enum ha_rkey_function find_flag);
+  int index_read_idx(byte * buf, uint idx, const byte * key,
+                     uint key_len, enum ha_rkey_function find_flag);
+  int index_next(byte * buf);
+  int index_prev(byte * buf);
+  int index_first(byte * buf);
+  int index_last(byte * buf);
+  int rnd_init(bool scan=1);
+  int rnd_next(byte *buf);
+  int rnd_pos(byte * buf, byte *pos);
+  int get_row(byte *buf);
+  void position(const byte *record);
+  void info(uint);
+  int extra(enum ha_extra_function operation);
+  int reset(void);
+  int external_lock(THD *thd, int lock_type);
+  ha_rows records_in_range(int inx, const byte *start_key,uint start_key_len,
+                           enum ha_rkey_function start_search_flag,
+                           const byte *end_key,uint end_key_len,
+                           enum ha_rkey_function end_search_flag);
+  int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info);
+
+  THR_LOCK_DATA **store_lock(THD *thd, THR_LOCK_DATA **to,
+                             enum thr_lock_type lock_type);
+};

sql/examples/ha_example.cc

@@ -596,6 +596,10 @@ THR_LOCK_DATA **ha_example::store_lock(THD *thd,
   shared references released. The variable name will just be the name of 
   the table. You will need to remove any files you have created at this point.
 
+  If you do not implement this, the default delete_table() is called from
+  handler.cc and it will delete all files with the file extentions returned
+  by bas_ext().
+
   Called from handler.cc by delete_table and  ha_create_table(). Only used
   during create if the table_flag HA_DROP_BEFORE_CREATE was specified for
   the storage engine.
@@ -610,6 +614,10 @@ int ha_example::delete_table(const char *name)
 /*
   Renames a table from one name to another from alter table call.
 
+  If you do not implement this, the default rename_table() is called from
+  handler.cc and it will delete all files with the file extentions returned
+  by bas_ext().
+
   Called from sql_table.cc by mysql_rename_table().
 */
 int ha_example::rename_table(const char * from, const char * to)

sql/examples/ha_example.h

@@ -21,6 +21,10 @@
   that you can implement.
 */
 
+#ifdef __GNUC__
+#pragma interface			/* gcc class implementation */
+#endif
+
 /*
   EXAMPLE_SHARE is a structure that will be shared amoung all open handlers
   The example implements the minimum of what you will probably need.
@@ -87,7 +91,6 @@ public:
     The next method will never be called if you do not implement indexes.
   */
   virtual double read_time(ha_rows rows) { return (double) rows /  20.0+1; }
-  virtual bool fast_key_read() { return 1;}
 
   /* 
     Everything below are methods that we implment in ha_example.cc.

sql/handler.cc

@@ -35,6 +35,9 @@
 #ifdef HAVE_EXAMPLE_DB
 #include "examples/ha_example.h"
 #endif
+#ifdef HAVE_ARCHIVE_DB
+#include "examples/ha_archive.h"
+#endif
 #ifdef HAVE_INNOBASE_DB
 #include "ha_innodb.h"
 #else
@@ -88,6 +91,8 @@ struct show_table_type_st sys_table_types[]=
    "Alias for NDBCLUSTER", DB_TYPE_NDBCLUSTER},
   {"EXAMPLE",&have_example_db,
    "Example storage engine", DB_TYPE_EXAMPLE_DB},
+  {"ARCHIVE",&have_archive_db,
+   "Archive storage engine", DB_TYPE_ARCHIVE_DB},
   {NullS, NULL, NullS, DB_TYPE_UNKNOWN}
 };
 
@@ -189,6 +194,10 @@ handler *get_new_handler(TABLE *table, enum db_type db_type)
   case DB_TYPE_EXAMPLE_DB:
     return new ha_example(table);
 #endif
+#ifdef HAVE_ARCHIVE_DB
+  case DB_TYPE_ARCHIVE_DB:
+    return new ha_archive(table);
+#endif
 #ifdef HAVE_NDBCLUSTER_DB
   case DB_TYPE_NDBCLUSTER:
     return new ha_ndbcluster(table);

sql/handler.h

@@ -158,7 +158,7 @@ enum db_type
   DB_TYPE_MRG_ISAM, DB_TYPE_MYISAM, DB_TYPE_MRG_MYISAM,
   DB_TYPE_BERKELEY_DB, DB_TYPE_INNODB, 
   DB_TYPE_GEMINI, DB_TYPE_NDBCLUSTER,
-  DB_TYPE_EXAMPLE_DB,
+  DB_TYPE_EXAMPLE_DB, DB_TYPE_ARCHIVE_DB,
 	       
   DB_TYPE_DEFAULT // Must be last
 };

sql/mysql_priv.h

@@ -915,7 +915,8 @@ extern struct my_option my_long_options[];
 
 /* optional things, have_* variables */
 
-extern SHOW_COMP_OPTION have_isam, have_innodb, have_berkeley_db, have_example_db;
+extern SHOW_COMP_OPTION have_isam, have_innodb, have_berkeley_db;
+extern SHOW_COMP_OPTION have_example_db, have_archive_db;
 extern SHOW_COMP_OPTION have_raid, have_openssl, have_symlink;
 extern SHOW_COMP_OPTION have_query_cache, have_berkeley_db, have_innodb;
 extern SHOW_COMP_OPTION have_geometry, have_rtree_keys;

sql/mysqld.cc

@@ -375,8 +375,8 @@ KEY_CACHE *sql_key_cache;
 CHARSET_INFO *system_charset_info, *files_charset_info ;
 CHARSET_INFO *national_charset_info, *table_alias_charset;
 
-SHOW_COMP_OPTION have_berkeley_db, have_innodb, have_isam, 
- have_ndbcluster, have_example_db;
+SHOW_COMP_OPTION have_berkeley_db, have_innodb, have_isam, have_ndbcluster, 
+  have_example_db, have_archive_db;
 SHOW_COMP_OPTION have_raid, have_openssl, have_symlink, have_query_cache;
 SHOW_COMP_OPTION have_geometry, have_rtree_keys;
 SHOW_COMP_OPTION have_crypt, have_compress;
@@ -3959,6 +3959,12 @@ Disable with --skip-bdb (will save memory).",
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"bootstrap", OPT_BOOTSTRAP, "Used by mysql installation scripts.", 0, 0, 0,
    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"character_set_server", 'C', "Set the default character set.",
+   (gptr*) &default_character_set_name, (gptr*) &default_character_set_name,
+   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+  {"collation_server", OPT_DEFAULT_COLLATION, "Set the default collation.",
+   (gptr*) &default_collation_name, (gptr*) &default_collation_name,
+   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
   {"console", OPT_CONSOLE, "Write error output on screen; Don't remove the console window on windows.",
    (gptr*) &opt_console, (gptr*) &opt_console, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
@@ -3992,10 +3998,10 @@ Disable with --skip-bdb (will save memory).",
    (gptr*) &des_key_file, (gptr*) &des_key_file, 0, GET_STR, REQUIRED_ARG,
    0, 0, 0, 0, 0, 0},
 #endif /* HAVE_OPENSSL */
-  {"default-character-set", 'C', "Set the default character set.",
+  {"default-character-set", 'C', "Set the default character set (Deprecated option, use character_set_server instead).",
    (gptr*) &default_character_set_name, (gptr*) &default_character_set_name,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
-  {"default-collation", OPT_DEFAULT_COLLATION, "Set the default collation.",
+  {"default-collation", OPT_DEFAULT_COLLATION, "Set the default collation (Deprecated option, use character_set_server instead).",
    (gptr*) &default_collation_name, (gptr*) &default_collation_name,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
   {"default-storage-engine", OPT_STORAGE_ENGINE,
@@ -5338,6 +5344,11 @@ static void mysql_init_variables(void)
 #else
   have_example_db= SHOW_OPTION_NO;
 #endif
+#ifdef HAVE_ARCHIVE_DB
+  have_archive_db= SHOW_OPTION_YES;
+#else
+  have_archive_db= SHOW_OPTION_NO;
+#endif
 #ifdef HAVE_NDBCLUSTER_DB
   have_ndbcluster=SHOW_OPTION_DISABLED;
 #else

sql/set_var.cc

@@ -638,6 +638,7 @@ struct show_var_st init_vars[]= {
   {"ft_query_expansion_limit",(char*) &ft_query_expansion_limit,    SHOW_LONG},
   {"ft_stopword_file",        (char*) &ft_stopword_file,            SHOW_CHAR_PTR},
   {sys_group_concat_max_len.name, (char*) &sys_group_concat_max_len,  SHOW_SYS},
+  {"have_archive",	      (char*) &have_archive_db,	            SHOW_HAVE},
   {"have_bdb",		      (char*) &have_berkeley_db,	    SHOW_HAVE},
   {"have_compress",	      (char*) &have_compress,		    SHOW_HAVE},
   {"have_crypt",	      (char*) &have_crypt,		    SHOW_HAVE},

strings/ctype-utf8.c

@@ -1524,8 +1524,12 @@ MY_UNICASE_INFO *uni_plane[256]={
 
 #ifdef HAVE_CHARSET_utf8
 
-/* These arrays are taken from usa7 implementation */
-
+/* 
+  We consider bytes with code more than 127 as a letter.
+  This garantees that word boundaries work fine with regular
+  expressions. Note, there is no need to mark byte 255  as a
+  letter, it is illegal byte in UTF8.
+*/
 static uchar ctype_utf8[] = {
     0,
    32, 32, 32, 32, 32, 32, 32, 32, 32, 40, 40, 40, 40, 40, 32, 32,
@@ -1536,16 +1540,18 @@ static uchar ctype_utf8[] = {
     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 16, 16, 16, 16, 16,
    16,130,130,130,130,130,130,  2,  2,  2,  2,  2,  2,  2,  2,  2,
     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, 16, 16, 16, 16, 32,
-    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
-    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
-    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
-    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
-    0,  0,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
+    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
+    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
+    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
+    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
+    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  0
 };
 
+/* The below are taken from usa7 implementation */
+
 static uchar to_lower_utf8[] = {
     0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
    16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,