MDEV-11605 Assertion `(longlong) thd->status_var.local_memory_used >= 0 ||...

MDEV-11605 Assertion `(longlong) thd->status_var.local_memory_used >= 0 || !debug_assert_on_not_freed_memory' failed in my_malloc_size_cb_func

MyISAM in compute_vcols() - which is used only in mi_check code -
was computing indexed vcols into an internally allocated buffer
(not record[0]) and the buffer was calculated to be long enough to fit
every keyseg (a keyseg knows where its value in a record buffer is
and the length of the value).

This logic didn' work for prefix keys, because the keyseg length is the
length of a prefix, but the record buffer needs to fit the complete
value of a vcol. In this bug MyISAM was writing a 2K varchar
into a buffer too short.

Also it didn't work for repair-with-keycache, because that code
recalculats all vcols, not only indexed ones.

So, the buffer size (MYISAM_SHARE::vreclength) should include all
vcols' full lengths. But it was calculated in mi_open and low-level
MyISAM code has no knowledge of vcols.

As a fix we now recalculate MYISAM_SHARE::vreclength in
ha_myisam::setup_vcols_for_repair() which is always called
before compute_vcols().

storage/myisam/ha_myisam.cc

@@ -892,14 +892,27 @@ int ha_myisam::write_row(uchar *buf)
 
 void ha_myisam::setup_vcols_for_repair(HA_CHECK *param)
 {
-  if (file->s->base.reclength < file->s->vreclength)
+  DBUG_ASSERT(file->s->base.reclength <= file->s->vreclength);
+  if (!table->vfield)
+    return;
+
+  if  (file->s->base.reclength == file->s->vreclength)
   {
-    param->fix_record= compute_vcols;
-    table->use_all_columns();
-    table->vcol_set= &table->s->all_set;
+    bool indexed_vcols= false;
+    ulong new_vreclength= file->s->vreclength;
+    for (Field **vf= table->vfield; *vf; vf++)
+    {
+      uint vf_end= (*vf)->offset(table->record[0]) + (*vf)->pack_length_in_rec();
+      set_if_bigger(new_vreclength, vf_end);
+      indexed_vcols|= (*vf)->flags & PART_KEY_FLAG;
+    }
+    if (!indexed_vcols)
+      return;
+    file->s->vreclength= new_vreclength;
   }
-  else
-    DBUG_ASSERT(file->s->base.reclength == file->s->vreclength);
+  param->fix_record= compute_vcols;
+  table->use_all_columns();
+  table->vcol_set= &table->s->all_set;
 }
 
 void ha_myisam::restore_vcos_after_repair()

storage/myisam/mi_open.c

@@ -336,7 +336,6 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
       for (i=0 ; i < keys ; i++)
       {
         MI_KEYDEF *keyinfo= share->keyinfo + i;
-        uint sp_segs;
         keyinfo->share= share;
         disk_pos=mi_keydef_read(disk_pos, keyinfo);
         disk_pos_assert(disk_pos + keyinfo->keysegs * HA_KEYSEG_SIZE, end_pos);
@@ -344,11 +343,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
           have_rtree=1;
         set_if_smaller(share->blocksize, keyinfo->block_length);
         keyinfo->seg= pos;
-        if (keyinfo->flag & HA_SPATIAL)
-          sp_segs= 2*SPDIMS;
-        else
-          sp_segs= 0;
-        for (j=0 ; j < keyinfo->keysegs; j++, pos++)
+        for (j=0 ; j < keyinfo->keysegs; j++,pos++)
 	{
 	  disk_pos=mi_keyseg_read(disk_pos, pos);
           if (pos->flag & HA_BLOB_PART &&
@@ -372,18 +367,11 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	  }
 	  else if (pos->type == HA_KEYTYPE_BINARY)
 	    pos->charset= &my_charset_bin;
-          if (j < keyinfo->keysegs - sp_segs)
-          {
-            uint real_length= pos->flag & HA_BLOB_PART ? pos->bit_start
-                                                       : pos->length;
-            if (pos->type == HA_KEYTYPE_BIT && pos->bit_length)
-              real_length--;
-            set_if_bigger(share->vreclength, pos->start + real_length);
-          }
 	}
         if (keyinfo->flag & HA_SPATIAL)
 	{
 #ifdef HAVE_SPATIAL
+          uint sp_segs= SPDIMS*2;
           keyinfo->seg= pos - sp_segs;
           DBUG_ASSERT(keyinfo->keysegs == sp_segs + 1);
           keyinfo->keysegs= sp_segs;