mi_check.c 146 KB
Newer Older
unknown's avatar
unknown committed
1
/* Copyright (C) 2000-2006 MySQL AB
2

unknown's avatar
unknown committed
3 4
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
unknown's avatar
unknown committed
5
   the Free Software Foundation; version 2 of the License.
6

unknown's avatar
unknown committed
7 8 9 10
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
11

unknown's avatar
unknown committed
12 13 14 15
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

16
/* Describe, check and repair of MyISAM tables */
unknown's avatar
unknown committed
17

18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
/*
  About checksum calculation.

  There are two types of checksums. Table checksum and row checksum.

  Row checksum is an additional byte at the end of dynamic length
  records. It must be calculated if the table is configured for them.
  Otherwise they must not be used. The variable
  MYISAM_SHARE::calc_checksum determines if row checksums are used.
  MI_INFO::checksum is used as temporary storage during row handling.
  For parallel repair we must assure that only one thread can use this
  variable. There is no problem on the write side as this is done by one
  thread only. But when checking a record after read this could go
  wrong. But since all threads read through a common read buffer, it is
  sufficient if only one thread checks it.

  Table checksum is an eight byte value in the header of the index file.
  It can be calculated even if row checksums are not used. The variable
  MI_CHECK::glob_crc is calculated over all records.
  MI_SORT_PARAM::calc_checksum determines if this should be done. This
  variable is not part of MI_CHECK because it must be set per thread for
  parallel repair. The global glob_crc must be changed by one thread
  only. And it is sufficient to calculate the checksum once only.
*/

43
#include "ftdefs.h"
unknown's avatar
unknown committed
44 45
#include <m_ctype.h>
#include <stdarg.h>
46
#include <my_getopt.h>
47
#ifdef HAVE_SYS_VADVISE_H
unknown's avatar
unknown committed
48 49 50 51 52
#include <sys/vadvise.h>
#endif
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
53
#include "rt_index.h"
unknown's avatar
unknown committed
54 55 56 57 58 59 60 61 62

#ifndef USE_RAID
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
#define my_raid_delete(A,B,C) my_delete(A,B)
#endif

	/* Functions defined in this file */

static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
63
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
64 65 66 67
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level);
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
static ha_checksum calc_checksum(ha_rows count);
68
static int writekeys(MI_SORT_PARAM *sort_param);
unknown's avatar
unknown committed
69 70
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file);
unknown's avatar
unknown committed
71 72 73 74
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
75
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
76
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
77 78
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
				uchar *key);
79
static int sort_insert_key(MI_SORT_PARAM  *sort_param,
unknown's avatar
unknown committed
80
                           reg1 SORT_KEY_BLOCKS *key_block,
unknown's avatar
unknown committed
81
			   uchar *key, my_off_t prev_block);
unknown's avatar
unknown committed
82
static int sort_delete_record(MI_SORT_PARAM *sort_param);
83
/*static int flush_pending_blocks(MI_CHECK *param);*/
unknown's avatar
unknown committed
84 85 86
static SORT_KEY_BLOCKS	*alloc_key_blocks(MI_CHECK *param, uint blocks,
					  uint buffer_length);
static ha_checksum mi_byte_checksum(const byte *buf, uint length);
unknown's avatar
unknown committed
87
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
unknown's avatar
unknown committed
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102

void myisamchk_init(MI_CHECK *param)
{
  bzero((gptr) param,sizeof(*param));
  param->opt_follow_links=1;
  param->keys_in_use= ~(ulonglong) 0;
  param->search_after_block=HA_OFFSET_ERROR;
  param->auto_increment_value= 0;
  param->use_buffers=USE_BUFFER_INIT;
  param->read_buffer_length=READ_BUFFER_INIT;
  param->write_buffer_length=READ_BUFFER_INIT;
  param->sort_buffer_length=SORT_BUFFER_INIT;
  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
103
  param->start_check_pos=0;
104
  param->max_record_length= LONGLONG_MAX;
105
  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
106
  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
unknown's avatar
unknown committed
107 108
}

109 110 111 112 113
	/* Check the status flags for the table */

int chk_status(MI_CHECK *param, register MI_INFO *info)
{
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
114

115 116 117 118 119 120
  if (mi_is_crashed_on_repair(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed and last repair failed");
  else if (mi_is_crashed(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed");
unknown's avatar
unknown committed
121
  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
122
  {
unknown's avatar
unknown committed
123 124
    /* Don't count this as a real warning, as check can correct this ! */
    uint save=param->warning_printed;
125
    mi_check_print_warning(param,
126 127 128
			   share->state.open_count==1 ? 
			   "%d client is using or hasn't closed the table properly" : 
			   "%d clients are using or haven't closed the table properly",
129
			   share->state.open_count);
unknown's avatar
unknown committed
130 131 132
    /* If this will be fixed by the check, forget the warning */
    if (param->testflag & T_UPDATE_STATE)
      param->warning_printed=save;
133 134 135 136
  }
  return 0;
}

unknown's avatar
unknown committed
137 138 139 140 141
	/* Check delete links */

int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
{
  reg2 ha_rows i;
142
  uint delete_link_length;
unknown's avatar
unknown committed
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
  my_off_t empty,next_link,old_link;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_del");

  LINT_INIT(old_link);
  param->record_checksum=0;
  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
		      info->s->rec_reflength+1);

  if (!(test_flag & T_SILENT))
    puts("- check record delete-chain");

  next_link=info->s->state.dellink;
  if (info->state->del == 0)
  {
    if (test_flag & T_VERBOSE)
    {
      puts("No recordlinks");
    }
  }
  else
  {
    if (test_flag & T_VERBOSE)
      printf("Recordlinks:    ");
    empty=0;
    for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
    {
unknown's avatar
unknown committed
170 171
      if (*killed_ptr(param))
        DBUG_RETURN(1);
unknown's avatar
unknown committed
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
      if (test_flag & T_VERBOSE)
	printf(" %9s",llstr(next_link,buff));
      if (next_link >= info->state->data_file_length)
	goto wrong;
      if (my_pread(info->dfile,(char*) buff,delete_link_length,
		   next_link,MYF(MY_NABP)))
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Can't read delete-link at filepos: %s",
		    llstr(next_link,buff));
	DBUG_RETURN(1);
      }
      if (*buff != '\0')
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Record at pos: %s is not remove-marked",
		    llstr(next_link,buff));
	goto wrong;
      }
      if (info->s->options & HA_OPTION_PACK_RECORD)
      {
	my_off_t prev_link=mi_sizekorr(buff+12);
	if (empty && prev_link != old_link)
	{
	  if (test_flag & T_VERBOSE) puts("");
	  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
	  goto wrong;
	}
	old_link=next_link;
	next_link=mi_sizekorr(buff+4);
	empty+=mi_uint3korr(buff+1);
      }
      else
      {
	param->record_checksum+=(ha_checksum) next_link;
	next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
	empty+=info->s->base.pack_reclength;
      }
    }
211 212
    if (test_flag & T_VERBOSE)
      puts("\n");
unknown's avatar
unknown committed
213 214 215
    if (empty != info->state->empty)
    {
      mi_check_print_warning(param,
216 217 218
			     "Found %s deleted space in delete link chain. Should be %s",
			     llstr(empty,buff2),
			     llstr(info->state->empty,buff));
unknown's avatar
unknown committed
219
    }
220 221 222 223 224
    if (next_link != HA_OFFSET_ERROR)
    {
      mi_check_print_error(param,
			   "Found more than the expected %s deleted rows in delete link chain",
			   llstr(info->state->del, buff));
unknown's avatar
unknown committed
225
      goto wrong;
226 227 228 229 230 231 232 233 234
    }
    if (i != 0)
    {
      mi_check_print_error(param,
			   "Found %s deleted rows in delete link chain. Should be %s",
			   llstr(info->state->del - i, buff2),
			   llstr(info->state->del, buff));
      goto wrong;
    }
unknown's avatar
unknown committed
235 236
  }
  DBUG_RETURN(0);
237

unknown's avatar
unknown committed
238
wrong:
unknown's avatar
unknown committed
239
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
240 241 242 243 244 245 246 247 248 249 250
  if (test_flag & T_VERBOSE) puts("");
  mi_check_print_error(param,"record delete-link-chain corrupted");
  DBUG_RETURN(1);
} /* chk_del */


	/* Check delete links in index file */

static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
{
  my_off_t next_link;
unknown's avatar
unknown committed
251
  uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
unknown's avatar
unknown committed
252
  ha_rows records;
253
  char llbuff[21], llbuff2[21], *buff;
unknown's avatar
unknown committed
254
  DBUG_ENTER("check_k_link");
255
  DBUG_PRINT("enter", ("block_size: %u", block_size));
unknown's avatar
unknown committed
256 257

  if (param->testflag & T_VERBOSE)
258
    printf("block_size %4u:", block_size); /* purecov: tested */
unknown's avatar
unknown committed
259 260 261 262 263

  next_link=info->s->state.key_del[nr];
  records= (ha_rows) (info->state->key_file_length / block_size);
  while (next_link != HA_OFFSET_ERROR && records > 0)
  {
unknown's avatar
unknown committed
264 265
    if (*killed_ptr(param))
      DBUG_RETURN(1);
unknown's avatar
unknown committed
266 267
    if (param->testflag & T_VERBOSE)
      printf("%16s",llstr(next_link,llbuff));
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287

    /* Key blocks must lay within the key file length entirely. */
    if (next_link + block_size > info->state->key_file_length)
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Invalid key block position: %s  "
                           "key block size: %u  file_length: %s",
                           llstr(next_link, llbuff), block_size,
                           llstr(info->state->key_file_length, llbuff2));
      DBUG_RETURN(1);
      /* purecov: end */
    }

    /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
    if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Mis-aligned key block: %s  "
                           "minimum key block length: %u",
                           llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
unknown's avatar
unknown committed
288
      DBUG_RETURN(1);
289 290 291 292 293 294 295 296
      /* purecov: end */
    }

    /*
      Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
      If the key cache block size is smaller than block_size, we can so
      avoid unecessary eviction of cache block.
    */
unknown's avatar
unknown committed
297
    if (!(buff=key_cache_read(info->s->key_cache,
298
                              info->s->kfile, next_link, DFLT_INIT_HITS,
299 300 301 302 303 304
                              (byte*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
                              MI_MIN_KEY_BLOCK_LENGTH, 1)))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "key cache read error for block: %s",
			   llstr(next_link,llbuff));
unknown's avatar
unknown committed
305
      DBUG_RETURN(1);
306 307
      /* purecov: end */
    }
unknown's avatar
unknown committed
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
    next_link=mi_sizekorr(buff);
    records--;
    param->key_file_blocks+=block_size;
  }
  if (param->testflag & T_VERBOSE)
  {
    if (next_link != HA_OFFSET_ERROR)
      printf("%16s\n",llstr(next_link,llbuff));
    else
      puts("");
  }
  DBUG_RETURN (next_link != HA_OFFSET_ERROR);
} /* check_k_link */


unknown's avatar
unknown committed
323
	/* Check sizes of files */
unknown's avatar
unknown committed
324 325 326 327 328 329 330 331 332 333

int chk_size(MI_CHECK *param, register MI_INFO *info)
{
  int error=0;
  register my_off_t skr,size;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_size");

  if (!(param->testflag & T_SILENT)) puts("- check file-size");

unknown's avatar
unknown committed
334 335 336
  /* The following is needed if called externally (not from myisamchk) */
  flush_key_blocks(info->s->key_cache,
		   info->s->kfile, FLUSH_FORCE_WRITE);
unknown's avatar
unknown committed
337

338
  size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
unknown's avatar
unknown committed
339 340
  if ((skr=(my_off_t) info->state->key_file_length) != size)
  {
341
    /* Don't give error if file generated by myisampack */
342
    if (skr > size && mi_is_any_key_active(info->s->state.key_map))
unknown's avatar
unknown committed
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
    {
      error=1;
      mi_check_print_error(param,
			   "Size of indexfile is: %-8s        Should be: %s",
			   llstr(size,buff), llstr(skr,buff2));
    }
    else
      mi_check_print_warning(param,
			     "Size of indexfile is: %-8s      Should be: %s",
			     llstr(size,buff), llstr(skr,buff2));
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->key_file_length) >
      ulonglong2double(info->s->base.margin_key_file_length)*0.9)
    mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
			   llstr(info->state->key_file_length,buff),
			   llstr(info->s->base.max_key_file_length-1,buff));

  size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
  skr=(my_off_t) info->state->data_file_length;
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    skr+= MEMMAP_EXTRA_MARGIN;
#ifdef USE_RELOC
  if (info->data_file_type == STATIC_RECORD &&
      skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
    skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
#endif
  if (skr != size)
  {
373
    info->state->data_file_length=size;	/* Skip other errors */
unknown's avatar
unknown committed
374 375 376
    if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
    {
      error=1;
377
      mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
unknown's avatar
unknown committed
378
		    llstr(size,buff), llstr(skr,buff2));
unknown's avatar
unknown committed
379
      param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
380 381 382 383
    }
    else
    {
      mi_check_print_warning(param,
384
			     "Size of datafile is: %-9s       Should be: %s",
unknown's avatar
unknown committed
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
			     llstr(size,buff), llstr(skr,buff2));
    }
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->data_file_length) >
      (ulonglong2double(info->s->base.max_data_file_length)*0.9))
    mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
			   llstr(info->state->data_file_length,buff),
			   llstr(info->s->base.max_data_file_length-1,buff2));
  DBUG_RETURN(error);
} /* chk_size */


	/* Check keys */

int chk_key(MI_CHECK *param, register MI_INFO *info)
{
403
  uint key,found_keys=0,full_text_keys=0,result=0;
unknown's avatar
unknown committed
404 405 406 407 408 409 410 411 412
  ha_rows keys;
  ha_checksum old_record_checksum,init_checksum;
  my_off_t all_keydata,all_totaldata,key_totlength,length;
  ulong   *rec_per_key_part;
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *keyinfo;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_key");

413 414 415 416
  if (!(param->testflag & T_SILENT))
    puts("- check key delete-chain");

  param->key_file_blocks=info->s->base.keystart;
417
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
418 419 420 421 422 423 424
    if (check_k_link(param,info,key))
    {
      if (param->testflag & T_VERBOSE) puts("");
      mi_check_print_error(param,"key delete-link-chain corrupted");
      DBUG_RETURN(-1);
    }

unknown's avatar
unknown committed
425 426 427 428 429 430 431 432 433 434 435 436 437 438
  if (!(param->testflag & T_SILENT)) puts("- check index reference");

  all_keydata=all_totaldata=key_totlength=0;
  old_record_checksum=0;
  init_checksum=param->record_checksum;
  if (!(share->options &
	(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
    old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
      share->base.pack_reclength;
  rec_per_key_part= param->rec_per_key_part;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
  {
    param->key_crc[key]=0;
439
    if (! mi_is_key_active(share->state.key_map, key))
440 441 442
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
443 444
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
445
	     keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
446
      continue;
447
    }
unknown's avatar
unknown committed
448 449 450
    found_keys++;

    param->record_checksum=init_checksum;
451
    
unknown's avatar
unknown committed
452
    bzero((char*) &param->unique_count,sizeof(param->unique_count));
453 454
    bzero((char*) &param->notnull_count,sizeof(param->notnull_count));

unknown's avatar
unknown committed
455 456
    if ((!(param->testflag & T_SILENT)))
      printf ("- check data record references index: %d\n",key+1);
unknown's avatar
unknown committed
457 458
    if (keyinfo->flag & HA_FULLTEXT)
      full_text_keys++;
unknown's avatar
unknown committed
459
    if (share->state.key_root[key] == HA_OFFSET_ERROR &&
460
	(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
461
      goto do_stat;
462 463
    if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
                           DFLT_INIT_HITS,info->buff,0))
unknown's avatar
unknown committed
464 465 466
    {
      mi_check_print_error(param,"Can't read indexpage from filepos: %s",
		  llstr(share->state.key_root[key],buff));
467 468 469 470
      if (!(param->testflag & T_INFO))
	DBUG_RETURN(-1);
      result= -1;
      continue;
unknown's avatar
unknown committed
471 472 473 474 475 476 477 478 479
    }
    param->key_file_blocks+=keyinfo->block_length;
    keys=0;
    param->keydata=param->totaldata=0;
    param->key_blocks=0;
    param->max_level=0;
    if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
		  &keys, param->key_crc+key,1))
      DBUG_RETURN(-1);
480
    if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
481 482 483 484 485
    {
      if (keys != info->state->records)
      {
	mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
		    llstr(info->state->records,buff2));
486
	if (!(param->testflag & T_INFO))
unknown's avatar
unknown committed
487
	DBUG_RETURN(-1);
488 489
	result= -1;
	continue;
unknown's avatar
unknown committed
490 491 492 493 494 495 496 497 498 499 500 501 502
      }
      if (found_keys - full_text_keys == 1 &&
	  ((share->options &
	    (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
	   (param->testflag & T_DONT_CHECK_CHECKSUM)))
	old_record_checksum=param->record_checksum;
      else if (old_record_checksum != param->record_checksum)
      {
	if (key)
	  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
		      key+1);
	else
	  mi_check_print_error(param,"Key 1 doesn't point at all records");
503 504 505 506
	if (!(param->testflag & T_INFO))
	  DBUG_RETURN(-1);
	result= -1;
	continue;
unknown's avatar
unknown committed
507 508 509 510
      }
    }
    if ((uint) share->base.auto_key -1 == key)
    {
unknown's avatar
unknown committed
511
      /* Check that auto_increment key is bigger than max key value */
512
      ulonglong auto_increment;
unknown's avatar
unknown committed
513 514
      info->lastinx=key;
      _mi_read_key_record(info, 0L, info->rec_buff);
515 516
      auto_increment= retrieve_auto_increment(info, info->rec_buff);
      if (auto_increment > info->s->state.auto_increment)
unknown's avatar
unknown committed
517
      {
518 519 520 521
        mi_check_print_warning(param, "Auto-increment value: %s is smaller "
                               "than max used value: %s",
                               llstr(info->s->state.auto_increment,buff2),
                               llstr(auto_increment, buff));
unknown's avatar
unknown committed
522 523 524
      }
      if (param->testflag & T_AUTO_INC)
      {
525 526 527 528
        set_if_bigger(info->s->state.auto_increment,
                      auto_increment);
        set_if_bigger(info->s->state.auto_increment,
                      param->auto_increment_value);
unknown's avatar
unknown committed
529
      }
unknown's avatar
unknown committed
530 531

      /* Check that there isn't a row with auto_increment = 0 in the table */
unknown's avatar
unknown committed
532
      mi_extra(info,HA_EXTRA_KEYREAD,0);
unknown's avatar
unknown committed
533
      bzero(info->lastkey,keyinfo->seg->length);
unknown's avatar
unknown committed
534
      if (!mi_rkey(info, info->rec_buff, key, (const byte*) info->lastkey,
unknown's avatar
unknown committed
535
		   (key_part_map)1, HA_READ_KEY_EXACT))
unknown's avatar
unknown committed
536 537 538
      {
	/* Don't count this as a real warning, as myisamchk can't correct it */
	uint save=param->warning_printed;
539 540
        mi_check_print_warning(param, "Found row where the auto_increment "
                               "column has the value 0");
unknown's avatar
unknown committed
541 542
	param->warning_printed=save;
      }
unknown's avatar
unknown committed
543
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
unknown's avatar
unknown committed
544 545 546 547 548 549 550 551 552 553 554 555
    }

    length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
    if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
      printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
	     key+1,
	     (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
	     (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
		    my_off_t2double(length)),
	     param->max_level);
    all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;

556
do_stat:
unknown's avatar
unknown committed
557 558
    if (param->testflag & T_STATISTICS)
      update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
559 560 561
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       param->notnull_count: NULL, 
                       (ulonglong)info->state->records);
unknown's avatar
unknown committed
562 563 564 565 566 567 568 569 570 571
  }
  if (param->testflag & T_INFO)
  {
    if (all_totaldata != 0L && found_keys > 0)
      printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
	     (int) (my_off_t2double(all_keydata)*100.0/
		    my_off_t2double(all_totaldata)),
	     (int) ((my_off_t2double(key_totlength) -
		     my_off_t2double(all_keydata))*100.0/
		     my_off_t2double(key_totlength)));
572
    else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
unknown's avatar
unknown committed
573 574 575 576 577 578 579 580 581
      puts("");
  }
  if (param->key_file_blocks != info->state->key_file_length &&
      param->keys_in_use != ~(ulonglong) 0)
    mi_check_print_warning(param, "Some data are unreferenced in keyfile");
  if (found_keys != full_text_keys)
    param->record_checksum=old_record_checksum-init_checksum;	/* Remove delete links */
  else
    param->record_checksum=0;
unknown's avatar
unknown committed
582
  DBUG_RETURN(result);
unknown's avatar
unknown committed
583 584
} /* chk_key */

unknown's avatar
unknown committed
585

586 587 588 589 590
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
                     my_off_t page, uchar *buff, ha_rows *keys,
                     ha_checksum *key_checksum, uint level)
{
  char llbuff[22],llbuff2[22];
591
  DBUG_ENTER("chk_index_down");
592

593 594 595 596 597
  /* Key blocks must lay within the key file length entirely. */
  if (page + keyinfo->block_length > info->state->key_file_length)
  {
    /* purecov: begin tested */
    /* Give it a chance to fit in the real file size. */
598 599
    my_off_t max_length= my_seek(info->s->kfile, 0L, MY_SEEK_END,
                                 MYF(MY_THREADSAFE));
600 601 602 603 604
    mi_check_print_error(param, "Invalid key block position: %s  "
                         "key block size: %u  file_length: %s",
                         llstr(page, llbuff), keyinfo->block_length,
                         llstr(info->state->key_file_length, llbuff2));
    if (page + keyinfo->block_length > max_length)
605
      goto err;
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
    /* Fix the remebered key file length. */
    info->state->key_file_length= (max_length &
                                   ~ (my_off_t) (keyinfo->block_length - 1));
    /* purecov: end */
  }

  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
  if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
  {
    /* purecov: begin tested */
    mi_check_print_error(param, "Mis-aligned key block: %s  "
                         "minimum key block length: %u",
                         llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
    goto err;
    /* purecov: end */
621
  }
622

623 624 625 626 627 628 629 630 631 632
  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
  {
    mi_check_print_error(param,"Can't read key from filepos: %s",
        llstr(page,llbuff));
    goto err;
  }
  param->key_file_blocks+=keyinfo->block_length;
  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
    goto err;

633 634 635
  DBUG_RETURN(0);

  /* purecov: begin tested */
636
err:
637 638
  DBUG_RETURN(1);
  /* purecov: end */
639
}
unknown's avatar
unknown committed
640

641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686

/*
  "Ignore NULLs" statistics collection method: process first index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_first()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      key      IN     Key values tuple

  DESCRIPTION
    Process the first index tuple - find out which prefix tuples don't
    contain NULLs, and update the array of notnull counters accordingly.
*/

static
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
                                    uchar *key)
{
  uint first_null, kp;
  first_null= ha_find_null(keyseg, key) - keyseg;
  /*
    All prefix tuples that don't include keypart_{first_null} are not-null
    tuples (and all others aren't), increment counters for them.
  */
  for (kp= 0; kp < first_null; kp++)
    notnull[kp]++;
}


/*
  "Ignore NULLs" statistics collection method: process next index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_next()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      prev_key IN     Previous key values tuple
      last_key IN     Next key values tuple

  DESCRIPTION
    Process the next index tuple:
    1. Find out which prefix tuples of last_key don't contain NULLs, and
       update the array of notnull counters accordingly.
687 688 689 690 691
    2. Find the first keypart number where the prev_key and last_key tuples
       are different(A), or last_key has NULL value(B), and return it, so the 
       caller can count number of unique tuples for each key prefix. We don't 
       need (B) to be counted, and that is compensated back in 
       update_key_parts().
692 693 694 695 696 697 698 699 700 701 702

  RETURN
    1 + number of first keypart where values differ or last_key tuple has NULL
*/

static
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
                                  uchar *prev_key, uchar *last_key)
{
  uint diffs[2];
  uint first_null_seg, kp;
703
  HA_KEYSEG *seg;
704

705 706 707 708 709 710 711 712
  /* 
     Find the first keypart where values are different or either of them is
     NULL. We get results in diffs array:
     diffs[0]= 1 + number of first different keypart
     diffs[1]=offset: (last_key + diffs[1]) points to first value in
                      last_key that is NULL or different from corresponding
                      value in prev_key.
  */
713
  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, 
714 715
             SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
  seg= keyseg + diffs[0] - 1;
716

717 718 719 720 721 722 723 724 725 726 727 728 729 730
  /* Find first NULL in last_key */
  first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
  for (kp= 0; kp < first_null_seg; kp++)
    notnull[kp]++;

  /* 
    Return 1+ number of first key part where values differ. Don't care if
    these were NULLs and not .... We compensate for that in
    update_key_parts.
  */
  return diffs[0];
}


unknown's avatar
unknown committed
731 732 733 734 735 736 737
	/* Check if index is ok */

static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level)
{
  int flag;
738
  uint used_length,comp_flag,nod_flag,key_length=0;
unknown's avatar
unknown committed
739 740
  uchar key[MI_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
  my_off_t next_page,record;
unknown's avatar
unknown committed
741
  char llbuff[22];
742
  uint diff_pos[2];
unknown's avatar
unknown committed
743 744 745
  DBUG_ENTER("chk_index");
  DBUG_DUMP("buff",(byte*) buff,mi_getint(buff));

746 747 748 749
  /* TODO: implement appropriate check for RTree keys */
  if (keyinfo->flag & HA_SPATIAL)
    DBUG_RETURN(0);

unknown's avatar
unknown committed
750 751
  if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
752
    mi_check_print_error(param,"Not enough memory for keyblock");
unknown's avatar
unknown committed
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
    DBUG_RETURN(-1);
  }

  if (keyinfo->flag & HA_NOSAME)
    comp_flag=SEARCH_FIND | SEARCH_UPDATE;	/* Not real duplicates */
  else
    comp_flag=SEARCH_SAME;			/* Keys in positionorder */
  nod_flag=mi_test_if_nod(buff);
  used_length=mi_getint(buff);
  keypos=buff+2+nod_flag;
  endpos=buff+used_length;

  param->keydata+=used_length; param->totaldata+=keyinfo->block_length;	/* INFO */
  param->key_blocks++;
  if (level > param->max_level)
    param->max_level=level;

  if (used_length > keyinfo->block_length)
  {
unknown's avatar
unknown committed
772 773
    mi_check_print_error(param,"Wrong pageinfo at page: %s",
			 llstr(page,llbuff));
unknown's avatar
unknown committed
774 775 776 777
    goto err;
  }
  for ( ;; )
  {
unknown's avatar
unknown committed
778 779
    if (*killed_ptr(param))
      goto err;
780 781
    memcpy((char*) info->lastkey,(char*) key,key_length);
    info->lastkey_length=key_length;
unknown's avatar
unknown committed
782 783 784
    if (nod_flag)
    {
      next_page=_mi_kpos(nod_flag,keypos);
785 786
      if (chk_index_down(param,info,keyinfo,next_page,
                         temp_buff,keys,key_checksum,level+1))
unknown's avatar
unknown committed
787 788 789 790 791 792 793 794 795 796 797 798
	goto err;
    }
    old_keypos=keypos;
    if (keypos >= endpos ||
	(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
      break;
    if (keypos > endpos)
    {
      mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
      goto err;
    }
    if ((*keys)++ &&
unknown's avatar
unknown committed
799
	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
800
			 comp_flag, diff_pos)) >=0)
unknown's avatar
unknown committed
801 802 803 804 805 806 807 808 809 810 811 812 813
    {
      DBUG_DUMP("old",(byte*) info->lastkey, info->lastkey_length);
      DBUG_DUMP("new",(byte*) key, key_length);
      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));

      if (comp_flag & SEARCH_FIND && flag == 0)
	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
      else
	mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
      goto err;
    }
    if (param->testflag & T_STATISTICS)
    {
814
      if (*keys != 1L)				/* not first_key */
unknown's avatar
unknown committed
815
      {
816 817 818
        if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
          ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
                     SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
819
                     diff_pos);
820 821
        else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
        {
822
          diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, 
823 824 825
                                                  param->notnull_count,
                                                  info->lastkey, key);
        }
826
	param->unique_count[diff_pos[0]-1]++;
unknown's avatar
unknown committed
827
      }
828 829 830 831 832
      else
      {  
        if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
          mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
                                         key);
unknown's avatar
unknown committed
833 834 835 836 837
      }
    }
    (*key_checksum)+= mi_byte_checksum((byte*) key,
				       key_length- info->s->rec_reflength);
    record= _mi_dpos(info,0,key+key_length);
838 839 840 841 842 843 844 845 846 847 848 849
    if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
    {
      uint off;
      int  subkeys;
      get_key_full_length_rdonly(off, key);
      subkeys=ft_sintXkorr(key+off);
      if (subkeys < 0)
      {
        ha_rows tmp_keys=0;
        if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
                           temp_buff,&tmp_keys,key_checksum,1))
          goto err;
850 851
        if (tmp_keys + subkeys)
        {
unknown's avatar
unknown committed
852 853 854 855 856
          mi_check_print_error(param,
                               "Number of words in the 2nd level tree "
                               "does not match the number in the header. "
                               "Parent word in on the page %s, offset %u",
                               llstr(page,llbuff), (uint) (old_keypos-buff));
857 858
          goto err;
        }
859 860 861 862 863
        (*keys)+=tmp_keys-1;
        continue;
      }
      /* fall through */
    }
unknown's avatar
unknown committed
864 865 866
    if (record >= info->state->data_file_length)
    {
#ifndef DBUG_OFF
unknown's avatar
unknown committed
867
      char llbuff2[22], llbuff3[22];
unknown's avatar
unknown committed
868 869 870 871 872
#endif
      mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
      DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
			 llstr(page,llbuff),llstr(record,llbuff2),
			 llstr(info->state->data_file_length,llbuff3)));
873
      DBUG_DUMP("key",(byte*) key,key_length);
unknown's avatar
unknown committed
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));
      goto err;
    }
    param->record_checksum+=(ha_checksum) record;
  }
  if (keypos != endpos)
  {
    mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
                llstr(page,llbuff), used_length, (keypos - buff));
    goto err;
  }
  my_afree((byte*) temp_buff);
  DBUG_RETURN(0);
 err:
  my_afree((byte*) temp_buff);
  DBUG_RETURN(1);
} /* chk_index */


	/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */

static ha_checksum calc_checksum(ha_rows count)
{
  ulonglong sum,a,b;
  DBUG_ENTER("calc_checksum");

  sum=0;
  a=count; b=count+1;
  if (a & 1)
    b>>=1;
  else
    a>>=1;
  while (b)
  {
    if (b & 1)
      sum+=a;
    a<<=1; b>>=1;
  }
  DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
  DBUG_RETURN((ha_checksum) sum);
} /* calc_checksum */


	/* Calc length of key in normal isam */

static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
{
  uint length;
unknown's avatar
unknown committed
922
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
  DBUG_ENTER("isam_key_length");

  length= info->s->rec_reflength;
  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
    length+= keyseg->length;

  DBUG_PRINT("exit",("length: %d",length));
  DBUG_RETURN(length);
} /* key_length */


	/* Check that record-link is ok */

int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
{
  int	error,got_error,flag;
  uint	key,left_length,b_type,field;
  ha_rows records,del_blocks;
  my_off_t used,empty,pos,splits,start_recpos,
	   del_length,link_used,start_block;
  byte	*record,*to;
  char llbuff[22],llbuff2[22],llbuff3[22];
  ha_checksum intern_record_checksum;
  ha_checksum key_checksum[MI_MAX_POSSIBLE_KEY];
  my_bool static_row_size;
  MI_KEYDEF *keyinfo;
  MI_BLOCK_INFO block_info;
  DBUG_ENTER("chk_data_link");

  if (!(param->testflag & T_SILENT))
  {
    if (extend)
      puts("- check records and index references");
    else
      puts("- check record links");
  }

unknown's avatar
unknown committed
960
  if (!(record= (byte*) my_malloc(info->s->base.pack_reclength,MYF(0))))
unknown's avatar
unknown committed
961
  {
962
    mi_check_print_error(param,"Not enough memory for record");
unknown's avatar
unknown committed
963 964 965 966 967 968 969
    DBUG_RETURN(-1);
  }
  records=del_blocks=0;
  used=link_used=splits=del_length=0;
  intern_record_checksum=param->glob_crc=0;
  LINT_INIT(left_length);  LINT_INIT(start_recpos);  LINT_INIT(to);
  got_error=error=0;
970
  empty=info->s->pack.header_length;
unknown's avatar
unknown committed
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986

  /* Check how to calculate checksum of rows */
  static_row_size=1;
  if (info->s->data_file_type == COMPRESSED_RECORD)
  {
    for (field=0 ; field < info->s->base.fields ; field++)
    {
      if (info->s->rec[field].base_type == FIELD_BLOB ||
	  info->s->rec[field].base_type == FIELD_VARCHAR)
      {
	static_row_size=0;
	break;
      }
    }
  }

987
  pos=my_b_tell(&param->read_cache);
unknown's avatar
unknown committed
988 989 990
  bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
  while (pos < info->state->data_file_length)
  {
unknown's avatar
unknown committed
991 992
    if (*killed_ptr(param))
      goto err2;
unknown's avatar
unknown committed
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
    switch (info->s->data_file_type) {
    case STATIC_RECORD:
      if (my_b_read(&param->read_cache,(byte*) record,
		    info->s->base.pack_reclength))
	goto err;
      start_recpos=pos;
      pos+=info->s->base.pack_reclength;
      splits++;
      if (*record == '\0')
      {
	del_blocks++;
	del_length+=info->s->base.pack_reclength;
	continue;					/* Record removed */
      }
      param->glob_crc+= mi_static_checksum(info,record);
      used+=info->s->base.pack_reclength;
      break;
    case DYNAMIC_RECORD:
      flag=block_info.second_read=0;
      block_info.next_filepos=pos;
      do
      {
	if (_mi_read_cache(&param->read_cache,(byte*) block_info.header,
1016 1017 1018
			   (start_block=block_info.next_filepos),
			   sizeof(block_info.header),
			   (flag ? 0 : READING_NEXT) | READING_HEADER))
unknown's avatar
unknown committed
1019 1020 1021
	  goto err;
	if (start_block & (MI_DYN_ALIGN_SIZE-1))
	{
1022 1023
	  mi_check_print_error(param,"Wrong aligned block at %s",
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
	  goto err2;
	}
	b_type=_mi_get_block_info(&block_info,-1,start_block);
	if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
		      BLOCK_FATAL_ERROR))
	{
	  if (b_type & BLOCK_SYNC_ERROR)
	  {
	    if (flag)
	    {
	      mi_check_print_error(param,"Unexpected byte: %d at link: %s",
			  (int) block_info.header[0],
			  llstr(start_block,llbuff));
	      goto err2;
	    }
	    pos=block_info.filepos+block_info.block_len;
	    goto next;
	  }
	  if (b_type & BLOCK_DELETED)
	  {
	    if (block_info.block_len < info->s->base.min_block_length)
	    {
1046 1047 1048
	      mi_check_print_error(param,
				   "Deleted block with impossible length %lu at %s",
				   block_info.block_len,llstr(pos,llbuff));
unknown's avatar
unknown committed
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
	      goto err2;
	    }
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >= info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      mi_check_print_error(param,"Delete link points outside datafile at %s",
			  llstr(pos,llbuff));
	      goto err2;
	    }
	    del_blocks++;
	    del_length+=block_info.block_len;
	    pos=block_info.filepos+block_info.block_len;
	    splits++;
	    goto next;
	  }
	  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1067 1068 1069
			       block_info.header[0],block_info.header[1],
			       block_info.header[2],
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1070 1071 1072 1073 1074
	  goto err2;
	}
	if (info->state->data_file_length < block_info.filepos+
	    block_info.block_len)
	{
1075 1076 1077
	  mi_check_print_error(param,
			       "Recordlink that points outside datafile at %s",
			       llstr(pos,llbuff));
unknown's avatar
unknown committed
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	  got_error=1;
	  break;
	}
	splits++;
	if (!flag++)				/* First block */
	{
	  start_recpos=pos;
	  pos=block_info.filepos+block_info.block_len;
	  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
	  {
1088 1089 1090
	    mi_check_print_error(param,"Found too long record (%lu) at %s",
				 (ulong) block_info.rec_len,
				 llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1091 1092 1093 1094 1095
	    got_error=1;
	    break;
	  }
	  if (info->s->base.blobs)
	  {
1096
	    if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1097
					&info->rec_buff)))
unknown's avatar
unknown committed
1098
	    {
1099 1100 1101 1102
	      mi_check_print_error(param,
				   "Not enough memory (%lu) for blob at %s",
				   (ulong) block_info.rec_len,
				   llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
	      got_error=1;
	      break;
	    }
	  }
	  else
	    to= info->rec_buff;
	  left_length=block_info.rec_len;
	}
	if (left_length < block_info.data_len)
	{
1113 1114 1115 1116 1117
	  mi_check_print_error(param,"Found too long record (%lu) at %s",
			       (ulong) block_info.data_len,
			       llstr(start_recpos,llbuff));
	  got_error=1;
	  break;
unknown's avatar
unknown committed
1118 1119
	}
	if (_mi_read_cache(&param->read_cache,(byte*) to,block_info.filepos,
1120 1121
			   (uint) block_info.data_len,
			   flag == 1 ? READING_NEXT : 0))
unknown's avatar
unknown committed
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
	  goto err;
	to+=block_info.data_len;
	link_used+= block_info.filepos-start_block;
	used+= block_info.filepos - start_block + block_info.data_len;
	empty+=block_info.block_len-block_info.data_len;
	left_length-=block_info.data_len;
	if (left_length)
	{
	  if (b_type & BLOCK_LAST)
	  {
1132 1133 1134 1135 1136
	    mi_check_print_error(param,
				 "Wrong record length %s of %s at %s",
				 llstr(block_info.rec_len-left_length,llbuff),
				 llstr(block_info.rec_len, llbuff2),
				 llstr(start_recpos,llbuff3));
unknown's avatar
unknown committed
1137 1138 1139 1140 1141
	    got_error=1;
	    break;
	  }
	  if (info->state->data_file_length < block_info.next_filepos)
	  {
1142 1143 1144
	    mi_check_print_error(param,
				 "Found next-recordlink that points outside datafile at %s",
				 llstr(block_info.filepos,llbuff));
unknown's avatar
unknown committed
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
	    got_error=1;
	    break;
	  }
	}
      } while (left_length);
      if (! got_error)
      {
	if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
	    MY_FILE_ERROR)
	{
1155 1156
	  mi_check_print_error(param,"Found wrong record at %s",
			       llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1157 1158 1159 1160 1161 1162 1163
	  got_error=1;
	}
	else
	{
	  info->checksum=mi_checksum(info,record);
	  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
	  {
1164 1165
	    if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
                              test(info->s->calc_checksum)))
unknown's avatar
unknown committed
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	    {
	      mi_check_print_error(param,"Found wrong packed record at %s",
			  llstr(start_recpos,llbuff));
	      got_error=1;
	    }
	  }
	  if (!got_error)
	    param->glob_crc+= info->checksum;
	}
      }
      else if (!flag)
	pos=block_info.filepos+block_info.block_len;
      break;
    case COMPRESSED_RECORD:
      if (_mi_read_cache(&param->read_cache,(byte*) block_info.header, pos,
1181
			 info->s->pack.ref_length, READING_NEXT))
unknown's avatar
unknown committed
1182 1183 1184
	goto err;
      start_recpos=pos;
      splits++;
1185 1186
      VOID(_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
                                   &info->rec_buff, -1, start_recpos));
unknown's avatar
unknown committed
1187 1188 1189 1190
      pos=block_info.filepos+block_info.rec_len;
      if (block_info.rec_len < (uint) info->s->min_pack_length ||
	  block_info.rec_len > (uint) info->s->max_pack_length)
      {
unknown's avatar
unknown committed
1191 1192 1193
	mi_check_print_error(param,
			     "Found block with wrong recordlength: %d at %s",
			     block_info.rec_len, llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1194 1195 1196 1197
	got_error=1;
	break;
      }
      if (_mi_read_cache(&param->read_cache,(byte*) info->rec_buff,
1198
			block_info.filepos, block_info.rec_len, READING_NEXT))
unknown's avatar
unknown committed
1199
	goto err;
1200 1201
      if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
                              info->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
1202
      {
unknown's avatar
unknown committed
1203 1204
	mi_check_print_error(param,"Found wrong record at %s",
			     llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	got_error=1;
      }
      if (static_row_size)
	param->glob_crc+= mi_static_checksum(info,record);
      else
	param->glob_crc+= mi_checksum(info,record);
      link_used+= (block_info.filepos - start_recpos);
      used+= (pos-start_recpos);
    } /* switch */
    if (! got_error)
    {
      intern_record_checksum+=(ha_checksum) start_recpos;
      records++;
      if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
      {
	printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
      }

      /* Check if keys match the record */

      for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
	   key++,keyinfo++)
      {
1228
        if (mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1229
	{
1230
          if(!(keyinfo->flag & HA_FULLTEXT))
unknown's avatar
unknown committed
1231 1232 1233 1234 1235 1236 1237 1238
	  {
	    uint key_length=_mi_make_key(info,key,info->lastkey,record,
					 start_recpos);
	    if (extend)
	    {
	      /* We don't need to lock the key tree here as we don't allow
		 concurrent threads when running myisamchk
	      */
1239 1240 1241
              int search_result=
#ifdef HAVE_RTREE_KEYS
                (keyinfo->flag & HA_SPATIAL) ?
1242
                rtree_find_first(info, key, info->lastkey, key_length,
1243
                                 MBR_EQUAL | MBR_DATA) : 
1244
#endif
1245 1246 1247 1248
                _mi_search(info,keyinfo,info->lastkey,key_length,
                           SEARCH_SAME, info->s->state.key_root[key]);
              if (search_result)
              {
1249 1250
                mi_check_print_error(param,"Record at: %10s  "
                                     "Can't find key for index: %2d",
1251 1252 1253 1254
                                     llstr(start_recpos,llbuff),key+1);
                if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                  goto err2;
              }
unknown's avatar
unknown committed
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
	    }
	    else
	      key_checksum[key]+=mi_byte_checksum((byte*) info->lastkey,
						  key_length);
	  }
	}
      }
    }
    else
    {
      got_error=0;
      if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
	goto err2;
    }
  next:;				/* Next record */
  }
  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
  if (records != info->state->records)
  {
    mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
		llstr(records,llbuff), llstr(info->state->records,llbuff2));
    error=1;
  }
  else if (param->record_checksum &&
	   param->record_checksum != intern_record_checksum)
  {
    mi_check_print_error(param,
			 "Keypointers and record positions doesn't match");
    error=1;
  }
1288
  else if (param->glob_crc != info->state->checksum &&
unknown's avatar
unknown committed
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	   (info->s->options &
	    (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
  {
    mi_check_print_warning(param,
			   "Record checksum is not the same as checksum stored in the index file\n");
    error=1;
  }
  else if (!extend)
  {
    for (key=0 ; key < info->s->base.keys;  key++)
    {
      if (key_checksum[key] != param->key_crc[key] &&
1301
          !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
1302 1303 1304 1305 1306 1307 1308 1309
      {
	mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
		    key+1);
	error=1;
      }
    }
  }

1310 1311 1312 1313 1314 1315 1316
  if (del_length != info->state->empty)
  {
    mi_check_print_warning(param,
			   "Found %s deleted space.   Should be %s",
			   llstr(del_length,llbuff2),
			   llstr(info->state->empty,llbuff));
  }
unknown's avatar
unknown committed
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
  if (used+empty+del_length != info->state->data_file_length)
  {
    mi_check_print_warning(param,
			   "Found %s record-data and %s unused data and %s deleted-data",
			   llstr(used,llbuff),llstr(empty,llbuff2),
			   llstr(del_length,llbuff3));
    mi_check_print_warning(param,
			   "Total %s, Should be: %s",
			   llstr((used+empty+del_length),llbuff),
			   llstr(info->state->data_file_length,llbuff2));
  }
  if (del_blocks != info->state->del)
  {
    mi_check_print_warning(param,
			   "Found %10s deleted blocks       Should be: %s",
			   llstr(del_blocks,llbuff),
			   llstr(info->state->del,llbuff2));
  }
  if (splits != info->s->state.split)
  {
    mi_check_print_warning(param,
			   "Found %10s parts                Should be: %s parts",
			   llstr(splits,llbuff),
			   llstr(info->s->state.split,llbuff2));
  }
  if (param->testflag & T_INFO)
  {
    if (param->warning_printed || param->error_printed)
      puts("");
    if (used != 0 && ! param->error_printed)
    {
      printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
unknown's avatar
unknown committed
1349
	     llstr(records,llbuff), (long)((used-link_used)/records),
unknown's avatar
unknown committed
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
	     (info->s->base.blobs ? 0.0 :
	      (ulonglong2double((ulonglong) info->s->base.reclength*records)-
	       my_off_t2double(used))/
	      ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
      printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
	     (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
	     (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
				      my_off_t2double(used)*100.0)),
	     ulonglong2double(splits - del_blocks) / records);
    }
    printf("Record blocks:%12s    Delete blocks:%10s\n",
	   llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
    printf("Record data:  %12s    Deleted data: %10s\n",
	   llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
    printf("Lost space:   %12s    Linkdata:     %10s\n",
	   llstr(empty,llbuff),llstr(link_used,llbuff2));
  }
unknown's avatar
unknown committed
1367
  my_free((gptr) record,MYF(0));
unknown's avatar
unknown committed
1368 1369
  DBUG_RETURN (error);
 err:
unknown's avatar
unknown committed
1370
  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
unknown's avatar
unknown committed
1371
 err2:
unknown's avatar
unknown committed
1372
  my_free((gptr) record,MYF(0));
unknown's avatar
unknown committed
1373
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
  DBUG_RETURN(1);
} /* chk_data_link */


	/* Recover old table by reading each record and writing all keys */
	/* Save new datafile-name in temp_filename */

int mi_repair(MI_CHECK *param, register MI_INFO *info,
	      my_string name, int rep_quick)
{
  int error,got_error;
  uint i;
  ha_rows start_records,new_header_length;
  my_off_t del;
  File new_file;
  MYISAM_SHARE *share=info->s;
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
1391 1392
  SORT_INFO sort_info;
  MI_SORT_PARAM sort_param;
unknown's avatar
unknown committed
1393 1394
  DBUG_ENTER("mi_repair");

unknown's avatar
unknown committed
1395 1396
  bzero((char *)&sort_info, sizeof(sort_info));
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
1397
  start_records=info->state->records;
1398 1399
  new_header_length=(param->testflag & T_UNPACK) ? 0L :
    share->pack.header_length;
unknown's avatar
unknown committed
1400 1401
  got_error=1;
  new_file= -1;
unknown's avatar
unknown committed
1402
  sort_param.sort_info=&sort_info;
1403

unknown's avatar
unknown committed
1404 1405
  if (!(param->testflag & T_SILENT))
  {
1406
    printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
1407 1408
    printf("Data records: %s\n", llstr(info->state->records,llbuff));
  }
1409
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
1410

1411 1412 1413
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
1414
  if (!param->using_global_keycache)
unknown's avatar
unknown committed
1415 1416
    VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
                        param->use_buffers, 0, 0));
unknown's avatar
unknown committed
1417 1418 1419 1420

  if (init_io_cache(&param->read_cache,info->dfile,
		    (uint) param->read_buffer_length,
		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1421 1422
  {
    bzero(&info->rec_cache,sizeof(info->rec_cache));
unknown's avatar
unknown committed
1423
    goto err;
1424
  }
unknown's avatar
unknown committed
1425 1426 1427 1428 1429 1430
  if (!rep_quick)
    if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
		      WRITE_CACHE, new_header_length, 1,
		      MYF(MY_WME | MY_WAIT_IF_FULL)))
      goto err;
  info->opt_flag|=WRITE_CACHE_USED;
unknown's avatar
unknown committed
1431
  if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength,
unknown's avatar
unknown committed
1432 1433
					   MYF(0))) ||
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
1434
  {
1435
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
1436 1437 1438 1439 1440
    goto err;
  }

  if (!rep_quick)
  {
1441 1442
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
1443
					   share->data_file_name, "",
unknown's avatar
unknown committed
1444
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
1455 1456
    if (new_header_length &&
        filecopy(param,new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
		 "datafile-header"))
      goto err;
    info->s->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
  }
unknown's avatar
unknown committed
1467 1468 1469 1470 1471 1472
  sort_info.info=info;
  sort_info.param = param;
  sort_param.read_cache=param->read_cache;
  sort_param.pos=sort_param.max_pos=share->pack.header_length;
  sort_param.filepos=new_header_length;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
1473
    my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
unknown's avatar
unknown committed
1474 1475
  sort_info.dupp=0;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
1476
  sort_param.master=1;
unknown's avatar
unknown committed
1477
  sort_info.max_records= ~(ha_rows) 0;
unknown's avatar
unknown committed
1478

unknown's avatar
unknown committed
1479
  set_data_file_type(&sort_info, share);
unknown's avatar
unknown committed
1480 1481 1482
  del=info->state->del;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;
1483 1484
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
1485
    sort_param.calc_checksum= 1;
1486

unknown's avatar
unknown committed
1487
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1488 1489 1490 1491 1492

  /*
    Clear all keys. Note that all key blocks allocated until now remain
    "dead" parts of the key file. (Bug #4692)
  */
unknown's avatar
unknown committed
1493 1494
  for (i=0 ; i < info->s->base.keys ; i++)
    share->state.key_root[i]= HA_OFFSET_ERROR;
1495 1496

  /* Drop the delete chain. */
1497
  for (i=0 ; i < share->state.header.max_block_size_index ; i++)
unknown's avatar
unknown committed
1498 1499
    share->state.key_del[i]=  HA_OFFSET_ERROR;

unknown's avatar
unknown committed
1500
  /*
1501 1502
    If requested, activate (enable) all keys in key_map. In this case,
    all indexes will be (re-)built.
1503
  */
1504
  if (param->testflag & T_CREATE_MISSING_KEYS)
1505
    mi_set_all_keys_active(share->state.key_map, share->base.keys);
1506

unknown's avatar
unknown committed
1507 1508 1509
  info->state->key_file_length=share->base.keystart;

  lock_memory(param);			/* Everything is alloced */
1510 1511

  /* Re-create all keys, which are set in key_map. */
unknown's avatar
unknown committed
1512
  while (!(error=sort_get_next_record(&sort_param)))
unknown's avatar
unknown committed
1513
  {
1514
    if (writekeys(&sort_param))
unknown's avatar
unknown committed
1515
    {
1516 1517
      if (my_errno != HA_ERR_FOUND_DUPP_KEY)
	goto err;
unknown's avatar
unknown committed
1518
      DBUG_DUMP("record",(byte*) sort_param.record,share->base.pack_reclength);
unknown's avatar
unknown committed
1519
      mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1520
			  info->errkey+1,
unknown's avatar
unknown committed
1521
			  llstr(sort_param.start_recpos,llbuff),
1522
			  llstr(info->dupp_key_pos,llbuff2));
unknown's avatar
unknown committed
1523 1524 1525
      if (param->testflag & T_VERBOSE)
      {
	VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
unknown's avatar
unknown committed
1526
			  sort_param.record,0L));
unknown's avatar
unknown committed
1527 1528 1529
	_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
		      USE_WHOLE_KEY);
      }
unknown's avatar
unknown committed
1530
      sort_info.dupp++;
1531
      if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
1532
      {
unknown's avatar
unknown committed
1533 1534
        param->testflag|=T_RETRY_WITHOUT_QUICK;
	param->error_printed=1;
unknown's avatar
unknown committed
1535 1536 1537 1538
	goto err;
      }
      continue;
    }
unknown's avatar
unknown committed
1539
    if (sort_write_record(&sort_param))
unknown's avatar
unknown committed
1540 1541
      goto err;
  }
unknown's avatar
unknown committed
1542
  if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
unknown's avatar
unknown committed
1543 1544 1545 1546 1547 1548 1549
      flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
    goto err;

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
1550
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
1551 1552 1553 1554 1555 1556 1557
  {
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);
    goto err;
  }

unknown's avatar
unknown committed
1558
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
1559 1560 1561 1562
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
1563 1564
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1565 1566
    goto err;
  }
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576
  if (param->testflag & T_SAFE_REPAIR)
  {
    /* Don't repair if we loosed more than one row */
    if (info->state->records+1 < start_records)
    {
      info->state->records=start_records;
      got_error=1;
      goto err;
    }
  }
unknown's avatar
unknown committed
1577 1578 1579 1580 1581

  if (!rep_quick)
  {
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
unknown's avatar
unknown committed
1582
    info->state->data_file_length=sort_param.filepos;
unknown's avatar
unknown committed
1583 1584 1585
    share->state.version=(ulong) time((time_t*) 0);	/* Force reopen */
  }
  else
1586
  {
unknown's avatar
unknown committed
1587
    info->state->data_file_length=sort_param.max_pos;
1588
  }
unknown's avatar
unknown committed
1589
  if (param->testflag & T_CALC_CHECKSUM)
1590
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
1591 1592 1593 1594 1595

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
1596
    if (sort_info.dupp)
unknown's avatar
unknown committed
1597 1598
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
1599
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
1600 1601 1602 1603 1604 1605 1606 1607
  }

  got_error=0;
  /* If invoked by external program that uses thr_lock */
  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
1608 1609 1610 1611 1612 1613 1614
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
1615
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1616 1617 1618
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
unknown's avatar
unknown committed
1619
	  mi_open_datafile(info,share,-1))
1620 1621 1622
	got_error=1;
    }
  }
unknown's avatar
unknown committed
1623 1624 1625 1626
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d for record at pos %s",my_errno,
unknown's avatar
unknown committed
1627
		  llstr(sort_param.start_recpos,llbuff));
unknown's avatar
unknown committed
1628 1629 1630 1631 1632
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
			  MYF(MY_WME)));
1633
      info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
unknown's avatar
unknown committed
1634
    }
1635
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
1636
  }
1637 1638
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1639 1640
  my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR));
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1641 1642 1643
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  VOID(end_io_cache(&info->rec_cache));
unknown's avatar
unknown committed
1644
  got_error|=flush_blocks(param, share->key_cache, share->kfile);
unknown's avatar
unknown committed
1645 1646 1647 1648
  if (!got_error && param->testflag & T_UNPACK)
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
unknown's avatar
unknown committed
1649
    share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
1650
  }
1651 1652
  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
			  STATE_NOT_ANALYZED);
unknown's avatar
unknown committed
1653
  DBUG_RETURN(got_error);
1654
}
unknown's avatar
unknown committed
1655 1656 1657 1658


/* Uppate keyfile when doing repair */

1659
static int writekeys(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
1660 1661
{
  register uint i;
1662 1663 1664 1665
  uchar    *key;
  MI_INFO  *info=   sort_param->sort_info->info;
  byte     *buff=   sort_param->record;
  my_off_t filepos= sort_param->filepos;
unknown's avatar
unknown committed
1666 1667 1668 1669 1670
  DBUG_ENTER("writekeys");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys ; i++)
  {
1671
    if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1672
    {
1673 1674
      if (info->s->keyinfo[i].flag & HA_FULLTEXT )
      {
1675 1676
        if (_mi_ft_add(info,i,(char*) key,buff,filepos))
	  goto err;
1677
      }
1678
#ifdef HAVE_SPATIAL
1679 1680 1681 1682 1683 1684
      else if (info->s->keyinfo[i].flag & HA_SPATIAL)
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
	if (rtree_insert(info, i, key, key_length))
	  goto err;
      }
1685
#endif /*HAVE_SPATIAL*/
1686 1687 1688
      else
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1689 1690
	if (_mi_ck_write(info,i,key,key_length))
	  goto err;
1691
      }
unknown's avatar
unknown committed
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
    }
  }
  DBUG_RETURN(0);

 err:
  if (my_errno == HA_ERR_FOUND_DUPP_KEY)
  {
    info->errkey=(int) i;			/* This key was found */
    while ( i-- > 0 )
    {
1702
      if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1703
      {
1704 1705
	if (info->s->keyinfo[i].flag & HA_FULLTEXT)
        {
1706 1707
          if (_mi_ft_del(info,i,(char*) key,buff,filepos))
	    break;
1708 1709 1710 1711
        }
        else
	{
	  uint key_length=_mi_make_key(info,i,key,buff,filepos);
1712 1713
	  if (_mi_ck_delete(info,i,key,key_length))
	    break;
1714
	}
unknown's avatar
unknown committed
1715 1716 1717
      }
    }
  }
1718
  /* Remove checksum that was added to glob_crc in sort_get_next_record */
1719 1720
  if (sort_param->calc_checksum)
    sort_param->sort_info->param->glob_crc-= info->checksum;
unknown's avatar
unknown committed
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
  DBUG_PRINT("error",("errno: %d",my_errno));
  DBUG_RETURN(-1);
} /* writekeys */


	/* Change all key-pointers that points to a records */

int movepoint(register MI_INFO *info, byte *record, my_off_t oldpos,
	      my_off_t newpos, uint prot_key)
{
  register uint i;
  uchar *key;
  uint key_length;
  DBUG_ENTER("movepoint");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys; i++)
  {
1739
    if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
    {
      key_length=_mi_make_key(info,i,key,record,oldpos);
      if (info->s->keyinfo[i].flag & HA_NOSAME)
      {					/* Change pointer direct */
	uint nod_flag;
	MI_KEYDEF *keyinfo;
	keyinfo=info->s->keyinfo+i;
	if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
		       (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
		       info->s->state.key_root[i]))
	  DBUG_RETURN(-1);
	nod_flag=mi_test_if_nod(info->buff);
	_mi_dpointer(info,info->int_keypos-nod_flag-
		     info->s->rec_reflength,newpos);
1754 1755
	if (_mi_write_keypage(info,keyinfo,info->last_keypage,
                              DFLT_INIT_HITS,info->buff))
unknown's avatar
unknown committed
1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
	  DBUG_RETURN(-1);
      }
      else
      {					/* Change old key to new */
	if (_mi_ck_delete(info,i,key,key_length))
	  DBUG_RETURN(-1);
	key_length=_mi_make_key(info,i,key,record,newpos);
	if (_mi_ck_write(info,i,key,key_length))
	  DBUG_RETURN(-1);
      }
    }
  }
  DBUG_RETURN(0);
} /* movepoint */


	/* Tell system that we want all memory for our cache */

void lock_memory(MI_CHECK *param __attribute__((unused)))
{
#ifdef SUN_OS				/* Key-cacheing thrases on sun 4.1 */
  if (param->opt_lock_memory)
  {
    int success = mlockall(MCL_CURRENT);	/* or plock(DATLOCK); */
    if (geteuid() == 0 && success != 0)
      mi_check_print_warning(param,
			     "Failed to lock memory. errno %d",my_errno);
  }
#endif
} /* lock_memory */


	/* Flush all changed blocks to disk */

unknown's avatar
unknown committed
1790
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
unknown's avatar
unknown committed
1791
{
1792
  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
unknown's avatar
unknown committed
1793 1794 1795 1796 1797
  {
    mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
    return(1);
  }
  if (!param->using_global_keycache)
1798
    end_key_cache(key_cache,1);
unknown's avatar
unknown committed
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
  return 0;
} /* flush_blocks */


	/* Sort index for more efficent reads */

int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name)
{
  reg2 uint key;
  reg1 MI_KEYDEF *keyinfo;
  File new_file;
  my_off_t index_pos[MI_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
1811
  uint r_locks,w_locks;
1812
  int old_lock;
unknown's avatar
unknown committed
1813
  MYISAM_SHARE *share=info->s;
1814
  MI_STATE_INFO old_state;
1815
  DBUG_ENTER("mi_sort_index");
unknown's avatar
unknown committed
1816

1817 1818 1819 1820
  /* cannot sort index files with R-tree indexes */
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       key++,keyinfo++)
    if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
unknown's avatar
unknown committed
1821
      DBUG_RETURN(0);
1822

unknown's avatar
unknown committed
1823 1824 1825
  if (!(param->testflag & T_SILENT))
    printf("- Sorting index for MyISAM-table '%s'\n",name);

1826 1827 1828 1829
  /* Get real path for index file */
  fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
  if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
				    "", INDEX_TMP_EXT,2+4),
unknown's avatar
unknown committed
1830 1831 1832 1833 1834 1835
			  0,param->tmpfile_createflag,MYF(0))) <= 0)
  {
    mi_check_print_error(param,"Can't create new tempfile: '%s'",
			 param->temp_filename);
    DBUG_RETURN(-1);
  }
unknown's avatar
unknown committed
1836 1837
  if (filecopy(param, new_file,share->kfile,0L,
	       (ulong) share->base.keystart, "headerblock"))
unknown's avatar
unknown committed
1838 1839
    goto err;

unknown's avatar
unknown committed
1840 1841
  param->new_file_pos=share->base.keystart;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
unknown's avatar
unknown committed
1842 1843
       key++,keyinfo++)
  {
1844
    if (! mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1845 1846
      continue;

unknown's avatar
unknown committed
1847
    if (share->state.key_root[key] != HA_OFFSET_ERROR)
unknown's avatar
unknown committed
1848
    {
1849
      index_pos[key]=param->new_file_pos;	/* Write first block here */
unknown's avatar
unknown committed
1850
      if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
unknown's avatar
unknown committed
1851 1852 1853 1854 1855 1856 1857 1858
			 new_file))
	goto err;
    }
    else
      index_pos[key]= HA_OFFSET_ERROR;		/* No blocks */
  }

  /* Flush key cache for this file if we are calling this outside myisamchk */
unknown's avatar
unknown committed
1859
  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
1860

unknown's avatar
unknown committed
1861
  share->state.version=(ulong) time((time_t*) 0);
1862 1863 1864 1865 1866
  old_state= share->state;			/* save state if not stored */
  r_locks=   share->r_locks;
  w_locks=   share->w_locks;
  old_lock=  info->lock_type;

1867
	/* Put same locks as old file */
1868
  share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
1869 1870 1871
  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
  VOID(my_close(share->kfile,MYF(MY_WME)));
  share->kfile = -1;
1872
  VOID(my_close(new_file,MYF(MY_WME)));
unknown's avatar
unknown committed
1873
  if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1874
			MYF(0)) ||
unknown's avatar
unknown committed
1875
      mi_open_keyfile(share))
1876
    goto err2;
1877
  info->lock_type= F_UNLCK;			/* Force mi_readinfo to lock */
unknown's avatar
unknown committed
1878
  _mi_readinfo(info,F_WRLCK,0);			/* Will lock the table */
1879 1880 1881
  info->lock_type=  old_lock;
  share->r_locks=   r_locks;
  share->w_locks=   w_locks;
1882
  share->tot_locks= r_locks+w_locks;
1883
  share->state=     old_state;			/* Restore old state */
unknown's avatar
unknown committed
1884 1885 1886 1887 1888

  info->state->key_file_length=param->new_file_pos;
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  for (key=0 ; key < info->s->base.keys ; key++)
    info->s->state.key_root[key]=index_pos[key];
1889
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
unknown's avatar
unknown committed
1890 1891
    info->s->state.key_del[key]=  HA_OFFSET_ERROR;

1892
  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
unknown's avatar
unknown committed
1893 1894 1895 1896
  DBUG_RETURN(0);

err:
  VOID(my_close(new_file,MYF(MY_WME)));
1897
err2:
unknown's avatar
unknown committed
1898 1899
  VOID(my_delete(param->temp_filename,MYF(MY_WME)));
  DBUG_RETURN(-1);
1900
} /* mi_sort_index */
unknown's avatar
unknown committed
1901 1902 1903 1904 1905 1906 1907


	 /* Sort records recursive using one index */

static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file)
{
1908
  uint length,nod_flag,used_length, key_length;
unknown's avatar
unknown committed
1909 1910 1911 1912 1913 1914
  uchar *buff,*keypos,*endpos;
  uchar key[MI_MAX_POSSIBLE_KEY_BUFF];
  my_off_t new_page_pos,next_page;
  char llbuff[22];
  DBUG_ENTER("sort_one_index");

1915 1916
  /* cannot walk over R-tree indices */
  DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
unknown's avatar
unknown committed
1917 1918 1919 1920 1921
  new_page_pos=param->new_file_pos;
  param->new_file_pos+=keyinfo->block_length;

  if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
1922
    mi_check_print_error(param,"Not enough memory for key block");
unknown's avatar
unknown committed
1923 1924
    DBUG_RETURN(-1);
  }
1925
  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
unknown's avatar
unknown committed
1926 1927 1928 1929 1930
  {
    mi_check_print_error(param,"Can't read key block from filepos: %s",
		llstr(pagepos,llbuff));
    goto err;
  }
1931
  if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
unknown's avatar
unknown committed
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941
  {
    used_length=mi_getint(buff);
    keypos=buff+2+nod_flag;
    endpos=buff+used_length;
    for ( ;; )
    {
      if (nod_flag)
      {
	next_page=_mi_kpos(nod_flag,keypos);
	_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
1942
	if (sort_one_index(param,info,keyinfo,next_page,new_file))
unknown's avatar
unknown committed
1943
	{
unknown's avatar
unknown committed
1944
	  DBUG_PRINT("error",
unknown's avatar
unknown committed
1945
		     ("From page: %ld, keyoffset: %lu  used_length: %d",
unknown's avatar
unknown committed
1946 1947
		      (ulong) pagepos, (ulong) (keypos - buff),
		      (int) used_length));
unknown's avatar
unknown committed
1948 1949 1950 1951 1952
	  DBUG_DUMP("buff",(byte*) buff,used_length);
	  goto err;
	}
      }
      if (keypos >= endpos ||
1953
	  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
unknown's avatar
unknown committed
1954
	break;
1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
      DBUG_ASSERT(keypos <= endpos);
      if (keyinfo->flag & HA_FULLTEXT)
      {
        uint off;
        int  subkeys;
        get_key_full_length_rdonly(off, key);
        subkeys=ft_sintXkorr(key+off);
        if (subkeys < 0)
        {
          next_page= _mi_dpos(info,0,key+key_length);
          _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
                       param->new_file_pos); /* Save new pos */
          if (sort_one_index(param,info,&info->s->ft2_keyinfo,
                             next_page,new_file))
            goto err;
        }
      }
unknown's avatar
unknown committed
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991
    }
  }

  /* Fill block with zero and write it to the new index file */
  length=mi_getint(buff);
  bzero((byte*) buff+length,keyinfo->block_length-length);
  if (my_pwrite(new_file,(byte*) buff,(uint) keyinfo->block_length,
		new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
  {
    mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
    goto err;
  }
  my_afree((gptr) buff);
  DBUG_RETURN(0);
err:
  my_afree((gptr) buff);
  DBUG_RETURN(1);
} /* sort_one_index */


1992 1993 1994 1995 1996 1997 1998 1999
	/*
	  Let temporary file replace old file.
	  This assumes that the new file was created in the same
	  directory as given by realpath(filename).
	  This will ensure that any symlinks that are used will still work.
	  Copy stats from old file to new file, deletes orignal and
	  changes new file name to old file name
	*/
unknown's avatar
unknown committed
2000 2001 2002

int change_to_newfile(const char * filename, const char * old_ext,
		      const char * new_ext,
2003 2004
		      uint raid_chunks __attribute__((unused)),
		      myf MyFlags)
unknown's avatar
unknown committed
2005 2006 2007 2008 2009 2010 2011
{
  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
#ifdef USE_RAID
  if (raid_chunks)
    return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
			 fn_format(new_filename,filename,"",new_ext,2+4),
			 raid_chunks,
2012
			 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
2013
#endif
2014 2015 2016 2017
  /* Get real path to filename */
  (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
  return my_redel(old_filename,
		  fn_format(new_filename,old_filename,"",new_ext,2+4),
2018
		  MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
} /* change_to_newfile */


	/* Locks a whole file */
	/* Gives an error-message if file can't be locked */

int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
	      const char *filetype, const char *filename)
{
  if (my_lock(file,lock_type,start,F_TO_EOF,
	      param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
	      MYF(MY_SEEK_NOT_DONE |  MY_DONT_WAIT)))
  {
    mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
    param->error_printed=2;		/* Don't give that data is crashed */
    return 1;
  }
  return 0;
} /* lock_file */


	/* Copy a block between two files */

int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
	     my_off_t length, const char *type)
{
  char tmp_buff[IO_SIZE],*buff;
  ulong buff_length;
  DBUG_ENTER("filecopy");

  buff_length=(ulong) min(param->write_buffer_length,length);
  if (!(buff=my_malloc(buff_length,MYF(0))))
  {
    buff=tmp_buff; buff_length=IO_SIZE;
  }

  VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
  while (length > buff_length)
  {
    if (my_read(from,(byte*) buff,buff_length,MYF(MY_NABP)) ||
	my_write(to,(byte*) buff,buff_length,param->myf_rw))
      goto err;
    length-= buff_length;
  }
  if (my_read(from,(byte*) buff,(uint) length,MYF(MY_NABP)) ||
      my_write(to,(byte*) buff,(uint) length,param->myf_rw))
    goto err;
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  DBUG_RETURN(0);
err:
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
		       type,my_errno);
  DBUG_RETURN(1);
}
unknown's avatar
unknown committed
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091


/*
  Repair table or given index using sorting

  SYNOPSIS
    mi_repair_by_sort()
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103

int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
		      const char * name, int rep_quick)
{
  int got_error;
  uint i;
  ulong length;
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
  MI_SORT_PARAM sort_param;
  MYISAM_SHARE *share=info->s;
2104
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2105 2106
  ulong   *rec_per_key_part;
  char llbuff[22];
unknown's avatar
unknown committed
2107
  SORT_INFO sort_info;
2108
  ulonglong key_map=share->state.key_map;
unknown's avatar
unknown committed
2109
  DBUG_ENTER("mi_repair_by_sort");
unknown's avatar
unknown committed
2110 2111 2112 2113 2114 2115 2116 2117

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
2118
    printf("- recovering (with sort) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
2119 2120
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2121
  param->testflag|=T_REP; /* for easy checking */
2122

2123 2124 2125
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
2126
  bzero((char*)&sort_info,sizeof(sort_info));
2127
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
2128
  if (!(sort_info.key_block=
unknown's avatar
unknown committed
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
	alloc_key_blocks(param,
			 (uint) param->sort_key_blocks,
			 share->base.max_key_block_length))
      || init_io_cache(&param->read_cache,info->dfile,
		       (uint) param->read_buffer_length,
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
      (! rep_quick &&
       init_io_cache(&info->rec_cache,info->dfile,
		     (uint) param->write_buffer_length,
		     WRITE_CACHE,new_header_length,1,
2139
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
unknown's avatar
unknown committed
2140
    goto err;
unknown's avatar
unknown committed
2141
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
unknown's avatar
unknown committed
2142 2143 2144
  info->opt_flag|=WRITE_CACHE_USED;
  info->rec_cache.file=info->dfile;		/* for sort_delete_record */

unknown's avatar
unknown committed
2145
  if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength,
unknown's avatar
unknown committed
2146 2147
					   MYF(0))) ||
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
2148
  {
2149
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
2150 2151 2152 2153
    goto err;
  }
  if (!rep_quick)
  {
2154 2155
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2156 2157
					   share->data_file_name, "",
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2168 2169
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2182 2183
  if (!(param->testflag & T_CREATE_MISSING_KEYS))
  {
unknown's avatar
unknown committed
2184 2185 2186 2187
    /*
      Flush key cache for this file if we are calling this outside
      myisamchk
    */
unknown's avatar
unknown committed
2188
    flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
2189
    /* Clear the pointers to the given rows */
2190 2191
    for (i=0 ; i < share->base.keys ; i++)
      share->state.key_root[i]= HA_OFFSET_ERROR;
2192
    for (i=0 ; i < share->state.header.max_block_size_index ; i++)
2193
      share->state.key_del[i]=  HA_OFFSET_ERROR;
unknown's avatar
unknown committed
2194
    info->state->key_file_length=share->base.keystart;
2195 2196
  }
  else
unknown's avatar
unknown committed
2197
  {
unknown's avatar
unknown committed
2198
    if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE))
unknown's avatar
unknown committed
2199
      goto err;
2200
    key_map= ~key_map;				/* Create the missing keys */
unknown's avatar
unknown committed
2201
  }
unknown's avatar
unknown committed
2202

unknown's avatar
unknown committed
2203 2204
  sort_info.info=info;
  sort_info.param = param;
unknown's avatar
unknown committed
2205

unknown's avatar
unknown committed
2206 2207 2208 2209 2210
  set_data_file_type(&sort_info, share);
  sort_param.filepos=new_header_length;
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
2211 2212
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

unknown's avatar
unknown committed
2213
  sort_param.wordlist=NULL;
2214
  init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2215

unknown's avatar
unknown committed
2216 2217 2218 2219 2220 2221
  if (share->data_file_type == DYNAMIC_RECORD)
    length=max(share->base.min_pack_length+1,share->base.min_block_length);
  else if (share->data_file_type == COMPRESSED_RECORD)
    length=share->base.min_block_length;
  else
    length=share->base.pack_reclength;
unknown's avatar
unknown committed
2222
  sort_info.max_records=
2223
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
unknown's avatar
unknown committed
2224
     (ha_rows) (sort_info.filelength/length+1));
unknown's avatar
unknown committed
2225 2226 2227
  sort_param.key_cmp=sort_key_cmp;
  sort_param.lock_in_memory=lock_memory;
  sort_param.tmpdir=param->tmpdir;
unknown's avatar
unknown committed
2228 2229
  sort_param.sort_info=&sort_info;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
2230
  sort_param.master =1;
2231
  
unknown's avatar
unknown committed
2232
  del=info->state->del;
2233 2234
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
2235
    sort_param.calc_checksum= 1;
unknown's avatar
unknown committed
2236 2237

  rec_per_key_part= param->rec_per_key_part;
unknown's avatar
unknown committed
2238 2239
  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
       rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
unknown's avatar
unknown committed
2240
  {
unknown's avatar
unknown committed
2241 2242
    sort_param.read_cache=param->read_cache;
    sort_param.keyinfo=share->keyinfo+sort_param.key;
2243
    sort_param.seg=sort_param.keyinfo->seg;
2244
    if (! mi_is_key_active(key_map, sort_param.key))
2245 2246 2247
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2248 2249
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2250
	     sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2251
      continue;
2252
    }
unknown's avatar
unknown committed
2253 2254

    if ((!(param->testflag & T_SILENT)))
unknown's avatar
unknown committed
2255 2256
      printf ("- Fixing index %d\n",sort_param.key+1);
    sort_param.max_pos=sort_param.pos=share->pack.header_length;
2257
    keyseg=sort_param.seg;
unknown's avatar
unknown committed
2258
    bzero((char*) sort_param.unique,sizeof(sort_param.unique));
unknown's avatar
unknown committed
2259
    sort_param.key_length=share->rec_reflength;
unknown's avatar
unknown committed
2260
    for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
unknown's avatar
unknown committed
2261
    {
unknown's avatar
unknown committed
2262 2263 2264
      sort_param.key_length+=keyseg[i].length;
      if (keyseg[i].flag & HA_SPACE_PACK)
	sort_param.key_length+=get_pack_length(keyseg[i].length);
2265
      if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2266 2267
	sort_param.key_length+=2 + test(keyseg[i].length >= 127);
      if (keyseg[i].flag & HA_NULL_PART)
unknown's avatar
unknown committed
2268 2269 2270 2271 2272
	sort_param.key_length++;
    }
    info->state->records=info->state->del=share->state.split=0;
    info->state->empty=0;

unknown's avatar
unknown committed
2273
    if (sort_param.keyinfo->flag & HA_FULLTEXT)
2274
    {
2275 2276
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param.keyinfo->seg->charset->mbmaxlen;
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
      /*
        fulltext indexes may have much more entries than the
        number of rows in the table. We estimate the number here.

        Note, built-in parser is always nr. 0 - see ftparser_call_initializer()
      */
      if (sort_param.keyinfo->ftparser_nr == 0)
      {
        /*
          for built-in parser the number of generated index entries
          cannot be larger than the size of the data file divided
          by the minimal word's length
        */
        sort_info.max_records=
          (ha_rows) (sort_info.filelength/ft_min_word_len+1);
      }
      else
      {
        /*
          for external plugin parser we cannot tell anything at all :(
          so, we'll use all the sort memory and start from ~10 buffpeks.
          (see _create_index_by_sort)
        */
        sort_info.max_records=
          10*param->sort_buffer_length/sort_param.key_length;
      }
2304 2305

      sort_param.key_read=sort_ft_key_read;
2306
      sort_param.key_write=sort_ft_key_write;
2307 2308
    }
    else
2309
    {
2310
      sort_param.key_read=sort_key_read;
2311 2312
      sort_param.key_write=sort_key_write;
    }
2313

unknown's avatar
unknown committed
2314 2315 2316
    if (_create_index_by_sort(&sort_param,
			      (my_bool) (!(param->testflag & T_VERBOSE)),
			      (uint) param->sort_buffer_length))
2317 2318
    {
      param->retry_repair=1;
unknown's avatar
unknown committed
2319
      goto err;
2320
    }
2321 2322
    /* No need to calculate checksum again. */
    sort_param.calc_checksum= 0;
2323
    free_root(&sort_param.wordroot, MYF(0));
unknown's avatar
unknown committed
2324 2325

    /* Set for next loop */
unknown's avatar
unknown committed
2326
    sort_info.max_records= (ha_rows) info->state->records;
unknown's avatar
unknown committed
2327 2328

    if (param->testflag & T_STATISTICS)
unknown's avatar
unknown committed
2329
      update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2330 2331
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       sort_param.notnull: NULL,(ulonglong) info->state->records);
2332
    mi_set_key_active(share->state.key_map, sort_param.key);
unknown's avatar
unknown committed
2333

unknown's avatar
unknown committed
2334
    if (sort_param.fix_datafile)
unknown's avatar
unknown committed
2335
    {
unknown's avatar
unknown committed
2336 2337
      param->read_cache.end_of_file=sort_param.filepos;
      if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
unknown's avatar
unknown committed
2338
	goto err;
2339 2340 2341 2342 2343 2344 2345 2346 2347
      if (param->testflag & T_SAFE_REPAIR)
      {
	/* Don't repair if we loosed more than one row */
	if (info->state->records+1 < start_records)
	{
	  info->state->records=start_records;
	  goto err;
	}
      }
unknown's avatar
unknown committed
2348 2349
      share->state.state.data_file_length = info->state->data_file_length=
	sort_param.filepos;
unknown's avatar
unknown committed
2350 2351 2352 2353
      /* Only whole records */
      share->state.version=(ulong) time((time_t*) 0);
      my_close(info->dfile,MYF(0));
      info->dfile=new_file;
unknown's avatar
unknown committed
2354
      share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
2355
      share->pack.header_length=(ulong) new_header_length;
unknown's avatar
unknown committed
2356
      sort_param.fix_datafile=0;
unknown's avatar
unknown committed
2357 2358
    }
    else
unknown's avatar
unknown committed
2359
      info->state->data_file_length=sort_param.max_pos;
unknown's avatar
unknown committed
2360 2361

    param->read_cache.file=info->dfile;		/* re-init read cache */
unknown's avatar
unknown committed
2362 2363
    reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
                    1,1);
unknown's avatar
unknown committed
2364 2365 2366 2367 2368 2369 2370
  }

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }

unknown's avatar
unknown committed
2371
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
2372 2373 2374 2375
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
2376 2377
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2378 2379 2380
    goto err;
  }

unknown's avatar
unknown committed
2381
  if (rep_quick & T_FORCE_UNIQUENESS)
unknown's avatar
unknown committed
2382 2383 2384 2385 2386 2387 2388 2389 2390
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
unknown's avatar
unknown committed
2391
    if (skr != sort_info.filelength && !info->s->base.raid_type)
2392
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
2393 2394 2395 2396
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
unknown's avatar
unknown committed
2397
  if (param->testflag & T_CALC_CHECKSUM)
2398
    info->state->checksum=param->glob_crc;
2399

2400
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
2401 2402 2403 2404 2405 2406 2407 2408
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
2409
    if (sort_info.dupp)
unknown's avatar
unknown committed
2410 2411
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
2412
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
2413 2414 2415 2416 2417 2418 2419
  }
  got_error=0;

  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
unknown's avatar
unknown committed
2420
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
2421 2422 2423 2424 2425 2426 2427 2428
  VOID(end_io_cache(&info->rec_cache));
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
2429
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2430 2431 2432
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
unknown's avatar
unknown committed
2433
	  mi_open_datafile(info,share,-1))
2434 2435 2436
	got_error=1;
    }
  }
unknown's avatar
unknown committed
2437 2438 2439 2440 2441 2442 2443
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
2444
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
unknown's avatar
unknown committed
2445
			  MYF(MY_WME)));
2446
      if (info->dfile == new_file)
2447
	info->dfile= -1;
unknown's avatar
unknown committed
2448
    }
2449
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
2450
  }
2451 2452 2453 2454
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

2455 2456
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2457
  my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR));
2458
  my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
2459
  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2460
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2461 2462
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2463
  if (!got_error && (param->testflag & T_UNPACK))
unknown's avatar
unknown committed
2464 2465 2466 2467 2468
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
2469
}
unknown's avatar
unknown committed
2470

unknown's avatar
unknown committed
2471 2472 2473 2474
/*
  Threaded repair of table using sorting

  SYNOPSIS
2475
    mi_repair_parallel()
unknown's avatar
unknown committed
2476 2477 2478 2479 2480 2481 2482 2483
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  DESCRIPTION
    Same as mi_repair_by_sort but do it multithreaded
    Each key is handled by a separate thread.
unknown's avatar
unknown committed
2484
    TODO: make a number of threads a parameter
unknown's avatar
unknown committed
2485

2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
    In parallel repair we use one thread per index. There are two modes:

    Quick

      Only the indexes are rebuilt. All threads share a read buffer.
      Every thread that needs fresh data in the buffer enters the shared
      cache lock. The last thread joining the lock reads the buffer from
      the data file and wakes all other threads.

    Non-quick

      The data file is rebuilt and all indexes are rebuilt to point to
      the new record positions. One thread is the master thread. It
      reads from the old data file and writes to the new data file. It
      also creates one of the indexes. The other threads read from a
      buffer which is filled by the master. If they need fresh data,
      they enter the shared cache lock. If the masters write buffer is
      full, it flushes it to the new data file and enters the shared
      cache lock too. When all threads joined in the lock, the master
      copies its write buffer to the read buffer for the other threads
      and wakes them.

unknown's avatar
unknown committed
2508 2509 2510 2511
  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2512

2513
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
unknown's avatar
unknown committed
2514
			const char * name, int rep_quick)
unknown's avatar
unknown committed
2515
{
2516 2517 2518
#ifndef THREAD
  return mi_repair_by_sort(param, info, name, rep_quick);
#else
unknown's avatar
unknown committed
2519
  int got_error;
unknown's avatar
unknown committed
2520
  uint i,key, total_key_length, istep;
unknown's avatar
unknown committed
2521
  ulong rec_length;
unknown's avatar
unknown committed
2522 2523 2524
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
unknown's avatar
unknown committed
2525
  MI_SORT_PARAM *sort_param=0;
unknown's avatar
unknown committed
2526 2527
  MYISAM_SHARE *share=info->s;
  ulong   *rec_per_key_part;
2528
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2529
  char llbuff[22];
2530
  IO_CACHE new_data_cache; /* For non-quick repair. */
unknown's avatar
unknown committed
2531 2532 2533
  IO_CACHE_SHARE io_share;
  SORT_INFO sort_info;
  ulonglong key_map=share->state.key_map;
2534
  pthread_attr_t thr_attr;
2535
  DBUG_ENTER("mi_repair_parallel");
unknown's avatar
unknown committed
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
    printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2547
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
2548

2549 2550 2551
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
  /*
    Quick repair (not touching data file, rebuilding indexes):
    {
      Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
    }

    Non-quick repair (rebuilding data file and indexes):
    {
      Master thread:

        Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
        Write cache is (MI_INFO   *info)->rec_cache  using new_file.

      Slave threads:

        Read  cache is new_data_cache synced to master rec_cache.

      The final assignment of the filedescriptor for rec_cache is done
      after the cache creation.

      Don't check file size on new_data_cache, as the resulting file size
      is not known yet.

      As rec_cache and new_data_cache are synced, write_buffer_length is
      used for the read cache 'new_data_cache'. Both start at the same
      position 'new_header_length'.
    }
  */
  DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
unknown's avatar
unknown committed
2581
  bzero((char*)&sort_info,sizeof(sort_info));
2582 2583 2584 2585
  /* Initialize pthread structures before goto err. */
  pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
  pthread_cond_init(&sort_info.cond, 0);

unknown's avatar
unknown committed
2586
  if (!(sort_info.key_block=
2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600
	alloc_key_blocks(param, (uint) param->sort_key_blocks,
			 share->base.max_key_block_length)) ||
      init_io_cache(&param->read_cache, info->dfile,
                    (uint) param->read_buffer_length,
                    READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
      (!rep_quick &&
       (init_io_cache(&info->rec_cache, info->dfile,
                      (uint) param->write_buffer_length,
                      WRITE_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
        init_io_cache(&new_data_cache, -1,
                      (uint) param->write_buffer_length,
                      READ_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
unknown's avatar
unknown committed
2601 2602 2603
    goto err;
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
  info->opt_flag|=WRITE_CACHE_USED;
2604
  info->rec_cache.file=info->dfile;         /* for sort_delete_record */
unknown's avatar
unknown committed
2605 2606 2607 2608 2609

  if (!rep_quick)
  {
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2610
					   share->data_file_name, "",
unknown's avatar
unknown committed
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
					   DATA_TMP_EXT,
					   2+4),
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2623 2624
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  if (!(param->testflag & T_CREATE_MISSING_KEYS))
  {
    /*
      Flush key cache for this file if we are calling this outside
      myisamchk
    */
unknown's avatar
unknown committed
2643
    flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
2644 2645 2646
    /* Clear the pointers to the given rows */
    for (i=0 ; i < share->base.keys ; i++)
      share->state.key_root[i]= HA_OFFSET_ERROR;
2647
    for (i=0 ; i < share->state.header.max_block_size_index ; i++)
unknown's avatar
unknown committed
2648 2649 2650 2651 2652
      share->state.key_del[i]=  HA_OFFSET_ERROR;
    info->state->key_file_length=share->base.keystart;
  }
  else
  {
unknown's avatar
unknown committed
2653
    if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE))
unknown's avatar
unknown committed
2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667
      goto err;
    key_map= ~key_map;				/* Create the missing keys */
  }

  sort_info.info=info;
  sort_info.param = param;

  set_data_file_type(&sort_info, share);
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

  if (share->data_file_type == DYNAMIC_RECORD)
unknown's avatar
unknown committed
2668
    rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
unknown's avatar
unknown committed
2669
  else if (share->data_file_type == COMPRESSED_RECORD)
unknown's avatar
unknown committed
2670
    rec_length=share->base.min_block_length;
unknown's avatar
unknown committed
2671
  else
unknown's avatar
unknown committed
2672
    rec_length=share->base.pack_reclength;
2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
  /*
    +1 below is required hack for parallel repair mode.
    The info->state->records value, that is compared later
    to sort_info.max_records and cannot exceed it, is
    increased in sort_key_write. In mi_repair_by_sort, sort_key_write
    is called after sort_key_read, where the comparison is performed,
    but in parallel mode master thread can call sort_key_write
    before some other repair thread calls sort_key_read.
    Furthermore I'm not even sure +1 would be enough.
    May be sort_info.max_records shold be always set to max value in
    parallel mode.
  */
unknown's avatar
unknown committed
2685
  sort_info.max_records=
2686
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
unknown's avatar
unknown committed
2687
     (ha_rows) (sort_info.filelength/rec_length+1));
unknown's avatar
unknown committed
2688 2689 2690 2691 2692 2693

  del=info->state->del;
  param->glob_crc=0;

  if (!(sort_param=(MI_SORT_PARAM *)
        my_malloc((uint) share->base.keys *
unknown's avatar
unknown committed
2694 2695
		  (sizeof(MI_SORT_PARAM) + share->base.pack_reclength),
		  MYF(MY_ZEROFILL))))
unknown's avatar
unknown committed
2696
  {
2697
    mi_check_print_error(param,"Not enough memory for key!");
unknown's avatar
unknown committed
2698 2699
    goto err;
  }
unknown's avatar
unknown committed
2700
  total_key_length=0;
unknown's avatar
unknown committed
2701 2702 2703 2704
  rec_per_key_part= param->rec_per_key_part;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;

unknown's avatar
unknown committed
2705 2706
  for (i=key=0, istep=1 ; key < share->base.keys ;
       rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
unknown's avatar
unknown committed
2707 2708 2709
  {
    sort_param[i].key=key;
    sort_param[i].keyinfo=share->keyinfo+key;
2710
    sort_param[i].seg=sort_param[i].keyinfo->seg;
2711
    if (! mi_is_key_active(key_map, key))
unknown's avatar
unknown committed
2712 2713 2714
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2715 2716
	     (char*) (share->state.rec_per_key_part+
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2717
	     sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2718
      istep=0;
unknown's avatar
unknown committed
2719 2720
      continue;
    }
unknown's avatar
unknown committed
2721
    istep=1;
unknown's avatar
unknown committed
2722 2723
    if ((!(param->testflag & T_SILENT)))
      printf ("- Fixing index %d\n",key+1);
2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
    {
      sort_param[i].key_read=sort_ft_key_read;
      sort_param[i].key_write=sort_ft_key_write;
    }
    else
    {
      sort_param[i].key_read=sort_key_read;
      sort_param[i].key_write=sort_key_write;
    }
unknown's avatar
unknown committed
2734 2735 2736 2737
    sort_param[i].key_cmp=sort_key_cmp;
    sort_param[i].lock_in_memory=lock_memory;
    sort_param[i].tmpdir=param->tmpdir;
    sort_param[i].sort_info=&sort_info;
2738
    sort_param[i].master=0;
unknown's avatar
unknown committed
2739
    sort_param[i].fix_datafile=0;
2740
    sort_param[i].calc_checksum= 0;
unknown's avatar
unknown committed
2741 2742 2743 2744

    sort_param[i].filepos=new_header_length;
    sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;

unknown's avatar
unknown committed
2745 2746
    sort_param[i].record= (((char *)(sort_param+share->base.keys))+
			   (share->base.pack_reclength * i));
unknown's avatar
unknown committed
2747 2748 2749 2750 2751
    if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
    {
      mi_check_print_error(param,"Not enough memory!");
      goto err;
    }
unknown's avatar
unknown committed
2752 2753

    sort_param[i].key_length=share->rec_reflength;
2754
    for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
unknown's avatar
unknown committed
2755
	 keyseg++)
unknown's avatar
unknown committed
2756 2757 2758 2759
    {
      sort_param[i].key_length+=keyseg->length;
      if (keyseg->flag & HA_SPACE_PACK)
        sort_param[i].key_length+=get_pack_length(keyseg->length);
2760
      if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2761 2762 2763 2764
        sort_param[i].key_length+=2 + test(keyseg->length >= 127);
      if (keyseg->flag & HA_NULL_PART)
        sort_param[i].key_length++;
    }
unknown's avatar
unknown committed
2765
    total_key_length+=sort_param[i].key_length;
unknown's avatar
unknown committed
2766 2767

    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2768 2769 2770 2771
    {
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2772
      init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2773
    }
unknown's avatar
unknown committed
2774 2775
  }
  sort_info.total_keys=i;
2776 2777
  sort_param[0].master= 1;
  sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2778
  sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
unknown's avatar
unknown committed
2779 2780

  sort_info.got_error=0;
unknown's avatar
unknown committed
2781
  pthread_mutex_lock(&sort_info.mutex);
unknown's avatar
unknown committed
2782

2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
  /*
    Initialize the I/O cache share for use with the read caches and, in
    case of non-quick repair, the write cache. When all threads join on
    the cache lock, the writer copies the write cache contents to the
    read caches.
  */
  if (i > 1)
  {
    if (rep_quick)
      init_io_cache_share(&param->read_cache, &io_share, NULL, i);
    else
      init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
  }
  else
    io_share.total_threads= 0; /* share not used */

2799 2800 2801
  (void) pthread_attr_init(&thr_attr);
  (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);

unknown's avatar
unknown committed
2802
  for (i=0 ; i < sort_info.total_keys ; i++)
unknown's avatar
unknown committed
2803
  {
2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816
    /*
      Copy the properly initialized IO_CACHE structure so that every
      thread has its own copy. In quick mode param->read_cache is shared
      for use by all threads. In non-quick mode all threads but the
      first copy the shared new_data_cache, which is synchronized to the
      write cache of the first thread. The first thread copies
      param->read_cache, which is not shared.
    */
    sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
                               new_data_cache);
    DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: 0x%lx",
                                  i, (long) &sort_param[i].read_cache));

unknown's avatar
unknown committed
2817 2818 2819 2820 2821 2822
    /*
      two approaches: the same amount of memory for each thread
      or the memory for the same number of keys for each thread...
      In the second one all the threads will fill their sort_buffers
      (and call write_keys) at the same time, putting more stress on i/o.
    */
unknown's avatar
unknown committed
2823
    sort_param[i].sortbuff_size=
unknown's avatar
unknown committed
2824
#ifndef USING_SECOND_APPROACH
unknown's avatar
unknown committed
2825 2826
      param->sort_buffer_length/sort_info.total_keys;
#else
unknown's avatar
unknown committed
2827
      param->sort_buffer_length*sort_param[i].key_length/total_key_length;
unknown's avatar
unknown committed
2828
#endif
2829 2830 2831
    if (pthread_create(&sort_param[i].thr, &thr_attr,
		       thr_find_all_keys,
		       (void *) (sort_param+i)))
unknown's avatar
unknown committed
2832 2833
    {
      mi_check_print_error(param,"Cannot start a repair thread");
2834 2835 2836 2837
      /* Cleanup: Detach from the share. Avoid others to be blocked. */
      if (io_share.total_threads)
        remove_io_thread(&sort_param[i].read_cache);
      DBUG_PRINT("error", ("Cannot start a repair thread"));
unknown's avatar
unknown committed
2838 2839 2840 2841 2842
      sort_info.got_error=1;
    }
    else
      sort_info.threads_running++;
  }
2843
  (void) pthread_attr_destroy(&thr_attr);
unknown's avatar
unknown committed
2844 2845 2846

  /* waiting for all threads to finish */
  while (sort_info.threads_running)
unknown's avatar
unknown committed
2847 2848
    pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
  pthread_mutex_unlock(&sort_info.mutex);
unknown's avatar
unknown committed
2849

2850
  if ((got_error= thr_write_keys(sort_param)))
unknown's avatar
unknown committed
2851 2852 2853 2854
  {
    param->retry_repair=1;
    goto err;
  }
unknown's avatar
unknown committed
2855
  got_error=1;				/* Assume the following may go wrong */
unknown's avatar
unknown committed
2856

unknown's avatar
unknown committed
2857
  if (sort_param[0].fix_datafile)
unknown's avatar
unknown committed
2858
  {
2859 2860 2861 2862 2863
    /*
      Append some nuls to the end of a memory mapped file. Destroy the
      write cache. The master thread did already detach from the share
      by remove_io_thread() in sort.c:thr_find_all_keys().
    */
unknown's avatar
unknown committed
2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874
    if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
      goto err;
    if (param->testflag & T_SAFE_REPAIR)
    {
      /* Don't repair if we loosed more than one row */
      if (info->state->records+1 < start_records)
      {
        info->state->records=start_records;
        goto err;
      }
    }
unknown's avatar
unknown committed
2875 2876
    share->state.state.data_file_length= info->state->data_file_length=
      sort_param->filepos;
unknown's avatar
unknown committed
2877 2878
    /* Only whole records */
    share->state.version=(ulong) time((time_t*) 0);
2879 2880 2881 2882 2883

    /*
      Exchange the data file descriptor of the table, so that we use the
      new file from now on.
     */
unknown's avatar
unknown committed
2884 2885
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
2886

unknown's avatar
unknown committed
2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912
    share->data_file_type=sort_info.new_data_file_type;
    share->pack.header_length=(ulong) new_header_length;
  }
  else
    info->state->data_file_length=sort_param->max_pos;

  if (rep_quick && del+sort_info.dupp != info->state->del)
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
    goto err;
  }

  if (rep_quick & T_FORCE_UNIQUENESS)
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
    if (skr != sort_info.filelength && !info->s->base.raid_type)
2913
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
2914 2915 2916 2917 2918
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
  if (param->testflag & T_CALC_CHECKSUM)
2919
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
2920

2921
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
2922
    mi_check_print_warning(param,
unknown's avatar
unknown committed
2923
			   "Can't change size of indexfile, error: %d", my_errno);
unknown's avatar
unknown committed
2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
    if (sort_info.dupp)
      mi_check_print_warning(param,
			     "%s records have been removed",
			     llstr(sort_info.dupp,llbuff));
  }
  got_error=0;

  if (&share->state.state != info->state)
2937
    memcpy(&share->state.state, info->state, sizeof(*info->state));
unknown's avatar
unknown committed
2938 2939

err:
unknown's avatar
unknown committed
2940
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
2941 2942 2943 2944 2945
  /*
    Destroy the write cache. The master thread did already detach from
    the share by remove_io_thread() or it was not yet started (if the
    error happend before creating the thread).
  */
unknown's avatar
unknown committed
2946
  VOID(end_io_cache(&info->rec_cache));
2947 2948 2949 2950 2951 2952 2953 2954
  /*
    Destroy the new data cache in case of non-quick repair. All slave
    threads did either detach from the share by remove_io_thread()
    already or they were not yet started (if the error happend before
    creating the threads).
  */
  if (!rep_quick)
    VOID(end_io_cache(&new_data_cache));
unknown's avatar
unknown committed
2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
	  mi_open_datafile(info,share,-1))
	got_error=1;
    }
  }
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
			  MYF(MY_WME)));
      if (info->dfile == new_file)
	info->dfile= -1;
    }
    mi_mark_crashed_on_repair(info);
  }
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

unknown's avatar
unknown committed
2988 2989
  pthread_cond_destroy (&sort_info.cond);
  pthread_mutex_destroy(&sort_info.mutex);
unknown's avatar
unknown committed
2990

2991
  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002
  my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
  my_free((gptr) sort_param,MYF(MY_ALLOW_ZERO_PTR));
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  if (!got_error && (param->testflag & T_UNPACK))
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
3003
#endif /* THREAD */
unknown's avatar
unknown committed
3004
}
unknown's avatar
unknown committed
3005 3006 3007

	/* Read next record and return next key */

unknown's avatar
unknown committed
3008
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
unknown's avatar
unknown committed
3009 3010
{
  int error;
unknown's avatar
unknown committed
3011 3012
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3013 3014
  DBUG_ENTER("sort_key_read");

unknown's avatar
unknown committed
3015
  if ((error=sort_get_next_record(sort_param)))
unknown's avatar
unknown committed
3016 3017 3018 3019
    DBUG_RETURN(error);
  if (info->state->records == sort_info->max_records)
  {
    mi_check_print_error(sort_info->param,
3020 3021
			 "Key %d - Found too many records; Can't continue",
                         sort_param->key+1);
unknown's avatar
unknown committed
3022 3023
    DBUG_RETURN(1);
  }
unknown's avatar
unknown committed
3024
  sort_param->real_key_length=
unknown's avatar
unknown committed
3025
    (info->s->rec_reflength+
unknown's avatar
unknown committed
3026 3027
     _mi_make_key(info, sort_param->key, (uchar*) key,
		  sort_param->record, sort_param->filepos));
unknown's avatar
unknown committed
3028 3029 3030 3031
#ifdef HAVE_purify
  bzero(key+sort_param->real_key_length,
	(sort_param->key_length-sort_param->real_key_length));
#endif
unknown's avatar
unknown committed
3032
  DBUG_RETURN(sort_write_record(sort_param));
unknown's avatar
unknown committed
3033 3034
} /* sort_key_read */

unknown's avatar
unknown committed
3035
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3036 3037
{
  int error;
unknown's avatar
unknown committed
3038 3039
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3040
  FT_WORD *wptr=0;
3041 3042
  DBUG_ENTER("sort_ft_key_read");

unknown's avatar
unknown committed
3043
  if (!sort_param->wordlist)
3044
  {
unknown's avatar
unknown committed
3045
    for (;;)
3046
    {
3047
      free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3048
      if ((error=sort_get_next_record(sort_param)))
3049
        DBUG_RETURN(error);
3050 3051
      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
                                    &sort_param->wordroot)))
3052
        DBUG_RETURN(1);
unknown's avatar
unknown committed
3053 3054
      if (wptr->pos)
        break;
unknown's avatar
unknown committed
3055
      error=sort_write_record(sort_param);
3056
    }
unknown's avatar
unknown committed
3057
    sort_param->wordptr=sort_param->wordlist=wptr;
3058 3059 3060 3061
  }
  else
  {
    error=0;
unknown's avatar
unknown committed
3062
    wptr=(FT_WORD*)(sort_param->wordptr);
3063 3064
  }

unknown's avatar
unknown committed
3065 3066 3067 3068 3069 3070 3071 3072
  sort_param->real_key_length=(info->s->rec_reflength+
			       _ft_make_key(info, sort_param->key,
					    key, wptr++, sort_param->filepos));
#ifdef HAVE_purify
  if (sort_param->key_length > sort_param->real_key_length)
    bzero(key+sort_param->real_key_length,
	  (sort_param->key_length-sort_param->real_key_length));
#endif
3073 3074
  if (!wptr->pos)
  {
3075
    free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3076
    sort_param->wordlist=0;
unknown's avatar
unknown committed
3077
    error=sort_write_record(sort_param);
3078 3079
  }
  else
unknown's avatar
unknown committed
3080
    sort_param->wordptr=(void*)wptr;
3081 3082 3083

  DBUG_RETURN(error);
} /* sort_ft_key_read */
unknown's avatar
unknown committed
3084

unknown's avatar
unknown committed
3085

3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115
/*
  Read next record from file using parameters in sort_info.

  SYNOPSIS
    sort_get_next_record()
      sort_param                Information about and for the sort process

  NOTE

    Dynamic Records With Non-Quick Parallel Repair

      For non-quick parallel repair we use a synchronized read/write
      cache. This means that one thread is the master who fixes the data
      file by reading each record from the old data file and writing it
      to the new data file. By doing this the records in the new data
      file are written contiguously. Whenever the write buffer is full,
      it is copied to the read buffer. The slaves read from the read
      buffer, which is not associated with a file. Thus read_cache.file
      is -1. When using _mi_read_cache(), the slaves must always set
      flag to READING_NEXT so that the function never tries to read from
      file. This is safe because the records are contiguous. There is no
      need to read outside the cache. This condition is evaluated in the
      variable 'parallel_flag' for quick reference. read_cache.file must
      be >= 0 in every other case.

  RETURN
    -1          end of file
    0           ok
    > 0         error
*/
unknown's avatar
unknown committed
3116

unknown's avatar
unknown committed
3117
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3118 3119
{
  int searching;
3120
  int parallel_flag;
unknown's avatar
unknown committed
3121 3122 3123 3124
  uint found_record,b_type,left_length;
  my_off_t pos;
  byte *to;
  MI_BLOCK_INFO block_info;
unknown's avatar
unknown committed
3125
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3126
  MI_CHECK *param=sort_info->param;
3127 3128
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3129 3130 3131
  char llbuff[22],llbuff2[22];
  DBUG_ENTER("sort_get_next_record");

unknown's avatar
unknown committed
3132 3133 3134
  if (*killed_ptr(param))
    DBUG_RETURN(1);

unknown's avatar
unknown committed
3135 3136 3137 3138
  switch (share->data_file_type) {
  case STATIC_RECORD:
    for (;;)
    {
unknown's avatar
unknown committed
3139
      if (my_b_read(&sort_param->read_cache,sort_param->record,
unknown's avatar
unknown committed
3140
		    share->base.pack_reclength))
3141
      {
unknown's avatar
unknown committed
3142
	if (sort_param->read_cache.error)
3143
	  param->out_flag |= O_DATA_LOST;
unknown's avatar
unknown committed
3144 3145
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3146
	DBUG_RETURN(-1);
3147
      }
unknown's avatar
unknown committed
3148 3149
      sort_param->start_recpos=sort_param->pos;
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3150
      {
unknown's avatar
unknown committed
3151
	sort_param->filepos=sort_param->pos;
3152 3153
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3154
      }
unknown's avatar
unknown committed
3155 3156
      sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
      if (*sort_param->record)
3157
      {
3158
	if (sort_param->calc_checksum)
3159
	  param->glob_crc+= (info->checksum=
unknown's avatar
unknown committed
3160
			     mi_static_checksum(info,sort_param->record));
unknown's avatar
unknown committed
3161
	DBUG_RETURN(0);
3162
      }
3163
      if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3164 3165 3166 3167 3168 3169 3170
      {
	info->state->del++;
	info->state->empty+=share->base.pack_reclength;
      }
    }
  case DYNAMIC_RECORD:
    LINT_INIT(to);
unknown's avatar
unknown committed
3171 3172
    pos=sort_param->pos;
    searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
3173
    parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
unknown's avatar
unknown committed
3174 3175 3176 3177 3178
    for (;;)
    {
      found_record=block_info.second_read= 0;
      left_length=1;
      if (searching)
unknown's avatar
unknown committed
3179
      {
unknown's avatar
unknown committed
3180
	pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3181
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3182
	sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3183
      }
unknown's avatar
unknown committed
3184 3185
      do
      {
unknown's avatar
unknown committed
3186 3187
	if (pos > sort_param->max_pos)
	  sort_param->max_pos=pos;
unknown's avatar
unknown committed
3188 3189 3190 3191 3192
	if (pos & (MI_DYN_ALIGN_SIZE-1))
	{
	  if ((param->testflag & T_VERBOSE) || searching == 0)
	    mi_check_print_info(param,"Wrong aligned block at %s",
				llstr(pos,llbuff));
unknown's avatar
unknown committed
3193
	  if (searching)
unknown's avatar
unknown committed
3194 3195 3196 3197 3198
	    goto try_next;
	}
	if (found_record && pos == param->search_after_block)
	  mi_check_print_info(param,"Block: %s used by record at %s",
		     llstr(param->search_after_block,llbuff),
unknown's avatar
unknown committed
3199 3200 3201
		     llstr(sort_param->start_recpos,llbuff2));
	if (_mi_read_cache(&sort_param->read_cache,
                           (byte*) block_info.header,pos,
3202 3203
			   MI_BLOCK_INFO_HEADER_LENGTH,
			   (! found_record ? READING_NEXT : 0) |
3204
                           parallel_flag | READING_HEADER))
unknown's avatar
unknown committed
3205 3206 3207 3208 3209
	{
	  if (found_record)
	  {
	    mi_check_print_info(param,
				"Can't read whole record at %s (errno: %d)",
unknown's avatar
unknown committed
3210
				llstr(sort_param->start_recpos,llbuff),errno);
unknown's avatar
unknown committed
3211 3212 3213 3214
	    goto try_next;
	  }
	  DBUG_RETURN(-1);
	}
unknown's avatar
unknown committed
3215
	if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3216 3217
	{
	  param->error_printed=1;
unknown's avatar
unknown committed
3218 3219
          param->retry_repair=1;
          param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3220 3221
	  DBUG_RETURN(1);	/* Something wrong with data */
	}
3222 3223 3224
	b_type=_mi_get_block_info(&block_info,-1,pos);
	if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
	   ((b_type & BLOCK_FIRST) &&
unknown's avatar
unknown committed
3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
	     (block_info.rec_len < (uint) share->base.min_pack_length ||
	      block_info.rec_len > (uint) share->base.max_pack_length)))
	{
	  uint i;
	  if (param->testflag & T_VERBOSE || searching == 0)
	    mi_check_print_info(param,
				"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
		       block_info.header[0],block_info.header[1],
		       block_info.header[2],llstr(pos,llbuff));
	  if (found_record)
	    goto try_next;
	  block_info.second_read=0;
	  searching=1;
	  /* Search after block in read header string */
	  for (i=MI_DYN_ALIGN_SIZE ;
	       i < MI_BLOCK_INFO_HEADER_LENGTH ;
	       i+= MI_DYN_ALIGN_SIZE)
	    if (block_info.header[i] >= 1 &&
		block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
	      break;
	  pos+=(ulong) i;
unknown's avatar
unknown committed
3246
	  sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280
	  continue;
	}
	if (b_type & BLOCK_DELETED)
	{
	  bool error=0;
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Deleted block with impossible length %u at %s",
				  block_info.block_len,llstr(pos,llbuff));
	    error=1;
	  }
	  else
	  {
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >=
		 info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      if (!searching)
		mi_check_print_info(param,
				    "Delete link points outside datafile at %s",
				    llstr(pos,llbuff));
	      error=1;
	    }
	  }
	  if (error)
	  {
	    if (found_record)
	      goto try_next;
	    searching=1;
3281
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3282
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301
	    block_info.second_read=0;
	    continue;
	  }
	}
	else
	{
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length ||
	      block_info.block_len > (uint) share->base.max_pack_length+
	      MI_SPLIT_LENGTH)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Found block with impossible length %u at %s; Skipped",
				  block_info.block_len+ (uint) (block_info.filepos-pos),
				  llstr(pos,llbuff));
	    if (found_record)
	      goto try_next;
	    searching=1;
3302
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3303
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3304 3305 3306 3307 3308 3309
	    block_info.second_read=0;
	    continue;
	  }
	}
	if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
	{
3310 3311
          if (!sort_param->fix_datafile && sort_param->master &&
              (b_type & BLOCK_DELETED))
unknown's avatar
unknown committed
3312 3313 3314 3315 3316 3317 3318 3319
	  {
	    info->state->empty+=block_info.block_len;
	    info->state->del++;
	    share->state.split++;
	  }
	  if (found_record)
	    goto try_next;
	  if (searching)
3320 3321
	  {
	    pos+=MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3322
	    sort_param->start_recpos=pos;
3323
	  }
unknown's avatar
unknown committed
3324 3325 3326 3327 3328 3329
	  else
	    pos=block_info.filepos+block_info.block_len;
	  block_info.second_read=0;
	  continue;
	}

3330
	if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3331
	  share->state.split++;
unknown's avatar
unknown committed
3332 3333
	if (! found_record++)
	{
unknown's avatar
unknown committed
3334 3335 3336 3337 3338 3339
	  sort_param->find_length=left_length=block_info.rec_len;
	  sort_param->start_recpos=pos;
	  if (!sort_param->fix_datafile)
	    sort_param->filepos=sort_param->start_recpos;
	  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
	    sort_param->pos=block_info.filepos+1;
unknown's avatar
unknown committed
3340
	  else
unknown's avatar
unknown committed
3341
	    sort_param->pos=block_info.filepos+block_info.block_len;
unknown's avatar
unknown committed
3342 3343
	  if (share->base.blobs)
	  {
3344
	    if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3345
				       &(sort_param->rec_buff))))
unknown's avatar
unknown committed
3346
	    {
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
	      if (param->max_record_length >= block_info.rec_len)
	      {
		mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
				     llstr(sort_param->start_recpos,llbuff),
				     (ulong) block_info.rec_len);
		DBUG_RETURN(1);
	      }
	      else
	      {
		mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
				    llstr(sort_param->start_recpos,llbuff),
				    (ulong) block_info.rec_len);
		goto try_next;
	      }
unknown's avatar
unknown committed
3361 3362 3363
	    }
	  }
	  else
3364
	    to= sort_param->rec_buff;
unknown's avatar
unknown committed
3365 3366 3367
	}
	if (left_length < block_info.data_len || ! block_info.data_len)
	{
3368 3369
	  mi_check_print_info(param,
			      "Found block with too small length at %s; Skipped",
unknown's avatar
unknown committed
3370
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3371 3372 3373
	  goto try_next;
	}
	if (block_info.filepos + block_info.data_len >
unknown's avatar
unknown committed
3374
	    sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3375
	{
3376 3377
	  mi_check_print_info(param,
			      "Found block that points outside data file at %s",
unknown's avatar
unknown committed
3378
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3379 3380
	  goto try_next;
	}
3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405
        /*
          Copy information that is already read. Avoid accessing data
          below the cache start. This could happen if the header
          streched over the end of the previous buffer contents.
        */
        {
          uint header_len= (uint) (block_info.filepos - pos);
          uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);

          if (prefetch_len > block_info.data_len)
            prefetch_len= block_info.data_len;
          if (prefetch_len)
          {
            memcpy(to, block_info.header + header_len, prefetch_len);
            block_info.filepos+= prefetch_len;
            block_info.data_len-= prefetch_len;
            left_length-= prefetch_len;
            to+= prefetch_len;
          }
        }
        if (block_info.data_len &&
            _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
                           block_info.data_len,
                           (found_record == 1 ? READING_NEXT : 0) |
                           parallel_flag))
unknown's avatar
unknown committed
3406
	{
3407 3408
	  mi_check_print_info(param,
			      "Read error for block at: %s (error: %d); Skipped",
3409
			      llstr(block_info.filepos,llbuff),my_errno);
unknown's avatar
unknown committed
3410 3411 3412 3413 3414 3415 3416 3417
	  goto try_next;
	}
	left_length-=block_info.data_len;
	to+=block_info.data_len;
	pos=block_info.next_filepos;
	if (pos == HA_OFFSET_ERROR && left_length)
	{
	  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
unknown's avatar
unknown committed
3418
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3419 3420
	  goto try_next;
	}
unknown's avatar
unknown committed
3421
	if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3422 3423
	{
	  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3424
			      llstr(pos,llbuff2),
unknown's avatar
unknown committed
3425
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3426 3427 3428 3429
	  goto try_next;
	}
      } while (left_length);

3430
      if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
unknown's avatar
unknown committed
3431
			 sort_param->find_length) != MY_FILE_ERROR)
unknown's avatar
unknown committed
3432
      {
unknown's avatar
unknown committed
3433
	if (sort_param->read_cache.error < 0)
unknown's avatar
unknown committed
3434
	  DBUG_RETURN(1);
3435 3436
	if (sort_param->calc_checksum)
	  info->checksum= mi_checksum(info, sort_param->record);
unknown's avatar
unknown committed
3437 3438
	if ((param->testflag & (T_EXTEND | T_REP)) || searching)
	{
unknown's avatar
unknown committed
3439
	  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3440 3441
                            sort_param->find_length,
                            (param->testflag & T_QUICK) &&
3442
                            sort_param->calc_checksum &&
3443
                            test(info->s->calc_checksum)))
unknown's avatar
unknown committed
3444 3445
	  {
	    mi_check_print_info(param,"Found wrong packed record at %s",
unknown's avatar
unknown committed
3446
				llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3447 3448 3449
	    goto try_next;
	  }
	}
3450
	if (sort_param->calc_checksum)
3451
	  param->glob_crc+= info->checksum;
unknown's avatar
unknown committed
3452 3453
	DBUG_RETURN(0);
      }
unknown's avatar
unknown committed
3454
      if (!searching)
3455 3456 3457
        mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
                            sort_param->key+1,
                            llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3458
    try_next:
unknown's avatar
unknown committed
3459
      pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3460 3461 3462
      searching=1;
    }
  case COMPRESSED_RECORD:
unknown's avatar
unknown committed
3463
    for (searching=0 ;; searching=1, sort_param->pos++)
unknown's avatar
unknown committed
3464
    {
unknown's avatar
unknown committed
3465 3466
      if (_mi_read_cache(&sort_param->read_cache,(byte*) block_info.header,
			 sort_param->pos,
3467
			 share->pack.ref_length,READING_NEXT))
unknown's avatar
unknown committed
3468
	DBUG_RETURN(-1);
unknown's avatar
unknown committed
3469
      if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3470 3471
      {
	param->error_printed=1;
unknown's avatar
unknown committed
3472 3473
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3474 3475
	DBUG_RETURN(1);		/* Something wrong with data */
      }
unknown's avatar
unknown committed
3476
      sort_param->start_recpos=sort_param->pos;
3477 3478
      if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
                                  &sort_param->rec_buff, -1, sort_param->pos))
unknown's avatar
unknown committed
3479 3480
	DBUG_RETURN(-1);
      if (!block_info.rec_len &&
unknown's avatar
unknown committed
3481 3482
	  sort_param->pos + MEMMAP_EXTRA_MARGIN ==
	  sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3483 3484 3485 3486 3487 3488
	DBUG_RETURN(-1);
      if (block_info.rec_len < (uint) share->min_pack_length ||
	  block_info.rec_len > (uint) share->max_pack_length)
      {
	if (! searching)
	  mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3489
			      block_info.rec_len,
unknown's avatar
unknown committed
3490
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3491 3492
	continue;
      }
3493
      if (_mi_read_cache(&sort_param->read_cache,(byte*) sort_param->rec_buff,
3494 3495
			 block_info.filepos, block_info.rec_len,
			 READING_NEXT))
unknown's avatar
unknown committed
3496 3497
      {
	if (! searching)
3498
	  mi_check_print_info(param,"Couldn't read whole record from %s",
unknown's avatar
unknown committed
3499
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3500 3501
	continue;
      }
3502 3503
      if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
                              sort_param->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
3504 3505
      {
	if (! searching)
3506
	  mi_check_print_info(param,"Found wrong record at %s",
unknown's avatar
unknown committed
3507
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3508 3509
	continue;
      }
unknown's avatar
unknown committed
3510
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3511
      {
unknown's avatar
unknown committed
3512
	sort_param->filepos=sort_param->pos;
3513 3514
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3515
      }
unknown's avatar
unknown committed
3516
      sort_param->max_pos=(sort_param->pos=block_info.filepos+
unknown's avatar
unknown committed
3517 3518
			 block_info.rec_len);
      info->packed_length=block_info.rec_len;
3519 3520 3521
      if (sort_param->calc_checksum)
	param->glob_crc+= (info->checksum=
                           mi_checksum(info, sort_param->record));
unknown's avatar
unknown committed
3522 3523 3524 3525 3526 3527 3528
      DBUG_RETURN(0);
    }
  }
  DBUG_RETURN(1);		/* Impossible */
}


3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
/*
  Write record to new file.

  SYNOPSIS
    sort_write_record()
      sort_param                Sort parameters.

  NOTE
    This is only called by a master thread if parallel repair is used.

  RETURN
    0           OK
    1           Error
*/
unknown's avatar
unknown committed
3543

unknown's avatar
unknown committed
3544
int sort_write_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3545 3546 3547 3548 3549 3550
{
  int flag;
  uint length;
  ulong block_length,reclength;
  byte *from;
  byte block_buff[8];
unknown's avatar
unknown committed
3551
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3552
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3553 3554
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3555 3556
  DBUG_ENTER("sort_write_record");

unknown's avatar
unknown committed
3557
  if (sort_param->fix_datafile)
unknown's avatar
unknown committed
3558 3559 3560
  {
    switch (sort_info->new_data_file_type) {
    case STATIC_RECORD:
unknown's avatar
unknown committed
3561
      if (my_b_write(&info->rec_cache,sort_param->record,
unknown's avatar
unknown committed
3562 3563 3564 3565 3566
		     share->base.pack_reclength))
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3567
      sort_param->filepos+=share->base.pack_reclength;
unknown's avatar
unknown committed
3568
      info->s->state.split++;
unknown's avatar
unknown committed
3569
      /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
unknown's avatar
unknown committed
3570 3571 3572
      break;
    case DYNAMIC_RECORD:
      if (! info->blobs)
3573
	from=sort_param->rec_buff;
unknown's avatar
unknown committed
3574 3575 3576 3577
      else
      {
	/* must be sure that local buffer is big enough */
	reclength=info->s->base.pack_reclength+
unknown's avatar
unknown committed
3578
	  _my_calc_total_blob_length(info,sort_param->record)+
unknown's avatar
unknown committed
3579 3580 3581 3582 3583
	  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
	  MI_DYN_DELETE_BLOCK_HEADER;
	if (sort_info->buff_length < reclength)
	{
	  if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3584 3585
					   MYF(MY_FREE_ON_ERROR |
					       MY_ALLOW_ZERO_PTR))))
unknown's avatar
unknown committed
3586 3587 3588 3589 3590
	    DBUG_RETURN(1);
	  sort_info->buff_length=reclength;
	}
	from=sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
      }
3591
      /* We can use info->checksum here as only one thread calls this. */
unknown's avatar
unknown committed
3592 3593
      info->checksum=mi_checksum(info,sort_param->record);
      reclength=_mi_rec_pack(info,from,sort_param->record);
unknown's avatar
unknown committed
3594
      flag=0;
unknown's avatar
unknown committed
3595 3596 3597
      /* sort_info->param->glob_crc+=info->checksum; */

      do
unknown's avatar
unknown committed
3598
      {
unknown's avatar
unknown committed
3599 3600 3601 3602 3603 3604 3605 3606
	block_length=reclength+ 3 + test(reclength >= (65520-3));
	if (block_length < share->base.min_block_length)
	  block_length=share->base.min_block_length;
	info->update|=HA_STATE_WRITE_AT_END;
	block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
	if (block_length > MI_MAX_BLOCK_LENGTH)
	  block_length=MI_MAX_BLOCK_LENGTH;
	if (_mi_write_part_record(info,0L,block_length,
unknown's avatar
unknown committed
3607
				  sort_param->filepos+block_length,
unknown's avatar
unknown committed
3608 3609 3610 3611 3612
				  &from,&reclength,&flag))
	{
	  mi_check_print_error(param,"%d when writing to datafile",my_errno);
	  DBUG_RETURN(1);
	}
unknown's avatar
unknown committed
3613
	sort_param->filepos+=block_length;
unknown's avatar
unknown committed
3614 3615
	info->s->state.split++;
      } while (reclength);
unknown's avatar
unknown committed
3616
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3617 3618 3619
      break;
    case COMPRESSED_RECORD:
      reclength=info->packed_length;
3620 3621
      length= save_pack_length((uint) share->pack.version, block_buff,
                               reclength);
unknown's avatar
unknown committed
3622
      if (info->s->base.blobs)
3623 3624
	length+= save_pack_length((uint) share->pack.version,
	                          block_buff + length, info->blob_length);
unknown's avatar
unknown committed
3625
      if (my_b_write(&info->rec_cache,block_buff,length) ||
3626
	  my_b_write(&info->rec_cache,(byte*) sort_param->rec_buff,reclength))
unknown's avatar
unknown committed
3627 3628 3629 3630
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3631
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3632
      sort_param->filepos+=reclength+length;
unknown's avatar
unknown committed
3633
      info->s->state.split++;
unknown's avatar
unknown committed
3634 3635 3636
      break;
    }
  }
3637
  if (sort_param->master)
unknown's avatar
unknown committed
3638
  {
3639 3640 3641 3642 3643
    info->state->records++;
    if ((param->testflag & T_WRITE_LOOP) &&
        (info->state->records % WRITE_COUNT) == 0)
    {
      char llbuff[22];
3644 3645
      printf("%s\r", llstr(info->state->records,llbuff));
      VOID(fflush(stdout));
3646
    }
unknown's avatar
unknown committed
3647 3648 3649 3650 3651 3652 3653
  }
  DBUG_RETURN(0);
} /* sort_write_record */


	/* Compare two keys from _create_index_by_sort */

unknown's avatar
unknown committed
3654 3655
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
			const void *b)
unknown's avatar
unknown committed
3656
{
3657
  uint not_used[2];
3658
  return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3659
		     USE_WHOLE_KEY, SEARCH_SAME, not_used));
unknown's avatar
unknown committed
3660 3661 3662
} /* sort_key_cmp */


unknown's avatar
unknown committed
3663
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
unknown's avatar
unknown committed
3664
{
3665
  uint diff_pos[2];
unknown's avatar
unknown committed
3666
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
3667
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3668 3669 3670 3671 3672
  MI_CHECK *param= sort_info->param;
  int cmp;

  if (sort_info->key_block->inited)
  {
3673
    cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
unknown's avatar
unknown committed
3674
		   (uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3675
		   diff_pos);
3676 3677 3678
    if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
      ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
                 (uchar*) a, USE_WHOLE_KEY, 
3679
                 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3680 3681
    else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
    {
3682 3683 3684 3685
      diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
                                                 sort_param->notnull,
                                                 sort_info->key_block->lastkey,
                                                 (uchar*)a);
3686
    }
3687
    sort_param->unique[diff_pos[0]-1]++;
unknown's avatar
unknown committed
3688 3689 3690
  }
  else
  {
3691
    cmp= -1;
3692 3693 3694
    if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
      mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
                                     (uchar*)a);
unknown's avatar
unknown committed
3695
  }
unknown's avatar
unknown committed
3696
  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
unknown's avatar
unknown committed
3697 3698 3699
  {
    sort_info->dupp++;
    sort_info->info->lastpos=get_record_for_key(sort_info->info,
3700
						sort_param->keyinfo,
unknown's avatar
unknown committed
3701
						(uchar*) a);
unknown's avatar
unknown committed
3702 3703 3704 3705
    mi_check_print_warning(param,
			   "Duplicate key for record at %10s against record at %10s",
			   llstr(sort_info->info->lastpos,llbuff),
			   llstr(get_record_for_key(sort_info->info,
unknown's avatar
unknown committed
3706
						    sort_param->keyinfo,
unknown's avatar
unknown committed
3707 3708 3709
						    sort_info->key_block->
						    lastkey),
				 llbuff2));
unknown's avatar
unknown committed
3710
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3711
    if (sort_info->param->testflag & T_VERBOSE)
3712
      _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
unknown's avatar
unknown committed
3713
    return (sort_delete_record(sort_param));
unknown's avatar
unknown committed
3714 3715 3716 3717 3718 3719 3720 3721 3722
  }
#ifndef DBUG_OFF
  if (cmp > 0)
  {
    mi_check_print_error(param,
			 "Internal error: Keys are not in order from sort");
    return(1);
  }
#endif
unknown's avatar
unknown committed
3723
  return (sort_insert_key(sort_param,sort_info->key_block,
unknown's avatar
unknown committed
3724
			  (uchar*) a, HA_OFFSET_ERROR));
unknown's avatar
unknown committed
3725 3726
} /* sort_key_write */

3727 3728 3729 3730 3731
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
{
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
  MYISAM_SHARE *share=sort_info->info->s;
unknown's avatar
unknown committed
3732 3733
  uint val_off, val_len;
  int error;
3734 3735 3736 3737 3738 3739 3740 3741
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  uchar *from, *to;

  val_len=share->ft2_keyinfo.keylength;
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
  to=ft_buf->lastkey+val_off;

  if (ft_buf->buf)
unknown's avatar
unknown committed
3742 3743 3744 3745
  {
    /* flushing first-level tree */
    error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			  HA_OFFSET_ERROR);
3746 3747 3748 3749 3750
    for (from=to+val_len;
         !error && from < ft_buf->buf;
         from+= val_len)
    {
      memcpy(to, from, val_len);
unknown's avatar
unknown committed
3751 3752
      error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			    HA_OFFSET_ERROR);
3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
    }
    return error;
  }
  /* flushing second-level tree keyblocks */
  error=flush_pending_blocks(sort_param);
  /* updating lastkey with second-level tree info */
  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
  _mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
      share->state.key_root[sort_param->key]);
  /* restoring first level tree data in sort_info/sort_param */
  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
  sort_param->keyinfo=share->keyinfo+sort_param->key;
  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
  /* writing lastkey in first-level tree */
  return error ? error :
                 sort_insert_key(sort_param,sort_info->key_block,
                                 ft_buf->lastkey,HA_OFFSET_ERROR);
}

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
{
  uint a_len, val_off, val_len, error;
  uchar *p;
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;

  val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
  get_key_full_length_rdonly(a_len, (uchar *)a);

  if (!ft_buf)
  {
    /*
      use two-level tree only if key_reflength fits in rec_reflength place
      and row format is NOT static - for _mi_dpointer not to garble offsets
     */
    if ((sort_info->info->s->base.key_reflength <=
         sort_info->info->s->base.rec_reflength) &&
        (sort_info->info->s->options &
          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
                                      sizeof(SORT_FT_BUF), MYF(MY_WME));

    if (!ft_buf)
    {
      sort_param->key_write=sort_key_write;
      return sort_key_write(sort_param, a);
    }
    sort_info->ft_buf=ft_buf;
    goto word_init_ft_buf; /* no need to duplicate the code */
  }
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3806
  if (mi_compare_text(sort_param->seg->charset,
3807
                      ((uchar *)a)+1,a_len-1,
3808
                      ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3809 3810 3811 3812 3813
  {
    if (!ft_buf->buf) /* store in second-level tree */
    {
      ft_buf->count++;
      return sort_insert_key(sort_param,key_block,
3814
                             ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3815 3816 3817
    }

    /* storing the key in the buffer. */
3818
    memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838
    ft_buf->buf+=val_len;
    if (ft_buf->buf < ft_buf->end)
      return 0;

    /* converting to two-level tree */
    p=ft_buf->lastkey+val_off;

    while (key_block->inited)
      key_block++;
    sort_info->key_block=key_block;
    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
    ft_buf->count=(ft_buf->buf - p)/val_len;

    /* flushing buffer to second-level tree */
    for (error=0; !error && p < ft_buf->buf; p+= val_len)
      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
    ft_buf->buf=0;
    return error;
  }

3839 3840 3841
  /* flushing buffer */
  if ((error=sort_ft_buf_flush(sort_param)))
    return error;
3842

3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856
word_init_ft_buf:
  a_len+=val_len;
  memcpy(ft_buf->lastkey, a, a_len);
  ft_buf->buf=ft_buf->lastkey+a_len;
  /*
    32 is just a safety margin here
    (at least max(val_len, sizeof(nod_flag)) should be there).
    May be better performance could be achieved if we'd put
      (sort_info->keyinfo->block_length-32)/XXX
      instead.
        TODO: benchmark the best value for XXX.
  */
  ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
  return 0;
3857
} /* sort_ft_key_write */
unknown's avatar
unknown committed
3858

3859

unknown's avatar
unknown committed
3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870
	/* get pointer to record from a key */

static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
				   uchar *key)
{
  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
} /* get_record_for_key */


	/* Insert a key in sort-key-blocks */

unknown's avatar
unknown committed
3871
static int sort_insert_key(MI_SORT_PARAM *sort_param,
unknown's avatar
unknown committed
3872 3873 3874 3875
			   register SORT_KEY_BLOCKS *key_block, uchar *key,
			   my_off_t prev_block)
{
  uint a_length,t_length,nod_flag;
3876
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
3877 3878 3879
  uchar *anc_buff,*lastkey;
  MI_KEY_PARAM s_temp;
  MI_INFO *info;
unknown's avatar
unknown committed
3880 3881 3882
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
  SORT_INFO *sort_info= sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3883 3884 3885 3886 3887 3888
  DBUG_ENTER("sort_insert_key");

  anc_buff=key_block->buff;
  info=sort_info->info;
  lastkey=key_block->lastkey;
  nod_flag= (key_block == sort_info->key_block ? 0 :
unknown's avatar
unknown committed
3889
	     info->s->base.key_reflength);
unknown's avatar
unknown committed
3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909

  if (!key_block->inited)
  {
    key_block->inited=1;
    if (key_block == sort_info->key_block_end)
    {
      mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
      DBUG_RETURN(1);
    }
    a_length=2+nod_flag;
    key_block->end_pos=anc_buff+2;
    lastkey=0;					/* No previous key in block */
  }
  else
    a_length=mi_getint(anc_buff);

	/* Save pointer to previous block */
  if (nod_flag)
    _mi_kpointer(info,key_block->end_pos,prev_block);

unknown's avatar
unknown committed
3910 3911 3912 3913
  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
				(uchar*) 0,lastkey,lastkey,key,
				 &s_temp);
  (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
unknown's avatar
unknown committed
3914 3915 3916
  a_length+=t_length;
  mi_putint(anc_buff,a_length,nod_flag);
  key_block->end_pos+=t_length;
unknown's avatar
unknown committed
3917
  if (a_length <= keyinfo->block_length)
unknown's avatar
unknown committed
3918
  {
unknown's avatar
unknown committed
3919
    VOID(_mi_move_key(keyinfo,key_block->lastkey,key));
unknown's avatar
unknown committed
3920 3921 3922 3923 3924 3925 3926
    key_block->last_length=a_length-t_length;
    DBUG_RETURN(0);
  }

	/* Fill block with end-zero and write filled block */
  mi_putint(anc_buff,key_block->last_length,nod_flag);
  bzero((byte*) anc_buff+key_block->last_length,
unknown's avatar
unknown committed
3927
	keyinfo->block_length- key_block->last_length);
3928
  key_file_length=info->state->key_file_length;
3929
  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3930 3931 3932 3933 3934
    DBUG_RETURN(1);

  /* If we read the page from the key cache, we have to write it back to it */
  if (key_file_length == info->state->key_file_length)
  {
3935
    if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
3936 3937 3938
      DBUG_RETURN(1);
  }
  else if (my_pwrite(info->s->kfile,(byte*) anc_buff,
unknown's avatar
unknown committed
3939
		     (uint) keyinfo->block_length,filepos, param->myf_rw))
unknown's avatar
unknown committed
3940 3941 3942 3943
    DBUG_RETURN(1);
  DBUG_DUMP("buff",(byte*) anc_buff,mi_getint(anc_buff));

	/* Write separator-key to block in next level */
unknown's avatar
unknown committed
3944
  if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
unknown's avatar
unknown committed
3945 3946 3947 3948
    DBUG_RETURN(1);

	/* clear old block and write new key in it */
  key_block->inited=0;
unknown's avatar
unknown committed
3949
  DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
unknown's avatar
unknown committed
3950 3951 3952 3953 3954
} /* sort_insert_key */


	/* Delete record when we found a duplicated key */

unknown's avatar
unknown committed
3955
static int sort_delete_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3956 3957 3958 3959
{
  uint i;
  int old_file,error;
  uchar *key;
unknown's avatar
unknown committed
3960 3961 3962
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3963 3964
  DBUG_ENTER("sort_delete_record");

3965
  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
3966 3967
  {
    mi_check_print_error(param,
unknown's avatar
unknown committed
3968
			 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
unknown's avatar
unknown committed
3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979
    DBUG_RETURN(1);
  }
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
  {
    mi_check_print_error(param,
			 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
    DBUG_RETURN(1);
  }

  old_file=info->dfile;
  info->dfile=info->rec_cache.file;
unknown's avatar
unknown committed
3980
  if (sort_info->current_key)
unknown's avatar
unknown committed
3981 3982
  {
    key=info->lastkey+info->s->base.max_key_length;
unknown's avatar
unknown committed
3983
    if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
unknown's avatar
unknown committed
3984 3985 3986 3987 3988 3989 3990
	error != HA_ERR_RECORD_DELETED)
    {
      mi_check_print_error(param,"Can't read record to be removed");
      info->dfile=old_file;
      DBUG_RETURN(1);
    }

unknown's avatar
unknown committed
3991
    for (i=0 ; i < sort_info->current_key ; i++)
unknown's avatar
unknown committed
3992
    {
unknown's avatar
unknown committed
3993
      uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
unknown's avatar
unknown committed
3994 3995 3996 3997 3998 3999 4000
      if (_mi_ck_delete(info,i,key,key_length))
      {
	mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
	info->dfile=old_file;
	DBUG_RETURN(1);
      }
    }
4001
    if (sort_param->calc_checksum)
unknown's avatar
unknown committed
4002
      param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
unknown's avatar
unknown committed
4003 4004 4005 4006 4007 4008 4009 4010 4011
  }
  error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
  info->dfile=old_file;				/* restore actual value */
  info->state->records--;
  DBUG_RETURN(error);
} /* sort_delete_record */

	/* Fix all pending blocks and flush everything to disk */

unknown's avatar
unknown committed
4012
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
4013 4014
{
  uint nod_flag,length;
4015
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
4016
  SORT_KEY_BLOCKS *key_block;
unknown's avatar
unknown committed
4017
  SORT_INFO *sort_info= sort_param->sort_info;
4018
  myf myf_rw=sort_info->param->myf_rw;
unknown's avatar
unknown committed
4019 4020
  MI_INFO *info=sort_info->info;
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
unknown's avatar
unknown committed
4021 4022 4023 4024 4025 4026 4027 4028 4029 4030
  DBUG_ENTER("flush_pending_blocks");

  filepos= HA_OFFSET_ERROR;			/* if empty file */
  nod_flag=0;
  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
  {
    key_block->inited=0;
    length=mi_getint(key_block->buff);
    if (nod_flag)
      _mi_kpointer(info,key_block->end_pos,filepos);
4031
    key_file_length=info->state->key_file_length;
unknown's avatar
unknown committed
4032
    bzero((byte*) key_block->buff+length, keyinfo->block_length-length);
4033
    if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4034 4035 4036 4037 4038
      DBUG_RETURN(1);

    /* If we read the page from the key cache, we have to write it back */
    if (key_file_length == info->state->key_file_length)
    {
4039 4040
      if (_mi_write_keypage(info, keyinfo, filepos,
                            DFLT_INIT_HITS, key_block->buff))
4041 4042 4043
	DBUG_RETURN(1);
    }
    else if (my_pwrite(info->s->kfile,(byte*) key_block->buff,
4044
		       (uint) keyinfo->block_length,filepos, myf_rw))
unknown's avatar
unknown committed
4045 4046 4047 4048
      DBUG_RETURN(1);
    DBUG_DUMP("buff",(byte*) key_block->buff,length);
    nod_flag=1;
  }
unknown's avatar
unknown committed
4049
  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
unknown's avatar
unknown committed
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065
  DBUG_RETURN(0);
} /* flush_pending_blocks */

	/* alloc space and pointers for key_blocks */

static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
                                         uint buffer_length)
{
  reg1 uint i;
  SORT_KEY_BLOCKS *block;
  DBUG_ENTER("alloc_key_blocks");

  if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
					    buffer_length+IO_SIZE)*blocks,
					   MYF(0))))
  {
4066
    mi_check_print_error(param,"Not enough memory for sort-key-blocks");
unknown's avatar
unknown committed
4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083
    return(0);
  }
  for (i=0 ; i < blocks ; i++)
  {
    block[i].inited=0;
    block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
  }
  DBUG_RETURN(block);
} /* alloc_key_blocks */


	/* Check if file is almost full */

int test_if_almost_full(MI_INFO *info)
{
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    return 0;
4084 4085 4086 4087
  return my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)) / 10 * 9 >
         (my_off_t) info->s->base.max_key_file_length ||
         my_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
         (my_off_t) info->s->base.max_data_file_length;
unknown's avatar
unknown committed
4088 4089 4090 4091 4092 4093 4094 4095 4096 4097
}

	/* Recreate table with bigger more alloced record-data */

int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
{
  int error;
  MI_INFO info;
  MYISAM_SHARE share;
  MI_KEYDEF *keyinfo,*key,*key_end;
unknown's avatar
unknown committed
4098
  HA_KEYSEG *keysegs,*keyseg;
unknown's avatar
unknown committed
4099 4100 4101 4102 4103 4104 4105
  MI_COLUMNDEF *recdef,*rec,*end;
  MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
  MI_STATUS_INFO status_info;
  uint unpack,key_parts;
  ha_rows max_records;
  ulonglong file_length,tmp_length;
  MI_CREATE_INFO create_info;
4106
  DBUG_ENTER("recreate_table");
unknown's avatar
unknown committed
4107 4108 4109 4110 4111 4112 4113 4114 4115

  error=1;					/* Default error */
  info= **org_info;
  status_info= (*org_info)->state[0];
  info.state= &status_info;
  share= *(*org_info)->s;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4116
    DBUG_RETURN(0);
unknown's avatar
unknown committed
4117 4118 4119 4120
  memcpy((byte*) keyinfo,(byte*) share.keyinfo,
	 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));

  key_parts= share.base.all_key_parts;
unknown's avatar
unknown committed
4121
  if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
unknown's avatar
unknown committed
4122 4123 4124
				       (key_parts+share.base.keys))))
  {
    my_afree((gptr) keyinfo);
4125
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4126 4127 4128 4129 4130 4131
  }
  if (!(recdef=(MI_COLUMNDEF*)
	my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
  {
    my_afree((gptr) keyinfo);
    my_afree((gptr) keysegs);
4132
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4133 4134 4135 4136 4137 4138 4139
  }
  if (!(uniquedef=(MI_UNIQUEDEF*)
	my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
  {
    my_afree((gptr) recdef);
    my_afree((gptr) keyinfo);
    my_afree((gptr) keysegs);
4140
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156
  }

  /* Copy the column definitions */
  memcpy((byte*) recdef,(byte*) share.rec,
	 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
  for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
  {
    if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
	rec->type != FIELD_BLOB &&
	rec->type != FIELD_VARCHAR &&
	rec->type != FIELD_CHECK)
      rec->type=(int) FIELD_NORMAL;
  }

  /* Change the new key to point at the saved key segments */
  memcpy((byte*) keysegs,(byte*) share.keyparts,
unknown's avatar
unknown committed
4157
	 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
unknown's avatar
unknown committed
4158 4159 4160 4161 4162 4163 4164 4165 4166 4167
				      share.state.header.uniques)));
  keyseg=keysegs;
  for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
  {
    key->seg=keyseg;
    for (; keyseg->type ; keyseg++)
    {
      if (param->language)
	keyseg->language=param->language;	/* change language */
    }
4168
    keyseg++;					/* Skip end pointer */
unknown's avatar
unknown committed
4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208
  }

  /* Copy the unique definitions and change them to point at the new key
     segments*/
  memcpy((byte*) uniquedef,(byte*) share.uniqueinfo,
	 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
  for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
       u_ptr != u_end ; u_ptr++)
  {
    u_ptr->seg=keyseg;
    keyseg+=u_ptr->keysegs+1;
  }
  if (share.options & HA_OPTION_COMPRESS_RECORD)
    share.base.records=max_records=info.state->records;
  else if (share.base.min_pack_length)
    max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
			   (ulong) share.base.min_pack_length);
  else
    max_records=0;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;

  file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
  tmp_length= file_length+file_length/10;
  set_if_bigger(file_length,param->max_data_file_length);
  set_if_bigger(file_length,tmp_length);
  set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);

  VOID(mi_close(*org_info));
  bzero((char*) &create_info,sizeof(create_info));
  create_info.max_rows=max(max_records,share.base.records);
  create_info.reloc_rows=share.base.reloc;
  create_info.old_options=(share.options |
			   (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));

  create_info.data_file_length=file_length;
  create_info.auto_increment=share.state.auto_increment;
  create_info.language = (param->language ? param->language :
			  share.state.header.language);
unknown's avatar
unknown committed
4209
  create_info.key_file_length=  status_info.key_file_length;
4210 4211 4212 4213 4214
  /*
    Allow for creating an auto_increment key. This has an effect only if
    an auto_increment key exists in the original table.
  */
  create_info.with_auto_increment= TRUE;
unknown's avatar
unknown committed
4215 4216 4217
  /* We don't have to handle symlinks here because we are using
     HA_DONT_TOUCH_DATA */
  if (mi_create(filename,
unknown's avatar
unknown committed
4218 4219 4220 4221 4222 4223 4224 4225 4226
		share.base.keys - share.state.header.uniques,
		keyinfo, share.base.fields, recdef,
		share.state.header.uniques, uniquedef,
		&create_info,
		HA_DONT_TOUCH_DATA))
  {
    mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
    goto end;
  }
unknown's avatar
unknown committed
4227
  *org_info=mi_open(filename,O_RDWR,
unknown's avatar
unknown committed
4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244
		    (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
		    (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
		    HA_OPEN_ABORT_IF_LOCKED);
  if (!*org_info)
  {
    mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
		my_errno);
    goto end;
  }
  /* We are modifing */
  (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
  VOID(_mi_readinfo(*org_info,F_WRLCK,0));
  (*org_info)->state->records=info.state->records;
  if (share.state.create_time)
    (*org_info)->s->state.create_time=share.state.create_time;
  (*org_info)->s->state.unique=(*org_info)->this_unique=
    share.state.unique;
4245
  (*org_info)->state->checksum=info.state->checksum;
unknown's avatar
unknown committed
4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258
  (*org_info)->state->del=info.state->del;
  (*org_info)->s->state.dellink=share.state.dellink;
  (*org_info)->state->empty=info.state->empty;
  (*org_info)->state->data_file_length=info.state->data_file_length;
  if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
			UPDATE_OPEN_COUNT))
    goto end;
  error=0;
end:
  my_afree((gptr) uniquedef);
  my_afree((gptr) keyinfo);
  my_afree((gptr) recdef);
  my_afree((gptr) keysegs);
4259
  DBUG_RETURN(error);
unknown's avatar
unknown committed
4260 4261 4262 4263 4264
}


	/* write suffix to data file if neaded */

unknown's avatar
unknown committed
4265
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
unknown's avatar
unknown committed
4266
{
unknown's avatar
unknown committed
4267 4268 4269
  MI_INFO *info=sort_info->info;

  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
unknown's avatar
unknown committed
4270 4271 4272 4273 4274
  {
    char buff[MEMMAP_EXTRA_MARGIN];
    bzero(buff,sizeof(buff));
    if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
    {
unknown's avatar
unknown committed
4275
      mi_check_print_error(sort_info->param,
unknown's avatar
unknown committed
4276
			   "%d when writing to datafile",my_errno);
unknown's avatar
unknown committed
4277 4278
      return 1;
    }
unknown's avatar
unknown committed
4279
    sort_info->param->read_cache.end_of_file+=sizeof(buff);
unknown's avatar
unknown committed
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296
  }
  return 0;
}

	/* Update state and myisamchk_time of indexfile */

int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
{
  MYISAM_SHARE *share=info->s;

  if (update & UPDATE_OPEN_COUNT)
  {
    share->state.open_count=0;
    share->global_changed=0;
  }
  if (update & UPDATE_STAT)
  {
4297
    uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
unknown's avatar
unknown committed
4298
    share->state.rec_per_key_rows=info->state->records;
4299
    share->state.changed&= ~STATE_NOT_ANALYZED;
unknown's avatar
unknown committed
4300
    if (info->state->records)
4301
    {
unknown's avatar
unknown committed
4302 4303 4304 4305 4306
      for (i=0; i<key_parts; i++)
      {
        if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
          share->state.changed|= STATE_NOT_ANALYZED;
      }
4307
    }
unknown's avatar
unknown committed
4308 4309 4310 4311 4312 4313 4314 4315 4316
  }
  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
  {
    if (update & UPDATE_TIME)
    {
      share->state.check_time= (long) time((time_t*) 0);
      if (!share->state.create_time)
	share->state.create_time=share->state.check_time;
    }
unknown's avatar
unknown committed
4317 4318 4319
    /*
      When tables are locked we haven't synched the share state and the
      real state for a while so we better do it here before synching
unknown's avatar
unknown committed
4320 4321
      the share state to disk. Only when table is write locked is it
      necessary to perform this synch.
unknown's avatar
unknown committed
4322
    */
unknown's avatar
unknown committed
4323 4324
    if (info->lock_type == F_WRLCK)
      share->state.state= *info->state;
unknown's avatar
unknown committed
4325 4326
    if (mi_state_info_write(share->kfile,&share->state,1+2))
      goto err;
unknown's avatar
unknown committed
4327
    share->changed=0;
unknown's avatar
unknown committed
4328 4329 4330 4331
  }
  {						/* Force update of status */
    int error;
    uint r_locks=share->r_locks,w_locks=share->w_locks;
4332
    share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
4333
    error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4334 4335 4336
    share->r_locks=r_locks;
    share->w_locks=w_locks;
    share->tot_locks=r_locks+w_locks;
unknown's avatar
unknown committed
4337 4338 4339 4340
    if (!error)
      return 0;
  }
err:
unknown's avatar
unknown committed
4341
  mi_check_print_error(param,"%d when updating keyfile",my_errno);
unknown's avatar
unknown committed
4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360
  return 1;
}

	/*
	  Update auto increment value for a table
	  When setting the 'repair_only' flag we only want to change the
	  old auto_increment value if its wrong (smaller than some given key).
	  The reason is that we shouldn't change the auto_increment value
	  for a table without good reason when only doing a repair; If the
	  user have inserted and deleted rows, the auto_increment value
	  may be bigger than the biggest current row and this is ok.

	  If repair_only is not set, we will update the flag to the value in
	  param->auto_increment is bigger than the biggest key.
	*/

void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
			       my_bool repair_only)
{
4361
  byte *record;
4362 4363
  DBUG_ENTER("update_auto_increment_key");

unknown's avatar
unknown committed
4364
  if (!info->s->base.auto_key ||
4365
      ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
unknown's avatar
unknown committed
4366
  {
unknown's avatar
unknown committed
4367 4368 4369 4370
    if (!(param->testflag & T_VERY_SILENT))
      mi_check_print_info(param,
			  "Table: %s doesn't have an auto increment key\n",
			  param->isam_file_name);
4371
    DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4372 4373
  }
  if (!(param->testflag & T_SILENT) &&
4374
      !(param->testflag & T_REP))
unknown's avatar
unknown committed
4375
    printf("Updating MyISAM file: %s\n", param->isam_file_name);
4376 4377 4378 4379 4380 4381 4382 4383
  /*
    We have to use an allocated buffer instead of info->rec_buff as 
    _mi_put_key_in_record() may use info->rec_buff
  */
  if (!(record= (byte*) my_malloc((uint) info->s->base.pack_reclength,
				  MYF(0))))
  {
    mi_check_print_error(param,"Not enough memory for extra record");
4384
    DBUG_VOID_RETURN;
4385 4386
  }

unknown's avatar
unknown committed
4387
  mi_extra(info,HA_EXTRA_KEYREAD,0);
4388
  if (mi_rlast(info, record, info->s->base.auto_key-1))
unknown's avatar
unknown committed
4389 4390 4391
  {
    if (my_errno != HA_ERR_END_OF_FILE)
    {
unknown's avatar
unknown committed
4392
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4393
      my_free((char*) record, MYF(0));
unknown's avatar
unknown committed
4394
      mi_check_print_error(param,"%d when reading last record",my_errno);
4395
      DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4396 4397 4398 4399 4400 4401
    }
    if (!repair_only)
      info->s->state.auto_increment=param->auto_increment_value;
  }
  else
  {
4402
    ulonglong auto_increment= retrieve_auto_increment(info, record);
unknown's avatar
unknown committed
4403
    set_if_bigger(info->s->state.auto_increment,auto_increment);
4404 4405
    if (!repair_only)
      set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
unknown's avatar
unknown committed
4406
  }
unknown's avatar
unknown committed
4407
  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4408
  my_free((char*) record, MYF(0));
unknown's avatar
unknown committed
4409
  update_state_info(param, info, UPDATE_AUTO_INC);
4410
  DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4411 4412
}

4413 4414 4415

/*
  Update statistics for each part of an index
4416

4417 4418
  SYNOPSIS
    update_key_parts()
4419
      keyinfo           IN  Index information (only key->keysegs used)
4420
      rec_per_key_part  OUT Store statistics here
4421 4422
      unique            IN  Array of (#distinct tuples)
      notnull_tuples    IN  Array of (#tuples), or NULL
4423
      records               Number of records in the table
4424 4425 4426 4427 4428 4429 4430

  DESCRIPTION
    This function is called produce index statistics values from unique and 
    notnull_tuples arrays after these arrays were produced with sequential
    index scan (the scan is done in two places: chk_index() and
    sort_key_write()).

4431 4432
    This function handles all 3 index statistics collection methods.

4433
    Unique is an array:
4434 4435 4436 4437 4438 4439 4440 4441 4442 4443
      unique[0]= (#different values of {keypart1}) - 1
      unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
      ...

    For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
      notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
      notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all 
                          keypart{i} are not NULL)
      ...
    For all other statistics collection methods notnull_tuples==NULL.
4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454

    Output is an array:
    rec_per_key_part[k] = 
     = E(#records in the table such that keypart_1=c_1 AND ... AND 
         keypart_k=c_k for arbitrary constants c_1 ... c_k) 
     
     = {assuming that values have uniform distribution and index contains all
        tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
        index tuples}
     
     = #tuples-in-the-index / #distinct-tuples-in-the-index.
4455 4456 4457 4458 4459 4460 4461 4462
    
    The #tuples-in-the-index and #distinct-tuples-in-the-index have different 
    meaning depending on which statistics collection method is used:
    
    MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
     NULLS_EQUAL            NULL == NULL           all tuples in table
     NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
     IGNORE_NULLS               n/a             tuples that don't have NULLs
4463
*/
unknown's avatar
unknown committed
4464

unknown's avatar
unknown committed
4465
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4466 4467
                      ulonglong *unique, ulonglong *notnull,
                      ulonglong records)
unknown's avatar
unknown committed
4468
{
4469 4470
  ulonglong count=0,tmp, unique_tuples;
  ulonglong tuples= records;
unknown's avatar
unknown committed
4471 4472 4473 4474
  uint parts;
  for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
  {
    count+=unique[parts];
4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490
    unique_tuples= count + 1;    
    if (notnull)
    {
      tuples= notnull[parts];
      /* 
        #(unique_tuples not counting tuples with NULLs) = 
          #(unique_tuples counting tuples with NULLs as different) - 
          #(tuples with NULLs)
      */
      unique_tuples -= (records - notnull[parts]);
    }
    
    if (unique_tuples == 0)
      tmp= 1;
    else if (count == 0)
      tmp= tuples; /* 1 unique tuple */
unknown's avatar
unknown committed
4491
    else
4492 4493 4494 4495 4496 4497
      tmp= (tuples + unique_tuples/2) / unique_tuples;

    /* 
      for some weird keys (e.g. FULLTEXT) tmp can be <1 here. 
      let's ensure it is not
    */
4498
    set_if_bigger(tmp,1);
unknown's avatar
unknown committed
4499 4500
    if (tmp >= (ulonglong) ~(ulong) 0)
      tmp=(ulonglong) ~(ulong) 0;
4501

unknown's avatar
unknown committed
4502 4503 4504 4505 4506 4507
    *rec_per_key_part=(ulong) tmp;
    rec_per_key_part++;
  }
}


unknown's avatar
unknown committed
4508
static ha_checksum mi_byte_checksum(const byte *buf, uint length)
unknown's avatar
unknown committed
4509 4510 4511 4512 4513 4514 4515 4516
{
  ha_checksum crc;
  const byte *end=buf+length;
  for (crc=0; buf != end; buf++)
    crc=((crc << 1) + *((uchar*) buf)) +
      test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
  return crc;
}
4517 4518 4519

static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
{
4520 4521
  uint key_maxlength=key->maxlength;
  if (key->flag & HA_FULLTEXT)
4522 4523 4524 4525 4526
  {
    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                  key->seg->charset->mbmaxlen;
    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
  }
4527 4528
  return (key->flag & HA_SPATIAL) ||
          (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4529
	  ((ulonglong) rows * key_maxlength >
4530
	   (ulonglong) myisam_max_temp_length));
4531 4532
}

4533 4534 4535 4536 4537 4538 4539 4540
/*
  Deactivate all not unique index that can be recreated fast
  These include packed keys on which sorting will use more temporary
  space than the max allowed file length or for which the unpacked keys
  will take much more space than packed keys.
  Note that 'rows' may be zero for the case when we don't know how many
  rows we will put into the file.
 */
4541

4542
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4543 4544
{
  MYISAM_SHARE *share=info->s;
4545 4546 4547 4548 4549 4550
  MI_KEYDEF    *key=share->keyinfo;
  uint          i;

  DBUG_ASSERT(info->state->records == 0 &&
              (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
  for (i=0 ; i < share->base.keys ; i++,key++)
4551
  {
4552 4553
    if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
        ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4554
    {
4555
      mi_clear_key_active(share->state.key_map, i);
4556
      info->update|= HA_STATE_CHANGED;
4557 4558 4559 4560 4561
    }
  }
}


4562 4563 4564 4565 4566
/*
  Return TRUE if we can use repair by sorting
  One can set the force argument to force to use sorting
  even if the temporary file would be quite big!
*/
4567

unknown's avatar
unknown committed
4568
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
unknown's avatar
unknown committed
4569
			    ulonglong key_map, my_bool force)
4570 4571 4572
{
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *key=share->keyinfo;
4573 4574 4575
  uint i;

  /*
unknown's avatar
unknown committed
4576
    mi_repair_by_sort only works if we have at least one key. If we don't
4577 4578
    have any keys, we should use the normal repair.
  */
4579
  if (! mi_is_any_key_active(key_map))
4580 4581 4582
    return FALSE;				/* Can't use sort */
  for (i=0 ; i < share->base.keys ; i++,key++)
  {
4583
    if (!force && mi_too_big_key_for_sort(key,rows))
4584 4585 4586 4587
      return FALSE;
  }
  return TRUE;
}
4588 4589 4590


static void
unknown's avatar
unknown committed
4591
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4592 4593
{
  if ((sort_info->new_data_file_type=share->data_file_type) ==
unknown's avatar
unknown committed
4594
      COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609
  {
    MYISAM_SHARE tmp;

    if (share->options & HA_OPTION_PACK_RECORD)
      sort_info->new_data_file_type = DYNAMIC_RECORD;
    else
      sort_info->new_data_file_type = STATIC_RECORD;

    /* Set delete_function for sort_delete_record() */
    memcpy((char*) &tmp, share, sizeof(*share));
    tmp.options= ~HA_OPTION_COMPRESS_RECORD;
    mi_setup_functions(&tmp);
    share->delete_record=tmp.delete_record;
  }
}