mi_check.c 155 KB
Newer Older
1 2
/*
   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
3

unknown's avatar
unknown committed
4 5
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
unknown's avatar
unknown committed
6
   the Free Software Foundation; version 2 of the License.
7

unknown's avatar
unknown committed
8 9 10 11
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
12

unknown's avatar
unknown committed
13 14
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
15
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
unknown's avatar
unknown committed
16

17
/* Describe, check and repair of MyISAM tables */
unknown's avatar
unknown committed
18

19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
/*
  About checksum calculation.

  There are two types of checksums. Table checksum and row checksum.

  Row checksum is an additional byte at the end of dynamic length
  records. It must be calculated if the table is configured for them.
  Otherwise they must not be used. The variable
  MYISAM_SHARE::calc_checksum determines if row checksums are used.
  MI_INFO::checksum is used as temporary storage during row handling.
  For parallel repair we must assure that only one thread can use this
  variable. There is no problem on the write side as this is done by one
  thread only. But when checking a record after read this could go
  wrong. But since all threads read through a common read buffer, it is
  sufficient if only one thread checks it.

  Table checksum is an eight byte value in the header of the index file.
  It can be calculated even if row checksums are not used. The variable
  MI_CHECK::glob_crc is calculated over all records.
  MI_SORT_PARAM::calc_checksum determines if this should be done. This
  variable is not part of MI_CHECK because it must be set per thread for
  parallel repair. The global glob_crc must be changed by one thread
  only. And it is sufficient to calculate the checksum once only.
*/

44
#include "ftdefs.h"
unknown's avatar
unknown committed
45 46
#include <m_ctype.h>
#include <stdarg.h>
47
#include <my_getopt.h>
48
#ifdef HAVE_SYS_VADVISE_H
unknown's avatar
unknown committed
49 50 51 52 53
#include <sys/vadvise.h>
#endif
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
54
#include "rt_index.h"
unknown's avatar
unknown committed
55 56 57 58

	/* Functions defined in this file */

static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
59
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
60 61 62 63
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level);
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
static ha_checksum calc_checksum(ha_rows count);
64
static int writekeys(MI_SORT_PARAM *sort_param);
unknown's avatar
unknown committed
65 66
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file);
unknown's avatar
unknown committed
67 68 69 70
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
71
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
72
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
73 74
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
				uchar *key);
75
static int sort_insert_key(MI_SORT_PARAM  *sort_param,
unknown's avatar
unknown committed
76
                           reg1 SORT_KEY_BLOCKS *key_block,
unknown's avatar
unknown committed
77
			   uchar *key, my_off_t prev_block);
unknown's avatar
unknown committed
78
static int sort_delete_record(MI_SORT_PARAM *sort_param);
79
/*static int flush_pending_blocks(MI_CHECK *param);*/
unknown's avatar
unknown committed
80 81
static SORT_KEY_BLOCKS	*alloc_key_blocks(MI_CHECK *param, uint blocks,
					  uint buffer_length);
82
static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
unknown's avatar
unknown committed
83
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
84
static HA_KEYSEG *ha_find_null(HA_KEYSEG *keyseg, uchar *a);
unknown's avatar
unknown committed
85 86 87

void myisamchk_init(MI_CHECK *param)
{
88
  bzero((uchar*) param,sizeof(*param));
unknown's avatar
unknown committed
89 90 91 92 93 94 95 96 97 98 99
  param->opt_follow_links=1;
  param->keys_in_use= ~(ulonglong) 0;
  param->search_after_block=HA_OFFSET_ERROR;
  param->auto_increment_value= 0;
  param->use_buffers=USE_BUFFER_INIT;
  param->read_buffer_length=READ_BUFFER_INIT;
  param->write_buffer_length=READ_BUFFER_INIT;
  param->sort_buffer_length=SORT_BUFFER_INIT;
  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
100
  param->start_check_pos=0;
101
  param->max_record_length= LONGLONG_MAX;
102
  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
103
  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
Sergey Vojtovich's avatar
Sergey Vojtovich committed
104
  param->need_print_msg_lock= 0;
unknown's avatar
unknown committed
105 106
}

107 108 109 110 111
	/* Check the status flags for the table */

int chk_status(MI_CHECK *param, register MI_INFO *info)
{
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
112

113 114 115 116 117 118
  if (mi_is_crashed_on_repair(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed and last repair failed");
  else if (mi_is_crashed(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed");
unknown's avatar
unknown committed
119
  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
120
  {
unknown's avatar
unknown committed
121 122
    /* Don't count this as a real warning, as check can correct this ! */
    uint save=param->warning_printed;
123
    mi_check_print_warning(param,
124 125 126
			   share->state.open_count==1 ? 
			   "%d client is using or hasn't closed the table properly" : 
			   "%d clients are using or haven't closed the table properly",
127
			   share->state.open_count);
unknown's avatar
unknown committed
128 129 130
    /* If this will be fixed by the check, forget the warning */
    if (param->testflag & T_UPDATE_STATE)
      param->warning_printed=save;
131 132 133 134
  }
  return 0;
}

unknown's avatar
unknown committed
135 136 137 138 139
	/* Check delete links */

int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
{
  reg2 ha_rows i;
140
  uint delete_link_length;
141
  my_off_t empty,next_link,UNINIT_VAR(old_link);
unknown's avatar
unknown committed
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
  char buff[22],buff2[22];
  DBUG_ENTER("chk_del");

  param->record_checksum=0;
  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
		      info->s->rec_reflength+1);

  if (!(test_flag & T_SILENT))
    puts("- check record delete-chain");

  next_link=info->s->state.dellink;
  if (info->state->del == 0)
  {
    if (test_flag & T_VERBOSE)
    {
      puts("No recordlinks");
    }
  }
  else
  {
    if (test_flag & T_VERBOSE)
      printf("Recordlinks:    ");
    empty=0;
    for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
    {
unknown's avatar
unknown committed
167 168
      if (*killed_ptr(param))
        DBUG_RETURN(1);
unknown's avatar
unknown committed
169 170 171 172
      if (test_flag & T_VERBOSE)
	printf(" %9s",llstr(next_link,buff));
      if (next_link >= info->state->data_file_length)
	goto wrong;
Marc Alff's avatar
Marc Alff committed
173 174
      if (mysql_file_pread(info->dfile, (uchar*) buff, delete_link_length,
                           next_link, MYF(MY_NABP)))
unknown's avatar
unknown committed
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Can't read delete-link at filepos: %s",
		    llstr(next_link,buff));
	DBUG_RETURN(1);
      }
      if (*buff != '\0')
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Record at pos: %s is not remove-marked",
		    llstr(next_link,buff));
	goto wrong;
      }
      if (info->s->options & HA_OPTION_PACK_RECORD)
      {
	my_off_t prev_link=mi_sizekorr(buff+12);
	if (empty && prev_link != old_link)
	{
	  if (test_flag & T_VERBOSE) puts("");
	  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
	  goto wrong;
	}
	old_link=next_link;
	next_link=mi_sizekorr(buff+4);
	empty+=mi_uint3korr(buff+1);
      }
      else
      {
	param->record_checksum+=(ha_checksum) next_link;
	next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
	empty+=info->s->base.pack_reclength;
      }
    }
208 209
    if (test_flag & T_VERBOSE)
      puts("\n");
unknown's avatar
unknown committed
210 211 212
    if (empty != info->state->empty)
    {
      mi_check_print_warning(param,
213 214 215
			     "Found %s deleted space in delete link chain. Should be %s",
			     llstr(empty,buff2),
			     llstr(info->state->empty,buff));
unknown's avatar
unknown committed
216
    }
217 218 219 220 221
    if (next_link != HA_OFFSET_ERROR)
    {
      mi_check_print_error(param,
			   "Found more than the expected %s deleted rows in delete link chain",
			   llstr(info->state->del, buff));
unknown's avatar
unknown committed
222
      goto wrong;
223 224 225 226 227 228 229 230 231
    }
    if (i != 0)
    {
      mi_check_print_error(param,
			   "Found %s deleted rows in delete link chain. Should be %s",
			   llstr(info->state->del - i, buff2),
			   llstr(info->state->del, buff));
      goto wrong;
    }
unknown's avatar
unknown committed
232 233
  }
  DBUG_RETURN(0);
234

unknown's avatar
unknown committed
235
wrong:
unknown's avatar
unknown committed
236
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
237 238 239 240 241 242 243 244 245 246 247
  if (test_flag & T_VERBOSE) puts("");
  mi_check_print_error(param,"record delete-link-chain corrupted");
  DBUG_RETURN(1);
} /* chk_del */


	/* Check delete links in index file */

static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
{
  my_off_t next_link;
unknown's avatar
unknown committed
248
  uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
unknown's avatar
unknown committed
249
  ha_rows records;
250 251
  char llbuff[21], llbuff2[21];
  uchar *buff;
unknown's avatar
unknown committed
252
  DBUG_ENTER("check_k_link");
253
  DBUG_PRINT("enter", ("block_size: %u", block_size));
unknown's avatar
unknown committed
254 255

  if (param->testflag & T_VERBOSE)
256
    printf("block_size %4u:", block_size); /* purecov: tested */
unknown's avatar
unknown committed
257 258 259 260 261

  next_link=info->s->state.key_del[nr];
  records= (ha_rows) (info->state->key_file_length / block_size);
  while (next_link != HA_OFFSET_ERROR && records > 0)
  {
unknown's avatar
unknown committed
262 263
    if (*killed_ptr(param))
      DBUG_RETURN(1);
unknown's avatar
unknown committed
264 265
    if (param->testflag & T_VERBOSE)
      printf("%16s",llstr(next_link,llbuff));
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285

    /* Key blocks must lay within the key file length entirely. */
    if (next_link + block_size > info->state->key_file_length)
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Invalid key block position: %s  "
                           "key block size: %u  file_length: %s",
                           llstr(next_link, llbuff), block_size,
                           llstr(info->state->key_file_length, llbuff2));
      DBUG_RETURN(1);
      /* purecov: end */
    }

    /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
    if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Mis-aligned key block: %s  "
                           "minimum key block length: %u",
                           llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
unknown's avatar
unknown committed
286
      DBUG_RETURN(1);
287 288 289 290 291 292 293 294
      /* purecov: end */
    }

    /*
      Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
      If the key cache block size is smaller than block_size, we can so
      avoid unecessary eviction of cache block.
    */
unknown's avatar
unknown committed
295
    if (!(buff=key_cache_read(info->s->key_cache,
296
                              info->s->kfile, next_link, DFLT_INIT_HITS,
297
                              (uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
298 299 300 301 302
                              MI_MIN_KEY_BLOCK_LENGTH, 1)))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "key cache read error for block: %s",
			   llstr(next_link,llbuff));
unknown's avatar
unknown committed
303
      DBUG_RETURN(1);
304 305
      /* purecov: end */
    }
unknown's avatar
unknown committed
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
    next_link=mi_sizekorr(buff);
    records--;
    param->key_file_blocks+=block_size;
  }
  if (param->testflag & T_VERBOSE)
  {
    if (next_link != HA_OFFSET_ERROR)
      printf("%16s\n",llstr(next_link,llbuff));
    else
      puts("");
  }
  DBUG_RETURN (next_link != HA_OFFSET_ERROR);
} /* check_k_link */


unknown's avatar
unknown committed
321
	/* Check sizes of files */
unknown's avatar
unknown committed
322 323 324 325 326 327 328 329 330 331

int chk_size(MI_CHECK *param, register MI_INFO *info)
{
  int error=0;
  register my_off_t skr,size;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_size");

  if (!(param->testflag & T_SILENT)) puts("- check file-size");

unknown's avatar
unknown committed
332 333 334
  /* The following is needed if called externally (not from myisamchk) */
  flush_key_blocks(info->s->key_cache,
		   info->s->kfile, FLUSH_FORCE_WRITE);
unknown's avatar
unknown committed
335

Marc Alff's avatar
Marc Alff committed
336
  size= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
unknown's avatar
unknown committed
337 338
  if ((skr=(my_off_t) info->state->key_file_length) != size)
  {
339
    /* Don't give error if file generated by myisampack */
340
    if (skr > size && mi_is_any_key_active(info->s->state.key_map))
unknown's avatar
unknown committed
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
    {
      error=1;
      mi_check_print_error(param,
			   "Size of indexfile is: %-8s        Should be: %s",
			   llstr(size,buff), llstr(skr,buff2));
    }
    else
      mi_check_print_warning(param,
			     "Size of indexfile is: %-8s      Should be: %s",
			     llstr(size,buff), llstr(skr,buff2));
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->key_file_length) >
      ulonglong2double(info->s->base.margin_key_file_length)*0.9)
    mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
			   llstr(info->state->key_file_length,buff),
			   llstr(info->s->base.max_key_file_length-1,buff));

Marc Alff's avatar
Marc Alff committed
360
  size= mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
unknown's avatar
unknown committed
361 362 363 364 365 366 367 368 369 370
  skr=(my_off_t) info->state->data_file_length;
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    skr+= MEMMAP_EXTRA_MARGIN;
#ifdef USE_RELOC
  if (info->data_file_type == STATIC_RECORD &&
      skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
    skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
#endif
  if (skr != size)
  {
371
    info->state->data_file_length=size;	/* Skip other errors */
unknown's avatar
unknown committed
372 373 374
    if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
    {
      error=1;
375
      mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
unknown's avatar
unknown committed
376
		    llstr(size,buff), llstr(skr,buff2));
unknown's avatar
unknown committed
377
      param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
378 379 380 381
    }
    else
    {
      mi_check_print_warning(param,
382
			     "Size of datafile is: %-9s       Should be: %s",
unknown's avatar
unknown committed
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
			     llstr(size,buff), llstr(skr,buff2));
    }
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->data_file_length) >
      (ulonglong2double(info->s->base.max_data_file_length)*0.9))
    mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
			   llstr(info->state->data_file_length,buff),
			   llstr(info->s->base.max_data_file_length-1,buff2));
  DBUG_RETURN(error);
} /* chk_size */


	/* Check keys */

int chk_key(MI_CHECK *param, register MI_INFO *info)
{
401
  uint key,found_keys=0,full_text_keys=0,result=0;
unknown's avatar
unknown committed
402 403 404 405 406 407 408 409 410
  ha_rows keys;
  ha_checksum old_record_checksum,init_checksum;
  my_off_t all_keydata,all_totaldata,key_totlength,length;
  ulong   *rec_per_key_part;
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *keyinfo;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_key");

411 412 413 414
  if (!(param->testflag & T_SILENT))
    puts("- check key delete-chain");

  param->key_file_blocks=info->s->base.keystart;
415
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
416 417 418 419 420 421 422
    if (check_k_link(param,info,key))
    {
      if (param->testflag & T_VERBOSE) puts("");
      mi_check_print_error(param,"key delete-link-chain corrupted");
      DBUG_RETURN(-1);
    }

unknown's avatar
unknown committed
423 424 425 426 427 428 429 430 431 432 433 434 435 436
  if (!(param->testflag & T_SILENT)) puts("- check index reference");

  all_keydata=all_totaldata=key_totlength=0;
  old_record_checksum=0;
  init_checksum=param->record_checksum;
  if (!(share->options &
	(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
    old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
      share->base.pack_reclength;
  rec_per_key_part= param->rec_per_key_part;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
  {
    param->key_crc[key]=0;
437
    if (! mi_is_key_active(share->state.key_map, key))
438 439 440
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
441 442
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
443
	     keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
444
      continue;
445
    }
unknown's avatar
unknown committed
446 447 448
    found_keys++;

    param->record_checksum=init_checksum;
449
    
unknown's avatar
unknown committed
450
    bzero((char*) &param->unique_count,sizeof(param->unique_count));
451 452
    bzero((char*) &param->notnull_count,sizeof(param->notnull_count));

unknown's avatar
unknown committed
453 454
    if ((!(param->testflag & T_SILENT)))
      printf ("- check data record references index: %d\n",key+1);
unknown's avatar
unknown committed
455
    if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
unknown's avatar
unknown committed
456
      full_text_keys++;
unknown's avatar
unknown committed
457
    if (share->state.key_root[key] == HA_OFFSET_ERROR &&
458
	(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
459
      goto do_stat;
460 461
    if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
                           DFLT_INIT_HITS,info->buff,0))
unknown's avatar
unknown committed
462 463 464
    {
      mi_check_print_error(param,"Can't read indexpage from filepos: %s",
		  llstr(share->state.key_root[key],buff));
465 466 467 468
      if (!(param->testflag & T_INFO))
	DBUG_RETURN(-1);
      result= -1;
      continue;
unknown's avatar
unknown committed
469 470 471 472 473 474 475 476 477
    }
    param->key_file_blocks+=keyinfo->block_length;
    keys=0;
    param->keydata=param->totaldata=0;
    param->key_blocks=0;
    param->max_level=0;
    if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
		  &keys, param->key_crc+key,1))
      DBUG_RETURN(-1);
478
    if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
479 480 481 482 483
    {
      if (keys != info->state->records)
      {
	mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
		    llstr(info->state->records,buff2));
484
	if (!(param->testflag & T_INFO))
unknown's avatar
unknown committed
485
	DBUG_RETURN(-1);
486 487
	result= -1;
	continue;
unknown's avatar
unknown committed
488 489 490 491 492 493 494 495 496 497 498 499 500
      }
      if (found_keys - full_text_keys == 1 &&
	  ((share->options &
	    (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
	   (param->testflag & T_DONT_CHECK_CHECKSUM)))
	old_record_checksum=param->record_checksum;
      else if (old_record_checksum != param->record_checksum)
      {
	if (key)
	  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
		      key+1);
	else
	  mi_check_print_error(param,"Key 1 doesn't point at all records");
501 502 503 504
	if (!(param->testflag & T_INFO))
	  DBUG_RETURN(-1);
	result= -1;
	continue;
unknown's avatar
unknown committed
505 506 507 508
      }
    }
    if ((uint) share->base.auto_key -1 == key)
    {
unknown's avatar
unknown committed
509
      /* Check that auto_increment key is bigger than max key value */
510
      ulonglong auto_increment;
unknown's avatar
unknown committed
511 512
      info->lastinx=key;
      _mi_read_key_record(info, 0L, info->rec_buff);
513 514
      auto_increment= retrieve_auto_increment(info, info->rec_buff);
      if (auto_increment > info->s->state.auto_increment)
unknown's avatar
unknown committed
515
      {
516 517 518 519
        mi_check_print_warning(param, "Auto-increment value: %s is smaller "
                               "than max used value: %s",
                               llstr(info->s->state.auto_increment,buff2),
                               llstr(auto_increment, buff));
unknown's avatar
unknown committed
520 521 522
      }
      if (param->testflag & T_AUTO_INC)
      {
523 524 525 526
        set_if_bigger(info->s->state.auto_increment,
                      auto_increment);
        set_if_bigger(info->s->state.auto_increment,
                      param->auto_increment_value);
unknown's avatar
unknown committed
527
      }
unknown's avatar
unknown committed
528 529

      /* Check that there isn't a row with auto_increment = 0 in the table */
unknown's avatar
unknown committed
530
      mi_extra(info,HA_EXTRA_KEYREAD,0);
unknown's avatar
unknown committed
531
      bzero(info->lastkey,keyinfo->seg->length);
532
      if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
unknown's avatar
unknown committed
533
		   (key_part_map)1, HA_READ_KEY_EXACT))
unknown's avatar
unknown committed
534 535 536
      {
	/* Don't count this as a real warning, as myisamchk can't correct it */
	uint save=param->warning_printed;
537 538
        mi_check_print_warning(param, "Found row where the auto_increment "
                               "column has the value 0");
unknown's avatar
unknown committed
539 540
	param->warning_printed=save;
      }
unknown's avatar
unknown committed
541
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
unknown's avatar
unknown committed
542 543 544 545 546 547 548 549 550 551 552 553
    }

    length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
    if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
      printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
	     key+1,
	     (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
	     (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
		    my_off_t2double(length)),
	     param->max_level);
    all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;

554
do_stat:
unknown's avatar
unknown committed
555 556
    if (param->testflag & T_STATISTICS)
      update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
557 558 559
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       param->notnull_count: NULL, 
                       (ulonglong)info->state->records);
unknown's avatar
unknown committed
560 561 562 563 564 565 566 567 568 569
  }
  if (param->testflag & T_INFO)
  {
    if (all_totaldata != 0L && found_keys > 0)
      printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
	     (int) (my_off_t2double(all_keydata)*100.0/
		    my_off_t2double(all_totaldata)),
	     (int) ((my_off_t2double(key_totlength) -
		     my_off_t2double(all_keydata))*100.0/
		     my_off_t2double(key_totlength)));
570
    else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
unknown's avatar
unknown committed
571 572 573 574 575 576 577 578 579
      puts("");
  }
  if (param->key_file_blocks != info->state->key_file_length &&
      param->keys_in_use != ~(ulonglong) 0)
    mi_check_print_warning(param, "Some data are unreferenced in keyfile");
  if (found_keys != full_text_keys)
    param->record_checksum=old_record_checksum-init_checksum;	/* Remove delete links */
  else
    param->record_checksum=0;
unknown's avatar
unknown committed
580
  DBUG_RETURN(result);
unknown's avatar
unknown committed
581 582
} /* chk_key */

unknown's avatar
unknown committed
583

584 585 586 587 588
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
                     my_off_t page, uchar *buff, ha_rows *keys,
                     ha_checksum *key_checksum, uint level)
{
  char llbuff[22],llbuff2[22];
589
  DBUG_ENTER("chk_index_down");
590

591 592 593 594 595
  /* Key blocks must lay within the key file length entirely. */
  if (page + keyinfo->block_length > info->state->key_file_length)
  {
    /* purecov: begin tested */
    /* Give it a chance to fit in the real file size. */
Marc Alff's avatar
Marc Alff committed
596 597
    my_off_t max_length= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END,
                                         MYF(MY_THREADSAFE));
598 599 600 601 602
    mi_check_print_error(param, "Invalid key block position: %s  "
                         "key block size: %u  file_length: %s",
                         llstr(page, llbuff), keyinfo->block_length,
                         llstr(info->state->key_file_length, llbuff2));
    if (page + keyinfo->block_length > max_length)
603
      goto err;
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
    /* Fix the remebered key file length. */
    info->state->key_file_length= (max_length &
                                   ~ (my_off_t) (keyinfo->block_length - 1));
    /* purecov: end */
  }

  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
  if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
  {
    /* purecov: begin tested */
    mi_check_print_error(param, "Mis-aligned key block: %s  "
                         "minimum key block length: %u",
                         llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
    goto err;
    /* purecov: end */
619
  }
620

621 622 623 624 625 626 627 628 629 630
  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
  {
    mi_check_print_error(param,"Can't read key from filepos: %s",
        llstr(page,llbuff));
    goto err;
  }
  param->key_file_blocks+=keyinfo->block_length;
  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
    goto err;

631 632 633
  DBUG_RETURN(0);

  /* purecov: begin tested */
634
err:
635 636
  DBUG_RETURN(1);
  /* purecov: end */
637
}
unknown's avatar
unknown committed
638

639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659

/*
  "Ignore NULLs" statistics collection method: process first index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_first()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      key      IN     Key values tuple

  DESCRIPTION
    Process the first index tuple - find out which prefix tuples don't
    contain NULLs, and update the array of notnull counters accordingly.
*/

static
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
                                    uchar *key)
{
  uint first_null, kp;
660
  first_null= (uint) (ha_find_null(keyseg, key) - keyseg);
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
  /*
    All prefix tuples that don't include keypart_{first_null} are not-null
    tuples (and all others aren't), increment counters for them.
  */
  for (kp= 0; kp < first_null; kp++)
    notnull[kp]++;
}


/*
  "Ignore NULLs" statistics collection method: process next index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_next()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      prev_key IN     Previous key values tuple
      last_key IN     Next key values tuple

  DESCRIPTION
    Process the next index tuple:
    1. Find out which prefix tuples of last_key don't contain NULLs, and
       update the array of notnull counters accordingly.
685 686 687 688 689
    2. Find the first keypart number where the prev_key and last_key tuples
       are different(A), or last_key has NULL value(B), and return it, so the 
       caller can count number of unique tuples for each key prefix. We don't 
       need (B) to be counted, and that is compensated back in 
       update_key_parts().
690 691 692 693 694 695 696 697 698 699 700

  RETURN
    1 + number of first keypart where values differ or last_key tuple has NULL
*/

static
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
                                  uchar *prev_key, uchar *last_key)
{
  uint diffs[2];
  uint first_null_seg, kp;
701
  HA_KEYSEG *seg;
702

703 704 705 706 707 708 709 710
  /* 
     Find the first keypart where values are different or either of them is
     NULL. We get results in diffs array:
     diffs[0]= 1 + number of first different keypart
     diffs[1]=offset: (last_key + diffs[1]) points to first value in
                      last_key that is NULL or different from corresponding
                      value in prev_key.
  */
711
  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, 
712 713
             SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
  seg= keyseg + diffs[0] - 1;
714

715
  /* Find first NULL in last_key */
716
  first_null_seg= (uint) (ha_find_null(seg, last_key + diffs[1]) - keyseg);
717 718 719 720 721 722 723 724 725 726 727 728
  for (kp= 0; kp < first_null_seg; kp++)
    notnull[kp]++;

  /* 
    Return 1+ number of first key part where values differ. Don't care if
    these were NULLs and not .... We compensate for that in
    update_key_parts.
  */
  return diffs[0];
}


unknown's avatar
unknown committed
729 730 731 732 733 734 735
	/* Check if index is ok */

static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level)
{
  int flag;
736
  uint used_length,comp_flag,nod_flag,key_length=0;
737
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
unknown's avatar
unknown committed
738
  my_off_t next_page,record;
unknown's avatar
unknown committed
739
  char llbuff[22];
740
  uint diff_pos[2];
unknown's avatar
unknown committed
741
  DBUG_ENTER("chk_index");
742
  DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
unknown's avatar
unknown committed
743

744 745 746 747
  /* TODO: implement appropriate check for RTree keys */
  if (keyinfo->flag & HA_SPATIAL)
    DBUG_RETURN(0);

unknown's avatar
unknown committed
748 749
  if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
750
    mi_check_print_error(param,"Not enough memory for keyblock");
unknown's avatar
unknown committed
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
    DBUG_RETURN(-1);
  }

  if (keyinfo->flag & HA_NOSAME)
    comp_flag=SEARCH_FIND | SEARCH_UPDATE;	/* Not real duplicates */
  else
    comp_flag=SEARCH_SAME;			/* Keys in positionorder */
  nod_flag=mi_test_if_nod(buff);
  used_length=mi_getint(buff);
  keypos=buff+2+nod_flag;
  endpos=buff+used_length;

  param->keydata+=used_length; param->totaldata+=keyinfo->block_length;	/* INFO */
  param->key_blocks++;
  if (level > param->max_level)
    param->max_level=level;

  if (used_length > keyinfo->block_length)
  {
unknown's avatar
unknown committed
770 771
    mi_check_print_error(param,"Wrong pageinfo at page: %s",
			 llstr(page,llbuff));
unknown's avatar
unknown committed
772 773 774 775
    goto err;
  }
  for ( ;; )
  {
unknown's avatar
unknown committed
776 777
    if (*killed_ptr(param))
      goto err;
778 779
    memcpy((char*) info->lastkey,(char*) key,key_length);
    info->lastkey_length=key_length;
unknown's avatar
unknown committed
780 781 782
    if (nod_flag)
    {
      next_page=_mi_kpos(nod_flag,keypos);
783 784
      if (chk_index_down(param,info,keyinfo,next_page,
                         temp_buff,keys,key_checksum,level+1))
unknown's avatar
unknown committed
785 786 787 788 789 790 791 792 793 794 795 796
	goto err;
    }
    old_keypos=keypos;
    if (keypos >= endpos ||
	(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
      break;
    if (keypos > endpos)
    {
      mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
      goto err;
    }
    if ((*keys)++ &&
unknown's avatar
unknown committed
797
	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
798
			 comp_flag, diff_pos)) >=0)
unknown's avatar
unknown committed
799
    {
800 801
      DBUG_DUMP("old",(uchar*) info->lastkey, info->lastkey_length);
      DBUG_DUMP("new",(uchar*) key, key_length);
802
      DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
unknown's avatar
unknown committed
803 804 805 806 807 808 809 810 811

      if (comp_flag & SEARCH_FIND && flag == 0)
	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
      else
	mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
      goto err;
    }
    if (param->testflag & T_STATISTICS)
    {
812
      if (*keys != 1L)				/* not first_key */
unknown's avatar
unknown committed
813
      {
814 815 816
        if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
          ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
                     SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
817
                     diff_pos);
818 819
        else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
        {
820
          diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, 
821 822 823
                                                  param->notnull_count,
                                                  info->lastkey, key);
        }
824
	param->unique_count[diff_pos[0]-1]++;
unknown's avatar
unknown committed
825
      }
826 827 828 829 830
      else
      {  
        if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
          mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
                                         key);
unknown's avatar
unknown committed
831 832
      }
    }
833
    (*key_checksum)+= mi_byte_checksum((uchar*) key,
unknown's avatar
unknown committed
834 835
				       key_length- info->s->rec_reflength);
    record= _mi_dpos(info,0,key+key_length);
836 837 838 839 840 841 842 843 844 845 846 847
    if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
    {
      uint off;
      int  subkeys;
      get_key_full_length_rdonly(off, key);
      subkeys=ft_sintXkorr(key+off);
      if (subkeys < 0)
      {
        ha_rows tmp_keys=0;
        if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
                           temp_buff,&tmp_keys,key_checksum,1))
          goto err;
848 849
        if (tmp_keys + subkeys)
        {
unknown's avatar
unknown committed
850 851 852 853 854
          mi_check_print_error(param,
                               "Number of words in the 2nd level tree "
                               "does not match the number in the header. "
                               "Parent word in on the page %s, offset %u",
                               llstr(page,llbuff), (uint) (old_keypos-buff));
855 856
          goto err;
        }
857 858 859 860 861
        (*keys)+=tmp_keys-1;
        continue;
      }
      /* fall through */
    }
unknown's avatar
unknown committed
862 863 864
    if (record >= info->state->data_file_length)
    {
#ifndef DBUG_OFF
unknown's avatar
unknown committed
865
      char llbuff2[22], llbuff3[22];
unknown's avatar
unknown committed
866 867 868 869 870
#endif
      mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
      DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
			 llstr(page,llbuff),llstr(record,llbuff2),
			 llstr(info->state->data_file_length,llbuff3)));
871
      DBUG_DUMP("key",(uchar*) key,key_length);
872
      DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
unknown's avatar
unknown committed
873 874 875 876 877 878 879 880 881 882
      goto err;
    }
    param->record_checksum+=(ha_checksum) record;
  }
  if (keypos != endpos)
  {
    mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
                llstr(page,llbuff), used_length, (keypos - buff));
    goto err;
  }
883
  my_afree((uchar*) temp_buff);
unknown's avatar
unknown committed
884 885
  DBUG_RETURN(0);
 err:
886
  my_afree((uchar*) temp_buff);
unknown's avatar
unknown committed
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
  DBUG_RETURN(1);
} /* chk_index */


	/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */

static ha_checksum calc_checksum(ha_rows count)
{
  ulonglong sum,a,b;
  DBUG_ENTER("calc_checksum");

  sum=0;
  a=count; b=count+1;
  if (a & 1)
    b>>=1;
  else
    a>>=1;
  while (b)
  {
    if (b & 1)
      sum+=a;
    a<<=1; b>>=1;
  }
  DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
  DBUG_RETURN((ha_checksum) sum);
} /* calc_checksum */


	/* Calc length of key in normal isam */

static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
{
  uint length;
unknown's avatar
unknown committed
920
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
  DBUG_ENTER("isam_key_length");

  length= info->s->rec_reflength;
  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
    length+= keyseg->length;

  DBUG_PRINT("exit",("length: %d",length));
  DBUG_RETURN(length);
} /* key_length */


	/* Check that record-link is ok */

int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
{
  int	error,got_error,flag;
937
  uint	key,UNINIT_VAR(left_length),b_type,field;
unknown's avatar
unknown committed
938
  ha_rows records,del_blocks;
939
  my_off_t used,empty,pos,splits,UNINIT_VAR(start_recpos),
unknown's avatar
unknown committed
940
	   del_length,link_used,start_block;
Staale Smedseng's avatar
Staale Smedseng committed
941
  uchar	*record= 0, *UNINIT_VAR(to);
unknown's avatar
unknown committed
942 943
  char llbuff[22],llbuff2[22],llbuff3[22];
  ha_checksum intern_record_checksum;
944
  ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
945 946 947 948 949 950 951 952 953 954 955 956 957
  my_bool static_row_size;
  MI_KEYDEF *keyinfo;
  MI_BLOCK_INFO block_info;
  DBUG_ENTER("chk_data_link");

  if (!(param->testflag & T_SILENT))
  {
    if (extend)
      puts("- check records and index references");
    else
      puts("- check record links");
  }

958
  if (!mi_alloc_rec_buff(info, -1, &record))
unknown's avatar
unknown committed
959
  {
960
    mi_check_print_error(param,"Not enough memory for record");
unknown's avatar
unknown committed
961 962 963 964 965 966
    DBUG_RETURN(-1);
  }
  records=del_blocks=0;
  used=link_used=splits=del_length=0;
  intern_record_checksum=param->glob_crc=0;
  got_error=error=0;
967
  empty=info->s->pack.header_length;
unknown's avatar
unknown committed
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983

  /* Check how to calculate checksum of rows */
  static_row_size=1;
  if (info->s->data_file_type == COMPRESSED_RECORD)
  {
    for (field=0 ; field < info->s->base.fields ; field++)
    {
      if (info->s->rec[field].base_type == FIELD_BLOB ||
	  info->s->rec[field].base_type == FIELD_VARCHAR)
      {
	static_row_size=0;
	break;
      }
    }
  }

984
  pos=my_b_tell(&param->read_cache);
unknown's avatar
unknown committed
985 986 987
  bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
  while (pos < info->state->data_file_length)
  {
unknown's avatar
unknown committed
988 989
    if (*killed_ptr(param))
      goto err2;
unknown's avatar
unknown committed
990 991
    switch (info->s->data_file_type) {
    case STATIC_RECORD:
992
      if (my_b_read(&param->read_cache,(uchar*) record,
unknown's avatar
unknown committed
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
		    info->s->base.pack_reclength))
	goto err;
      start_recpos=pos;
      pos+=info->s->base.pack_reclength;
      splits++;
      if (*record == '\0')
      {
	del_blocks++;
	del_length+=info->s->base.pack_reclength;
	continue;					/* Record removed */
      }
      param->glob_crc+= mi_static_checksum(info,record);
      used+=info->s->base.pack_reclength;
      break;
    case DYNAMIC_RECORD:
      flag=block_info.second_read=0;
      block_info.next_filepos=pos;
      do
      {
1012
	if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header,
1013 1014 1015
			   (start_block=block_info.next_filepos),
			   sizeof(block_info.header),
			   (flag ? 0 : READING_NEXT) | READING_HEADER))
unknown's avatar
unknown committed
1016 1017 1018
	  goto err;
	if (start_block & (MI_DYN_ALIGN_SIZE-1))
	{
1019 1020
	  mi_check_print_error(param,"Wrong aligned block at %s",
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
	  goto err2;
	}
	b_type=_mi_get_block_info(&block_info,-1,start_block);
	if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
		      BLOCK_FATAL_ERROR))
	{
	  if (b_type & BLOCK_SYNC_ERROR)
	  {
	    if (flag)
	    {
	      mi_check_print_error(param,"Unexpected byte: %d at link: %s",
			  (int) block_info.header[0],
			  llstr(start_block,llbuff));
	      goto err2;
	    }
	    pos=block_info.filepos+block_info.block_len;
	    goto next;
	  }
	  if (b_type & BLOCK_DELETED)
	  {
	    if (block_info.block_len < info->s->base.min_block_length)
	    {
1043 1044 1045
	      mi_check_print_error(param,
				   "Deleted block with impossible length %lu at %s",
				   block_info.block_len,llstr(pos,llbuff));
unknown's avatar
unknown committed
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
	      goto err2;
	    }
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >= info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      mi_check_print_error(param,"Delete link points outside datafile at %s",
			  llstr(pos,llbuff));
	      goto err2;
	    }
	    del_blocks++;
	    del_length+=block_info.block_len;
	    pos=block_info.filepos+block_info.block_len;
	    splits++;
	    goto next;
	  }
	  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1064 1065 1066
			       block_info.header[0],block_info.header[1],
			       block_info.header[2],
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1067 1068 1069 1070 1071
	  goto err2;
	}
	if (info->state->data_file_length < block_info.filepos+
	    block_info.block_len)
	{
1072 1073 1074
	  mi_check_print_error(param,
			       "Recordlink that points outside datafile at %s",
			       llstr(pos,llbuff));
unknown's avatar
unknown committed
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	  got_error=1;
	  break;
	}
	splits++;
	if (!flag++)				/* First block */
	{
	  start_recpos=pos;
	  pos=block_info.filepos+block_info.block_len;
	  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
	  {
1085 1086 1087
	    mi_check_print_error(param,"Found too long record (%lu) at %s",
				 (ulong) block_info.rec_len,
				 llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1088 1089 1090 1091 1092
	    got_error=1;
	    break;
	  }
	  if (info->s->base.blobs)
	  {
1093
	    if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1094
					&info->rec_buff)))
unknown's avatar
unknown committed
1095
	    {
1096 1097 1098 1099
	      mi_check_print_error(param,
				   "Not enough memory (%lu) for blob at %s",
				   (ulong) block_info.rec_len,
				   llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
	      got_error=1;
	      break;
	    }
	  }
	  else
	    to= info->rec_buff;
	  left_length=block_info.rec_len;
	}
	if (left_length < block_info.data_len)
	{
1110 1111 1112 1113 1114
	  mi_check_print_error(param,"Found too long record (%lu) at %s",
			       (ulong) block_info.data_len,
			       llstr(start_recpos,llbuff));
	  got_error=1;
	  break;
unknown's avatar
unknown committed
1115
	}
1116
	if (_mi_read_cache(&param->read_cache,(uchar*) to,block_info.filepos,
1117 1118
			   (uint) block_info.data_len,
			   flag == 1 ? READING_NEXT : 0))
unknown's avatar
unknown committed
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
	  goto err;
	to+=block_info.data_len;
	link_used+= block_info.filepos-start_block;
	used+= block_info.filepos - start_block + block_info.data_len;
	empty+=block_info.block_len-block_info.data_len;
	left_length-=block_info.data_len;
	if (left_length)
	{
	  if (b_type & BLOCK_LAST)
	  {
1129 1130 1131 1132 1133
	    mi_check_print_error(param,
				 "Wrong record length %s of %s at %s",
				 llstr(block_info.rec_len-left_length,llbuff),
				 llstr(block_info.rec_len, llbuff2),
				 llstr(start_recpos,llbuff3));
unknown's avatar
unknown committed
1134 1135 1136 1137 1138
	    got_error=1;
	    break;
	  }
	  if (info->state->data_file_length < block_info.next_filepos)
	  {
1139 1140 1141
	    mi_check_print_error(param,
				 "Found next-recordlink that points outside datafile at %s",
				 llstr(block_info.filepos,llbuff));
unknown's avatar
unknown committed
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	    got_error=1;
	    break;
	  }
	}
      } while (left_length);
      if (! got_error)
      {
	if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
	    MY_FILE_ERROR)
	{
1152 1153
	  mi_check_print_error(param,"Found wrong record at %s",
			       llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1154 1155 1156 1157 1158 1159 1160
	  got_error=1;
	}
	else
	{
	  info->checksum=mi_checksum(info,record);
	  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
	  {
1161 1162
	    if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
                              test(info->s->calc_checksum)))
unknown's avatar
unknown committed
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
	    {
	      mi_check_print_error(param,"Found wrong packed record at %s",
			  llstr(start_recpos,llbuff));
	      got_error=1;
	    }
	  }
	  if (!got_error)
	    param->glob_crc+= info->checksum;
	}
      }
      else if (!flag)
	pos=block_info.filepos+block_info.block_len;
      break;
    case COMPRESSED_RECORD:
1177
      if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header, pos,
1178
			 info->s->pack.ref_length, READING_NEXT))
unknown's avatar
unknown committed
1179 1180 1181
	goto err;
      start_recpos=pos;
      splits++;
Konstantin Osipov's avatar
Konstantin Osipov committed
1182 1183
      (void) _mi_pack_get_block_info(info, &info->bit_buff, &block_info,
                                   &info->rec_buff, -1, start_recpos);
unknown's avatar
unknown committed
1184 1185 1186 1187
      pos=block_info.filepos+block_info.rec_len;
      if (block_info.rec_len < (uint) info->s->min_pack_length ||
	  block_info.rec_len > (uint) info->s->max_pack_length)
      {
unknown's avatar
unknown committed
1188 1189 1190
	mi_check_print_error(param,
			     "Found block with wrong recordlength: %d at %s",
			     block_info.rec_len, llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1191 1192 1193
	got_error=1;
	break;
      }
1194
      if (_mi_read_cache(&param->read_cache,(uchar*) info->rec_buff,
1195
			block_info.filepos, block_info.rec_len, READING_NEXT))
unknown's avatar
unknown committed
1196
	goto err;
1197 1198
      if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
                              info->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
1199
      {
unknown's avatar
unknown committed
1200 1201
	mi_check_print_error(param,"Found wrong record at %s",
			     llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1202 1203 1204 1205 1206 1207 1208 1209
	got_error=1;
      }
      if (static_row_size)
	param->glob_crc+= mi_static_checksum(info,record);
      else
	param->glob_crc+= mi_checksum(info,record);
      link_used+= (block_info.filepos - start_recpos);
      used+= (pos-start_recpos);
1210
      break;
1211 1212
    case BLOCK_RECORD:
      assert(0);                                /* Impossible */
unknown's avatar
unknown committed
1213 1214 1215 1216 1217 1218 1219
    } /* switch */
    if (! got_error)
    {
      intern_record_checksum+=(ha_checksum) start_recpos;
      records++;
      if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
      {
Konstantin Osipov's avatar
Konstantin Osipov committed
1220
	printf("%s\r", llstr(records,llbuff)); (void) fflush(stdout);
unknown's avatar
unknown committed
1221 1222 1223 1224 1225 1226 1227
      }

      /* Check if keys match the record */

      for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
	   key++,keyinfo++)
      {
1228
        if (mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1229
	{
1230
          if(!(keyinfo->flag & HA_FULLTEXT))
unknown's avatar
unknown committed
1231 1232 1233 1234 1235 1236 1237 1238
	  {
	    uint key_length=_mi_make_key(info,key,info->lastkey,record,
					 start_recpos);
	    if (extend)
	    {
	      /* We don't need to lock the key tree here as we don't allow
		 concurrent threads when running myisamchk
	      */
1239 1240 1241
              int search_result=
#ifdef HAVE_RTREE_KEYS
                (keyinfo->flag & HA_SPATIAL) ?
1242
                rtree_find_first(info, key, info->lastkey, key_length,
1243
                                 MBR_EQUAL | MBR_DATA) : 
1244
#endif
1245 1246 1247 1248
                _mi_search(info,keyinfo,info->lastkey,key_length,
                           SEARCH_SAME, info->s->state.key_root[key]);
              if (search_result)
              {
1249 1250
                mi_check_print_error(param,"Record at: %10s  "
                                     "Can't find key for index: %2d",
1251 1252 1253 1254
                                     llstr(start_recpos,llbuff),key+1);
                if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                  goto err2;
              }
unknown's avatar
unknown committed
1255 1256
	    }
	    else
1257
	      key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
unknown's avatar
unknown committed
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
						  key_length);
	  }
	}
      }
    }
    else
    {
      got_error=0;
      if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
	goto err2;
    }
  next:;				/* Next record */
  }
  if (param->testflag & T_WRITE_LOOP)
  {
Konstantin Osipov's avatar
Konstantin Osipov committed
1273
    (void) fputs("          \r",stdout); (void) fflush(stdout);
unknown's avatar
unknown committed
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
  }
  if (records != info->state->records)
  {
    mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
		llstr(records,llbuff), llstr(info->state->records,llbuff2));
    error=1;
  }
  else if (param->record_checksum &&
	   param->record_checksum != intern_record_checksum)
  {
    mi_check_print_error(param,
			 "Keypointers and record positions doesn't match");
    error=1;
  }
1288
  else if (param->glob_crc != info->state->checksum &&
unknown's avatar
unknown committed
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	   (info->s->options &
	    (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
  {
    mi_check_print_warning(param,
			   "Record checksum is not the same as checksum stored in the index file\n");
    error=1;
  }
  else if (!extend)
  {
    for (key=0 ; key < info->s->base.keys;  key++)
    {
      if (key_checksum[key] != param->key_crc[key] &&
1301
          !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
1302 1303 1304 1305 1306 1307 1308 1309
      {
	mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
		    key+1);
	error=1;
      }
    }
  }

1310 1311 1312 1313 1314 1315 1316
  if (del_length != info->state->empty)
  {
    mi_check_print_warning(param,
			   "Found %s deleted space.   Should be %s",
			   llstr(del_length,llbuff2),
			   llstr(info->state->empty,llbuff));
  }
unknown's avatar
unknown committed
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
  if (used+empty+del_length != info->state->data_file_length)
  {
    mi_check_print_warning(param,
			   "Found %s record-data and %s unused data and %s deleted-data",
			   llstr(used,llbuff),llstr(empty,llbuff2),
			   llstr(del_length,llbuff3));
    mi_check_print_warning(param,
			   "Total %s, Should be: %s",
			   llstr((used+empty+del_length),llbuff),
			   llstr(info->state->data_file_length,llbuff2));
  }
  if (del_blocks != info->state->del)
  {
    mi_check_print_warning(param,
			   "Found %10s deleted blocks       Should be: %s",
			   llstr(del_blocks,llbuff),
			   llstr(info->state->del,llbuff2));
  }
  if (splits != info->s->state.split)
  {
    mi_check_print_warning(param,
1338
			   "Found %10s key parts. Should be: %s",
unknown's avatar
unknown committed
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
			   llstr(splits,llbuff),
			   llstr(info->s->state.split,llbuff2));
  }
  if (param->testflag & T_INFO)
  {
    if (param->warning_printed || param->error_printed)
      puts("");
    if (used != 0 && ! param->error_printed)
    {
      printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
unknown's avatar
unknown committed
1349
	     llstr(records,llbuff), (long)((used-link_used)/records),
unknown's avatar
unknown committed
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
	     (info->s->base.blobs ? 0.0 :
	      (ulonglong2double((ulonglong) info->s->base.reclength*records)-
	       my_off_t2double(used))/
	      ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
      printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
	     (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
	     (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
				      my_off_t2double(used)*100.0)),
	     ulonglong2double(splits - del_blocks) / records);
    }
    printf("Record blocks:%12s    Delete blocks:%10s\n",
	   llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
    printf("Record data:  %12s    Deleted data: %10s\n",
	   llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
    printf("Lost space:   %12s    Linkdata:     %10s\n",
	   llstr(empty,llbuff),llstr(link_used,llbuff2));
  }
1367
  my_free(mi_get_rec_buff_ptr(info, record));
unknown's avatar
unknown committed
1368 1369
  DBUG_RETURN (error);
 err:
unknown's avatar
unknown committed
1370
  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
unknown's avatar
unknown committed
1371
 err2:
1372
  my_free(mi_get_rec_buff_ptr(info, record));
unknown's avatar
unknown committed
1373
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1374 1375 1376 1377
  DBUG_RETURN(1);
} /* chk_data_link */


1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
/**
  @brief Drop all indexes

  @param[in]    param           check parameters
  @param[in]    info            MI_INFO handle
  @param[in]    force           if to force drop all indexes

  @return       status
    @retval     0               OK
    @retval     != 0            Error

  @note
    Once allocated, index blocks remain part of the key file forever.
    When indexes are disabled, no block is freed. When enabling indexes,
    no block is freed either. The new indexes are create from new
    blocks. (Bug #4692)

    Before recreating formerly disabled indexes, the unused blocks
    must be freed. There are two options to do this:
    - Follow the tree of disabled indexes, add all blocks to the
      deleted blocks chain. Would require a lot of random I/O.
    - Drop all blocks by clearing all index root pointers and all
      delete chain pointers and resetting key_file_length to the end
      of the index file header. This requires to recreate all indexes,
      even those that may still be intact.
    The second method is probably faster in most cases.

    When disabling indexes, MySQL disables either all indexes or all
    non-unique indexes. When MySQL [re-]enables disabled indexes
    (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
    index file, or there are no non-unique indexes. In the latter case,
    mi_repair*() would not be called as there would be no disabled
    indexes.

    If there would be more unique indexes than disabled (non-unique)
    indexes, we could do the first method. But this is not implemented
    yet. By now we drop and recreate all indexes when repair is called.

    However, there is an exception. Sometimes MySQL disables non-unique
    indexes when the table is empty (e.g. when copying a table in
    mysql_alter_table()). When enabling the non-unique indexes, they
    are still empty. So there is no index block that can be lost. This
    optimization is implemented in this function.

    Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
    recreate all enabled indexes unconditonally. We do not change the
    key_map. Otherwise we invert the key map temporarily (outside of
    this function) and recreate the then "seemingly" enabled indexes.
    When we cannot use the optimization, and drop all indexes, we
    pretend that all indexes were disabled. By the inversion, we will
    then recrate all indexes.
*/

static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force)
{
  MYISAM_SHARE *share= info->s;
  MI_STATE_INFO *state= &share->state;
  uint i;
  int error;
  DBUG_ENTER("mi_drop_all_indexes");

  /*
    If any of the disabled indexes has a key block assigned, we must
    drop and recreate all indexes to avoid losing index blocks.

    If we want to recreate disabled indexes only _and_ all of these
    indexes are empty, we don't need to recreate the existing indexes.
  */
  if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
  {
    DBUG_PRINT("repair", ("creating missing indexes"));
    for (i= 0; i < share->base.keys; i++)
    {
      DBUG_PRINT("repair", ("index #: %u  key_root: 0x%lx  active: %d",
                            i, (long) state->key_root[i],
                            mi_is_key_active(state->key_map, i)));
      if ((state->key_root[i] != HA_OFFSET_ERROR) &&
          !mi_is_key_active(state->key_map, i))
      {
        /*
          This index has at least one key block and it is disabled.
          We would lose its block(s) if would just recreate it.
          So we need to drop and recreate all indexes.
        */
        DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
        break;
      }
    }
    if (i >= share->base.keys)
    {
      /*
        All of the disabled indexes are empty. We can just recreate them.
        Flush dirty blocks of this index file from key cache and remove
        all blocks of this index file from key cache.
      */
      DBUG_PRINT("repair", ("all disabled are empty: create missing"));
      error= flush_key_blocks(share->key_cache, share->kfile,
                              FLUSH_FORCE_WRITE);
      goto end;
    }
    /*
      We do now drop all indexes and declare them disabled. With the
      T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
      disabled indexes and enable them.
    */
    mi_clear_all_keys_active(state->key_map);
    DBUG_PRINT("repair", ("declared all indexes disabled"));
  }

  /* Remove all key blocks of this index file from key cache. */
  if ((error= flush_key_blocks(share->key_cache, share->kfile,
                               FLUSH_IGNORE_CHANGED)))
1490
    goto end; /* purecov: inspected */
1491 1492 1493 1494 1495 1496

  /* Clear index root block pointers. */
  for (i= 0; i < share->base.keys; i++)
    state->key_root[i]= HA_OFFSET_ERROR;

  /* Clear the delete chains. */
1497
  for (i= 0; i < state->header.max_block_size_index; i++)
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
    state->key_del[i]= HA_OFFSET_ERROR;

  /* Reset index file length to end of index file header. */
  info->state->key_file_length= share->base.keystart;

  DBUG_PRINT("repair", ("dropped all indexes"));
  /* error= 0; set by last (error= flush_key_bocks()). */

 end:
  DBUG_RETURN(error);
}


unknown's avatar
unknown committed
1511 1512 1513 1514
	/* Recover old table by reading each record and writing all keys */
	/* Save new datafile-name in temp_filename */

int mi_repair(MI_CHECK *param, register MI_INFO *info,
1515
	      char * name, int rep_quick)
unknown's avatar
unknown committed
1516 1517 1518 1519 1520 1521 1522
{
  int error,got_error;
  ha_rows start_records,new_header_length;
  my_off_t del;
  File new_file;
  MYISAM_SHARE *share=info->s;
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
1523 1524
  SORT_INFO sort_info;
  MI_SORT_PARAM sort_param;
unknown's avatar
unknown committed
1525 1526
  DBUG_ENTER("mi_repair");

unknown's avatar
unknown committed
1527 1528
  bzero((char *)&sort_info, sizeof(sort_info));
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
1529
  start_records=info->state->records;
1530 1531
  new_header_length=(param->testflag & T_UNPACK) ? 0L :
    share->pack.header_length;
unknown's avatar
unknown committed
1532 1533
  got_error=1;
  new_file= -1;
unknown's avatar
unknown committed
1534
  sort_param.sort_info=&sort_info;
1535

unknown's avatar
unknown committed
1536 1537
  if (!(param->testflag & T_SILENT))
  {
1538
    printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
1539 1540
    printf("Data records: %s\n", llstr(info->state->records,llbuff));
  }
1541
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
1542

1543 1544 1545
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

1546 1547
  DBUG_ASSERT(param->use_buffers < SIZE_T_MAX);

unknown's avatar
unknown committed
1548
  if (!param->using_global_keycache)
Konstantin Osipov's avatar
Konstantin Osipov committed
1549 1550
    (void) init_key_cache(dflt_key_cache, param->key_cache_block_size,
                        param->use_buffers, 0, 0);
unknown's avatar
unknown committed
1551 1552 1553 1554

  if (init_io_cache(&param->read_cache,info->dfile,
		    (uint) param->read_buffer_length,
		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1555 1556
  {
    bzero(&info->rec_cache,sizeof(info->rec_cache));
unknown's avatar
unknown committed
1557
    goto err;
1558
  }
unknown's avatar
unknown committed
1559 1560 1561 1562 1563 1564
  if (!rep_quick)
    if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
		      WRITE_CACHE, new_header_length, 1,
		      MYF(MY_WME | MY_WAIT_IF_FULL)))
      goto err;
  info->opt_flag|=WRITE_CACHE_USED;
1565
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
unknown's avatar
unknown committed
1566
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
1567
  {
1568
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
1569 1570 1571 1572 1573
    goto err;
  }

  if (!rep_quick)
  {
1574
    /* Get real path for data file */
1575 1576 1577 1578 1579 1580
    if ((new_file= mysql_file_create(mi_key_file_datatmp,
                                     fn_format(param->temp_filename,
                                               share->data_file_name, "",
                                               DATA_TMP_EXT, 2+4),
                                     0, param->tmpfile_createflag,
                                     MYF(0))) < 0)
unknown's avatar
unknown committed
1581 1582 1583 1584 1585
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
1586 1587
    if (new_header_length &&
        filecopy(param,new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
		 "datafile-header"))
      goto err;
    info->s->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
  }
unknown's avatar
unknown committed
1598 1599 1600 1601 1602 1603
  sort_info.info=info;
  sort_info.param = param;
  sort_param.read_cache=param->read_cache;
  sort_param.pos=sort_param.max_pos=share->pack.header_length;
  sort_param.filepos=new_header_length;
  param->read_cache.end_of_file=sort_info.filelength=
Marc Alff's avatar
Marc Alff committed
1604
    mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
unknown's avatar
unknown committed
1605 1606
  sort_info.dupp=0;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
1607
  sort_param.master=1;
unknown's avatar
unknown committed
1608
  sort_info.max_records= ~(ha_rows) 0;
unknown's avatar
unknown committed
1609

unknown's avatar
unknown committed
1610
  set_data_file_type(&sort_info, share);
unknown's avatar
unknown committed
1611 1612 1613
  del=info->state->del;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;
1614 1615
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
1616
    sort_param.calc_checksum= 1;
1617

unknown's avatar
unknown committed
1618
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1619

1620
  /* This function always recreates all enabled indexes. */
1621
  if (param->testflag & T_CREATE_MISSING_KEYS)
1622
    mi_set_all_keys_active(share->state.key_map, share->base.keys);
1623
  mi_drop_all_indexes(param, info, TRUE);
unknown's avatar
unknown committed
1624 1625

  lock_memory(param);			/* Everything is alloced */
1626 1627

  /* Re-create all keys, which are set in key_map. */
unknown's avatar
unknown committed
1628
  while (!(error=sort_get_next_record(&sort_param)))
unknown's avatar
unknown committed
1629
  {
1630
    if (writekeys(&sort_param))
unknown's avatar
unknown committed
1631
    {
1632 1633
      if (my_errno != HA_ERR_FOUND_DUPP_KEY)
	goto err;
1634
      DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
unknown's avatar
unknown committed
1635
      mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1636
			  info->errkey+1,
unknown's avatar
unknown committed
1637
			  llstr(sort_param.start_recpos,llbuff),
1638
			  llstr(info->dupp_key_pos,llbuff2));
unknown's avatar
unknown committed
1639 1640
      if (param->testflag & T_VERBOSE)
      {
Konstantin Osipov's avatar
Konstantin Osipov committed
1641 1642
	(void) _mi_make_key(info,(uint) info->errkey,info->lastkey,
			  sort_param.record,0L);
unknown's avatar
unknown committed
1643 1644 1645
	_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
		      USE_WHOLE_KEY);
      }
unknown's avatar
unknown committed
1646
      sort_info.dupp++;
1647
      if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
1648
      {
unknown's avatar
unknown committed
1649 1650
        param->testflag|=T_RETRY_WITHOUT_QUICK;
	param->error_printed=1;
unknown's avatar
unknown committed
1651 1652 1653 1654
	goto err;
      }
      continue;
    }
unknown's avatar
unknown committed
1655
    if (sort_write_record(&sort_param))
unknown's avatar
unknown committed
1656 1657
      goto err;
  }
unknown's avatar
unknown committed
1658
  if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
unknown's avatar
unknown committed
1659 1660 1661 1662 1663
      flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
    goto err;

  if (param->testflag & T_WRITE_LOOP)
  {
Konstantin Osipov's avatar
Konstantin Osipov committed
1664
    (void) fputs("          \r",stdout); (void) fflush(stdout);
unknown's avatar
unknown committed
1665
  }
Marc Alff's avatar
Marc Alff committed
1666
  if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
unknown's avatar
unknown committed
1667 1668 1669 1670 1671 1672 1673
  {
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);
    goto err;
  }

unknown's avatar
unknown committed
1674
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
1675 1676 1677 1678
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
1679 1680
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1681 1682
    goto err;
  }
1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
  if (param->testflag & T_SAFE_REPAIR)
  {
    /* Don't repair if we loosed more than one row */
    if (info->state->records+1 < start_records)
    {
      info->state->records=start_records;
      got_error=1;
      goto err;
    }
  }
unknown's avatar
unknown committed
1693 1694 1695

  if (!rep_quick)
  {
Marc Alff's avatar
Marc Alff committed
1696
    mysql_file_close(info->dfile, MYF(0));
unknown's avatar
unknown committed
1697
    info->dfile=new_file;
unknown's avatar
unknown committed
1698
    info->state->data_file_length=sort_param.filepos;
unknown's avatar
unknown committed
1699 1700 1701
    share->state.version=(ulong) time((time_t*) 0);	/* Force reopen */
  }
  else
1702
  {
unknown's avatar
unknown committed
1703
    info->state->data_file_length=sort_param.max_pos;
1704
  }
unknown's avatar
unknown committed
1705
  if (param->testflag & T_CALC_CHECKSUM)
1706
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
1707 1708 1709 1710 1711

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
1712
    if (sort_info.dupp)
unknown's avatar
unknown committed
1713 1714
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
1715
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
1716 1717 1718 1719 1720 1721 1722 1723
  }

  got_error=0;
  /* If invoked by external program that uses thr_lock */
  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
1724 1725 1726 1727 1728
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
1729
      mysql_file_close(new_file, MYF(0));
1730
      info->dfile=new_file= -1;
1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
      /*
        On Windows, the old data file cannot be deleted if it is either
        open, or memory mapped. Closing the file won't remove the memory
        map implicilty on Windows. We closed the data file, but we keep
        the MyISAM table open. A memory map will be closed on the final
        mi_close() only. So we need to unmap explicitly here. After
        renaming the new file under the hook, we couldn't use the map of
        the old file any more anyway.
      */
      if (info->s->file_map)
      {
        (void) my_munmap((char*) info->s->file_map,
                         (size_t) info->s->mmaped_length);
        info->s->file_map= NULL;
      }
1746
      if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT,
1747 1748
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
1749
	  mi_open_datafile(info,share,name,-1))
1750
	got_error=1;
1751 1752

      param->retry_repair= 0;
1753 1754
    }
  }
unknown's avatar
unknown committed
1755 1756 1757 1758
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d for record at pos %s",my_errno,
unknown's avatar
unknown committed
1759
		  llstr(sort_param.start_recpos,llbuff));
unknown's avatar
unknown committed
1760 1761
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
1762
      (void) mysql_file_close(new_file, MYF(0));
1763 1764
      (void) mysql_file_delete(mi_key_file_datatmp,
                               param->temp_filename, MYF(MY_WME));
1765
      info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
unknown's avatar
unknown committed
1766
    }
1767
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
1768
  }
1769 1770 1771
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
  my_free(mi_get_rec_buff_ptr(info, sort_param.record));
  my_free(sort_info.buff);
Konstantin Osipov's avatar
Konstantin Osipov committed
1772
  (void) end_io_cache(&param->read_cache);
unknown's avatar
unknown committed
1773
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
Konstantin Osipov's avatar
Konstantin Osipov committed
1774
  (void) end_io_cache(&info->rec_cache);
unknown's avatar
unknown committed
1775
  got_error|=flush_blocks(param, share->key_cache, share->kfile);
unknown's avatar
unknown committed
1776 1777 1778 1779
  if (!got_error && param->testflag & T_UNPACK)
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
unknown's avatar
unknown committed
1780
    share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
1781
  }
1782 1783
  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
			  STATE_NOT_ANALYZED);
unknown's avatar
unknown committed
1784
  DBUG_RETURN(got_error);
1785
}
unknown's avatar
unknown committed
1786 1787 1788 1789


/* Uppate keyfile when doing repair */

1790
static int writekeys(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
1791 1792
{
  register uint i;
1793 1794
  uchar    *key;
  MI_INFO  *info=   sort_param->sort_info->info;
1795
  uchar    *buff=   sort_param->record;
1796
  my_off_t filepos= sort_param->filepos;
unknown's avatar
unknown committed
1797 1798 1799 1800 1801
  DBUG_ENTER("writekeys");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys ; i++)
  {
1802
    if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1803
    {
1804 1805
      if (info->s->keyinfo[i].flag & HA_FULLTEXT )
      {
1806
        if (_mi_ft_add(info, i, key, buff, filepos))
1807
	  goto err;
1808
      }
1809
#ifdef HAVE_SPATIAL
1810 1811 1812 1813 1814 1815
      else if (info->s->keyinfo[i].flag & HA_SPATIAL)
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
	if (rtree_insert(info, i, key, key_length))
	  goto err;
      }
1816
#endif /*HAVE_SPATIAL*/
1817 1818 1819
      else
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1820 1821
	if (_mi_ck_write(info,i,key,key_length))
	  goto err;
1822
      }
unknown's avatar
unknown committed
1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
    }
  }
  DBUG_RETURN(0);

 err:
  if (my_errno == HA_ERR_FOUND_DUPP_KEY)
  {
    info->errkey=(int) i;			/* This key was found */
    while ( i-- > 0 )
    {
1833
      if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1834
      {
1835 1836
	if (info->s->keyinfo[i].flag & HA_FULLTEXT)
        {
1837
          if (_mi_ft_del(info,i, key,buff,filepos))
1838
	    break;
1839 1840 1841 1842
        }
        else
	{
	  uint key_length=_mi_make_key(info,i,key,buff,filepos);
1843 1844
	  if (_mi_ck_delete(info,i,key,key_length))
	    break;
1845
	}
unknown's avatar
unknown committed
1846 1847 1848
      }
    }
  }
1849
  /* Remove checksum that was added to glob_crc in sort_get_next_record */
1850 1851
  if (sort_param->calc_checksum)
    sort_param->sort_info->param->glob_crc-= info->checksum;
unknown's avatar
unknown committed
1852 1853 1854 1855 1856 1857 1858
  DBUG_PRINT("error",("errno: %d",my_errno));
  DBUG_RETURN(-1);
} /* writekeys */


	/* Change all key-pointers that points to a records */

1859
int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
unknown's avatar
unknown committed
1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
	      my_off_t newpos, uint prot_key)
{
  register uint i;
  uchar *key;
  uint key_length;
  DBUG_ENTER("movepoint");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys; i++)
  {
1870
    if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
    {
      key_length=_mi_make_key(info,i,key,record,oldpos);
      if (info->s->keyinfo[i].flag & HA_NOSAME)
      {					/* Change pointer direct */
	uint nod_flag;
	MI_KEYDEF *keyinfo;
	keyinfo=info->s->keyinfo+i;
	if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
		       (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
		       info->s->state.key_root[i]))
	  DBUG_RETURN(-1);
	nod_flag=mi_test_if_nod(info->buff);
	_mi_dpointer(info,info->int_keypos-nod_flag-
		     info->s->rec_reflength,newpos);
1885 1886
	if (_mi_write_keypage(info,keyinfo,info->last_keypage,
                              DFLT_INIT_HITS,info->buff))
unknown's avatar
unknown committed
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	  DBUG_RETURN(-1);
      }
      else
      {					/* Change old key to new */
	if (_mi_ck_delete(info,i,key,key_length))
	  DBUG_RETURN(-1);
	key_length=_mi_make_key(info,i,key,record,newpos);
	if (_mi_ck_write(info,i,key,key_length))
	  DBUG_RETURN(-1);
      }
    }
  }
  DBUG_RETURN(0);
} /* movepoint */


	/* Tell system that we want all memory for our cache */

void lock_memory(MI_CHECK *param __attribute__((unused)))
{
#ifdef SUN_OS				/* Key-cacheing thrases on sun 4.1 */
  if (param->opt_lock_memory)
  {
    int success = mlockall(MCL_CURRENT);	/* or plock(DATLOCK); */
    if (geteuid() == 0 && success != 0)
      mi_check_print_warning(param,
			     "Failed to lock memory. errno %d",my_errno);
  }
#endif
} /* lock_memory */


	/* Flush all changed blocks to disk */

unknown's avatar
unknown committed
1921
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
unknown's avatar
unknown committed
1922
{
1923
  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
unknown's avatar
unknown committed
1924 1925 1926 1927 1928
  {
    mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
    return(1);
  }
  if (!param->using_global_keycache)
1929
    end_key_cache(key_cache,1);
unknown's avatar
unknown committed
1930 1931 1932 1933 1934 1935
  return 0;
} /* flush_blocks */


	/* Sort index for more efficent reads */

1936
int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
unknown's avatar
unknown committed
1937 1938 1939 1940
{
  reg2 uint key;
  reg1 MI_KEYDEF *keyinfo;
  File new_file;
1941
  my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
1942
  uint r_locks,w_locks;
1943
  int old_lock;
unknown's avatar
unknown committed
1944
  MYISAM_SHARE *share=info->s;
1945
  MI_STATE_INFO old_state;
1946
  DBUG_ENTER("mi_sort_index");
unknown's avatar
unknown committed
1947

1948 1949 1950 1951
  /* cannot sort index files with R-tree indexes */
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       key++,keyinfo++)
    if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
unknown's avatar
unknown committed
1952
      DBUG_RETURN(0);
1953

unknown's avatar
unknown committed
1954 1955 1956
  if (!(param->testflag & T_SILENT))
    printf("- Sorting index for MyISAM-table '%s'\n",name);

1957 1958
  /* Get real path for index file */
  fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
Marc Alff's avatar
Marc Alff committed
1959 1960 1961 1962 1963
  if ((new_file= mysql_file_create(mi_key_file_datatmp,
                                   fn_format(param->temp_filename,
                                             param->temp_filename,
                                             "", INDEX_TMP_EXT, 2+4),
                                   0, param->tmpfile_createflag, MYF(0))) <= 0)
unknown's avatar
unknown committed
1964 1965 1966 1967 1968
  {
    mi_check_print_error(param,"Can't create new tempfile: '%s'",
			 param->temp_filename);
    DBUG_RETURN(-1);
  }
unknown's avatar
unknown committed
1969 1970
  if (filecopy(param, new_file,share->kfile,0L,
	       (ulong) share->base.keystart, "headerblock"))
unknown's avatar
unknown committed
1971 1972
    goto err;

unknown's avatar
unknown committed
1973 1974
  param->new_file_pos=share->base.keystart;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
unknown's avatar
unknown committed
1975 1976
       key++,keyinfo++)
  {
1977
    if (! mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1978 1979
      continue;

unknown's avatar
unknown committed
1980
    if (share->state.key_root[key] != HA_OFFSET_ERROR)
unknown's avatar
unknown committed
1981
    {
1982
      index_pos[key]=param->new_file_pos;	/* Write first block here */
unknown's avatar
unknown committed
1983
      if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
unknown's avatar
unknown committed
1984 1985 1986 1987 1988 1989 1990 1991
			 new_file))
	goto err;
    }
    else
      index_pos[key]= HA_OFFSET_ERROR;		/* No blocks */
  }

  /* Flush key cache for this file if we are calling this outside myisamchk */
unknown's avatar
unknown committed
1992
  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
1993

unknown's avatar
unknown committed
1994
  share->state.version=(ulong) time((time_t*) 0);
1995 1996 1997 1998 1999
  old_state= share->state;			/* save state if not stored */
  r_locks=   share->r_locks;
  w_locks=   share->w_locks;
  old_lock=  info->lock_type;

2000
	/* Put same locks as old file */
2001
  share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
2002
  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
Marc Alff's avatar
Marc Alff committed
2003
  (void) mysql_file_close(share->kfile, MYF(MY_WME));
unknown's avatar
unknown committed
2004
  share->kfile = -1;
Marc Alff's avatar
Marc Alff committed
2005
  (void) mysql_file_close(new_file, MYF(MY_WME));
2006
  if (change_to_newfile(share->index_file_name, MI_NAME_IEXT, INDEX_TMP_EXT,
2007
			MYF(0)) ||
unknown's avatar
unknown committed
2008
      mi_open_keyfile(share))
2009
    goto err2;
2010
  info->lock_type= F_UNLCK;			/* Force mi_readinfo to lock */
unknown's avatar
unknown committed
2011
  _mi_readinfo(info,F_WRLCK,0);			/* Will lock the table */
2012 2013 2014
  info->lock_type=  old_lock;
  share->r_locks=   r_locks;
  share->w_locks=   w_locks;
2015
  share->tot_locks= r_locks+w_locks;
2016
  share->state=     old_state;			/* Restore old state */
unknown's avatar
unknown committed
2017 2018 2019 2020 2021

  info->state->key_file_length=param->new_file_pos;
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  for (key=0 ; key < info->s->base.keys ; key++)
    info->s->state.key_root[key]=index_pos[key];
2022
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
unknown's avatar
unknown committed
2023 2024
    info->s->state.key_del[key]=  HA_OFFSET_ERROR;

2025
  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
unknown's avatar
unknown committed
2026 2027 2028
  DBUG_RETURN(0);

err:
Marc Alff's avatar
Marc Alff committed
2029
  (void) mysql_file_close(new_file, MYF(MY_WME));
2030
err2:
Marc Alff's avatar
Marc Alff committed
2031 2032
  (void) mysql_file_delete(mi_key_file_datatmp,
                           param->temp_filename, MYF(MY_WME));
unknown's avatar
unknown committed
2033
  DBUG_RETURN(-1);
2034
} /* mi_sort_index */
unknown's avatar
unknown committed
2035 2036 2037 2038 2039 2040 2041


	 /* Sort records recursive using one index */

static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file)
{
2042
  uint length,nod_flag,used_length, key_length;
unknown's avatar
unknown committed
2043
  uchar *buff,*keypos,*endpos;
2044
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
unknown's avatar
unknown committed
2045 2046 2047 2048
  my_off_t new_page_pos,next_page;
  char llbuff[22];
  DBUG_ENTER("sort_one_index");

2049 2050
  /* cannot walk over R-tree indices */
  DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
unknown's avatar
unknown committed
2051 2052 2053 2054 2055
  new_page_pos=param->new_file_pos;
  param->new_file_pos+=keyinfo->block_length;

  if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
2056
    mi_check_print_error(param,"Not enough memory for key block");
unknown's avatar
unknown committed
2057 2058
    DBUG_RETURN(-1);
  }
2059
  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
unknown's avatar
unknown committed
2060 2061 2062 2063 2064
  {
    mi_check_print_error(param,"Can't read key block from filepos: %s",
		llstr(pagepos,llbuff));
    goto err;
  }
2065
  if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
unknown's avatar
unknown committed
2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
  {
    used_length=mi_getint(buff);
    keypos=buff+2+nod_flag;
    endpos=buff+used_length;
    for ( ;; )
    {
      if (nod_flag)
      {
	next_page=_mi_kpos(nod_flag,keypos);
	_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2076
	if (sort_one_index(param,info,keyinfo,next_page,new_file))
unknown's avatar
unknown committed
2077
	{
unknown's avatar
unknown committed
2078
	  DBUG_PRINT("error",
unknown's avatar
unknown committed
2079
		     ("From page: %ld, keyoffset: %lu  used_length: %d",
unknown's avatar
unknown committed
2080 2081
		      (ulong) pagepos, (ulong) (keypos - buff),
		      (int) used_length));
2082
	  DBUG_DUMP("buff",(uchar*) buff,used_length);
unknown's avatar
unknown committed
2083 2084 2085 2086
	  goto err;
	}
      }
      if (keypos >= endpos ||
2087
	  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
unknown's avatar
unknown committed
2088
	break;
2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105
      DBUG_ASSERT(keypos <= endpos);
      if (keyinfo->flag & HA_FULLTEXT)
      {
        uint off;
        int  subkeys;
        get_key_full_length_rdonly(off, key);
        subkeys=ft_sintXkorr(key+off);
        if (subkeys < 0)
        {
          next_page= _mi_dpos(info,0,key+key_length);
          _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
                       param->new_file_pos); /* Save new pos */
          if (sort_one_index(param,info,&info->s->ft2_keyinfo,
                             next_page,new_file))
            goto err;
        }
      }
unknown's avatar
unknown committed
2106 2107 2108 2109 2110
    }
  }

  /* Fill block with zero and write it to the new index file */
  length=mi_getint(buff);
2111
  bzero((uchar*) buff+length,keyinfo->block_length-length);
Marc Alff's avatar
Marc Alff committed
2112 2113
  if (mysql_file_pwrite(new_file, (uchar*) buff, (uint) keyinfo->block_length,
                        new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL)))
unknown's avatar
unknown committed
2114 2115 2116 2117
  {
    mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
    goto err;
  }
2118
  my_afree((uchar*) buff);
unknown's avatar
unknown committed
2119 2120
  DBUG_RETURN(0);
err:
2121
  my_afree((uchar*) buff);
unknown's avatar
unknown committed
2122 2123 2124 2125
  DBUG_RETURN(1);
} /* sort_one_index */


2126 2127 2128 2129 2130 2131 2132 2133
	/*
	  Let temporary file replace old file.
	  This assumes that the new file was created in the same
	  directory as given by realpath(filename).
	  This will ensure that any symlinks that are used will still work.
	  Copy stats from old file to new file, deletes orignal and
	  changes new file name to old file name
	*/
unknown's avatar
unknown committed
2134 2135

int change_to_newfile(const char * filename, const char * old_ext,
2136
                      const char * new_ext, myf MyFlags)
unknown's avatar
unknown committed
2137 2138
{
  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
2139 2140 2141 2142
  /* Get real path to filename */
  (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
  return my_redel(old_filename,
		  fn_format(new_filename,old_filename,"",new_ext,2+4),
2143
		  MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
} /* change_to_newfile */


	/* Locks a whole file */
	/* Gives an error-message if file can't be locked */

int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
	      const char *filetype, const char *filename)
{
  if (my_lock(file,lock_type,start,F_TO_EOF,
	      param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
	      MYF(MY_SEEK_NOT_DONE |  MY_DONT_WAIT)))
  {
    mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
    param->error_printed=2;		/* Don't give that data is crashed */
    return 1;
  }
  return 0;
} /* lock_file */


	/* Copy a block between two files */

int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
	     my_off_t length, const char *type)
{
  char tmp_buff[IO_SIZE],*buff;
  ulong buff_length;
  DBUG_ENTER("filecopy");

  buff_length=(ulong) min(param->write_buffer_length,length);
  if (!(buff=my_malloc(buff_length,MYF(0))))
  {
    buff=tmp_buff; buff_length=IO_SIZE;
  }

Marc Alff's avatar
Marc Alff committed
2180
  mysql_file_seek(from, start, MY_SEEK_SET, MYF(0));
unknown's avatar
unknown committed
2181 2182
  while (length > buff_length)
  {
Marc Alff's avatar
Marc Alff committed
2183 2184
    if (mysql_file_read(from, (uchar*) buff, buff_length, MYF(MY_NABP)) ||
        mysql_file_write(to, (uchar*) buff, buff_length, param->myf_rw))
unknown's avatar
unknown committed
2185 2186 2187
      goto err;
    length-= buff_length;
  }
Marc Alff's avatar
Marc Alff committed
2188 2189
  if (mysql_file_read(from, (uchar*) buff, (uint) length, MYF(MY_NABP)) ||
      mysql_file_write(to, (uchar*) buff, (uint) length, param->myf_rw))
unknown's avatar
unknown committed
2190 2191
    goto err;
  if (buff != tmp_buff)
2192
    my_free(buff);
unknown's avatar
unknown committed
2193 2194 2195
  DBUG_RETURN(0);
err:
  if (buff != tmp_buff)
2196
    my_free(buff);
unknown's avatar
unknown committed
2197 2198 2199 2200
  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
		       type,my_errno);
  DBUG_RETURN(1);
}
unknown's avatar
unknown committed
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216


/*
  Repair table or given index using sorting

  SYNOPSIS
    mi_repair_by_sort()
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228

int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
		      const char * name, int rep_quick)
{
  int got_error;
  uint i;
  ulong length;
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
  MI_SORT_PARAM sort_param;
  MYISAM_SHARE *share=info->s;
2229
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2230 2231
  ulong   *rec_per_key_part;
  char llbuff[22];
unknown's avatar
unknown committed
2232
  SORT_INFO sort_info;
2233
  ulonglong UNINIT_VAR(key_map);
unknown's avatar
unknown committed
2234
  DBUG_ENTER("mi_repair_by_sort");
unknown's avatar
unknown committed
2235 2236 2237 2238 2239 2240 2241 2242

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
2243
    printf("- recovering (with sort) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
2244 2245
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2246
  param->testflag|=T_REP; /* for easy checking */
2247

2248 2249 2250
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
2251
  bzero((char*)&sort_info,sizeof(sort_info));
2252
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
2253
  if (!(sort_info.key_block=
unknown's avatar
unknown committed
2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
	alloc_key_blocks(param,
			 (uint) param->sort_key_blocks,
			 share->base.max_key_block_length))
      || init_io_cache(&param->read_cache,info->dfile,
		       (uint) param->read_buffer_length,
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
      (! rep_quick &&
       init_io_cache(&info->rec_cache,info->dfile,
		     (uint) param->write_buffer_length,
		     WRITE_CACHE,new_header_length,1,
2264
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
unknown's avatar
unknown committed
2265
    goto err;
unknown's avatar
unknown committed
2266
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
unknown's avatar
unknown committed
2267 2268 2269
  info->opt_flag|=WRITE_CACHE_USED;
  info->rec_cache.file=info->dfile;		/* for sort_delete_record */

2270
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
unknown's avatar
unknown committed
2271
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
2272
  {
2273
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
2274 2275 2276 2277
    goto err;
  }
  if (!rep_quick)
  {
2278
    /* Get real path for data file */
2279 2280 2281 2282 2283 2284
    if ((new_file= mysql_file_create(mi_key_file_datatmp,
                                     fn_format(param->temp_filename,
                                               share->data_file_name, "",
                                               DATA_TMP_EXT, 2+4),
                                     0, param->tmpfile_createflag,
                                     MYF(0))) < 0)
unknown's avatar
unknown committed
2285 2286 2287 2288 2289
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2290 2291
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2304 2305 2306 2307 2308

  /* Optionally drop indexes and optionally modify the key_map. */
  mi_drop_all_indexes(param, info, FALSE);
  key_map= share->state.key_map;
  if (param->testflag & T_CREATE_MISSING_KEYS)
unknown's avatar
unknown committed
2309
  {
2310 2311
    /* Invert the copied key_map to recreate all disabled indexes. */
    key_map= ~key_map;
unknown's avatar
unknown committed
2312
  }
unknown's avatar
unknown committed
2313

unknown's avatar
unknown committed
2314 2315
  sort_info.info=info;
  sort_info.param = param;
unknown's avatar
unknown committed
2316

unknown's avatar
unknown committed
2317 2318 2319 2320 2321
  set_data_file_type(&sort_info, share);
  sort_param.filepos=new_header_length;
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
Marc Alff's avatar
Marc Alff committed
2322
    mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
unknown's avatar
unknown committed
2323

unknown's avatar
unknown committed
2324
  sort_param.wordlist=NULL;
2325
  init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2326

unknown's avatar
unknown committed
2327 2328 2329 2330 2331 2332
  if (share->data_file_type == DYNAMIC_RECORD)
    length=max(share->base.min_pack_length+1,share->base.min_block_length);
  else if (share->data_file_type == COMPRESSED_RECORD)
    length=share->base.min_block_length;
  else
    length=share->base.pack_reclength;
unknown's avatar
unknown committed
2333
  sort_info.max_records=
2334
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
unknown's avatar
unknown committed
2335
     (ha_rows) (sort_info.filelength/length+1));
unknown's avatar
unknown committed
2336 2337 2338
  sort_param.key_cmp=sort_key_cmp;
  sort_param.lock_in_memory=lock_memory;
  sort_param.tmpdir=param->tmpdir;
unknown's avatar
unknown committed
2339 2340
  sort_param.sort_info=&sort_info;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
2341
  sort_param.master =1;
2342
  
unknown's avatar
unknown committed
2343
  del=info->state->del;
2344 2345
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
2346
    sort_param.calc_checksum= 1;
unknown's avatar
unknown committed
2347 2348

  rec_per_key_part= param->rec_per_key_part;
unknown's avatar
unknown committed
2349 2350
  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
       rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
unknown's avatar
unknown committed
2351
  {
unknown's avatar
unknown committed
2352 2353
    sort_param.read_cache=param->read_cache;
    sort_param.keyinfo=share->keyinfo+sort_param.key;
2354
    sort_param.seg=sort_param.keyinfo->seg;
2355 2356 2357 2358
    /*
      Skip this index if it is marked disabled in the copied
      (and possibly inverted) key_map.
    */
2359
    if (! mi_is_key_active(key_map, sort_param.key))
2360 2361 2362
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2363 2364
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2365
	     sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2366 2367
      DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
                            sort_param.key));
unknown's avatar
unknown committed
2368
      continue;
2369
    }
unknown's avatar
unknown committed
2370 2371

    if ((!(param->testflag & T_SILENT)))
unknown's avatar
unknown committed
2372 2373
      printf ("- Fixing index %d\n",sort_param.key+1);
    sort_param.max_pos=sort_param.pos=share->pack.header_length;
2374
    keyseg=sort_param.seg;
unknown's avatar
unknown committed
2375
    bzero((char*) sort_param.unique,sizeof(sort_param.unique));
unknown's avatar
unknown committed
2376
    sort_param.key_length=share->rec_reflength;
unknown's avatar
unknown committed
2377
    for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
unknown's avatar
unknown committed
2378
    {
unknown's avatar
unknown committed
2379 2380 2381
      sort_param.key_length+=keyseg[i].length;
      if (keyseg[i].flag & HA_SPACE_PACK)
	sort_param.key_length+=get_pack_length(keyseg[i].length);
2382
      if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2383 2384
	sort_param.key_length+=2 + test(keyseg[i].length >= 127);
      if (keyseg[i].flag & HA_NULL_PART)
unknown's avatar
unknown committed
2385 2386 2387 2388 2389
	sort_param.key_length++;
    }
    info->state->records=info->state->del=share->state.split=0;
    info->state->empty=0;

unknown's avatar
unknown committed
2390
    if (sort_param.keyinfo->flag & HA_FULLTEXT)
2391
    {
2392 2393
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param.keyinfo->seg->charset->mbmaxlen;
2394 2395 2396 2397 2398
      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
      /*
        fulltext indexes may have much more entries than the
        number of rows in the table. We estimate the number here.
      */
2399
      if (sort_param.keyinfo->parser == &ft_default_parser)
2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
      {
        /*
          for built-in parser the number of generated index entries
          cannot be larger than the size of the data file divided
          by the minimal word's length
        */
        sort_info.max_records=
          (ha_rows) (sort_info.filelength/ft_min_word_len+1);
      }
      else
      {
        /*
          for external plugin parser we cannot tell anything at all :(
          so, we'll use all the sort memory and start from ~10 buffpeks.
          (see _create_index_by_sort)
        */
2416 2417 2418
        sort_info.max_records= 10 *
                               max(param->sort_buffer_length, MIN_SORT_BUFFER) /
                               sort_param.key_length;
2419
      }
2420 2421

      sort_param.key_read=sort_ft_key_read;
2422
      sort_param.key_write=sort_ft_key_write;
2423 2424
    }
    else
2425
    {
2426
      sort_param.key_read=sort_key_read;
2427 2428
      sort_param.key_write=sort_key_write;
    }
2429

unknown's avatar
unknown committed
2430 2431 2432
    if (_create_index_by_sort(&sort_param,
			      (my_bool) (!(param->testflag & T_VERBOSE)),
			      (uint) param->sort_buffer_length))
2433 2434
    {
      param->retry_repair=1;
unknown's avatar
unknown committed
2435
      goto err;
2436
    }
2437 2438
    /* No need to calculate checksum again. */
    sort_param.calc_checksum= 0;
2439
    free_root(&sort_param.wordroot, MYF(0));
unknown's avatar
unknown committed
2440 2441

    /* Set for next loop */
unknown's avatar
unknown committed
2442
    sort_info.max_records= (ha_rows) info->state->records;
unknown's avatar
unknown committed
2443 2444

    if (param->testflag & T_STATISTICS)
unknown's avatar
unknown committed
2445
      update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2446
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2447 2448 2449
                       sort_param.notnull: NULL,
                       (ulonglong) info->state->records);
    /* Enable this index in the permanent (not the copied) key_map. */
2450
    mi_set_key_active(share->state.key_map, sort_param.key);
2451
    DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
unknown's avatar
unknown committed
2452

unknown's avatar
unknown committed
2453
    if (sort_param.fix_datafile)
unknown's avatar
unknown committed
2454
    {
unknown's avatar
unknown committed
2455 2456
      param->read_cache.end_of_file=sort_param.filepos;
      if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
unknown's avatar
unknown committed
2457
	goto err;
2458 2459 2460 2461 2462 2463 2464 2465 2466
      if (param->testflag & T_SAFE_REPAIR)
      {
	/* Don't repair if we loosed more than one row */
	if (info->state->records+1 < start_records)
	{
	  info->state->records=start_records;
	  goto err;
	}
      }
unknown's avatar
unknown committed
2467 2468
      share->state.state.data_file_length = info->state->data_file_length=
	sort_param.filepos;
unknown's avatar
unknown committed
2469 2470
      /* Only whole records */
      share->state.version=(ulong) time((time_t*) 0);
Marc Alff's avatar
Marc Alff committed
2471
      mysql_file_close(info->dfile, MYF(0));
unknown's avatar
unknown committed
2472
      info->dfile=new_file;
unknown's avatar
unknown committed
2473
      share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
2474
      share->pack.header_length=(ulong) new_header_length;
unknown's avatar
unknown committed
2475
      sort_param.fix_datafile=0;
unknown's avatar
unknown committed
2476 2477
    }
    else
unknown's avatar
unknown committed
2478
      info->state->data_file_length=sort_param.max_pos;
unknown's avatar
unknown committed
2479 2480

    param->read_cache.file=info->dfile;		/* re-init read cache */
unknown's avatar
unknown committed
2481 2482
    reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
                    1,1);
unknown's avatar
unknown committed
2483 2484 2485 2486
  }

  if (param->testflag & T_WRITE_LOOP)
  {
Konstantin Osipov's avatar
Konstantin Osipov committed
2487
    (void) fputs("          \r",stdout); (void) fflush(stdout);
unknown's avatar
unknown committed
2488 2489
  }

unknown's avatar
unknown committed
2490
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
2491 2492 2493 2494
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
2495 2496
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2497 2498 2499
    goto err;
  }

unknown's avatar
unknown committed
2500
  if (rep_quick & T_FORCE_UNIQUENESS)
unknown's avatar
unknown committed
2501 2502 2503 2504 2505 2506 2507 2508 2509
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
2510
    if (skr != sort_info.filelength)
Marc Alff's avatar
Marc Alff committed
2511
      if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
unknown's avatar
unknown committed
2512 2513 2514 2515
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
unknown's avatar
unknown committed
2516
  if (param->testflag & T_CALC_CHECKSUM)
2517
    info->state->checksum=param->glob_crc;
2518

Marc Alff's avatar
Marc Alff committed
2519
  if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
unknown's avatar
unknown committed
2520 2521 2522 2523 2524 2525 2526 2527
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
2528
    if (sort_info.dupp)
unknown's avatar
unknown committed
2529 2530
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
2531
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
2532 2533 2534 2535 2536 2537 2538
  }
  got_error=0;

  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
unknown's avatar
unknown committed
2539
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
Konstantin Osipov's avatar
Konstantin Osipov committed
2540
  (void) end_io_cache(&info->rec_cache);
2541 2542 2543 2544 2545
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
2546
      mysql_file_close(new_file, MYF(0));
2547
      info->dfile=new_file= -1;
2548
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, DATA_TMP_EXT,
2549 2550
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
2551
	  mi_open_datafile(info,share,name,-1))
2552 2553 2554
	got_error=1;
    }
  }
unknown's avatar
unknown committed
2555 2556 2557 2558 2559 2560
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
2561
      (void) mysql_file_close(new_file, MYF(0));
2562 2563
      (void) mysql_file_delete(mi_key_file_datatmp,
                               param->temp_filename, MYF(MY_WME));
2564 2565 2566
      if (info->dfile == new_file) /* Retry with key cache */
        if (unlikely(mi_open_datafile(info, share, name, -1)))
          param->retry_repair= 0; /* Safety */
unknown's avatar
unknown committed
2567
    }
2568
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
2569
  }
2570 2571 2572 2573
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

2574 2575 2576 2577 2578
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
  my_free(mi_get_rec_buff_ptr(info, sort_param.record));
  my_free(sort_info.key_block);
  my_free(sort_info.ft_buf);
  my_free(sort_info.buff);
Konstantin Osipov's avatar
Konstantin Osipov committed
2579
  (void) end_io_cache(&param->read_cache);
unknown's avatar
unknown committed
2580
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2581
  if (!got_error && (param->testflag & T_UNPACK))
unknown's avatar
unknown committed
2582 2583 2584 2585 2586
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
2587
}
unknown's avatar
unknown committed
2588

unknown's avatar
unknown committed
2589 2590 2591 2592
/*
  Threaded repair of table using sorting

  SYNOPSIS
2593
    mi_repair_parallel()
unknown's avatar
unknown committed
2594 2595 2596 2597 2598 2599 2600 2601
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  DESCRIPTION
    Same as mi_repair_by_sort but do it multithreaded
    Each key is handled by a separate thread.
unknown's avatar
unknown committed
2602
    TODO: make a number of threads a parameter
unknown's avatar
unknown committed
2603

2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
    In parallel repair we use one thread per index. There are two modes:

    Quick

      Only the indexes are rebuilt. All threads share a read buffer.
      Every thread that needs fresh data in the buffer enters the shared
      cache lock. The last thread joining the lock reads the buffer from
      the data file and wakes all other threads.

    Non-quick

      The data file is rebuilt and all indexes are rebuilt to point to
      the new record positions. One thread is the master thread. It
      reads from the old data file and writes to the new data file. It
      also creates one of the indexes. The other threads read from a
      buffer which is filled by the master. If they need fresh data,
      they enter the shared cache lock. If the masters write buffer is
      full, it flushes it to the new data file and enters the shared
      cache lock too. When all threads joined in the lock, the master
      copies its write buffer to the read buffer for the other threads
      and wakes them.

unknown's avatar
unknown committed
2626 2627 2628 2629
  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2630

2631
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
unknown's avatar
unknown committed
2632
			const char * name, int rep_quick)
unknown's avatar
unknown committed
2633 2634
{
  int got_error;
unknown's avatar
unknown committed
2635
  uint i,key, total_key_length, istep;
unknown's avatar
unknown committed
2636
  ulong rec_length;
unknown's avatar
unknown committed
2637 2638 2639
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
unknown's avatar
unknown committed
2640
  MI_SORT_PARAM *sort_param=0;
unknown's avatar
unknown committed
2641 2642
  MYISAM_SHARE *share=info->s;
  ulong   *rec_per_key_part;
2643
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2644
  char llbuff[22];
2645
  IO_CACHE new_data_cache; /* For non-quick repair. */
unknown's avatar
unknown committed
2646 2647
  IO_CACHE_SHARE io_share;
  SORT_INFO sort_info;
2648
  ulonglong UNINIT_VAR(key_map);
2649
  pthread_attr_t thr_attr;
2650
  ulong max_pack_reclength;
2651
  DBUG_ENTER("mi_repair_parallel");
unknown's avatar
unknown committed
2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
    printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2663
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
2664

2665 2666 2667
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696
  /*
    Quick repair (not touching data file, rebuilding indexes):
    {
      Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
    }

    Non-quick repair (rebuilding data file and indexes):
    {
      Master thread:

        Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
        Write cache is (MI_INFO   *info)->rec_cache  using new_file.

      Slave threads:

        Read  cache is new_data_cache synced to master rec_cache.

      The final assignment of the filedescriptor for rec_cache is done
      after the cache creation.

      Don't check file size on new_data_cache, as the resulting file size
      is not known yet.

      As rec_cache and new_data_cache are synced, write_buffer_length is
      used for the read cache 'new_data_cache'. Both start at the same
      position 'new_header_length'.
    }
  */
  DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
unknown's avatar
unknown committed
2697
  bzero((char*)&sort_info,sizeof(sort_info));
2698
  /* Initialize pthread structures before goto err. */
Marc Alff's avatar
Marc Alff committed
2699 2700 2701 2702 2703
  mysql_mutex_init(mi_key_mutex_MI_SORT_INFO_mutex,
                   &sort_info.mutex, MY_MUTEX_INIT_FAST);
  mysql_cond_init(mi_key_cond_MI_SORT_INFO_cond, &sort_info.cond, 0);
  mysql_mutex_init(mi_key_mutex_MI_CHECK_print_msg,
                   &param->print_msg_mutex, MY_MUTEX_INIT_FAST);
Sergey Vojtovich's avatar
Sergey Vojtovich committed
2704
  param->need_print_msg_lock= 1;
2705

unknown's avatar
unknown committed
2706
  if (!(sort_info.key_block=
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720
	alloc_key_blocks(param, (uint) param->sort_key_blocks,
			 share->base.max_key_block_length)) ||
      init_io_cache(&param->read_cache, info->dfile,
                    (uint) param->read_buffer_length,
                    READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
      (!rep_quick &&
       (init_io_cache(&info->rec_cache, info->dfile,
                      (uint) param->write_buffer_length,
                      WRITE_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
        init_io_cache(&new_data_cache, -1,
                      (uint) param->write_buffer_length,
                      READ_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
unknown's avatar
unknown committed
2721 2722 2723
    goto err;
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
  info->opt_flag|=WRITE_CACHE_USED;
2724
  info->rec_cache.file=info->dfile;         /* for sort_delete_record */
unknown's avatar
unknown committed
2725 2726 2727 2728

  if (!rep_quick)
  {
    /* Get real path for data file */
2729 2730 2731 2732 2733 2734
    if ((new_file= mysql_file_create(mi_key_file_datatmp,
                                     fn_format(param->temp_filename,
                                               share->data_file_name, "",
                                               DATA_TMP_EXT, 2+4),
                                     0, param->tmpfile_createflag,
                                     MYF(0))) < 0)
unknown's avatar
unknown committed
2735 2736 2737 2738 2739
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2740 2741
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2754 2755 2756 2757 2758

  /* Optionally drop indexes and optionally modify the key_map. */
  mi_drop_all_indexes(param, info, FALSE);
  key_map= share->state.key_map;
  if (param->testflag & T_CREATE_MISSING_KEYS)
unknown's avatar
unknown committed
2759
  {
2760 2761
    /* Invert the copied key_map to recreate all disabled indexes. */
    key_map= ~key_map;
unknown's avatar
unknown committed
2762 2763 2764 2765 2766 2767 2768 2769 2770
  }

  sort_info.info=info;
  sort_info.param = param;

  set_data_file_type(&sort_info, share);
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
Marc Alff's avatar
Marc Alff committed
2771
    mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
unknown's avatar
unknown committed
2772 2773

  if (share->data_file_type == DYNAMIC_RECORD)
unknown's avatar
unknown committed
2774
    rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
unknown's avatar
unknown committed
2775
  else if (share->data_file_type == COMPRESSED_RECORD)
unknown's avatar
unknown committed
2776
    rec_length=share->base.min_block_length;
unknown's avatar
unknown committed
2777
  else
unknown's avatar
unknown committed
2778
    rec_length=share->base.pack_reclength;
2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790
  /*
    +1 below is required hack for parallel repair mode.
    The info->state->records value, that is compared later
    to sort_info.max_records and cannot exceed it, is
    increased in sort_key_write. In mi_repair_by_sort, sort_key_write
    is called after sort_key_read, where the comparison is performed,
    but in parallel mode master thread can call sort_key_write
    before some other repair thread calls sort_key_read.
    Furthermore I'm not even sure +1 would be enough.
    May be sort_info.max_records shold be always set to max value in
    parallel mode.
  */
unknown's avatar
unknown committed
2791
  sort_info.max_records=
2792
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
unknown's avatar
unknown committed
2793
     (ha_rows) (sort_info.filelength/rec_length+1));
unknown's avatar
unknown committed
2794 2795 2796

  del=info->state->del;
  param->glob_crc=0;
2797 2798 2799 2800
  /* for compressed tables */
  max_pack_reclength= share->base.pack_reclength;
  if (share->options & HA_OPTION_COMPRESS_RECORD)
    set_if_bigger(max_pack_reclength, share->max_pack_length);
unknown's avatar
unknown committed
2801 2802
  if (!(sort_param=(MI_SORT_PARAM *)
        my_malloc((uint) share->base.keys *
2803
		  (sizeof(MI_SORT_PARAM) + max_pack_reclength),
unknown's avatar
unknown committed
2804
		  MYF(MY_ZEROFILL))))
unknown's avatar
unknown committed
2805
  {
2806
    mi_check_print_error(param,"Not enough memory for key!");
unknown's avatar
unknown committed
2807 2808
    goto err;
  }
unknown's avatar
unknown committed
2809
  total_key_length=0;
unknown's avatar
unknown committed
2810 2811 2812 2813
  rec_per_key_part= param->rec_per_key_part;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;

unknown's avatar
unknown committed
2814 2815
  for (i=key=0, istep=1 ; key < share->base.keys ;
       rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
unknown's avatar
unknown committed
2816 2817 2818
  {
    sort_param[i].key=key;
    sort_param[i].keyinfo=share->keyinfo+key;
2819
    sort_param[i].seg=sort_param[i].keyinfo->seg;
2820 2821 2822 2823
    /*
      Skip this index if it is marked disabled in the copied
      (and possibly inverted) key_map.
    */
2824
    if (! mi_is_key_active(key_map, key))
unknown's avatar
unknown committed
2825 2826 2827
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2828 2829
	     (char*) (share->state.rec_per_key_part+
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2830
	     sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2831
      istep=0;
unknown's avatar
unknown committed
2832 2833
      continue;
    }
unknown's avatar
unknown committed
2834
    istep=1;
unknown's avatar
unknown committed
2835 2836
    if ((!(param->testflag & T_SILENT)))
      printf ("- Fixing index %d\n",key+1);
2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
    {
      sort_param[i].key_read=sort_ft_key_read;
      sort_param[i].key_write=sort_ft_key_write;
    }
    else
    {
      sort_param[i].key_read=sort_key_read;
      sort_param[i].key_write=sort_key_write;
    }
unknown's avatar
unknown committed
2847 2848 2849 2850
    sort_param[i].key_cmp=sort_key_cmp;
    sort_param[i].lock_in_memory=lock_memory;
    sort_param[i].tmpdir=param->tmpdir;
    sort_param[i].sort_info=&sort_info;
2851
    sort_param[i].master=0;
unknown's avatar
unknown committed
2852
    sort_param[i].fix_datafile=0;
2853
    sort_param[i].calc_checksum= 0;
unknown's avatar
unknown committed
2854 2855 2856 2857

    sort_param[i].filepos=new_header_length;
    sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;

2858
    sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2859
			   (max_pack_reclength * i));
unknown's avatar
unknown committed
2860 2861 2862 2863 2864
    if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
    {
      mi_check_print_error(param,"Not enough memory!");
      goto err;
    }
unknown's avatar
unknown committed
2865 2866

    sort_param[i].key_length=share->rec_reflength;
2867
    for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
unknown's avatar
unknown committed
2868
	 keyseg++)
unknown's avatar
unknown committed
2869 2870 2871 2872
    {
      sort_param[i].key_length+=keyseg->length;
      if (keyseg->flag & HA_SPACE_PACK)
        sort_param[i].key_length+=get_pack_length(keyseg->length);
2873
      if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2874 2875 2876 2877
        sort_param[i].key_length+=2 + test(keyseg->length >= 127);
      if (keyseg->flag & HA_NULL_PART)
        sort_param[i].key_length++;
    }
unknown's avatar
unknown committed
2878
    total_key_length+=sort_param[i].key_length;
unknown's avatar
unknown committed
2879 2880

    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2881 2882 2883 2884
    {
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2885
      init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2886
    }
unknown's avatar
unknown committed
2887 2888
  }
  sort_info.total_keys=i;
2889 2890
  sort_param[0].master= 1;
  sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2891
  sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
unknown's avatar
unknown committed
2892

2893 2894 2895
  if (!ftparser_alloc_param(info))
    goto err;

unknown's avatar
unknown committed
2896
  sort_info.got_error=0;
Marc Alff's avatar
Marc Alff committed
2897
  mysql_mutex_lock(&sort_info.mutex);
unknown's avatar
unknown committed
2898

2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
  /*
    Initialize the I/O cache share for use with the read caches and, in
    case of non-quick repair, the write cache. When all threads join on
    the cache lock, the writer copies the write cache contents to the
    read caches.
  */
  if (i > 1)
  {
    if (rep_quick)
      init_io_cache_share(&param->read_cache, &io_share, NULL, i);
    else
      init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
  }
  else
    io_share.total_threads= 0; /* share not used */

2915 2916 2917
  (void) pthread_attr_init(&thr_attr);
  (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);

unknown's avatar
unknown committed
2918
  for (i=0 ; i < sort_info.total_keys ; i++)
unknown's avatar
unknown committed
2919
  {
2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
    /*
      Copy the properly initialized IO_CACHE structure so that every
      thread has its own copy. In quick mode param->read_cache is shared
      for use by all threads. In non-quick mode all threads but the
      first copy the shared new_data_cache, which is synchronized to the
      write cache of the first thread. The first thread copies
      param->read_cache, which is not shared.
    */
    sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
                               new_data_cache);
    DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: 0x%lx",
                                  i, (long) &sort_param[i].read_cache));

unknown's avatar
unknown committed
2933 2934 2935 2936 2937 2938
    /*
      two approaches: the same amount of memory for each thread
      or the memory for the same number of keys for each thread...
      In the second one all the threads will fill their sort_buffers
      (and call write_keys) at the same time, putting more stress on i/o.
    */
unknown's avatar
unknown committed
2939
    sort_param[i].sortbuff_size=
unknown's avatar
unknown committed
2940
#ifndef USING_SECOND_APPROACH
unknown's avatar
unknown committed
2941 2942
      param->sort_buffer_length/sort_info.total_keys;
#else
unknown's avatar
unknown committed
2943
      param->sort_buffer_length*sort_param[i].key_length/total_key_length;
unknown's avatar
unknown committed
2944
#endif
Marc Alff's avatar
Marc Alff committed
2945 2946 2947 2948
    if (mysql_thread_create(mi_key_thread_find_all_keys,
                            &sort_param[i].thr, &thr_attr,
                            thr_find_all_keys,
                            (void *) (sort_param+i)))
unknown's avatar
unknown committed
2949 2950
    {
      mi_check_print_error(param,"Cannot start a repair thread");
2951 2952 2953 2954
      /* Cleanup: Detach from the share. Avoid others to be blocked. */
      if (io_share.total_threads)
        remove_io_thread(&sort_param[i].read_cache);
      DBUG_PRINT("error", ("Cannot start a repair thread"));
unknown's avatar
unknown committed
2955 2956 2957 2958 2959
      sort_info.got_error=1;
    }
    else
      sort_info.threads_running++;
  }
2960
  (void) pthread_attr_destroy(&thr_attr);
unknown's avatar
unknown committed
2961 2962 2963

  /* waiting for all threads to finish */
  while (sort_info.threads_running)
Marc Alff's avatar
Marc Alff committed
2964 2965
    mysql_cond_wait(&sort_info.cond, &sort_info.mutex);
  mysql_mutex_unlock(&sort_info.mutex);
unknown's avatar
unknown committed
2966

2967
  if ((got_error= thr_write_keys(sort_param)))
unknown's avatar
unknown committed
2968 2969 2970 2971
  {
    param->retry_repair=1;
    goto err;
  }
unknown's avatar
unknown committed
2972
  got_error=1;				/* Assume the following may go wrong */
unknown's avatar
unknown committed
2973

unknown's avatar
unknown committed
2974
  if (sort_param[0].fix_datafile)
unknown's avatar
unknown committed
2975
  {
2976 2977 2978 2979 2980
    /*
      Append some nuls to the end of a memory mapped file. Destroy the
      write cache. The master thread did already detach from the share
      by remove_io_thread() in sort.c:thr_find_all_keys().
    */
unknown's avatar
unknown committed
2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991
    if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
      goto err;
    if (param->testflag & T_SAFE_REPAIR)
    {
      /* Don't repair if we loosed more than one row */
      if (info->state->records+1 < start_records)
      {
        info->state->records=start_records;
        goto err;
      }
    }
unknown's avatar
unknown committed
2992 2993
    share->state.state.data_file_length= info->state->data_file_length=
      sort_param->filepos;
unknown's avatar
unknown committed
2994 2995
    /* Only whole records */
    share->state.version=(ulong) time((time_t*) 0);
2996 2997 2998 2999 3000

    /*
      Exchange the data file descriptor of the table, so that we use the
      new file from now on.
     */
Marc Alff's avatar
Marc Alff committed
3001
    mysql_file_close(info->dfile, MYF(0));
unknown's avatar
unknown committed
3002
    info->dfile=new_file;
3003

unknown's avatar
unknown committed
3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028
    share->data_file_type=sort_info.new_data_file_type;
    share->pack.header_length=(ulong) new_header_length;
  }
  else
    info->state->data_file_length=sort_param->max_pos;

  if (rep_quick && del+sort_info.dupp != info->state->del)
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
    goto err;
  }

  if (rep_quick & T_FORCE_UNIQUENESS)
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
3029
    if (skr != sort_info.filelength)
Marc Alff's avatar
Marc Alff committed
3030
      if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
unknown's avatar
unknown committed
3031 3032 3033 3034 3035
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
  if (param->testflag & T_CALC_CHECKSUM)
3036
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
3037

Marc Alff's avatar
Marc Alff committed
3038
  if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
unknown's avatar
unknown committed
3039
    mi_check_print_warning(param,
unknown's avatar
unknown committed
3040
			   "Can't change size of indexfile, error: %d", my_errno);
unknown's avatar
unknown committed
3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
    if (sort_info.dupp)
      mi_check_print_warning(param,
			     "%s records have been removed",
			     llstr(sort_info.dupp,llbuff));
  }
  got_error=0;

  if (&share->state.state != info->state)
3054
    memcpy(&share->state.state, info->state, sizeof(*info->state));
unknown's avatar
unknown committed
3055 3056

err:
unknown's avatar
unknown committed
3057
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
3058 3059 3060 3061 3062
  /*
    Destroy the write cache. The master thread did already detach from
    the share by remove_io_thread() or it was not yet started (if the
    error happend before creating the thread).
  */
Konstantin Osipov's avatar
Konstantin Osipov committed
3063
  (void) end_io_cache(&info->rec_cache);
3064 3065 3066 3067 3068 3069 3070
  /*
    Destroy the new data cache in case of non-quick repair. All slave
    threads did either detach from the share by remove_io_thread()
    already or they were not yet started (if the error happend before
    creating the threads).
  */
  if (!rep_quick)
Konstantin Osipov's avatar
Konstantin Osipov committed
3071
    (void) end_io_cache(&new_data_cache);
unknown's avatar
unknown committed
3072 3073 3074 3075 3076
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
3077
      mysql_file_close(new_file, MYF(0));
unknown's avatar
unknown committed
3078
      info->dfile=new_file= -1;
3079
      if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT,
unknown's avatar
unknown committed
3080 3081
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
3082
	  mi_open_datafile(info,share,name,-1))
unknown's avatar
unknown committed
3083 3084 3085 3086 3087 3088 3089 3090 3091
	got_error=1;
    }
  }
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
Marc Alff's avatar
Marc Alff committed
3092
      (void) mysql_file_close(new_file, MYF(0));
3093 3094
      (void) mysql_file_delete(mi_key_file_datatmp,
                               param->temp_filename, MYF(MY_WME));
3095 3096 3097
      if (info->dfile == new_file) /* Retry with key cache */
        if (unlikely(mi_open_datafile(info, share, name, -1)))
          param->retry_repair= 0; /* Safety */
unknown's avatar
unknown committed
3098 3099 3100 3101 3102 3103 3104
    }
    mi_mark_crashed_on_repair(info);
  }
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

Marc Alff's avatar
Marc Alff committed
3105 3106 3107
  mysql_cond_destroy(&sort_info.cond);
  mysql_mutex_destroy(&sort_info.mutex);
  mysql_mutex_destroy(&param->print_msg_mutex);
Sergey Vojtovich's avatar
Sergey Vojtovich committed
3108
  param->need_print_msg_lock= 0;
unknown's avatar
unknown committed
3109

3110 3111 3112 3113
  my_free(sort_info.ft_buf);
  my_free(sort_info.key_block);
  my_free(sort_param);
  my_free(sort_info.buff);
Konstantin Osipov's avatar
Konstantin Osipov committed
3114
  (void) end_io_cache(&param->read_cache);
unknown's avatar
unknown committed
3115 3116 3117 3118 3119 3120 3121 3122
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  if (!got_error && (param->testflag & T_UNPACK))
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
}
unknown's avatar
unknown committed
3123 3124 3125

	/* Read next record and return next key */

unknown's avatar
unknown committed
3126
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
unknown's avatar
unknown committed
3127 3128
{
  int error;
unknown's avatar
unknown committed
3129 3130
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3131 3132
  DBUG_ENTER("sort_key_read");

unknown's avatar
unknown committed
3133
  if ((error=sort_get_next_record(sort_param)))
unknown's avatar
unknown committed
3134 3135 3136 3137
    DBUG_RETURN(error);
  if (info->state->records == sort_info->max_records)
  {
    mi_check_print_error(sort_info->param,
3138 3139
			 "Key %d - Found too many records; Can't continue",
                         sort_param->key+1);
unknown's avatar
unknown committed
3140 3141
    DBUG_RETURN(1);
  }
unknown's avatar
unknown committed
3142
  sort_param->real_key_length=
unknown's avatar
unknown committed
3143
    (info->s->rec_reflength+
unknown's avatar
unknown committed
3144 3145
     _mi_make_key(info, sort_param->key, (uchar*) key,
		  sort_param->record, sort_param->filepos));
unknown's avatar
unknown committed
3146 3147 3148 3149
#ifdef HAVE_purify
  bzero(key+sort_param->real_key_length,
	(sort_param->key_length-sort_param->real_key_length));
#endif
unknown's avatar
unknown committed
3150
  DBUG_RETURN(sort_write_record(sort_param));
unknown's avatar
unknown committed
3151 3152
} /* sort_key_read */

unknown's avatar
unknown committed
3153
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3154 3155
{
  int error;
unknown's avatar
unknown committed
3156 3157
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3158
  FT_WORD *wptr=0;
3159 3160
  DBUG_ENTER("sort_ft_key_read");

unknown's avatar
unknown committed
3161
  if (!sort_param->wordlist)
3162
  {
unknown's avatar
unknown committed
3163
    for (;;)
3164
    {
3165
      free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3166
      if ((error=sort_get_next_record(sort_param)))
3167
        DBUG_RETURN(error);
3168 3169
      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
                                    &sort_param->wordroot)))
3170
        DBUG_RETURN(1);
unknown's avatar
unknown committed
3171 3172
      if (wptr->pos)
        break;
unknown's avatar
unknown committed
3173
      error=sort_write_record(sort_param);
3174
    }
unknown's avatar
unknown committed
3175
    sort_param->wordptr=sort_param->wordlist=wptr;
3176 3177 3178 3179
  }
  else
  {
    error=0;
unknown's avatar
unknown committed
3180
    wptr=(FT_WORD*)(sort_param->wordptr);
3181 3182
  }

unknown's avatar
unknown committed
3183 3184 3185 3186 3187 3188 3189 3190
  sort_param->real_key_length=(info->s->rec_reflength+
			       _ft_make_key(info, sort_param->key,
					    key, wptr++, sort_param->filepos));
#ifdef HAVE_purify
  if (sort_param->key_length > sort_param->real_key_length)
    bzero(key+sort_param->real_key_length,
	  (sort_param->key_length-sort_param->real_key_length));
#endif
3191 3192
  if (!wptr->pos)
  {
3193
    free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3194
    sort_param->wordlist=0;
unknown's avatar
unknown committed
3195
    error=sort_write_record(sort_param);
3196 3197
  }
  else
unknown's avatar
unknown committed
3198
    sort_param->wordptr=(void*)wptr;
3199 3200 3201

  DBUG_RETURN(error);
} /* sort_ft_key_read */
unknown's avatar
unknown committed
3202

unknown's avatar
unknown committed
3203

3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233
/*
  Read next record from file using parameters in sort_info.

  SYNOPSIS
    sort_get_next_record()
      sort_param                Information about and for the sort process

  NOTE

    Dynamic Records With Non-Quick Parallel Repair

      For non-quick parallel repair we use a synchronized read/write
      cache. This means that one thread is the master who fixes the data
      file by reading each record from the old data file and writing it
      to the new data file. By doing this the records in the new data
      file are written contiguously. Whenever the write buffer is full,
      it is copied to the read buffer. The slaves read from the read
      buffer, which is not associated with a file. Thus read_cache.file
      is -1. When using _mi_read_cache(), the slaves must always set
      flag to READING_NEXT so that the function never tries to read from
      file. This is safe because the records are contiguous. There is no
      need to read outside the cache. This condition is evaluated in the
      variable 'parallel_flag' for quick reference. read_cache.file must
      be >= 0 in every other case.

  RETURN
    -1          end of file
    0           ok
    > 0         error
*/
unknown's avatar
unknown committed
3234

unknown's avatar
unknown committed
3235
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3236 3237
{
  int searching;
3238
  int parallel_flag;
unknown's avatar
unknown committed
3239 3240
  uint found_record,b_type,left_length;
  my_off_t pos;
Staale Smedseng's avatar
Staale Smedseng committed
3241
  uchar *UNINIT_VAR(to);
unknown's avatar
unknown committed
3242
  MI_BLOCK_INFO block_info;
unknown's avatar
unknown committed
3243
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3244
  MI_CHECK *param=sort_info->param;
3245 3246
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3247 3248 3249
  char llbuff[22],llbuff2[22];
  DBUG_ENTER("sort_get_next_record");

unknown's avatar
unknown committed
3250 3251 3252
  if (*killed_ptr(param))
    DBUG_RETURN(1);

unknown's avatar
unknown committed
3253 3254 3255 3256
  switch (share->data_file_type) {
  case STATIC_RECORD:
    for (;;)
    {
unknown's avatar
unknown committed
3257
      if (my_b_read(&sort_param->read_cache,sort_param->record,
unknown's avatar
unknown committed
3258
		    share->base.pack_reclength))
3259
      {
unknown's avatar
unknown committed
3260
	if (sort_param->read_cache.error)
3261
	  param->out_flag |= O_DATA_LOST;
unknown's avatar
unknown committed
3262 3263
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3264
	DBUG_RETURN(-1);
3265
      }
unknown's avatar
unknown committed
3266 3267
      sort_param->start_recpos=sort_param->pos;
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3268
      {
unknown's avatar
unknown committed
3269
	sort_param->filepos=sort_param->pos;
3270 3271
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3272
      }
unknown's avatar
unknown committed
3273 3274
      sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
      if (*sort_param->record)
3275
      {
3276
	if (sort_param->calc_checksum)
3277
	  param->glob_crc+= (info->checksum=
unknown's avatar
unknown committed
3278
			     mi_static_checksum(info,sort_param->record));
unknown's avatar
unknown committed
3279
	DBUG_RETURN(0);
3280
      }
3281
      if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3282 3283 3284 3285 3286 3287 3288
      {
	info->state->del++;
	info->state->empty+=share->base.pack_reclength;
      }
    }
  case DYNAMIC_RECORD:
    LINT_INIT(to);
unknown's avatar
unknown committed
3289 3290
    pos=sort_param->pos;
    searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
3291
    parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
unknown's avatar
unknown committed
3292 3293 3294 3295 3296
    for (;;)
    {
      found_record=block_info.second_read= 0;
      left_length=1;
      if (searching)
unknown's avatar
unknown committed
3297
      {
unknown's avatar
unknown committed
3298
	pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3299
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3300
	sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3301
      }
unknown's avatar
unknown committed
3302 3303
      do
      {
unknown's avatar
unknown committed
3304 3305
	if (pos > sort_param->max_pos)
	  sort_param->max_pos=pos;
unknown's avatar
unknown committed
3306 3307 3308 3309 3310
	if (pos & (MI_DYN_ALIGN_SIZE-1))
	{
	  if ((param->testflag & T_VERBOSE) || searching == 0)
	    mi_check_print_info(param,"Wrong aligned block at %s",
				llstr(pos,llbuff));
unknown's avatar
unknown committed
3311
	  if (searching)
unknown's avatar
unknown committed
3312 3313 3314 3315 3316
	    goto try_next;
	}
	if (found_record && pos == param->search_after_block)
	  mi_check_print_info(param,"Block: %s used by record at %s",
		     llstr(param->search_after_block,llbuff),
unknown's avatar
unknown committed
3317 3318
		     llstr(sort_param->start_recpos,llbuff2));
	if (_mi_read_cache(&sort_param->read_cache,
3319
                           (uchar*) block_info.header,pos,
3320 3321
			   MI_BLOCK_INFO_HEADER_LENGTH,
			   (! found_record ? READING_NEXT : 0) |
3322
                           parallel_flag | READING_HEADER))
unknown's avatar
unknown committed
3323 3324 3325 3326 3327
	{
	  if (found_record)
	  {
	    mi_check_print_info(param,
				"Can't read whole record at %s (errno: %d)",
unknown's avatar
unknown committed
3328
				llstr(sort_param->start_recpos,llbuff),errno);
unknown's avatar
unknown committed
3329 3330 3331 3332
	    goto try_next;
	  }
	  DBUG_RETURN(-1);
	}
unknown's avatar
unknown committed
3333
	if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3334 3335
	{
	  param->error_printed=1;
unknown's avatar
unknown committed
3336 3337
          param->retry_repair=1;
          param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3338 3339
	  DBUG_RETURN(1);	/* Something wrong with data */
	}
3340 3341 3342
	b_type=_mi_get_block_info(&block_info,-1,pos);
	if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
	   ((b_type & BLOCK_FIRST) &&
unknown's avatar
unknown committed
3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363
	     (block_info.rec_len < (uint) share->base.min_pack_length ||
	      block_info.rec_len > (uint) share->base.max_pack_length)))
	{
	  uint i;
	  if (param->testflag & T_VERBOSE || searching == 0)
	    mi_check_print_info(param,
				"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
		       block_info.header[0],block_info.header[1],
		       block_info.header[2],llstr(pos,llbuff));
	  if (found_record)
	    goto try_next;
	  block_info.second_read=0;
	  searching=1;
	  /* Search after block in read header string */
	  for (i=MI_DYN_ALIGN_SIZE ;
	       i < MI_BLOCK_INFO_HEADER_LENGTH ;
	       i+= MI_DYN_ALIGN_SIZE)
	    if (block_info.header[i] >= 1 &&
		block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
	      break;
	  pos+=(ulong) i;
unknown's avatar
unknown committed
3364
	  sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3365 3366 3367 3368
	  continue;
	}
	if (b_type & BLOCK_DELETED)
	{
3369
	  my_bool error=0;
unknown's avatar
unknown committed
3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Deleted block with impossible length %u at %s",
				  block_info.block_len,llstr(pos,llbuff));
	    error=1;
	  }
	  else
	  {
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >=
		 info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      if (!searching)
		mi_check_print_info(param,
				    "Delete link points outside datafile at %s",
				    llstr(pos,llbuff));
	      error=1;
	    }
	  }
	  if (error)
	  {
	    if (found_record)
	      goto try_next;
	    searching=1;
3399
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3400
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419
	    block_info.second_read=0;
	    continue;
	  }
	}
	else
	{
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length ||
	      block_info.block_len > (uint) share->base.max_pack_length+
	      MI_SPLIT_LENGTH)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Found block with impossible length %u at %s; Skipped",
				  block_info.block_len+ (uint) (block_info.filepos-pos),
				  llstr(pos,llbuff));
	    if (found_record)
	      goto try_next;
	    searching=1;
3420
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3421
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3422 3423 3424 3425 3426 3427
	    block_info.second_read=0;
	    continue;
	  }
	}
	if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
	{
3428 3429
          if (!sort_param->fix_datafile && sort_param->master &&
              (b_type & BLOCK_DELETED))
unknown's avatar
unknown committed
3430 3431 3432 3433 3434 3435 3436 3437
	  {
	    info->state->empty+=block_info.block_len;
	    info->state->del++;
	    share->state.split++;
	  }
	  if (found_record)
	    goto try_next;
	  if (searching)
3438 3439
	  {
	    pos+=MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3440
	    sort_param->start_recpos=pos;
3441
	  }
unknown's avatar
unknown committed
3442 3443 3444 3445 3446 3447
	  else
	    pos=block_info.filepos+block_info.block_len;
	  block_info.second_read=0;
	  continue;
	}

3448
	if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3449
	  share->state.split++;
unknown's avatar
unknown committed
3450 3451
	if (! found_record++)
	{
unknown's avatar
unknown committed
3452 3453 3454 3455 3456 3457
	  sort_param->find_length=left_length=block_info.rec_len;
	  sort_param->start_recpos=pos;
	  if (!sort_param->fix_datafile)
	    sort_param->filepos=sort_param->start_recpos;
	  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
	    sort_param->pos=block_info.filepos+1;
unknown's avatar
unknown committed
3458
	  else
unknown's avatar
unknown committed
3459
	    sort_param->pos=block_info.filepos+block_info.block_len;
unknown's avatar
unknown committed
3460 3461
	  if (share->base.blobs)
	  {
3462
	    if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3463
				       &(sort_param->rec_buff))))
unknown's avatar
unknown committed
3464
	    {
3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
	      if (param->max_record_length >= block_info.rec_len)
	      {
		mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
				     llstr(sort_param->start_recpos,llbuff),
				     (ulong) block_info.rec_len);
		DBUG_RETURN(1);
	      }
	      else
	      {
		mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
				    llstr(sort_param->start_recpos,llbuff),
				    (ulong) block_info.rec_len);
		goto try_next;
	      }
unknown's avatar
unknown committed
3479 3480 3481
	    }
	  }
	  else
3482
	    to= sort_param->rec_buff;
unknown's avatar
unknown committed
3483 3484 3485
	}
	if (left_length < block_info.data_len || ! block_info.data_len)
	{
3486 3487
	  mi_check_print_info(param,
			      "Found block with too small length at %s; Skipped",
unknown's avatar
unknown committed
3488
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3489 3490 3491
	  goto try_next;
	}
	if (block_info.filepos + block_info.data_len >
unknown's avatar
unknown committed
3492
	    sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3493
	{
3494 3495
	  mi_check_print_info(param,
			      "Found block that points outside data file at %s",
unknown's avatar
unknown committed
3496
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3497 3498
	  goto try_next;
	}
3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523
        /*
          Copy information that is already read. Avoid accessing data
          below the cache start. This could happen if the header
          streched over the end of the previous buffer contents.
        */
        {
          uint header_len= (uint) (block_info.filepos - pos);
          uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);

          if (prefetch_len > block_info.data_len)
            prefetch_len= block_info.data_len;
          if (prefetch_len)
          {
            memcpy(to, block_info.header + header_len, prefetch_len);
            block_info.filepos+= prefetch_len;
            block_info.data_len-= prefetch_len;
            left_length-= prefetch_len;
            to+= prefetch_len;
          }
        }
        if (block_info.data_len &&
            _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
                           block_info.data_len,
                           (found_record == 1 ? READING_NEXT : 0) |
                           parallel_flag))
unknown's avatar
unknown committed
3524
	{
3525 3526
	  mi_check_print_info(param,
			      "Read error for block at: %s (error: %d); Skipped",
3527
			      llstr(block_info.filepos,llbuff),my_errno);
unknown's avatar
unknown committed
3528 3529 3530 3531 3532 3533 3534 3535
	  goto try_next;
	}
	left_length-=block_info.data_len;
	to+=block_info.data_len;
	pos=block_info.next_filepos;
	if (pos == HA_OFFSET_ERROR && left_length)
	{
	  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
unknown's avatar
unknown committed
3536
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3537 3538
	  goto try_next;
	}
unknown's avatar
unknown committed
3539
	if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3540 3541
	{
	  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3542
			      llstr(pos,llbuff2),
unknown's avatar
unknown committed
3543
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3544 3545 3546 3547
	  goto try_next;
	}
      } while (left_length);

3548
      if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
unknown's avatar
unknown committed
3549
			 sort_param->find_length) != MY_FILE_ERROR)
unknown's avatar
unknown committed
3550
      {
unknown's avatar
unknown committed
3551
	if (sort_param->read_cache.error < 0)
unknown's avatar
unknown committed
3552
	  DBUG_RETURN(1);
3553 3554
	if (sort_param->calc_checksum)
	  info->checksum= mi_checksum(info, sort_param->record);
unknown's avatar
unknown committed
3555 3556
	if ((param->testflag & (T_EXTEND | T_REP)) || searching)
	{
unknown's avatar
unknown committed
3557
	  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3558 3559
                            sort_param->find_length,
                            (param->testflag & T_QUICK) &&
3560
                            sort_param->calc_checksum &&
3561
                            test(info->s->calc_checksum)))
unknown's avatar
unknown committed
3562 3563
	  {
	    mi_check_print_info(param,"Found wrong packed record at %s",
unknown's avatar
unknown committed
3564
				llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3565 3566 3567
	    goto try_next;
	  }
	}
3568
	if (sort_param->calc_checksum)
3569
	  param->glob_crc+= info->checksum;
unknown's avatar
unknown committed
3570 3571
	DBUG_RETURN(0);
      }
unknown's avatar
unknown committed
3572
      if (!searching)
3573 3574 3575
        mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
                            sort_param->key+1,
                            llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3576
    try_next:
unknown's avatar
unknown committed
3577
      pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3578 3579 3580
      searching=1;
    }
  case COMPRESSED_RECORD:
unknown's avatar
unknown committed
3581
    for (searching=0 ;; searching=1, sort_param->pos++)
unknown's avatar
unknown committed
3582
    {
3583
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
unknown's avatar
unknown committed
3584
			 sort_param->pos,
3585
			 share->pack.ref_length,READING_NEXT))
unknown's avatar
unknown committed
3586
	DBUG_RETURN(-1);
unknown's avatar
unknown committed
3587
      if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3588 3589
      {
	param->error_printed=1;
unknown's avatar
unknown committed
3590 3591
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3592 3593
	DBUG_RETURN(1);		/* Something wrong with data */
      }
unknown's avatar
unknown committed
3594
      sort_param->start_recpos=sort_param->pos;
3595 3596
      if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
                                  &sort_param->rec_buff, -1, sort_param->pos))
unknown's avatar
unknown committed
3597 3598
	DBUG_RETURN(-1);
      if (!block_info.rec_len &&
unknown's avatar
unknown committed
3599 3600
	  sort_param->pos + MEMMAP_EXTRA_MARGIN ==
	  sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3601 3602 3603 3604 3605 3606
	DBUG_RETURN(-1);
      if (block_info.rec_len < (uint) share->min_pack_length ||
	  block_info.rec_len > (uint) share->max_pack_length)
      {
	if (! searching)
	  mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3607
			      block_info.rec_len,
unknown's avatar
unknown committed
3608
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3609 3610
	continue;
      }
3611
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3612 3613
			 block_info.filepos, block_info.rec_len,
			 READING_NEXT))
unknown's avatar
unknown committed
3614 3615
      {
	if (! searching)
3616
	  mi_check_print_info(param,"Couldn't read whole record from %s",
unknown's avatar
unknown committed
3617
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3618 3619
	continue;
      }
3620 3621
      if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
                              sort_param->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
3622 3623
      {
	if (! searching)
3624
	  mi_check_print_info(param,"Found wrong record at %s",
unknown's avatar
unknown committed
3625
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3626 3627
	continue;
      }
unknown's avatar
unknown committed
3628
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3629
      {
unknown's avatar
unknown committed
3630
	sort_param->filepos=sort_param->pos;
3631 3632
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3633
      }
unknown's avatar
unknown committed
3634
      sort_param->max_pos=(sort_param->pos=block_info.filepos+
unknown's avatar
unknown committed
3635 3636
			 block_info.rec_len);
      info->packed_length=block_info.rec_len;
3637 3638 3639
      if (sort_param->calc_checksum)
	param->glob_crc+= (info->checksum=
                           mi_checksum(info, sort_param->record));
unknown's avatar
unknown committed
3640 3641
      DBUG_RETURN(0);
    }
3642 3643
  case BLOCK_RECORD:
    assert(0);                                  /* Impossible */
unknown's avatar
unknown committed
3644
  }
3645
  DBUG_RETURN(1);                               /* Impossible */
unknown's avatar
unknown committed
3646 3647 3648
}


3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662
/*
  Write record to new file.

  SYNOPSIS
    sort_write_record()
      sort_param                Sort parameters.

  NOTE
    This is only called by a master thread if parallel repair is used.

  RETURN
    0           OK
    1           Error
*/
unknown's avatar
unknown committed
3663

unknown's avatar
unknown committed
3664
int sort_write_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3665 3666 3667 3668
{
  int flag;
  uint length;
  ulong block_length,reclength;
3669 3670
  uchar *from;
  uchar block_buff[8];
unknown's avatar
unknown committed
3671
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3672
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3673 3674
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3675 3676
  DBUG_ENTER("sort_write_record");

unknown's avatar
unknown committed
3677
  if (sort_param->fix_datafile)
unknown's avatar
unknown committed
3678 3679 3680
  {
    switch (sort_info->new_data_file_type) {
    case STATIC_RECORD:
unknown's avatar
unknown committed
3681
      if (my_b_write(&info->rec_cache,sort_param->record,
unknown's avatar
unknown committed
3682 3683 3684 3685 3686
		     share->base.pack_reclength))
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3687
      sort_param->filepos+=share->base.pack_reclength;
unknown's avatar
unknown committed
3688
      info->s->state.split++;
unknown's avatar
unknown committed
3689
      /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
unknown's avatar
unknown committed
3690 3691 3692
      break;
    case DYNAMIC_RECORD:
      if (! info->blobs)
3693
	from=sort_param->rec_buff;
unknown's avatar
unknown committed
3694 3695 3696 3697
      else
      {
	/* must be sure that local buffer is big enough */
	reclength=info->s->base.pack_reclength+
unknown's avatar
unknown committed
3698
	  _my_calc_total_blob_length(info,sort_param->record)+
unknown's avatar
unknown committed
3699 3700 3701 3702 3703
	  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
	  MI_DYN_DELETE_BLOCK_HEADER;
	if (sort_info->buff_length < reclength)
	{
	  if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3704 3705
					   MYF(MY_FREE_ON_ERROR |
					       MY_ALLOW_ZERO_PTR))))
unknown's avatar
unknown committed
3706 3707 3708
	    DBUG_RETURN(1);
	  sort_info->buff_length=reclength;
	}
3709
	from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
unknown's avatar
unknown committed
3710
      }
3711
      /* We can use info->checksum here as only one thread calls this. */
unknown's avatar
unknown committed
3712 3713
      info->checksum=mi_checksum(info,sort_param->record);
      reclength=_mi_rec_pack(info,from,sort_param->record);
unknown's avatar
unknown committed
3714
      flag=0;
unknown's avatar
unknown committed
3715 3716 3717
      /* sort_info->param->glob_crc+=info->checksum; */

      do
unknown's avatar
unknown committed
3718
      {
unknown's avatar
unknown committed
3719 3720 3721 3722 3723 3724 3725 3726
	block_length=reclength+ 3 + test(reclength >= (65520-3));
	if (block_length < share->base.min_block_length)
	  block_length=share->base.min_block_length;
	info->update|=HA_STATE_WRITE_AT_END;
	block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
	if (block_length > MI_MAX_BLOCK_LENGTH)
	  block_length=MI_MAX_BLOCK_LENGTH;
	if (_mi_write_part_record(info,0L,block_length,
unknown's avatar
unknown committed
3727
				  sort_param->filepos+block_length,
unknown's avatar
unknown committed
3728 3729 3730 3731 3732
				  &from,&reclength,&flag))
	{
	  mi_check_print_error(param,"%d when writing to datafile",my_errno);
	  DBUG_RETURN(1);
	}
unknown's avatar
unknown committed
3733
	sort_param->filepos+=block_length;
unknown's avatar
unknown committed
3734 3735
	info->s->state.split++;
      } while (reclength);
unknown's avatar
unknown committed
3736
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3737 3738 3739
      break;
    case COMPRESSED_RECORD:
      reclength=info->packed_length;
3740 3741
      length= save_pack_length((uint) share->pack.version, block_buff,
                               reclength);
unknown's avatar
unknown committed
3742
      if (info->s->base.blobs)
3743 3744
	length+= save_pack_length((uint) share->pack.version,
	                          block_buff + length, info->blob_length);
unknown's avatar
unknown committed
3745
      if (my_b_write(&info->rec_cache,block_buff,length) ||
3746
	  my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
unknown's avatar
unknown committed
3747 3748 3749 3750
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3751
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3752
      sort_param->filepos+=reclength+length;
unknown's avatar
unknown committed
3753
      info->s->state.split++;
unknown's avatar
unknown committed
3754
      break;
3755 3756
    case BLOCK_RECORD:
      assert(0);                                  /* Impossible */
unknown's avatar
unknown committed
3757 3758
    }
  }
3759
  if (sort_param->master)
unknown's avatar
unknown committed
3760
  {
3761 3762 3763 3764 3765
    info->state->records++;
    if ((param->testflag & T_WRITE_LOOP) &&
        (info->state->records % WRITE_COUNT) == 0)
    {
      char llbuff[22];
3766
      printf("%s\r", llstr(info->state->records,llbuff));
Konstantin Osipov's avatar
Konstantin Osipov committed
3767
      (void) fflush(stdout);
3768
    }
unknown's avatar
unknown committed
3769 3770 3771 3772 3773 3774 3775
  }
  DBUG_RETURN(0);
} /* sort_write_record */


	/* Compare two keys from _create_index_by_sort */

unknown's avatar
unknown committed
3776 3777
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
			const void *b)
unknown's avatar
unknown committed
3778
{
3779
  uint not_used[2];
3780
  return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3781
		     USE_WHOLE_KEY, SEARCH_SAME, not_used));
unknown's avatar
unknown committed
3782 3783 3784
} /* sort_key_cmp */


unknown's avatar
unknown committed
3785
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
unknown's avatar
unknown committed
3786
{
3787
  uint diff_pos[2];
unknown's avatar
unknown committed
3788
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
3789
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3790 3791 3792 3793 3794
  MI_CHECK *param= sort_info->param;
  int cmp;

  if (sort_info->key_block->inited)
  {
3795
    cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
unknown's avatar
unknown committed
3796
		   (uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3797
		   diff_pos);
3798 3799 3800
    if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
      ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
                 (uchar*) a, USE_WHOLE_KEY, 
3801
                 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3802 3803
    else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
    {
3804 3805 3806 3807
      diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
                                                 sort_param->notnull,
                                                 sort_info->key_block->lastkey,
                                                 (uchar*)a);
3808
    }
3809
    sort_param->unique[diff_pos[0]-1]++;
unknown's avatar
unknown committed
3810 3811 3812
  }
  else
  {
3813
    cmp= -1;
3814 3815 3816
    if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
      mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
                                     (uchar*)a);
unknown's avatar
unknown committed
3817
  }
unknown's avatar
unknown committed
3818
  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
unknown's avatar
unknown committed
3819 3820 3821
  {
    sort_info->dupp++;
    sort_info->info->lastpos=get_record_for_key(sort_info->info,
3822
						sort_param->keyinfo,
unknown's avatar
unknown committed
3823
						(uchar*) a);
unknown's avatar
unknown committed
3824 3825 3826 3827
    mi_check_print_warning(param,
			   "Duplicate key for record at %10s against record at %10s",
			   llstr(sort_info->info->lastpos,llbuff),
			   llstr(get_record_for_key(sort_info->info,
unknown's avatar
unknown committed
3828
						    sort_param->keyinfo,
unknown's avatar
unknown committed
3829 3830 3831
						    sort_info->key_block->
						    lastkey),
				 llbuff2));
unknown's avatar
unknown committed
3832
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3833
    if (sort_info->param->testflag & T_VERBOSE)
3834
      _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
unknown's avatar
unknown committed
3835
    return (sort_delete_record(sort_param));
unknown's avatar
unknown committed
3836 3837 3838 3839 3840 3841 3842 3843 3844
  }
#ifndef DBUG_OFF
  if (cmp > 0)
  {
    mi_check_print_error(param,
			 "Internal error: Keys are not in order from sort");
    return(1);
  }
#endif
unknown's avatar
unknown committed
3845
  return (sort_insert_key(sort_param,sort_info->key_block,
unknown's avatar
unknown committed
3846
			  (uchar*) a, HA_OFFSET_ERROR));
unknown's avatar
unknown committed
3847 3848
} /* sort_key_write */

3849 3850 3851 3852 3853
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
{
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
  MYISAM_SHARE *share=sort_info->info->s;
unknown's avatar
unknown committed
3854 3855
  uint val_off, val_len;
  int error;
3856 3857 3858 3859 3860 3861 3862 3863
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  uchar *from, *to;

  val_len=share->ft2_keyinfo.keylength;
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
  to=ft_buf->lastkey+val_off;

  if (ft_buf->buf)
unknown's avatar
unknown committed
3864 3865 3866 3867
  {
    /* flushing first-level tree */
    error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			  HA_OFFSET_ERROR);
3868 3869 3870 3871 3872
    for (from=to+val_len;
         !error && from < ft_buf->buf;
         from+= val_len)
    {
      memcpy(to, from, val_len);
unknown's avatar
unknown committed
3873 3874
      error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			    HA_OFFSET_ERROR);
3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901
    }
    return error;
  }
  /* flushing second-level tree keyblocks */
  error=flush_pending_blocks(sort_param);
  /* updating lastkey with second-level tree info */
  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
  _mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
      share->state.key_root[sort_param->key]);
  /* restoring first level tree data in sort_info/sort_param */
  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
  sort_param->keyinfo=share->keyinfo+sort_param->key;
  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
  /* writing lastkey in first-level tree */
  return error ? error :
                 sort_insert_key(sort_param,sort_info->key_block,
                                 ft_buf->lastkey,HA_OFFSET_ERROR);
}

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
{
  uint a_len, val_off, val_len, error;
  uchar *p;
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;

3902
  val_len= HA_FT_WLEN + sort_info->info->s->rec_reflength;
3903 3904 3905 3906 3907 3908 3909 3910 3911
  get_key_full_length_rdonly(a_len, (uchar *)a);

  if (!ft_buf)
  {
    /*
      use two-level tree only if key_reflength fits in rec_reflength place
      and row format is NOT static - for _mi_dpointer not to garble offsets
     */
    if ((sort_info->info->s->base.key_reflength <=
3912
         sort_info->info->s->rec_reflength) &&
3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927
        (sort_info->info->s->options &
          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
                                      sizeof(SORT_FT_BUF), MYF(MY_WME));

    if (!ft_buf)
    {
      sort_param->key_write=sort_key_write;
      return sort_key_write(sort_param, a);
    }
    sort_info->ft_buf=ft_buf;
    goto word_init_ft_buf; /* no need to duplicate the code */
  }
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3928
  if (ha_compare_text(sort_param->seg->charset,
3929
                      ((uchar *)a)+1,a_len-1,
3930
                      ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3931 3932 3933 3934 3935
  {
    if (!ft_buf->buf) /* store in second-level tree */
    {
      ft_buf->count++;
      return sort_insert_key(sort_param,key_block,
3936
                             ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3937 3938 3939
    }

    /* storing the key in the buffer. */
3940
    memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951
    ft_buf->buf+=val_len;
    if (ft_buf->buf < ft_buf->end)
      return 0;

    /* converting to two-level tree */
    p=ft_buf->lastkey+val_off;

    while (key_block->inited)
      key_block++;
    sort_info->key_block=key_block;
    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
3952
    ft_buf->count=(uint) (ft_buf->buf - p)/val_len;
3953 3954 3955 3956 3957 3958 3959 3960

    /* flushing buffer to second-level tree */
    for (error=0; !error && p < ft_buf->buf; p+= val_len)
      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
    ft_buf->buf=0;
    return error;
  }

3961 3962 3963
  /* flushing buffer */
  if ((error=sort_ft_buf_flush(sort_param)))
    return error;
3964

3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978
word_init_ft_buf:
  a_len+=val_len;
  memcpy(ft_buf->lastkey, a, a_len);
  ft_buf->buf=ft_buf->lastkey+a_len;
  /*
    32 is just a safety margin here
    (at least max(val_len, sizeof(nod_flag)) should be there).
    May be better performance could be achieved if we'd put
      (sort_info->keyinfo->block_length-32)/XXX
      instead.
        TODO: benchmark the best value for XXX.
  */
  ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
  return 0;
3979
} /* sort_ft_key_write */
unknown's avatar
unknown committed
3980

3981

unknown's avatar
unknown committed
3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992
	/* get pointer to record from a key */

static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
				   uchar *key)
{
  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
} /* get_record_for_key */


	/* Insert a key in sort-key-blocks */

unknown's avatar
unknown committed
3993
static int sort_insert_key(MI_SORT_PARAM *sort_param,
unknown's avatar
unknown committed
3994 3995 3996 3997
			   register SORT_KEY_BLOCKS *key_block, uchar *key,
			   my_off_t prev_block)
{
  uint a_length,t_length,nod_flag;
3998
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
3999 4000 4001
  uchar *anc_buff,*lastkey;
  MI_KEY_PARAM s_temp;
  MI_INFO *info;
unknown's avatar
unknown committed
4002 4003 4004
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
  SORT_INFO *sort_info= sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
4005 4006 4007 4008 4009 4010
  DBUG_ENTER("sort_insert_key");

  anc_buff=key_block->buff;
  info=sort_info->info;
  lastkey=key_block->lastkey;
  nod_flag= (key_block == sort_info->key_block ? 0 :
unknown's avatar
unknown committed
4011
	     info->s->base.key_reflength);
unknown's avatar
unknown committed
4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031

  if (!key_block->inited)
  {
    key_block->inited=1;
    if (key_block == sort_info->key_block_end)
    {
      mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
      DBUG_RETURN(1);
    }
    a_length=2+nod_flag;
    key_block->end_pos=anc_buff+2;
    lastkey=0;					/* No previous key in block */
  }
  else
    a_length=mi_getint(anc_buff);

	/* Save pointer to previous block */
  if (nod_flag)
    _mi_kpointer(info,key_block->end_pos,prev_block);

unknown's avatar
unknown committed
4032 4033 4034 4035
  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
				(uchar*) 0,lastkey,lastkey,key,
				 &s_temp);
  (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
unknown's avatar
unknown committed
4036 4037 4038
  a_length+=t_length;
  mi_putint(anc_buff,a_length,nod_flag);
  key_block->end_pos+=t_length;
unknown's avatar
unknown committed
4039
  if (a_length <= keyinfo->block_length)
unknown's avatar
unknown committed
4040
  {
Konstantin Osipov's avatar
Konstantin Osipov committed
4041
    (void) _mi_move_key(keyinfo,key_block->lastkey,key);
unknown's avatar
unknown committed
4042 4043 4044 4045 4046 4047
    key_block->last_length=a_length-t_length;
    DBUG_RETURN(0);
  }

	/* Fill block with end-zero and write filled block */
  mi_putint(anc_buff,key_block->last_length,nod_flag);
4048
  bzero((uchar*) anc_buff+key_block->last_length,
unknown's avatar
unknown committed
4049
	keyinfo->block_length- key_block->last_length);
4050
  key_file_length=info->state->key_file_length;
4051
  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4052 4053 4054 4055 4056
    DBUG_RETURN(1);

  /* If we read the page from the key cache, we have to write it back to it */
  if (key_file_length == info->state->key_file_length)
  {
4057
    if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4058 4059
      DBUG_RETURN(1);
  }
Marc Alff's avatar
Marc Alff committed
4060 4061 4062
  else if (mysql_file_pwrite(info->s->kfile, (uchar*) anc_buff,
                             (uint) keyinfo->block_length, filepos,
                             param->myf_rw))
unknown's avatar
unknown committed
4063
    DBUG_RETURN(1);
4064
  DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
unknown's avatar
unknown committed
4065 4066

	/* Write separator-key to block in next level */
unknown's avatar
unknown committed
4067
  if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
unknown's avatar
unknown committed
4068 4069 4070 4071
    DBUG_RETURN(1);

	/* clear old block and write new key in it */
  key_block->inited=0;
unknown's avatar
unknown committed
4072
  DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
unknown's avatar
unknown committed
4073 4074 4075 4076 4077
} /* sort_insert_key */


	/* Delete record when we found a duplicated key */

unknown's avatar
unknown committed
4078
static int sort_delete_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
4079 4080 4081 4082
{
  uint i;
  int old_file,error;
  uchar *key;
unknown's avatar
unknown committed
4083 4084 4085
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
4086 4087
  DBUG_ENTER("sort_delete_record");

4088
  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
4089 4090
  {
    mi_check_print_error(param,
unknown's avatar
unknown committed
4091
			 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
unknown's avatar
unknown committed
4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102
    DBUG_RETURN(1);
  }
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
  {
    mi_check_print_error(param,
			 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
    DBUG_RETURN(1);
  }

  old_file=info->dfile;
  info->dfile=info->rec_cache.file;
unknown's avatar
unknown committed
4103
  if (sort_info->current_key)
unknown's avatar
unknown committed
4104 4105
  {
    key=info->lastkey+info->s->base.max_key_length;
unknown's avatar
unknown committed
4106
    if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
unknown's avatar
unknown committed
4107 4108 4109 4110 4111 4112 4113
	error != HA_ERR_RECORD_DELETED)
    {
      mi_check_print_error(param,"Can't read record to be removed");
      info->dfile=old_file;
      DBUG_RETURN(1);
    }

unknown's avatar
unknown committed
4114
    for (i=0 ; i < sort_info->current_key ; i++)
unknown's avatar
unknown committed
4115
    {
unknown's avatar
unknown committed
4116
      uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
unknown's avatar
unknown committed
4117 4118 4119 4120 4121 4122 4123
      if (_mi_ck_delete(info,i,key,key_length))
      {
	mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
	info->dfile=old_file;
	DBUG_RETURN(1);
      }
    }
4124
    if (sort_param->calc_checksum)
unknown's avatar
unknown committed
4125
      param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
unknown's avatar
unknown committed
4126 4127 4128 4129 4130 4131 4132 4133 4134
  }
  error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
  info->dfile=old_file;				/* restore actual value */
  info->state->records--;
  DBUG_RETURN(error);
} /* sort_delete_record */

	/* Fix all pending blocks and flush everything to disk */

unknown's avatar
unknown committed
4135
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
4136 4137
{
  uint nod_flag,length;
4138
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
4139
  SORT_KEY_BLOCKS *key_block;
unknown's avatar
unknown committed
4140
  SORT_INFO *sort_info= sort_param->sort_info;
4141
  myf myf_rw=sort_info->param->myf_rw;
unknown's avatar
unknown committed
4142 4143
  MI_INFO *info=sort_info->info;
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
unknown's avatar
unknown committed
4144 4145 4146 4147 4148 4149 4150 4151 4152 4153
  DBUG_ENTER("flush_pending_blocks");

  filepos= HA_OFFSET_ERROR;			/* if empty file */
  nod_flag=0;
  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
  {
    key_block->inited=0;
    length=mi_getint(key_block->buff);
    if (nod_flag)
      _mi_kpointer(info,key_block->end_pos,filepos);
4154
    key_file_length=info->state->key_file_length;
4155
    bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4156
    if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4157 4158 4159 4160 4161
      DBUG_RETURN(1);

    /* If we read the page from the key cache, we have to write it back */
    if (key_file_length == info->state->key_file_length)
    {
4162 4163
      if (_mi_write_keypage(info, keyinfo, filepos,
                            DFLT_INIT_HITS, key_block->buff))
4164 4165
	DBUG_RETURN(1);
    }
Marc Alff's avatar
Marc Alff committed
4166 4167
    else if (mysql_file_pwrite(info->s->kfile, (uchar*) key_block->buff,
                               (uint) keyinfo->block_length, filepos, myf_rw))
unknown's avatar
unknown committed
4168
      DBUG_RETURN(1);
4169
    DBUG_DUMP("buff",(uchar*) key_block->buff,length);
unknown's avatar
unknown committed
4170 4171
    nod_flag=1;
  }
unknown's avatar
unknown committed
4172
  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
unknown's avatar
unknown committed
4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
  DBUG_RETURN(0);
} /* flush_pending_blocks */

	/* alloc space and pointers for key_blocks */

static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
                                         uint buffer_length)
{
  reg1 uint i;
  SORT_KEY_BLOCKS *block;
  DBUG_ENTER("alloc_key_blocks");

  if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
					    buffer_length+IO_SIZE)*blocks,
					   MYF(0))))
  {
4189
    mi_check_print_error(param,"Not enough memory for sort-key-blocks");
unknown's avatar
unknown committed
4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206
    return(0);
  }
  for (i=0 ; i < blocks ; i++)
  {
    block[i].inited=0;
    block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
  }
  DBUG_RETURN(block);
} /* alloc_key_blocks */


	/* Check if file is almost full */

int test_if_almost_full(MI_INFO *info)
{
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    return 0;
Marc Alff's avatar
Marc Alff committed
4207 4208
  return mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END,
                         MYF(MY_THREADSAFE)) / 10 * 9 >
4209
         (my_off_t) info->s->base.max_key_file_length ||
Marc Alff's avatar
Marc Alff committed
4210 4211
         mysql_file_seek(info->dfile, 0L, MY_SEEK_END,
                         MYF(0)) / 10 * 9 >
4212
         (my_off_t) info->s->base.max_data_file_length;
unknown's avatar
unknown committed
4213 4214 4215 4216 4217 4218 4219 4220 4221 4222
}

	/* Recreate table with bigger more alloced record-data */

int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
{
  int error;
  MI_INFO info;
  MYISAM_SHARE share;
  MI_KEYDEF *keyinfo,*key,*key_end;
unknown's avatar
unknown committed
4223
  HA_KEYSEG *keysegs,*keyseg;
unknown's avatar
unknown committed
4224 4225 4226 4227 4228 4229 4230
  MI_COLUMNDEF *recdef,*rec,*end;
  MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
  MI_STATUS_INFO status_info;
  uint unpack,key_parts;
  ha_rows max_records;
  ulonglong file_length,tmp_length;
  MI_CREATE_INFO create_info;
4231
  DBUG_ENTER("recreate_table");
unknown's avatar
unknown committed
4232 4233 4234 4235 4236 4237 4238 4239 4240

  error=1;					/* Default error */
  info= **org_info;
  status_info= (*org_info)->state[0];
  info.state= &status_info;
  share= *(*org_info)->s;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4241
    DBUG_RETURN(0);
4242
  memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
unknown's avatar
unknown committed
4243 4244 4245
	 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));

  key_parts= share.base.all_key_parts;
unknown's avatar
unknown committed
4246
  if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
unknown's avatar
unknown committed
4247 4248
				       (key_parts+share.base.keys))))
  {
4249
    my_afree((uchar*) keyinfo);
4250
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4251 4252 4253 4254
  }
  if (!(recdef=(MI_COLUMNDEF*)
	my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
  {
4255 4256
    my_afree((uchar*) keyinfo);
    my_afree((uchar*) keysegs);
4257
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4258 4259 4260 4261
  }
  if (!(uniquedef=(MI_UNIQUEDEF*)
	my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
  {
4262 4263 4264
    my_afree((uchar*) recdef);
    my_afree((uchar*) keyinfo);
    my_afree((uchar*) keysegs);
4265
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4266 4267 4268
  }

  /* Copy the column definitions */
4269
  memcpy((uchar*) recdef,(uchar*) share.rec,
unknown's avatar
unknown committed
4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280
	 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
  for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
  {
    if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
	rec->type != FIELD_BLOB &&
	rec->type != FIELD_VARCHAR &&
	rec->type != FIELD_CHECK)
      rec->type=(int) FIELD_NORMAL;
  }

  /* Change the new key to point at the saved key segments */
4281
  memcpy((uchar*) keysegs,(uchar*) share.keyparts,
unknown's avatar
unknown committed
4282
	 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
unknown's avatar
unknown committed
4283 4284 4285 4286 4287 4288 4289 4290 4291 4292
				      share.state.header.uniques)));
  keyseg=keysegs;
  for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
  {
    key->seg=keyseg;
    for (; keyseg->type ; keyseg++)
    {
      if (param->language)
	keyseg->language=param->language;	/* change language */
    }
4293
    keyseg++;					/* Skip end pointer */
unknown's avatar
unknown committed
4294 4295 4296 4297
  }

  /* Copy the unique definitions and change them to point at the new key
     segments*/
4298
  memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
unknown's avatar
unknown committed
4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309
	 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
  for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
       u_ptr != u_end ; u_ptr++)
  {
    u_ptr->seg=keyseg;
    keyseg+=u_ptr->keysegs+1;
  }
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;

Marc Alff's avatar
Marc Alff committed
4310
  file_length=(ulonglong) mysql_file_seek(info.dfile, 0L, MY_SEEK_END, MYF(0));
unknown's avatar
unknown committed
4311 4312 4313 4314
  tmp_length= file_length+file_length/10;
  set_if_bigger(file_length,param->max_data_file_length);
  set_if_bigger(file_length,tmp_length);
  set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);
4315 4316
 
  if (share.options & HA_OPTION_COMPRESS_RECORD)
4317
    share.base.records= max_records= info.state->records;
4318 4319 4320 4321
  else if (!(share.options & HA_OPTION_PACK_RECORD))
    max_records= (ha_rows) (file_length / share.base.pack_reclength);
  else
    max_records= 0;
unknown's avatar
unknown committed
4322

Konstantin Osipov's avatar
Konstantin Osipov committed
4323
  (void) mi_close(*org_info);
unknown's avatar
unknown committed
4324
  bzero((char*) &create_info,sizeof(create_info));
4325
  create_info.max_rows= max_records;
unknown's avatar
unknown committed
4326 4327 4328 4329 4330 4331 4332 4333
  create_info.reloc_rows=share.base.reloc;
  create_info.old_options=(share.options |
			   (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));

  create_info.data_file_length=file_length;
  create_info.auto_increment=share.state.auto_increment;
  create_info.language = (param->language ? param->language :
			  share.state.header.language);
unknown's avatar
unknown committed
4334
  create_info.key_file_length=  status_info.key_file_length;
4335 4336 4337 4338 4339
  /*
    Allow for creating an auto_increment key. This has an effect only if
    an auto_increment key exists in the original table.
  */
  create_info.with_auto_increment= TRUE;
unknown's avatar
unknown committed
4340 4341 4342
  /* We don't have to handle symlinks here because we are using
     HA_DONT_TOUCH_DATA */
  if (mi_create(filename,
unknown's avatar
unknown committed
4343 4344 4345 4346 4347 4348 4349 4350 4351
		share.base.keys - share.state.header.uniques,
		keyinfo, share.base.fields, recdef,
		share.state.header.uniques, uniquedef,
		&create_info,
		HA_DONT_TOUCH_DATA))
  {
    mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
    goto end;
  }
unknown's avatar
unknown committed
4352
  *org_info=mi_open(filename,O_RDWR,
unknown's avatar
unknown committed
4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363
		    (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
		    (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
		    HA_OPEN_ABORT_IF_LOCKED);
  if (!*org_info)
  {
    mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
		my_errno);
    goto end;
  }
  /* We are modifing */
  (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
Konstantin Osipov's avatar
Konstantin Osipov committed
4364
  (void) _mi_readinfo(*org_info,F_WRLCK,0);
unknown's avatar
unknown committed
4365 4366 4367 4368 4369
  (*org_info)->state->records=info.state->records;
  if (share.state.create_time)
    (*org_info)->s->state.create_time=share.state.create_time;
  (*org_info)->s->state.unique=(*org_info)->this_unique=
    share.state.unique;
4370
  (*org_info)->state->checksum=info.state->checksum;
unknown's avatar
unknown committed
4371 4372 4373 4374 4375 4376 4377 4378 4379
  (*org_info)->state->del=info.state->del;
  (*org_info)->s->state.dellink=share.state.dellink;
  (*org_info)->state->empty=info.state->empty;
  (*org_info)->state->data_file_length=info.state->data_file_length;
  if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
			UPDATE_OPEN_COUNT))
    goto end;
  error=0;
end:
4380 4381 4382 4383
  my_afree((uchar*) uniquedef);
  my_afree((uchar*) keyinfo);
  my_afree((uchar*) recdef);
  my_afree((uchar*) keysegs);
4384
  DBUG_RETURN(error);
unknown's avatar
unknown committed
4385 4386 4387 4388 4389
}


	/* write suffix to data file if neaded */

unknown's avatar
unknown committed
4390
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
unknown's avatar
unknown committed
4391
{
unknown's avatar
unknown committed
4392 4393 4394
  MI_INFO *info=sort_info->info;

  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
unknown's avatar
unknown committed
4395
  {
4396
    uchar buff[MEMMAP_EXTRA_MARGIN];
unknown's avatar
unknown committed
4397 4398 4399
    bzero(buff,sizeof(buff));
    if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
    {
unknown's avatar
unknown committed
4400
      mi_check_print_error(sort_info->param,
unknown's avatar
unknown committed
4401
			   "%d when writing to datafile",my_errno);
unknown's avatar
unknown committed
4402 4403
      return 1;
    }
unknown's avatar
unknown committed
4404
    sort_info->param->read_cache.end_of_file+=sizeof(buff);
unknown's avatar
unknown committed
4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421
  }
  return 0;
}

	/* Update state and myisamchk_time of indexfile */

int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
{
  MYISAM_SHARE *share=info->s;

  if (update & UPDATE_OPEN_COUNT)
  {
    share->state.open_count=0;
    share->global_changed=0;
  }
  if (update & UPDATE_STAT)
  {
4422
    uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
unknown's avatar
unknown committed
4423
    share->state.rec_per_key_rows=info->state->records;
4424
    share->state.changed&= ~STATE_NOT_ANALYZED;
unknown's avatar
unknown committed
4425
    if (info->state->records)
4426
    {
unknown's avatar
unknown committed
4427 4428 4429 4430 4431
      for (i=0; i<key_parts; i++)
      {
        if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
          share->state.changed|= STATE_NOT_ANALYZED;
      }
4432
    }
unknown's avatar
unknown committed
4433 4434 4435 4436 4437 4438 4439 4440 4441
  }
  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
  {
    if (update & UPDATE_TIME)
    {
      share->state.check_time= (long) time((time_t*) 0);
      if (!share->state.create_time)
	share->state.create_time=share->state.check_time;
    }
unknown's avatar
unknown committed
4442 4443 4444
    /*
      When tables are locked we haven't synched the share state and the
      real state for a while so we better do it here before synching
unknown's avatar
unknown committed
4445 4446
      the share state to disk. Only when table is write locked is it
      necessary to perform this synch.
unknown's avatar
unknown committed
4447
    */
unknown's avatar
unknown committed
4448 4449
    if (info->lock_type == F_WRLCK)
      share->state.state= *info->state;
unknown's avatar
unknown committed
4450 4451
    if (mi_state_info_write(share->kfile,&share->state,1+2))
      goto err;
unknown's avatar
unknown committed
4452
    share->changed=0;
unknown's avatar
unknown committed
4453 4454 4455 4456
  }
  {						/* Force update of status */
    int error;
    uint r_locks=share->r_locks,w_locks=share->w_locks;
4457
    share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
4458
    error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4459 4460 4461
    share->r_locks=r_locks;
    share->w_locks=w_locks;
    share->tot_locks=r_locks+w_locks;
unknown's avatar
unknown committed
4462 4463 4464 4465
    if (!error)
      return 0;
  }
err:
unknown's avatar
unknown committed
4466
  mi_check_print_error(param,"%d when updating keyfile",my_errno);
unknown's avatar
unknown committed
4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485
  return 1;
}

	/*
	  Update auto increment value for a table
	  When setting the 'repair_only' flag we only want to change the
	  old auto_increment value if its wrong (smaller than some given key).
	  The reason is that we shouldn't change the auto_increment value
	  for a table without good reason when only doing a repair; If the
	  user have inserted and deleted rows, the auto_increment value
	  may be bigger than the biggest current row and this is ok.

	  If repair_only is not set, we will update the flag to the value in
	  param->auto_increment is bigger than the biggest key.
	*/

void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
			       my_bool repair_only)
{
4486
  uchar *record= 0;
4487 4488
  DBUG_ENTER("update_auto_increment_key");

unknown's avatar
unknown committed
4489
  if (!info->s->base.auto_key ||
4490
      ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
unknown's avatar
unknown committed
4491
  {
unknown's avatar
unknown committed
4492 4493 4494 4495
    if (!(param->testflag & T_VERY_SILENT))
      mi_check_print_info(param,
			  "Table: %s doesn't have an auto increment key\n",
			  param->isam_file_name);
4496
    DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4497 4498
  }
  if (!(param->testflag & T_SILENT) &&
4499
      !(param->testflag & T_REP))
unknown's avatar
unknown committed
4500
    printf("Updating MyISAM file: %s\n", param->isam_file_name);
4501 4502 4503 4504
  /*
    We have to use an allocated buffer instead of info->rec_buff as 
    _mi_put_key_in_record() may use info->rec_buff
  */
4505
  if (!mi_alloc_rec_buff(info, -1, &record))
4506 4507
  {
    mi_check_print_error(param,"Not enough memory for extra record");
4508
    DBUG_VOID_RETURN;
4509 4510
  }

unknown's avatar
unknown committed
4511
  mi_extra(info,HA_EXTRA_KEYREAD,0);
4512
  if (mi_rlast(info, record, info->s->base.auto_key-1))
unknown's avatar
unknown committed
4513 4514 4515
  {
    if (my_errno != HA_ERR_END_OF_FILE)
    {
unknown's avatar
unknown committed
4516
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4517
      my_free(mi_get_rec_buff_ptr(info, record));
unknown's avatar
unknown committed
4518
      mi_check_print_error(param,"%d when reading last record",my_errno);
4519
      DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4520 4521 4522 4523 4524 4525
    }
    if (!repair_only)
      info->s->state.auto_increment=param->auto_increment_value;
  }
  else
  {
4526
    ulonglong auto_increment= retrieve_auto_increment(info, record);
unknown's avatar
unknown committed
4527
    set_if_bigger(info->s->state.auto_increment,auto_increment);
4528 4529
    if (!repair_only)
      set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
unknown's avatar
unknown committed
4530
  }
unknown's avatar
unknown committed
4531
  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4532
  my_free(mi_get_rec_buff_ptr(info, record));
unknown's avatar
unknown committed
4533
  update_state_info(param, info, UPDATE_AUTO_INC);
4534
  DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4535 4536
}

4537 4538 4539

/*
  Update statistics for each part of an index
4540

4541 4542
  SYNOPSIS
    update_key_parts()
4543
      keyinfo           IN  Index information (only key->keysegs used)
4544
      rec_per_key_part  OUT Store statistics here
4545 4546
      unique            IN  Array of (#distinct tuples)
      notnull_tuples    IN  Array of (#tuples), or NULL
4547
      records               Number of records in the table
4548 4549 4550 4551 4552 4553 4554

  DESCRIPTION
    This function is called produce index statistics values from unique and 
    notnull_tuples arrays after these arrays were produced with sequential
    index scan (the scan is done in two places: chk_index() and
    sort_key_write()).

4555 4556
    This function handles all 3 index statistics collection methods.

4557
    Unique is an array:
4558 4559 4560 4561 4562 4563 4564 4565 4566 4567
      unique[0]= (#different values of {keypart1}) - 1
      unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
      ...

    For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
      notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
      notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all 
                          keypart{i} are not NULL)
      ...
    For all other statistics collection methods notnull_tuples==NULL.
4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578

    Output is an array:
    rec_per_key_part[k] = 
     = E(#records in the table such that keypart_1=c_1 AND ... AND 
         keypart_k=c_k for arbitrary constants c_1 ... c_k) 
     
     = {assuming that values have uniform distribution and index contains all
        tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
        index tuples}
     
     = #tuples-in-the-index / #distinct-tuples-in-the-index.
4579 4580 4581 4582 4583 4584 4585 4586
    
    The #tuples-in-the-index and #distinct-tuples-in-the-index have different 
    meaning depending on which statistics collection method is used:
    
    MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
     NULLS_EQUAL            NULL == NULL           all tuples in table
     NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
     IGNORE_NULLS               n/a             tuples that don't have NULLs
4587
*/
unknown's avatar
unknown committed
4588

unknown's avatar
unknown committed
4589
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4590 4591
                      ulonglong *unique, ulonglong *notnull,
                      ulonglong records)
unknown's avatar
unknown committed
4592
{
4593 4594
  ulonglong count=0,tmp, unique_tuples;
  ulonglong tuples= records;
unknown's avatar
unknown committed
4595 4596 4597 4598
  uint parts;
  for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
  {
    count+=unique[parts];
4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614
    unique_tuples= count + 1;    
    if (notnull)
    {
      tuples= notnull[parts];
      /* 
        #(unique_tuples not counting tuples with NULLs) = 
          #(unique_tuples counting tuples with NULLs as different) - 
          #(tuples with NULLs)
      */
      unique_tuples -= (records - notnull[parts]);
    }
    
    if (unique_tuples == 0)
      tmp= 1;
    else if (count == 0)
      tmp= tuples; /* 1 unique tuple */
unknown's avatar
unknown committed
4615
    else
4616 4617 4618 4619 4620 4621
      tmp= (tuples + unique_tuples/2) / unique_tuples;

    /* 
      for some weird keys (e.g. FULLTEXT) tmp can be <1 here. 
      let's ensure it is not
    */
4622
    set_if_bigger(tmp,1);
unknown's avatar
unknown committed
4623 4624
    if (tmp >= (ulonglong) ~(ulong) 0)
      tmp=(ulonglong) ~(ulong) 0;
4625

unknown's avatar
unknown committed
4626 4627 4628 4629 4630 4631
    *rec_per_key_part=(ulong) tmp;
    rec_per_key_part++;
  }
}


4632
static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
unknown's avatar
unknown committed
4633 4634
{
  ha_checksum crc;
4635
  const uchar *end=buf+length;
unknown's avatar
unknown committed
4636 4637 4638 4639 4640
  for (crc=0; buf != end; buf++)
    crc=((crc << 1) + *((uchar*) buf)) +
      test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
  return crc;
}
4641 4642 4643

static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
{
4644 4645
  uint key_maxlength=key->maxlength;
  if (key->flag & HA_FULLTEXT)
4646 4647 4648 4649 4650
  {
    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                  key->seg->charset->mbmaxlen;
    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
  }
4651 4652
  return (key->flag & HA_SPATIAL) ||
          (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4653
	  ((ulonglong) rows * key_maxlength > myisam_max_temp_length));
4654 4655
}

4656 4657 4658 4659 4660 4661 4662 4663
/*
  Deactivate all not unique index that can be recreated fast
  These include packed keys on which sorting will use more temporary
  space than the max allowed file length or for which the unpacked keys
  will take much more space than packed keys.
  Note that 'rows' may be zero for the case when we don't know how many
  rows we will put into the file.
 */
4664

4665
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4666 4667
{
  MYISAM_SHARE *share=info->s;
4668 4669 4670 4671 4672 4673
  MI_KEYDEF    *key=share->keyinfo;
  uint          i;

  DBUG_ASSERT(info->state->records == 0 &&
              (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
  for (i=0 ; i < share->base.keys ; i++,key++)
4674
  {
4675 4676
    if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
        ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4677
    {
4678
      mi_clear_key_active(share->state.key_map, i);
4679
      info->update|= HA_STATE_CHANGED;
4680 4681 4682 4683 4684
    }
  }
}


4685 4686 4687 4688 4689
/*
  Return TRUE if we can use repair by sorting
  One can set the force argument to force to use sorting
  even if the temporary file would be quite big!
*/
4690

unknown's avatar
unknown committed
4691
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
unknown's avatar
unknown committed
4692
			    ulonglong key_map, my_bool force)
4693 4694 4695
{
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *key=share->keyinfo;
4696 4697 4698
  uint i;

  /*
unknown's avatar
unknown committed
4699
    mi_repair_by_sort only works if we have at least one key. If we don't
4700 4701
    have any keys, we should use the normal repair.
  */
4702
  if (! mi_is_any_key_active(key_map))
4703 4704 4705
    return FALSE;				/* Can't use sort */
  for (i=0 ; i < share->base.keys ; i++,key++)
  {
4706
    if (!force && mi_too_big_key_for_sort(key,rows))
4707 4708 4709 4710
      return FALSE;
  }
  return TRUE;
}
4711 4712 4713


static void
unknown's avatar
unknown committed
4714
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4715 4716
{
  if ((sort_info->new_data_file_type=share->data_file_type) ==
unknown's avatar
unknown committed
4717
      COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732
  {
    MYISAM_SHARE tmp;

    if (share->options & HA_OPTION_PACK_RECORD)
      sort_info->new_data_file_type = DYNAMIC_RECORD;
    else
      sort_info->new_data_file_type = STATIC_RECORD;

    /* Set delete_function for sort_delete_record() */
    memcpy((char*) &tmp, share, sizeof(*share));
    tmp.options= ~HA_OPTION_COMPRESS_RECORD;
    mi_setup_functions(&tmp);
    share->delete_record=tmp.delete_record;
  }
}
4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821

/*
  Find the first NULL value in index-suffix values tuple

  SYNOPSIS
    ha_find_null()
      keyseg     Array of keyparts for key suffix
      a          Key suffix value tuple

  DESCRIPTION
    Find the first NULL value in index-suffix values tuple.

  TODO
    Consider optimizing this function or its use so we don't search for
    NULL values in completely NOT NULL index suffixes.

  RETURN
    First key part that has NULL as value in values tuple, or the last key
    part (with keyseg->type==HA_TYPE_END) if values tuple doesn't contain
    NULLs.
*/

static HA_KEYSEG *ha_find_null(HA_KEYSEG *keyseg, uchar *a)
{
  for (; (enum ha_base_keytype) keyseg->type != HA_KEYTYPE_END; keyseg++)
  {
    uchar *end;
    if (keyseg->null_bit)
    {
      if (!*a++)
        return keyseg;
    }
    end= a+ keyseg->length;

    switch ((enum ha_base_keytype) keyseg->type) {
    case HA_KEYTYPE_TEXT:
    case HA_KEYTYPE_BINARY:
    case HA_KEYTYPE_BIT:
      if (keyseg->flag & HA_SPACE_PACK)
      {
        int a_length;
        get_key_length(a_length, a);
        a += a_length;
        break;
      }
      else
        a= end;
      break;
    case HA_KEYTYPE_VARTEXT1:
    case HA_KEYTYPE_VARTEXT2:
    case HA_KEYTYPE_VARBINARY1:
    case HA_KEYTYPE_VARBINARY2:
      {
        int a_length;
        get_key_length(a_length, a);
        a+= a_length;
        break;
      }
    case HA_KEYTYPE_NUM:
      if (keyseg->flag & HA_SPACE_PACK)
      {
        int alength= *a++;
        end= a+alength;
      }
      a= end;
      break;
    case HA_KEYTYPE_INT8:
    case HA_KEYTYPE_SHORT_INT:
    case HA_KEYTYPE_USHORT_INT:
    case HA_KEYTYPE_LONG_INT:
    case HA_KEYTYPE_ULONG_INT:
    case HA_KEYTYPE_INT24:
    case HA_KEYTYPE_UINT24:
#ifdef HAVE_LONG_LONG
    case HA_KEYTYPE_LONGLONG:
    case HA_KEYTYPE_ULONGLONG:
#endif
    case HA_KEYTYPE_FLOAT:
    case HA_KEYTYPE_DOUBLE:
      a= end;
      break;
    case HA_KEYTYPE_END:                        /* purecov: inspected */
      /* keep compiler happy */
      DBUG_ASSERT(0);
      break;
    }
  }
  return keyseg;
}